lib/repo: Add a new private API for bare content writes

This lowers into the commit core what the static delta code
was doing, and improves the API.

The bigger picture issue is that for writing large files, our current "pull" API
where the caller provides a `GInputStream` is very awkward in some scenarios.
For example, we have a whole "libarchive input stream" that is a ~200 line
GObject that boils down to wrapping `archive_read_data()`.

This came more to a head when I was working on rpm-ostree jigdo since I had to
copy that object.

One step we can take after this is to further split `write_content_object()`
into a "write symlink or archive object" versus "write bare content object"
(it already has a mess of conditionals) and teach the latter case to call
this.

The eventual goal here is to make this API public.

Closes: #1355
Approved by: jlebon

3 files changed, 161 insertions, 133 deletions

diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index e43a0fa7..e283702d 100644
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -405,46 +405,116 @@ add_size_index_to_metadata (OstreeRepo        *self,
   return g_variant_ref_sink (g_variant_builder_end (builder));
 }
 
+typedef struct {
+  gboolean initialized;
+  GLnxTmpfile tmpf;
+  char *expected_checksum;
+  OtChecksum checksum;
+  guint64 content_len;
+  guint64 bytes_written;
+  guint uid;
+  guint gid;
+  guint mode;
+  GVariant *xattrs;
+} OstreeRealRepoBareContent;
+G_STATIC_ASSERT (sizeof (OstreeRepoBareContent) >= sizeof (OstreeRealRepoBareContent));
+
 /* Create a tmpfile for writing a bare file.  Currently just used
  * by the static delta code, but will likely later be extended
  * to be used also by the dfd_iter commit path.
  */
 gboolean
-_ostree_repo_open_content_bare (OstreeRepo          *self,
-                                const char          *checksum,
-                                guint64              content_len,
-                                GLnxTmpfile         *out_tmpf,
-                                GCancellable        *cancellable,
-                                GError             **error)
-{
-  return glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC,
-                                        out_tmpf, error);
+_ostree_repo_bare_content_open (OstreeRepo            *self,
+                                const char            *expected_checksum,
+                                guint64                content_len,
+                                guint                  uid,
+                                guint                  gid,
+                                guint                  mode,
+                                GVariant              *xattrs,
+                                OstreeRepoBareContent *out_regwrite,
+                                GCancellable          *cancellable,
+                                GError               **error)
+{
+  OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) out_regwrite;
+  g_assert (!real->initialized);
+  real->initialized = TRUE;
+  g_assert (S_ISREG (mode));
+  if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC,
+                                      &real->tmpf, error))
+    return FALSE;
+  ot_checksum_init (&real->checksum);
+  real->expected_checksum = g_strdup (expected_checksum);
+  real->content_len = content_len;
+  real->bytes_written = 0;
+  real->uid = uid;
+  real->gid = gid;
+  real->mode = mode;
+  real->xattrs = xattrs ? g_variant_ref (xattrs) : NULL;
+
+  /* Initialize the checksum with the header info */
+  g_autoptr(GFileInfo) finfo = _ostree_mode_uidgid_to_gfileinfo (mode, uid, gid);
+  g_autoptr(GBytes) header = _ostree_file_header_new (finfo, xattrs);
+  ot_checksum_update_bytes (&real->checksum, header);
+
+  return TRUE;
 }
 
-/* Used by static deltas, which have a separate "push" flow for
- * regfile objects distinct from the "pull" model used by
- * write_content_object().
- */
 gboolean
-_ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
-                                          const char          *checksum,
-                                          GLnxTmpfile         *tmpf,
-                                          guint32              uid,
-                                          guint32              gid,
-                                          guint32              mode,
-                                          GVariant            *xattrs,
-                                          GCancellable        *cancellable,
-                                          GError             **error)
-{
-  /* I don't think this is necessary, but a similar check was here previously,
-   * keeping it for extra redundancy.
-   */
-  if (!tmpf->initialized || tmpf->fd == -1)
-    return TRUE;
+_ostree_repo_bare_content_write (OstreeRepo                 *repo,
+                                 OstreeRepoBareContent      *barewrite,
+                                 const guint8               *buf,
+                                 size_t                      len,
+                                 GCancellable               *cancellable,
+                                 GError                    **error)
+{
+  OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) barewrite;
+  g_assert (real->initialized);
+  ot_checksum_update (&real->checksum, buf, len);
+  if (glnx_loop_write (real->tmpf.fd, buf, len) < 0)
+    return glnx_throw_errno_prefix (error, "write");
+  return TRUE;
+}
+
+gboolean
+_ostree_repo_bare_content_commit (OstreeRepo                 *self,
+                                  OstreeRepoBareContent      *barewrite,
+                                  char                       *checksum_buf,
+                                  size_t                      buflen,
+                                  GCancellable               *cancellable,
+                                  GError                    **error)
+{
+  OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) barewrite;
+  g_assert (real->initialized);
+  ot_checksum_get_hexdigest (&real->checksum, checksum_buf, buflen);
+
+  if (real->expected_checksum &&
+      !_ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE,
+                                        real->expected_checksum, checksum_buf,
+                                        error))
+    return FALSE;
+
+  if (!commit_loose_regfile_object (self, checksum_buf,
+                                    &real->tmpf, real->uid, real->gid,
+                                    real->mode, real->xattrs,
+                                    cancellable, error))
+    return FALSE;
 
-  return commit_loose_regfile_object (self, checksum,
-                                      tmpf, uid, gid, mode, xattrs,
-                                      cancellable, error);
+  /* Let's have a guarantee that after commit the object is cleaned up */
+  _ostree_repo_bare_content_cleanup (barewrite);
+  return TRUE;
+}
+
+void
+_ostree_repo_bare_content_cleanup (OstreeRepoBareContent *regwrite)
+{
+  OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) regwrite;
+  if (!real->initialized)
+    return;
+  glnx_tmpfile_clear (&real->tmpf);
+  ot_checksum_clear (&real->checksum);
+  g_clear_pointer (&real->expected_checksum, (GDestroyNotify)g_free);
+  g_clear_pointer (&real->xattrs, (GDestroyNotify)g_variant_unref);
+  real->initialized = FALSE;
 }
 
 /* Allocate an O_TMPFILE, write everything from @input to it, but
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index d8d16e1a..58c104b9 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -365,24 +365,41 @@ _ostree_repo_commit_tmpf_final (OstreeRepo        *self,
                                 GCancellable      *cancellable,
                                 GError           **error);
 
+typedef struct {
+  gboolean initialized;
+  gpointer opaque0[10];
+  guint opaque1[10];
+} OstreeRepoBareContent;
+void _ostree_repo_bare_content_cleanup (OstreeRepoBareContent *regwrite);
+G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OstreeRepoBareContent, _ostree_repo_bare_content_cleanup)
+
 gboolean
-_ostree_repo_open_content_bare (OstreeRepo          *self,
-                                const char          *checksum,
-                                guint64              content_len,
-                                GLnxTmpfile         *out_tmpf,
+_ostree_repo_bare_content_open (OstreeRepo            *self,
+                                const char            *checksum,
+                                guint64                content_len,
+                                guint                  uid,
+                                guint                  gid,
+                                guint                  mode,
+                                GVariant              *xattrs,
+                                OstreeRepoBareContent *out_regwrite,
                                 GCancellable        *cancellable,
                                 GError             **error);
 
 gboolean
-_ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
-                                          const char          *checksum,
-                                          GLnxTmpfile         *tmpf,
-                                          guint32              uid,
-                                          guint32              gid,
-                                          guint32              mode,
-                                          GVariant            *xattrs,
-                                          GCancellable        *cancellable,
-                                          GError             **error);
+_ostree_repo_bare_content_write (OstreeRepo                 *repo,
+                                 OstreeRepoBareContent      *barewrite,
+                                 const guint8               *buf,
+                                 size_t                      len,
+                                 GCancellable               *cancellable,
+                                 GError                    **error);
+
+gboolean
+_ostree_repo_bare_content_commit (OstreeRepo                 *self,
+                                  OstreeRepoBareContent      *barewrite,
+                                  char                       *checksum_buf,
+                                  size_t                      buflen,
+                                  GCancellable               *cancellable,
+                                  GError                    **error);
 
 gboolean
 _ostree_repo_load_file_bare (OstreeRepo         *self,
diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c
index 4545b39f..3b9fd49f 100644
--- a/src/libostree/ostree-repo-static-delta-processing.c
+++ b/src/libostree/ostree-repo-static-delta-processing.c
@@ -55,11 +55,9 @@ typedef struct {
   GError        **async_error;
 
   OstreeObjectType output_objtype;
-  GLnxTmpfile      tmpf;
   guint64          content_size;
-  GOutputStream   *content_out;
-  OtChecksum       content_checksum;
   char             checksum[OSTREE_SHA256_STRING_LEN+1];
+  OstreeRepoBareContent content_out;
   char             *read_source_object;
   int               read_source_fd;
   gboolean        have_obj;
@@ -278,9 +276,7 @@ _ostree_static_delta_part_execute (OstreeRepo      *repo,
 
   ret = TRUE;
  out:
-  glnx_tmpfile_clear (&state->tmpf);
-  g_clear_object (&state->content_out);
-  ot_checksum_clear (&state->content_checksum);
+  _ostree_repo_bare_content_cleanup (&state->content_out);
   return ret;
 }
 
@@ -379,29 +375,6 @@ validate_ofs (StaticDeltaExecutionState  *state,
 }
 
 static gboolean
-content_out_write (OstreeRepo                 *repo,
-                   StaticDeltaExecutionState  *state,
-                   const guint8*               buf,
-                   gsize                       len,
-                   GCancellable               *cancellable,
-                   GError                    **error)
-{
-  gsize bytes_written;
-
-  if (state->content_checksum.initialized)
-    ot_checksum_update (&state->content_checksum, buf, len);
-
-  /* Ignore bytes_written since we discard partial content */
-  if (!g_output_stream_write_all (state->content_out,
-                                  buf, len,
-                                  &bytes_written,
-                                  cancellable, error))
-    return FALSE;
-
-  return TRUE;
-}
-
-static gboolean
 do_content_open_generic (OstreeRepo                 *repo,
                          StaticDeltaExecutionState  *state,
                          GCancellable               *cancellable,
@@ -485,33 +458,15 @@ dispatch_bspatch (OstreeRepo                 *repo,
                    &stream) < 0)
         return FALSE;
 
-      if (!content_out_write (repo, state, buf, state->content_size,
-                              cancellable, error))
+      if (!_ostree_repo_bare_content_write (repo, &state->content_out,
+                                            buf, state->content_size,
+                                            cancellable, error))
         return FALSE;
     }
 
   return TRUE;
 }
 
-/* Before, we had a distinction between "trusted" and "untrusted" deltas
- * which we've decided wasn't a good idea.  Now, we always checksum the content.
- * Compare with what ostree_checksum_file_from_input() is doing too.
- */
-static gboolean
-handle_untrusted_content_checksum (OstreeRepo                 *repo,
-                                   StaticDeltaExecutionState  *state,
-                                   GCancellable               *cancellable,
-                                   GError                    **error)
-{
-  g_autoptr(GFileInfo) finfo = _ostree_mode_uidgid_to_gfileinfo (state->mode, state->uid, state->gid);
-  g_autoptr(GBytes) header = _ostree_file_header_new (finfo, state->xattrs);
-
-  ot_checksum_init (&state->content_checksum);
-  ot_checksum_update_bytes (&state->content_checksum, header);
-
-  return TRUE;
-}
-
 static gboolean
 dispatch_open_splice_and_close (OstreeRepo                 *repo,
                                 StaticDeltaExecutionState  *state,
@@ -589,20 +544,18 @@ dispatch_open_splice_and_close (OstreeRepo                 *repo,
 
           if (!state->have_obj)
             {
-              if (!_ostree_repo_open_content_bare (repo, state->checksum,
+              if (!_ostree_repo_bare_content_open (repo, state->checksum,
                                                    state->content_size,
-                                                   &state->tmpf,
+                                                   state->uid, state->gid, state->mode,
+                                                   state->xattrs,
+                                                   &state->content_out,
                                                    cancellable, error))
                 goto out;
 
-              state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
-              if (!handle_untrusted_content_checksum (repo, state, cancellable, error))
-                goto out;
-
-              if (!content_out_write (repo, state,
-                                      state->payload_data + content_offset,
-                                      state->content_size,
-                                      cancellable, error))
+              if (!_ostree_repo_bare_content_write (repo, &state->content_out,
+                                                    state->payload_data + content_offset,
+                                                    state->content_size,
+                                                    cancellable, error))
                 goto out;
             }
         }
@@ -691,17 +644,15 @@ dispatch_open (OstreeRepo                 *repo,
 
   if (!state->have_obj)
     {
-      if (!_ostree_repo_open_content_bare (repo, state->checksum,
+      if (!_ostree_repo_bare_content_open (repo, state->checksum,
                                            state->content_size,
-                                           &state->tmpf,
+                                           state->uid, state->gid, state->mode,
+                                           state->xattrs,
+                                           &state->content_out,
                                            cancellable, error))
         return FALSE;
-      state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
     }
 
-  if (!handle_untrusted_content_checksum (repo, state, cancellable, error))
-    return FALSE;
-
   return TRUE;
 }
 
@@ -740,8 +691,9 @@ dispatch_write (OstreeRepo                 *repo,
               if (G_UNLIKELY (bytes_read == 0))
                 return glnx_throw (error, "Unexpected EOF reading object %s", state->read_source_object);
 
-              if (!content_out_write (repo, state, (guint8*)buf, bytes_read,
-                                      cancellable, error))
+              if (!_ostree_repo_bare_content_write (repo, &state->content_out,
+                                                    (guint8*)buf, bytes_read,
+                                                    cancellable, error))
                 return FALSE;
 
               content_size -= bytes_read;
@@ -753,8 +705,9 @@ dispatch_write (OstreeRepo                 *repo,
           if (!validate_ofs (state, content_offset, content_size, error))
             return FALSE;
 
-          if (!content_out_write (repo, state, state->payload_data + content_offset, content_size,
-                                  cancellable, error))
+          if (!_ostree_repo_bare_content_write (repo, &state->content_out,
+                                                state->payload_data + content_offset, content_size,
+                                                cancellable, error))
             return FALSE;
         }
     }
@@ -818,34 +771,22 @@ dispatch_close (OstreeRepo                 *repo,
 {
   GLNX_AUTO_PREFIX_ERROR("opcode close", error);
 
-  if (state->content_out)
+  if (state->content_out.initialized)
     {
-      if (!g_output_stream_flush (state->content_out, cancellable, error))
+      char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+      if (!_ostree_repo_bare_content_commit (repo, &state->content_out, actual_checksum,
+                                             sizeof (actual_checksum),
+                                             cancellable, error))
         return FALSE;
 
-      if (state->content_checksum.initialized)
-        {
-          char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
-          ot_checksum_get_hexdigest (&state->content_checksum, actual_checksum, sizeof (actual_checksum));
-
-          if (strcmp (actual_checksum, state->checksum) != 0)
-            return glnx_throw (error, "Corrupted object %s (actual checksum is %s)",
-                               state->checksum, actual_checksum);
-        }
-
-      if (!_ostree_repo_commit_trusted_content_bare (repo, state->checksum, &state->tmpf,
-                                                     state->uid, state->gid, state->mode,
-                                                     state->xattrs,
-                                                     cancellable, error))
-        return FALSE;
-      g_clear_object (&state->content_out);
+      g_assert_cmpstr (state->checksum, ==, actual_checksum);
     }
 
   if (!dispatch_unset_read_source (repo, state, cancellable, error))
     return FALSE;
 
   g_clear_pointer (&state->xattrs, g_variant_unref);
-  ot_checksum_clear (&state->content_checksum);
+  _ostree_repo_bare_content_cleanup (&state->content_out);
 
   state->checksum_index++;
   state->output_target = NULL;