core: Clean up staging API and internals

Cleanly separate metadata/content APIs, rather than defaulting to
raw streams.  This helps most use cases.

Also, drop support for staging content without knowing the total
length.  This complicated the code, and for things like streaming
HTTP, we should be able to figure this out from Content-Length.

4 files changed, 181 insertions, 206 deletions

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 05041b45..30cb3e49 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -824,22 +824,17 @@ commit_loose_object_trusted (OstreeRepo        *self,
   return ret;
 }
 
-typedef enum {
-  OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<0)
-} OstreeRepoStageFlags;
-
 static gboolean
-stage_object_internal (OstreeRepo         *self,
-                       OstreeRepoStageFlags flags,
-                       OstreeObjectType    objtype,
-                       GInputStream       *input,
-                       guint64             file_object_length,
-                       const char         *expected_checksum,
-                       guchar            **out_csum,
-                       GCancellable       *cancellable,
-                       GError            **error)
+stage_object (OstreeRepo         *self,
+              OstreeObjectType    objtype,
+              const char         *expected_checksum,
+              GInputStream       *input,
+              guint64             file_object_length,
+              guchar            **out_csum,
+              GCancellable       *cancellable,
+              GError            **error)
 {
- gboolean ret = FALSE;
+  gboolean ret = FALSE;
   const char *actual_checksum;
   gboolean do_commit;
   ot_lobj GFileInfo *temp_info = NULL;
@@ -854,6 +849,20 @@ stage_object_internal (OstreeRepo         *self,
   gboolean staged_archive_file = FALSE;
   gboolean temp_file_is_regular;
 
+  g_return_val_if_fail (self->in_transaction, FALSE);
+  
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  g_assert (expected_checksum || out_csum);
+
+  if (expected_checksum)
+    {
+      if (!repo_find_object (self, objtype, expected_checksum, &stored_path,
+                             cancellable, error))
+        goto out;
+    }
+
   if (out_csum)
     {
       checksum = g_checksum_new (G_CHECKSUM_SHA256);
@@ -861,8 +870,7 @@ stage_object_internal (OstreeRepo         *self,
         checksum_input = ostree_checksum_input_stream_new (input, checksum);
     }
 
-  if (objtype == OSTREE_OBJECT_TYPE_FILE
-      && (flags & OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID) > 0)
+  if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
       ot_lobj GInputStream *file_input = NULL;
       ot_lobj GFileInfo *file_info = NULL;
@@ -1039,50 +1047,6 @@ stage_object_internal (OstreeRepo         *self,
 }
 
 static gboolean
-stage_object (OstreeRepo         *self,
-              OstreeRepoStageFlags flags,
-              OstreeObjectType    objtype,
-              GInputStream       *input,
-              guint64             file_object_length,
-              const char         *expected_checksum,
-              guchar            **out_csum,
-              GCancellable       *cancellable,
-              GError            **error)
-{
-  gboolean ret = FALSE;
-  ot_lobj GFile *stored_path = NULL;
-  ot_lfree guchar *ret_csum = NULL;
-
-  g_return_val_if_fail (self->in_transaction, FALSE);
-  
-  if (g_cancellable_set_error_if_cancelled (cancellable, error))
-    return FALSE;
-
-  g_assert (expected_checksum || out_csum);
-
-  if (expected_checksum)
-    {
-      if (!repo_find_object (self, objtype, expected_checksum, &stored_path,
-                             cancellable, error))
-        goto out;
-    }
-
-  if (stored_path == NULL)
-    {
-      if (!stage_object_internal (self, flags, objtype, input,
-                                  file_object_length, expected_checksum,
-                                  out_csum ? &ret_csum : NULL,
-                                  cancellable, error))
-        goto out;
-    }
-
-  ret = TRUE;
-  ot_transfer_out_value(out_csum, &ret_csum);
- out:
-  return ret;
-}
-
-static gboolean
 get_loose_object_dirs (OstreeRepo       *self,
                        GPtrArray       **out_object_dirs,
                        GCancellable     *cancellable,
@@ -1322,26 +1286,56 @@ ostree_repo_abort_transaction (OstreeRepo     *self,
   return ret;
 }
 
-static gboolean
-stage_metadata_object (OstreeRepo         *self,
-                       OstreeObjectType    type,
-                       GVariant           *variant,
-                       guchar            **out_csum,
-                       GCancellable       *cancellable,
-                       GError            **error)
+/**
+ * ostree_repo_stage_metadata:
+ * 
+ * Store the metadata object @variant.  Return the checksum
+ * as @out_csum.
+ *
+ * If @expected_checksum is not %NULL, verify it against the
+ * computed checksum.
+ */
+gboolean
+ostree_repo_stage_metadata (OstreeRepo         *self,
+                            OstreeObjectType    type,
+                            const char         *expected_checksum,
+                            GVariant           *variant,
+                            guchar            **out_csum,
+                            GCancellable       *cancellable,
+                            GError            **error)
 {
-  gboolean ret = FALSE;
   ot_lobj GInputStream *input = NULL;
+  ot_lvariant GVariant *normalized = NULL;
 
-  input = ot_variant_read (variant);
+  normalized = g_variant_get_normal_form (variant);
+  input = ot_variant_read (normalized);
   
-  if (!stage_object (self, 0, type, input, 0, NULL,
-                     out_csum, cancellable, error))
-    goto out;
+  return stage_object (self, type, expected_checksum, input, 0, out_csum,
+                       cancellable, error);
+}
 
-  ret = TRUE;
- out:
-  return ret;
+/**
+ * ostree_repo_stage_metadata_trusted:
+ * 
+ * Store the metadata object @variant; the provided @checksum
+ * is trusted.
+ */
+gboolean
+ostree_repo_stage_metadata_trusted (OstreeRepo         *self,
+                                    OstreeObjectType    type,
+                                    const char         *checksum,
+                                    GVariant           *variant,
+                                    GCancellable       *cancellable,
+                                    GError            **error)
+{
+  ot_lobj GInputStream *input = NULL;
+  ot_lvariant GVariant *normalized = NULL;
+
+  normalized = g_variant_get_normal_form (variant);
+  input = ot_variant_read (normalized);
+  
+  return stage_object (self, type, checksum, input, 0, NULL,
+                       cancellable, error);
 }
 
 static gboolean
@@ -1352,7 +1346,6 @@ stage_directory_meta (OstreeRepo   *self,
                       GCancellable *cancellable,
                       GError      **error)
 {
-  gboolean ret = FALSE;
   ot_lvariant GVariant *dirmeta = NULL;
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
@@ -1360,13 +1353,8 @@ stage_directory_meta (OstreeRepo   *self,
 
   dirmeta = ostree_create_directory_metadata (file_info, xattrs);
   
-  if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_DIR_META, 
-                              dirmeta, out_csum, cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  return ret;
+  return ostree_repo_stage_metadata (self, OSTREE_OBJECT_TYPE_DIR_META, NULL,
+                                     dirmeta, out_csum, cancellable, error);
 }
 
 GFile *
@@ -1384,77 +1372,49 @@ ostree_repo_get_object_path (OstreeRepo  *self,
   return ret;
 }
 
+/**
+ * ostree_repo_stage_content_trusted:
+ *
+ * Store the content object streamed as @object_input, with total
+ * length @length.  The given @checksum will be treated as trusted.
+ *
+ * This function should be used when importing file objects from local
+ * disk, for example.
+ */
 gboolean      
-ostree_repo_stage_object_trusted (OstreeRepo   *self,
-                                  OstreeObjectType objtype,
-                                  const char   *checksum,
-                                  GInputStream     *object_input,
-                                  GCancellable *cancellable,
-                                  GError      **error)
-{
-  int flags = 0;
-  return stage_object (self, flags, objtype,
-                       object_input, 0, checksum, NULL,
+ostree_repo_stage_content_trusted (OstreeRepo       *self,
+                                   const char       *checksum,
+                                   GInputStream     *object_input,
+                                   guint64           length,
+                                   GCancellable     *cancellable,
+                                   GError          **error)
+{
+  return stage_object (self, OSTREE_OBJECT_TYPE_FILE, checksum,
+                       object_input, length, NULL,
                        cancellable, error);
 }
 
+/**
+ * ostree_repo_stage_content:
+ *
+ * Store the content object streamed as @object_input,
+ * with total length @length.  The actual checksum will
+ * be returned as @out_csum.
+ */
 gboolean
-ostree_repo_stage_object (OstreeRepo       *self,
-                          OstreeObjectType  objtype,
-                          const char       *expected_checksum,
-                          GInputStream     *object_input,
-                          GCancellable     *cancellable,
-                          GError          **error)
+ostree_repo_stage_content (OstreeRepo       *self,
+                           const char       *expected_checksum,
+                           GInputStream     *object_input,
+                           guint64           length,
+                           guchar          **out_csum,
+                           GCancellable     *cancellable,
+                           GError          **error)
 {
-  gboolean ret = FALSE;
-  ot_lfree guchar *actual_csum = NULL;
-  
-  if (!stage_object (self, 0, objtype, 
-                     object_input, 0, expected_checksum, &actual_csum,
-                     cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  return ret;
-}
-
-gboolean      
-ostree_repo_stage_file_object_trusted (OstreeRepo       *self,
-                                       const char       *checksum,
-                                       GInputStream     *object_input,
-                                       guint64           length,
-                                       GCancellable     *cancellable,
-                                       GError          **error)
-{
-  int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
-  return stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE,
-                       object_input, length, checksum, NULL,
+  return stage_object (self, OSTREE_OBJECT_TYPE_FILE, expected_checksum,
+                       object_input, length, out_csum,
                        cancellable, error);
 }
 
-gboolean
-ostree_repo_stage_file_object (OstreeRepo       *self,
-                               const char       *expected_checksum,
-                               GInputStream     *object_input,
-                               guint64           length,
-                               GCancellable     *cancellable,
-                               GError          **error)
-{
-  gboolean ret = FALSE;
-  int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
-  ot_lfree guchar *actual_csum = NULL;
-  
-  if (!stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE, 
-                     object_input, length, expected_checksum, &actual_csum,
-                     cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  return ret;
-}
-
 static GVariant *
 create_empty_gvariant_dict (void)
 {
@@ -1658,8 +1618,9 @@ ostree_repo_stage_commit (OstreeRepo *self,
                           ostree_checksum_to_bytes_v (root_contents_checksum),
                           ostree_checksum_to_bytes_v (root_metadata_checksum));
   g_variant_ref_sink (commit);
-  if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_COMMIT,
-                              commit, &commit_csum, cancellable, error))
+  if (!ostree_repo_stage_metadata (self, OSTREE_OBJECT_TYPE_COMMIT, NULL,
+                                   commit, &commit_csum,
+                                   cancellable, error))
     goto out;
 
   ret_commit = ostree_checksum_from_bytes (commit_csum);
@@ -1932,9 +1893,8 @@ stage_directory_to_mtree_internal (OstreeRepo           *self,
                                                               &file_object_input, &file_obj_length,
                                                               cancellable, error))
                         goto out;
-                      if (!stage_object (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID,
-                                         OSTREE_OBJECT_TYPE_FILE, file_object_input, file_obj_length,
-                                         NULL, &child_file_csum, cancellable, error))
+                      if (!ostree_repo_stage_content (self, NULL, file_object_input, file_obj_length,
+                                                      &child_file_csum, cancellable, error))
                         goto out;
 
                       g_free (tmp_checksum);
@@ -2041,9 +2001,9 @@ ostree_repo_stage_mtree (OstreeRepo           *self,
                                                          dir_contents_checksums,
                                                          dir_metadata_checksums);
       
-      if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_DIR_TREE,
-                                  serialized_tree, &contents_csum,
-                                  cancellable, error))
+      if (!ostree_repo_stage_metadata (self, OSTREE_OBJECT_TYPE_DIR_TREE, NULL,
+                                       serialized_tree, &contents_csum,
+                                       cancellable, error))
         goto out;
       ret_contents_checksum = ostree_checksum_from_bytes (contents_csum);
     }
@@ -2140,9 +2100,8 @@ import_libarchive_entry_file (OstreeRepo           *self,
                                           &file_object_input, &length, cancellable, error))
     goto out;
   
-  if (!stage_object (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID, OSTREE_OBJECT_TYPE_FILE,
-                     file_object_input, length, NULL, out_csum,
-                     cancellable, error))
+  if (!ostree_repo_stage_content (self, NULL, file_object_input, length, out_csum,
+                                  cancellable, error))
     goto out;
 
   ret = TRUE;
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 151ea3af..2f31bef0 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -94,33 +94,35 @@ gboolean      ostree_repo_has_object (OstreeRepo           *self,
                                       GCancellable         *cancellable,
                                       GError              **error);
 
-gboolean      ostree_repo_stage_object (OstreeRepo       *self,
-                                        OstreeObjectType  objtype,
-                                        const char       *expected_checksum,
-                                        GInputStream     *content,
-                                        GCancellable     *cancellable,
-                                        GError          **error);
-
-gboolean      ostree_repo_stage_file_object (OstreeRepo       *self,
-                                             const char       *expected_checksum,
-                                             GInputStream     *content,
-                                             guint64           content_length,
-                                             GCancellable     *cancellable,
-                                             GError          **error);
-
-gboolean      ostree_repo_stage_object_trusted (OstreeRepo   *self,
-                                                OstreeObjectType objtype,
-                                                const char   *checksum,
-                                                GInputStream     *content,
-                                                GCancellable *cancellable,
-                                                GError      **error);
-
-gboolean      ostree_repo_stage_file_object_trusted (OstreeRepo   *self,
-                                                     const char   *checksum,
-                                                     GInputStream     *content,
-                                                     guint64           content_length,
-                                                     GCancellable *cancellable,
-                                                     GError      **error);
+gboolean      ostree_repo_stage_metadata (OstreeRepo        *self,
+                                          OstreeObjectType   objtype,
+                                          const char        *expected_checksum,
+                                          GVariant          *object,
+                                          guchar           **out_csum,
+                                          GCancellable      *cancellable,
+                                          GError           **error);
+
+gboolean      ostree_repo_stage_content (OstreeRepo       *self,
+                                         const char       *expected_checksum,
+                                         GInputStream     *content,
+                                         guint64           content_length,
+                                         guchar          **out_csum,
+                                         GCancellable     *cancellable,
+                                         GError          **error);
+
+gboolean      ostree_repo_stage_metadata_trusted (OstreeRepo        *self,
+                                                  OstreeObjectType   objtype,
+                                                  const char        *checksum,
+                                                  GVariant          *object,
+                                                  GCancellable      *cancellable,
+                                                  GError           **error);
+
+gboolean      ostree_repo_stage_content_trusted (OstreeRepo       *self,
+                                                 const char       *checksum,
+                                                 GInputStream     *content,
+                                                 guint64           content_length,
+                                                 GCancellable     *cancellable,
+                                                 GError          **error);
 
 gboolean      ostree_repo_resolve_rev (OstreeRepo  *self,
                                        const char  *rev,
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index d1934b47..7122f5b0 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -359,7 +359,6 @@ fetch_and_store_metadata (OtPullData          *pull_data,
   gboolean is_stored;
   ot_lvariant GVariant *ret_variant = NULL;
   ot_lobj GFile *temp_path = NULL;
-  ot_lobj GInputStream *input = NULL;
   ot_lvariant GVariant *metadata = NULL;
 
   g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype));
@@ -370,15 +369,32 @@ fetch_and_store_metadata (OtPullData          *pull_data,
       
   if (!is_stored)
     {
+      ot_lvariant GVariant *tmp_metadata = NULL;
+      const GVariantType *vtype;
+  
       if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
         goto out;
-      
-      input = (GInputStream*)g_file_read (temp_path, cancellable, error);
-      if (!input)
-        goto out;
 
-      if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input,
-                                     cancellable, error))
+      switch (objtype)
+        {
+        case OSTREE_OBJECT_TYPE_DIR_TREE:
+          vtype = OSTREE_TREE_GVARIANT_FORMAT;
+          break;
+        case OSTREE_OBJECT_TYPE_DIR_META:
+          vtype = OSTREE_DIRMETA_GVARIANT_FORMAT;
+          break;
+        case OSTREE_OBJECT_TYPE_COMMIT:
+          vtype = OSTREE_COMMIT_GVARIANT_FORMAT;
+          break;
+        default:
+          g_assert_not_reached ();
+        }
+
+      if (!ot_util_variant_map (temp_path, vtype, FALSE, &tmp_metadata, error))
+        goto out;
+      
+      if (!ostree_repo_stage_metadata (pull_data->repo, objtype, checksum, tmp_metadata, NULL,
+                                       cancellable, error))
         goto out;
     }
 
@@ -664,9 +680,9 @@ content_fetch_on_checksum_complete (GObject        *object,
       goto out;
     }
 
-  if (!ostree_repo_stage_file_object_trusted (data->pull_data->repo, checksum,
-                                              file_object_input, length,
-                                              cancellable, error))
+  if (!ostree_repo_stage_content_trusted (data->pull_data->repo, checksum,
+                                          file_object_input, length,
+                                          cancellable, error))
     goto out;
 
  out:
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index 7d6e05d0..8b7304db 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -45,17 +45,16 @@ import_one_object (OtLocalCloneData *data,
                    GError        **error)
 {
   gboolean ret = FALSE;
-  ot_lobj GFileInfo *file_info = NULL;
   ot_lobj GFile *content_path = NULL;
   ot_lobj GFileInfo *archive_info = NULL;
-  ot_lvariant GVariant *metadata = NULL;
-  ot_lvariant GVariant *xattrs = NULL;
-  ot_lobj GInputStream *input = NULL;
 
   if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
-      ot_lobj GInputStream *file_object = NULL;
       guint64 length;
+      ot_lobj GInputStream *file_object = NULL;
+      ot_lobj GInputStream *input = NULL;
+      ot_lobj GFileInfo *file_info = NULL;
+      ot_lvariant GVariant *xattrs = NULL;
 
       if (!ostree_repo_load_file (data->src_repo, checksum,
                                   &input, &file_info, &xattrs,
@@ -67,22 +66,21 @@ import_one_object (OtLocalCloneData *data,
                                               cancellable, error))
         goto out;
 
-      if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum,
-                                                  file_object, length,
-                                                  cancellable, error))
+      if (!ostree_repo_stage_content_trusted (data->dest_repo, checksum,
+                                              file_object, length,
+                                              cancellable, error))
         goto out;
     }
   else
     {
+      ot_lvariant GVariant *metadata = NULL;
+
       if (!ostree_repo_load_variant (data->src_repo, objtype, checksum, &metadata,
                                      error))
         goto out;
 
-      input = ot_variant_read (metadata);
-
-      if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype,
-                                             checksum, input,
-                                             cancellable, error))
+      if (!ostree_repo_stage_metadata_trusted (data->dest_repo, objtype, checksum, metadata,
+                                               cancellable, error))
         goto out;
     }