pull: Implement a metadata-only pull modewip/thin-commits

This just fetches the top level commit object. The intention is to
allow this minimal pre-fetch to be used to determine the size of an
upcoming pull in advance.  Invoking pull again without the
METADATA_ONLY flag fetches the content.

Prior to metadata-only pulls, having the top level object was assumed
to mean you had the whole tree - this is not the case anymore, so we
explicitly store the commits named as "committhin".

7 files changed, 278 insertions, 43 deletions

diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index 9c399387..e652f381 100644
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -300,6 +300,7 @@ write_object (OstreeRepo         *self,
               const char         *expected_checksum,
               GInputStream       *input,
               guint64             file_object_length,
+              gboolean            is_thin_commit,
               guchar            **out_csum,
               GCancellable       *cancellable,
               GError            **error)
@@ -491,6 +492,17 @@ write_object (OstreeRepo         *self,
 
   if (do_commit)
     {
+      /* Override the loose path here for thin commits; we name them
+       * as "committhin", even though we checked for just "commit"
+       * above.  This is mildly lame in that attempting to store thin
+       * commits will rewrite the existing object, but it doesn't
+       * really matter.  Trying to do better would result in even
+       * *more* conditionals in this already heavily conditional
+       * function.
+       */
+      if (objtype == OSTREE_OBJECT_TYPE_COMMIT && is_thin_commit)
+        _ostree_loose_path_with_suffix (loose_objpath, actual_checksum, objtype, self->mode, "thin");
+          
       if (!commit_loose_object_trusted (self, objtype, loose_objpath,
                                         temp_file, temp_filename,
                                         is_symlink, file_info,
@@ -977,7 +989,7 @@ ostree_repo_write_metadata (OstreeRepo         *self,
   normalized = g_variant_get_normal_form (object);
   input = ot_variant_read (normalized);
 
-  return write_object (self, objtype, expected_checksum, input, 0, out_csum,
+  return write_object (self, objtype, expected_checksum, input, 0, FALSE, out_csum,
                        cancellable, error);
 }
 
@@ -1004,7 +1016,7 @@ ostree_repo_write_metadata_stream_trusted (OstreeRepo        *self,
                                            GError           **error)
 {
   /* Ignore provided length for now */
-  return write_object (self, objtype, checksum, object_input, 0, NULL,
+  return write_object (self, objtype, checksum, object_input, 0, FALSE, NULL,
                        cancellable, error);
 }
 
@@ -1034,13 +1046,48 @@ ostree_repo_write_metadata_trusted (OstreeRepo         *self,
   normalized = g_variant_get_normal_form (variant);
   input = ot_variant_read (normalized);
 
-  return write_object (self, type, checksum, input, 0, NULL,
+  return write_object (self, type, checksum, input, 0, FALSE, NULL,
+                       cancellable, error);
+}
+
+/**
+ * ostree_repo_write_commit_trusted:
+ * @self: Repo
+ * @checksum: Checksum for new commit object
+ * @variant: Commit object
+ * @is_thin_commit: If %TRUE, the commit is "thin"; i.e. other metadata/content is not stored
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Store the commit object @variant; the provided @checksum is
+ * trusted.
+ *
+ * Set @is_thin_commit if the other metadata and content objects
+ * referenced by @object will not also be stored.
+ */
+gboolean
+ostree_repo_write_commit_trusted (OstreeRepo        *self,
+                                  const char        *checksum,
+                                  GVariant          *variant,
+                                  gboolean           is_thin_commit,
+                                  GCancellable      *cancellable,
+                                  GError           **error)
+{
+  gs_unref_object GInputStream *input = NULL;
+  gs_unref_variant GVariant *normalized = NULL;
+
+  normalized = g_variant_get_normal_form (variant);
+  input = ot_variant_read (normalized);
+
+  return write_object (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, input,
+                       0, is_thin_commit, NULL,
                        cancellable, error);
 }
 
 typedef struct {
   OstreeRepo *repo;
   OstreeObjectType objtype;
+  gboolean is_thin_commit;
   char *expected_checksum;
   GVariant *object;
   GCancellable *cancellable;
@@ -1069,12 +1116,17 @@ write_metadata_thread (GSimpleAsyncResult  *res,
 {
   GError *error = NULL;
   WriteMetadataAsyncData *data;
+  gs_unref_object GInputStream *input = NULL;
+  gs_unref_variant GVariant *normalized = NULL;
 
   data = g_simple_async_result_get_op_res_gpointer (res);
-  if (!ostree_repo_write_metadata (data->repo, data->objtype, data->expected_checksum,
-                                   data->object,
-                                   &data->result_csum,
-                                   cancellable, &error))
+
+  normalized = g_variant_get_normal_form (data->object);
+  input = ot_variant_read (normalized);
+
+  if (!write_object (data->repo, data->objtype, data->expected_checksum,
+                     input, 0, data->is_thin_commit, &data->result_csum,
+                     cancellable, &error))
     g_simple_async_result_take_error (res, error);
 }
 
@@ -1140,6 +1192,52 @@ ostree_repo_write_metadata_finish (OstreeRepo        *self,
   return TRUE;
 }
 
+/**
+ * ostree_repo_write_commit_async:
+ * @self: Repo
+ * @expected_checksum: (allow-none): If provided, validate content against this checksum
+ * @object: Commit object
+ * @is_thin_commit: If %TRUE, the commit is "thin"; i.e. other metadata/content is not stored
+ * @cancellable: Cancellable
+ * @callback: Invoked when metadata is writed
+ * @user_data: Data for @callback
+ *
+ * Asynchronously store the commit object @object.  If provided, the
+ * checksum @expected_checksum will be verified.
+ *
+ * Set @is_thin_commit if the other metadata and content objects
+ * referenced by @object will not also be stored.
+ */
+void
+ostree_repo_write_commit_async (OstreeRepo              *self,
+                                 const char              *expected_checksum,
+                                 GVariant                *object,
+                                 gboolean                 is_thin_commit,
+                                 GCancellable            *cancellable,
+                                 GAsyncReadyCallback      callback,
+                                 gpointer                 user_data)
+{
+  WriteMetadataAsyncData *asyncdata;
+
+  asyncdata = g_new0 (WriteMetadataAsyncData, 1);
+  asyncdata->is_thin_commit = is_thin_commit;
+  asyncdata->repo = g_object_ref (self);
+  asyncdata->objtype = OSTREE_OBJECT_TYPE_COMMIT;
+  asyncdata->expected_checksum = g_strdup (expected_checksum);
+  asyncdata->object = g_variant_ref (object);
+  asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
+
+  asyncdata->result = g_simple_async_result_new ((GObject*) self,
+                                                 callback, user_data,
+                                                 ostree_repo_write_metadata_async);
+
+  g_simple_async_result_set_op_res_gpointer (asyncdata->result, asyncdata,
+                                             write_metadata_async_data_free);
+  g_simple_async_result_run_in_thread (asyncdata->result, write_metadata_thread, G_PRIORITY_DEFAULT, cancellable);
+  g_object_unref (asyncdata->result);
+
+}
+
 gboolean
 _ostree_repo_write_directory_meta (OstreeRepo   *self,
                                    GFileInfo    *file_info,
@@ -1201,7 +1299,7 @@ ostree_repo_write_content_trusted (OstreeRepo       *self,
                                    GError          **error)
 {
   return write_object (self, OSTREE_OBJECT_TYPE_FILE, checksum,
-                       object_input, length, NULL,
+                       object_input, length, FALSE, NULL,
                        cancellable, error);
 }
 
@@ -1229,7 +1327,7 @@ ostree_repo_write_content (OstreeRepo       *self,
                            GError          **error)
 {
   return write_object (self, OSTREE_OBJECT_TYPE_FILE, expected_checksum,
-                       object_input, length, out_csum,
+                       object_input, length, FALSE, out_csum,
                        cancellable, error);
 }
 
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 0a3b0a01..45f26989 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -137,5 +137,6 @@ _ostree_repo_commit_modifier_apply (OstreeRepo               *self,
                                     GFileInfo                *file_info,
                                     GFileInfo               **out_modified_info);
 
+
 G_END_DECLS
 
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 271d44da..65d543c6 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -693,9 +693,19 @@ meta_fetch_on_complete (GObject           *object,
                                 FALSE, &metadata, error))
         goto out;
       
-      ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata,
-                                        pull_data->cancellable,
-                                        on_metadata_writed, fetch_data);
+      if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+        {
+          gboolean is_thin = (pull_data->flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
+          ostree_repo_write_commit_async (pull_data->repo, checksum, metadata, is_thin,
+                                          pull_data->cancellable,
+                                          on_metadata_writed, fetch_data);
+        }
+      else
+        {
+          ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata,
+                                            pull_data->cancellable,
+                                            on_metadata_writed, fetch_data);
+        }
       pull_data->n_outstanding_metadata_write_requests++;
     }
 
@@ -743,24 +753,28 @@ scan_commit_object (OtPullData         *pull_data,
     }
 #endif
 
-  if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
-                                 &commit, error))
+  if (!ostree_repo_load_commit (pull_data->repo, checksum, &commit, NULL,
+                                cancellable, error))
     goto out;
 
   /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
   g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
   g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
 
-  if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
-                                 OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
-                                 cancellable, error))
-    goto out;
+  // If this is a commit only pull, don't grab the top dirtree/dirmeta:
+  if (!(pull_data->flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY))
+    {
+      if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
+                                     OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
+                                     cancellable, error))
+        goto out;
+
+      if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
+                                     OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
+                                     cancellable, error))
+        goto out;
+    }
 
-  if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
-                                 OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
-                                 cancellable, error))
-    goto out;
-  
   ret = TRUE;
  out:
   if (iter)
@@ -781,6 +795,7 @@ scan_one_metadata_object (OtPullData         *pull_data,
   gs_free char *tmp_checksum = NULL;
   gboolean is_requested;
   gboolean is_stored;
+  gboolean is_thin_commit = FALSE;
 
   tmp_checksum = ostree_checksum_from_bytes (csum);
   object = ostree_object_name_serialize (tmp_checksum, objtype);
@@ -789,9 +804,19 @@ scan_one_metadata_object (OtPullData         *pull_data,
     return TRUE;
 
   is_requested = g_hash_table_lookup (pull_data->requested_metadata, tmp_checksum) != NULL;
-  if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
-                               cancellable, error))
-    goto out;
+  if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+    {
+      if (!ostree_repo_load_commit (pull_data->repo, tmp_checksum,
+                                    NULL, &is_thin_commit,
+                                    cancellable, error))
+        goto out;
+    }
+  else
+    {
+      if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
+                                   cancellable, error))
+        goto out;
+    }
 
   if (!is_stored && !is_requested)
     {
@@ -809,7 +834,13 @@ scan_one_metadata_object (OtPullData         *pull_data,
     }
   else if (is_stored)
     {
-      if (pull_data->transaction_resuming || is_requested)
+      /* Scan if it was already stored,
+         and one of the following:
+         - Resuming an interrupted pull
+         - This object was explicitly requested
+         - The commit object is thin, and we might want to make it non-thin
+      */
+      if (pull_data->transaction_resuming || is_requested || is_thin_commit)
         {
           switch (objtype)
             {
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index eb5bc537..353324fd 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -810,6 +810,7 @@ load_metadata_internal (OstreeRepo       *self,
                         const char       *sha256,
                         gboolean          error_if_not_found,
                         GVariant        **out_variant,
+                        gboolean          check_thin,
                         GInputStream    **out_stream,
                         guint64          *out_size,
                         GCancellable     *cancellable,
@@ -823,7 +824,10 @@ load_metadata_internal (OstreeRepo       *self,
 
   g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE);
 
-  _ostree_loose_path (loose_path_buf, sha256, objtype, self->mode);
+  if (objtype == OSTREE_OBJECT_TYPE_COMMIT && check_thin)
+    _ostree_loose_path_with_suffix (loose_path_buf, sha256, objtype, self->mode, "thin");
+  else
+    _ostree_loose_path (loose_path_buf, sha256, objtype, self->mode);
 
   if (!openat_allow_noent (self->objects_dir_fd, loose_path_buf, &fd,
                            cancellable, error))
@@ -1112,7 +1116,7 @@ ostree_repo_load_object_stream (OstreeRepo         *self,
 
   if (OSTREE_OBJECT_TYPE_IS_META (objtype))
     {
-      if (!load_metadata_internal (self, objtype, checksum, TRUE, NULL,
+      if (!load_metadata_internal (self, objtype, checksum, TRUE, NULL, FALSE,
                                    &ret_input, &size,
                                    cancellable, error))
         goto out;
@@ -1325,7 +1329,7 @@ ostree_repo_load_variant_if_exists (OstreeRepo       *self,
                                     GError          **error)
 {
   return load_metadata_internal (self, objtype, sha256, FALSE,
-                                 out_variant, NULL, NULL, NULL, error);
+                                 out_variant, FALSE, NULL, NULL, NULL, error);
 }
 
 /**
@@ -1347,7 +1351,55 @@ ostree_repo_load_variant (OstreeRepo       *self,
                           GError          **error)
 {
   return load_metadata_internal (self, objtype, sha256, TRUE,
-                                 out_variant, NULL, NULL, NULL, error);
+                                 out_variant, FALSE, NULL, NULL, NULL, error);
+}
+
+/**
+ * ostree_repo_load_commit:
+ * @self: Repo
+ * @sha256: Checksum string
+ * @out_variant: (out) (transfer full): Metadata object
+ * @out_is_thin: (out): Whether or not the commit is "thin"; i.e. only storing the commit itself
+ * @error: Error
+ *
+ * Load the metadata object @sha256 of type @objtype, storing the
+ * result in @out_variant.
+ */
+gboolean
+ostree_repo_load_commit (OstreeRepo  *self,
+                         const char    *sha256, 
+                         GVariant     **out_variant,
+                         gboolean      *out_is_thin,
+                         GCancellable  *cancellable,
+                         GError       **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_variant GVariant *ret_variant = NULL;
+
+  if (!load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, sha256, FALSE,
+                               &ret_variant, FALSE, NULL, NULL,
+                               cancellable, error))
+    goto out;
+
+  if (ret_variant != NULL)
+    {
+      if (out_is_thin)
+        *out_is_thin = FALSE;
+    }
+  else
+    {
+      if (!load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, sha256, TRUE,
+                                   &ret_variant, TRUE, NULL, NULL,
+                                   cancellable, error))
+        goto out;
+      if (out_is_thin)
+        *out_is_thin = TRUE;
+    }
+
+  ret = TRUE;
+  gs_transfer_out_value (out_variant, &ret_variant);
+ out:
+  return ret;
 }
 
 /**
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 6a97f6b8..7ced5a16 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -155,6 +155,15 @@ gboolean      ostree_repo_write_metadata_finish (OstreeRepo        *self,
                                                  guchar           **out_csum,
                                                  GError           **error);
 
+
+void          ostree_repo_write_commit_async (OstreeRepo              *self,
+                                              const char              *expected_checksum,
+                                              GVariant                *object,
+                                              gboolean                 is_thin_commit,
+                                              GCancellable            *cancellable,
+                                              GAsyncReadyCallback      callback,
+                                              gpointer                 user_data);
+
 gboolean      ostree_repo_write_content (OstreeRepo       *self,
                                          const char       *expected_checksum,
                                          GInputStream     *object_input,
@@ -163,6 +172,13 @@ gboolean      ostree_repo_write_content (OstreeRepo       *self,
                                          GCancellable     *cancellable,
                                          GError          **error);
 
+gboolean      ostree_repo_write_commit_trusted (OstreeRepo        *self,
+                                                const char        *checksum,
+                                                GVariant          *variant,
+                                                gboolean           is_thin_commit,
+                                                GCancellable      *cancellable,
+                                                GError           **error);
+
 gboolean      ostree_repo_write_metadata_trusted (OstreeRepo        *self,
                                                   OstreeObjectType   objtype,
                                                   const char        *checksum,
@@ -210,6 +226,13 @@ gboolean      ostree_repo_list_refs (OstreeRepo       *self,
                                      GCancellable     *cancellable,
                                      GError          **error);
 
+gboolean      ostree_repo_load_commit (OstreeRepo  *self,
+                                       const char    *sha256, 
+                                       GVariant     **out_variant,
+                                       gboolean      *out_is_thin,
+                                       GCancellable  *cancellable,
+                                       GError       **error);
+
 gboolean      ostree_repo_load_variant (OstreeRepo  *self,
                                         OstreeObjectType objtype,
                                         const char    *sha256, 
@@ -453,9 +476,11 @@ gboolean ostree_repo_prune (OstreeRepo        *self,
 /**
  * OstreeRepoPullFlags:
  * @OSTREE_REPO_PULL_FLAGS_NONE: No special options for pull
+ * @OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY: Only fetch the commit object
  */
 typedef enum {
-  OSTREE_REPO_PULL_FLAGS_NONE
+  OSTREE_REPO_PULL_FLAGS_NONE     = 0x00,
+  OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY = 0x01,
 } OstreeRepoPullFlags;
 
 gboolean ostree_repo_pull (OstreeRepo             *self,
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index 0feeddf2..504b1f67 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -30,9 +30,11 @@
 #include "otutil.h"
 
 static char *opt_remote;
+gboolean opt_commit_only;
 
 static GOptionEntry options[] = {
   { "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" },
+  { "commit-only", 'm', 0, G_OPTION_ARG_NONE, &opt_commit_only, "Download only the commit", NULL },
   { NULL }
 };
 
@@ -61,16 +63,17 @@ import_one_object (OtLocalCloneData *data,
                    GError        **error)
 {
   gboolean ret = FALSE;
-  guint64 length;
-  gs_unref_object GInputStream *object = NULL;
-  
-  if (!ostree_repo_load_object_stream (data->src_repo, objtype, checksum,
-                                       &object, &length,
-                                       cancellable, error))
-    goto out;
 
   if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
+      gs_unref_object GInputStream *object = NULL;
+      guint64 length;
+
+      if (!ostree_repo_load_object_stream (data->src_repo, objtype, checksum,
+                                           &object, &length,
+                                           cancellable, error))
+        goto out;
+
       if (!ostree_repo_write_content_trusted (data->dest_repo, checksum,
                                               object, length,
                                               cancellable, error))
@@ -78,9 +81,16 @@ import_one_object (OtLocalCloneData *data,
     }
   else
     {
+      gs_unref_variant GVariant *metadata = NULL;
+        
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
         {
+          gboolean is_thin = FALSE;
           gs_unref_variant GVariant *detached_meta = NULL;
+
+          if (!ostree_repo_load_commit (data->src_repo, checksum, &metadata, &is_thin,
+                                        cancellable, error))
+            goto out;
           
           if (!ostree_repo_read_commit_detached_metadata (data->src_repo,
                                                           checksum, &detached_meta,
@@ -94,11 +104,23 @@ import_one_object (OtLocalCloneData *data,
                                                                cancellable, error))
                 goto out;
             }
+
+          if (!ostree_repo_write_commit_trusted (data->dest_repo,
+                                                 checksum, metadata, is_thin,
+                                                 cancellable, error))
+            goto out;
+        }
+      else
+        {
+          if (!ostree_repo_load_variant (data->src_repo, objtype, checksum, &metadata,
+                                         error))
+            goto out;
+
+          if (!ostree_repo_write_metadata_trusted (data->dest_repo, objtype,
+                                                   checksum, metadata,
+                                                   cancellable, error))
+            goto out;
         }
-      if (!ostree_repo_write_metadata_stream_trusted (data->dest_repo, objtype,
-                                                      checksum, object, length,
-                                                      cancellable, error))
-        goto out;
     }
 
   g_atomic_int_inc (&data->n_objects_copied);
diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c
index 67305fdc..99628f4f 100644
--- a/src/ostree/ot-builtin-pull.c
+++ b/src/ostree/ot-builtin-pull.c
@@ -27,7 +27,10 @@
 #include "ostree.h"
 #include "otutil.h"
 
+gboolean opt_commit_only;
+
 static GOptionEntry options[] = {
+  { "commit-only", 'm', 0, G_OPTION_ARG_NONE, &opt_commit_only, "Download only the commit", NULL },
   { NULL }
 };
 
@@ -64,6 +67,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *canc
       g_ptr_array_add (refs_to_fetch, NULL);
     }
 
+  if (opt_commit_only)
+    pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY;
+
   console = gs_console_get ();
   if (console)
     progress = ostree_async_progress_new_and_connect (ot_common_pull_progress, console);