src/ostree: Add --commit-only option to ostree prune

Recently we have noticed exceedingly long execution times
for multiple invocations of ostree prune. This is a result of
calculating full reachability on each invocation.

The --commit-only flag provides an alternative strategy. It will only
traverse and delete commit objects to avoid the more expensive
reachability calculations. This allows us to chain multiple --commit-only
commands cheaply, and then follow with a more expensive ostree prune
invocation at the end to clean up orphaned meta and content objects.

7 files changed, 167 insertions, 59 deletions

diff --git a/Makefile-libostree.am b/Makefile-libostree.am
index 02ae9c6a..9b48a308 100644
--- a/Makefile-libostree.am
+++ b/Makefile-libostree.am
@@ -171,9 +171,9 @@ endif # USE_GPGME
 symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
 
 # Uncomment this include when adding new development symbols.
-# if BUILDOPT_IS_DEVEL_BUILD
-# symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
-# endif
+if BUILDOPT_IS_DEVEL_BUILD
+symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
+endif
 
 # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
 wl_versionscript_arg = -Wl,--version-script=
diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt
index aa74c839..577ee808 100644
--- a/apidoc/ostree-sections.txt
+++ b/apidoc/ostree-sections.txt
@@ -451,6 +451,7 @@ ostree_repo_traverse_parents_get_commits
 ostree_repo_traverse_commit
 ostree_repo_traverse_commit_union
 ostree_repo_traverse_commit_union_with_parents
+ostree_repo_traverse_commit_with_flags
 ostree_repo_commit_traverse_iter_cleanup
 ostree_repo_commit_traverse_iter_clear
 ostree_repo_commit_traverse_iter_get_dir
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym
index 9168db73..74f9b9a9 100644
--- a/src/libostree/libostree-devel.sym
+++ b/src/libostree/libostree-devel.sym
@@ -20,6 +20,11 @@
    - uncomment the include in Makefile-libostree.am
 */
 
+LIBOSTREE_2022.2 {
+global:
+  ostree_repo_traverse_commit_with_flags;
+} LIBOSTREE_2021.5;
+
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the year.  This is just a copy/paste
  * source.  Replace $LASTSTABLE with the last stable version, and $NEWVERSION
diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c
index 175765fd..0c702dc9 100644
--- a/src/libostree/ostree-repo-prune.c
+++ b/src/libostree/ostree-repo-prune.c
@@ -48,6 +48,10 @@ maybe_prune_loose_object (OtPruneData           *data,
   OstreeObjectType objtype;
 
   ostree_object_name_deserialize (key, &checksum, &objtype);
+  /* Return if we only want to delete commits and this object is not a commit object. */
+  gboolean commit_only = flags & OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY;
+  if (commit_only && (objtype != OSTREE_OBJECT_TYPE_COMMIT))
+    goto exit;
 
   if (g_hash_table_lookup_extended (data->reachable, key, NULL, NULL))
     reachable = TRUE;
@@ -125,7 +129,11 @@ maybe_prune_loose_object (OtPruneData           *data,
       else
         data->n_reachable_content++;
     }
-
+    if (commit_only && (objtype != OSTREE_OBJECT_TYPE_COMMIT))
+    {
+      g_debug ("Keeping object (not commit) %s.%s", checksum,
+               ostree_object_type_to_string (objtype));
+    }
   return TRUE;
 }
 
@@ -299,25 +307,13 @@ repo_prune_internal (OstreeRepo        *self,
   return TRUE;
 }
 
-/**
- * ostree_repo_traverse_reachable_refs:
- * @self: Repo
- * @depth: Depth of traversal
- * @reachable: (element-type GVariant GVariant): Set of reachable objects (will be modified)
- * @cancellable: Cancellable
- * @error: Error
- *
- * Add all commit objects directly reachable via a ref to @reachable.
- *
- * Locking: shared
- * Since: 2018.6
- */
-gboolean
-ostree_repo_traverse_reachable_refs (OstreeRepo *self,
-                                     guint       depth,
-                                     GHashTable *reachable,
-                                     GCancellable *cancellable,
-                                     GError      **error)
+static gboolean
+traverse_reachable_internal (OstreeRepo                    *self,
+                             OstreeRepoCommitTraverseFlags  flags,
+                             guint                          depth,
+                             GHashTable                    *reachable,
+                             GCancellable                  *cancellable,
+                             GError                       **error)
 {
   g_autoptr(OstreeRepoAutoLock) lock =
     ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_SHARED, cancellable, error);
@@ -334,8 +330,8 @@ ostree_repo_traverse_reachable_refs (OstreeRepo *self,
   GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum)
     {
       g_debug ("Finding objects to keep for commit %s", checksum);
-      if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
-                                              cancellable, error))
+      if (!ostree_repo_traverse_commit_with_flags (self, flags, checksum, depth, reachable,
+                                                    NULL, cancellable, error))
         return FALSE;
     }
 
@@ -349,8 +345,8 @@ ostree_repo_traverse_reachable_refs (OstreeRepo *self,
   GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum)
     {
       g_debug ("Finding objects to keep for commit %s", checksum);
-      if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
-                                              cancellable, error))
+      if (!ostree_repo_traverse_commit_with_flags (self, flags, checksum, depth, reachable,
+                                                    NULL, cancellable, error))
         return FALSE;
     }
 
@@ -358,6 +354,32 @@ ostree_repo_traverse_reachable_refs (OstreeRepo *self,
 }
 
 /**
+ * ostree_repo_traverse_reachable_refs:
+ * @self: Repo
+ * @depth: Depth of traversal
+ * @reachable: (element-type GVariant GVariant): Set of reachable objects (will be modified)
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Add all commit objects directly reachable via a ref to @reachable.
+ *
+ * Locking: shared
+ * Since: 2018.6
+ */
+gboolean
+ostree_repo_traverse_reachable_refs (OstreeRepo *self,
+                                     guint       depth,
+                                     GHashTable *reachable,
+                                     GCancellable *cancellable,
+                                     GError      **error)
+{
+  return traverse_reachable_internal (self, 
+                                      OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
+                                      depth, reachable,
+                                      cancellable, error);
+}
+
+/**
  * ostree_repo_prune:
  * @self: Repo
  * @flags: Options controlling prune process
@@ -401,6 +423,7 @@ ostree_repo_prune (OstreeRepo        *self,
 
   g_autoptr(GHashTable) objects = NULL;
   gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
+  gboolean commit_only = flags & OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY;
 
   g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable ();
 
@@ -409,9 +432,15 @@ ostree_repo_prune (OstreeRepo        *self,
    * the deletion.
    */
 
+  OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE;
+  if (commit_only)
+    traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
+
   if (refs_only)
     {
-      if (!ostree_repo_traverse_reachable_refs (self, depth, reachable, cancellable, error))
+      if (!traverse_reachable_internal (self, traverse_flags,
+                                        depth, reachable,
+                                        cancellable, error))
         return FALSE;
     }
 
@@ -432,8 +461,8 @@ ostree_repo_prune (OstreeRepo        *self,
             continue;
 
           g_debug ("Finding objects to keep for commit %s", checksum);
-          if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
-                                                  cancellable, error))
+          if (!ostree_repo_traverse_commit_with_flags (self, traverse_flags, checksum, depth, reachable,
+                                                       NULL, cancellable, error))
             return FALSE;
         }
     }
diff --git a/src/libostree/ostree-repo-traverse.c b/src/libostree/ostree-repo-traverse.c
index c5c204d7..5efed100 100644
--- a/src/libostree/ostree-repo-traverse.c
+++ b/src/libostree/ostree-repo-traverse.c
@@ -542,8 +542,9 @@ traverse_dirtree (OstreeRepo           *repo,
 }
 
 /**
- * ostree_repo_traverse_commit_union_with_parents: (skip)
+ * ostree_repo_traverse_commit_with_flags: (skip)
  * @repo: Repo
+ * @flags: change traversal behaviour according to these flags
  * @commit_checksum: ASCII SHA256 checksum
  * @maxdepth: Traverse this many parent commits, -1 for unlimited
  * @inout_reachable: Set of reachable objects
@@ -561,15 +562,17 @@ traverse_dirtree (OstreeRepo           *repo,
  * Since: 2018.5
  */
 gboolean
-ostree_repo_traverse_commit_union_with_parents (OstreeRepo      *repo,
-                                                const char      *commit_checksum,
-                                                int              maxdepth,
-                                                GHashTable      *inout_reachable,
-                                                GHashTable      *inout_parents,
-                                                GCancellable    *cancellable,
-                                                GError         **error)
+ostree_repo_traverse_commit_with_flags (OstreeRepo                     *repo,
+                                        OstreeRepoCommitTraverseFlags   flags,
+                                        const char                     *commit_checksum,
+                                        int                             maxdepth,
+                                        GHashTable                     *inout_reachable,
+                                        GHashTable                     *inout_parents,
+                                        GCancellable                   *cancellable,
+                                        GError                        **error)
 {
   g_autofree char *tmp_checksum = NULL;
+  gboolean commit_only = flags & OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
 
   while (TRUE)
     {
@@ -603,16 +606,20 @@ ostree_repo_traverse_commit_union_with_parents (OstreeRepo      *repo,
 
       g_hash_table_add (inout_reachable, g_variant_ref (key));
 
-      g_debug ("Traversing commit %s", commit_checksum);
-      ostree_cleanup_repo_commit_traverse_iter
-        OstreeRepoCommitTraverseIter iter = { 0, };
-      if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit,
-                                                         OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
-                                                         error))
-        return FALSE;
-
-      if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error))
-        return FALSE;
+      /* Save time by skipping traversal of non-commit objects */
+      if (!commit_only) 
+        {
+          g_debug ("Traversing commit %s", commit_checksum);
+          ostree_cleanup_repo_commit_traverse_iter
+            OstreeRepoCommitTraverseIter iter = { 0, };
+          if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit,
+                                                            OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
+                                                            error))
+            return FALSE;
+
+          if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error))
+            return FALSE;
+        } 
 
       gboolean recurse = FALSE;
       if (maxdepth == -1 || maxdepth > 0)
@@ -635,6 +642,39 @@ ostree_repo_traverse_commit_union_with_parents (OstreeRepo      *repo,
 }
 
 /**
+ * ostree_repo_traverse_commit_union_with_parents: (skip)
+ * @repo: Repo
+ * @commit_checksum: ASCII SHA256 checksum
+ * @maxdepth: Traverse this many parent commits, -1 for unlimited
+ * @inout_reachable: Set of reachable objects
+ * @inout_parents: Map from object to parent object
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Update the set @inout_reachable containing all objects reachable
+ * from @commit_checksum, traversing @maxdepth parent commits.
+ *
+ * Additionally this constructs a mapping from each object to the parents
+ * of the object, which can be used to track which commits an object
+ * belongs to.
+ *
+ * Since: 2018.5
+ */
+gboolean
+ostree_repo_traverse_commit_union_with_parents (OstreeRepo      *repo,
+                                                const char      *commit_checksum,
+                                                int              maxdepth,
+                                                GHashTable      *inout_reachable,
+                                                GHashTable      *inout_parents,
+                                                GCancellable    *cancellable,
+                                                GError         **error)
+{
+  return ostree_repo_traverse_commit_with_flags(repo, OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
+                                                commit_checksum, maxdepth, inout_reachable, inout_parents,
+                                                cancellable, error);
+}
+
+/**
  * ostree_repo_traverse_commit_union: (skip)
  * @repo: Repo
  * @commit_checksum: ASCII SHA256 checksum
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 8a5c3b33..98571170 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -1118,6 +1118,16 @@ typedef enum {
   OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0,
 } OstreeStaticDeltaIndexFlags;
 
+/**
+ * OstreeRepoCommitTraverseFlags:
+ * @OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE: No special options for traverse
+ * @OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY: Traverse and retrieve only commit objects.  (Since: 2022.2)
+ */
+typedef enum {
+  OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0),
+  OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY = (1 << 1),
+} OstreeRepoCommitTraverseFlags;
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_static_delta_reindex (OstreeRepo                  *repo,
                                            OstreeStaticDeltaIndexFlags flags,
@@ -1171,6 +1181,7 @@ gboolean ostree_repo_traverse_commit_union (OstreeRepo         *repo,
                                             GHashTable         *inout_reachable,
                                             GCancellable       *cancellable,
                                             GError            **error);
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo         *repo,
                                                          const char         *commit_checksum,
@@ -1180,6 +1191,16 @@ gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo         *rep
                                                          GCancellable       *cancellable,
                                                          GError            **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_traverse_commit_with_flags (OstreeRepo                     *repo,
+                                                 OstreeRepoCommitTraverseFlags   flags,
+                                                 const char                     *commit_checksum,
+                                                 int                             maxdepth,
+                                                 GHashTable                     *inout_reachable,
+                                                 GHashTable                     *inout_parents,
+                                                 GCancellable                   *cancellable,
+                                                 GError                        **error);
+
 struct _OstreeRepoCommitTraverseIter {
   gboolean initialized;
   /* 4 byte hole on 64 bit */
@@ -1189,10 +1210,6 @@ struct _OstreeRepoCommitTraverseIter {
 
 typedef struct _OstreeRepoCommitTraverseIter OstreeRepoCommitTraverseIter;
 
-typedef enum {
-  OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0)
-} OstreeRepoCommitTraverseFlags;
-
 _OSTREE_PUBLIC
 gboolean
 ostree_repo_commit_traverse_iter_init_commit (OstreeRepoCommitTraverseIter    *iter,
@@ -1245,11 +1262,13 @@ void ostree_repo_commit_traverse_iter_cleanup (void *p);
  * @OSTREE_REPO_PRUNE_FLAGS_NONE: No special options for pruning
  * @OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE: Don't actually delete objects
  * @OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY: Do not traverse individual commit objects, only follow refs
+ * @OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY: Only traverse commit objects.  (Since 2022.2)
  */
 typedef enum {
   OSTREE_REPO_PRUNE_FLAGS_NONE = 0,
   OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE = (1 << 0),
   OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY = (1 << 1),
+  OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY = (1 << 2),
 } OstreeRepoPruneFlags;
 
 _OSTREE_PUBLIC
diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c
index d133bbd8..b2dd407a 100644
--- a/src/ostree/ot-builtin-prune.c
+++ b/src/ostree/ot-builtin-prune.c
@@ -35,6 +35,7 @@ static char *opt_delete_commit;
 static char *opt_keep_younger_than;
 static char **opt_retain_branch_depth;
 static char **opt_only_branches;
+static gboolean opt_commit_only;
 
 /* ATTENTION:
  * Please remember to update the bash-completion script (bash/ostree) and
@@ -50,6 +51,7 @@ static GOptionEntry options[] = {
   { "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" },
   { "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" },
   { "only-branch", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_only_branches, "Only prune BRANCH (may be specified multiple times)", "BRANCH" },
+  { "commit-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Only traverse and delete commit objects.", NULL },
   { NULL }
 };
 
@@ -99,12 +101,15 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
                             GCancellable *cancellable, GError **error)
 {
   g_autofree char *next_checksum = g_strdup (checksum);
+  OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE;
+  if (opt_commit_only)
+    traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
 
   /* This is the first commit in our loop, which has a ref pointing to it. We
    * don't want to auto-prune it.
    */
-  if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable,
-                                          cancellable, error))
+  if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, checksum, 0, reachable,
+                                                NULL, cancellable, error))
     return FALSE;
 
   while (TRUE)
@@ -121,8 +126,8 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
       if (commit_timestamp >= ts->tv_sec)
         {
           /* It's newer, traverse it */
-          if (!ostree_repo_traverse_commit_union (repo, next_checksum, 0, reachable,
-                                                  cancellable, error))
+          if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, next_checksum, 0, reachable,
+                                                        NULL, cancellable, error))
             return FALSE;
 
           g_free (next_checksum);
@@ -183,6 +188,8 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
   if (opt_no_prune)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE;
+  if (opt_commit_only)
+    pruneflags |= OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY;
 
   /* If no newer more complex options are specified, drop down to the original
    * prune API - both to avoid code duplication, and to keep it run from the
@@ -285,6 +292,10 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
       /* Traverse each ref, and gather all objects pointed to by it up to a
        * specific depth (if configured).
        */
+      OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE;
+      if (opt_commit_only)
+        /** We can avoid looking at all objects if --commit-only is specified **/
+        traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
       g_hash_table_iter_init (&hash_iter, all_refs);
       while (g_hash_table_iter_next (&hash_iter, &key, &value))
         {
@@ -316,8 +327,8 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
                                   the global default */
 
           g_debug ("Finding objects to keep for commit %s", checksum);
-          if (!ostree_repo_traverse_commit_union (repo, checksum, depth, reachable,
-                                                  cancellable, error))
+          if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, checksum, depth, reachable,
+                                                        NULL, cancellable, error))
             return FALSE;
         }
 
@@ -333,7 +344,10 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
     }
 
   g_autofree char *formatted_freed_size = g_format_size_full (objsize_total, 0);
-  g_print ("Total objects: %u\n", n_objects_total);
+  if (opt_commit_only) 
+    g_print("Total (commit only) objects: %u\n", n_objects_total);
+  else
+    g_print ("Total objects: %u\n", n_objects_total);
   if (n_objects_pruned == 0)
     g_print ("No unreachable objects\n");
   else if (pruneflags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)