Add support for more selective pruning

There are use cases for having a single repo with branches
with different lifecycles; a simple example of what I was
trying to do in CentOS Atomic Host work is have "stable"
and "devel" branches, were we want to prune devel, but
retain *all* of stable.

This patch is split into two parts - first we add a low level "delete all
objects not in this set" API, and change the current prune API
to use this.

Next, we move more logic into the "ostree prune" command. This paves the way for
demonstrating how more sophisticated algorithms/logic could be developed outside
of the ostree core.

Also, the --keep-younger-than logic already lived in the commandline, so it
makes sense to keep extending it there.

Closes: https://github.com/ostreedev/ostree/issues/604

Closes: #646
Approved by: jlebon

1 files changed, 158 insertions, 80 deletions

diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c
index d226c84b..853c051f 100644
--- a/src/ostree/ot-builtin-prune.c
+++ b/src/ostree/ot-builtin-prune.c
@@ -34,6 +34,7 @@ static gint opt_depth = -1;
 static gboolean opt_refs_only;
 static char *opt_delete_commit;
 static char *opt_keep_younger_than;
+static char **opt_retain_branch_depth;
 
 static GOptionEntry options[] = {
   { "no-prune", 0, 0, G_OPTION_ARG_NONE, &opt_no_prune, "Only display unreachable objects; don't delete", NULL },
@@ -42,6 +43,7 @@ static GOptionEntry options[] = {
   { "delete-commit", 0, 0, G_OPTION_ARG_STRING, &opt_delete_commit, "Specify a commit to delete", "COMMIT" },
   { "keep-younger-than", 0, 0, G_OPTION_ARG_STRING, &opt_keep_younger_than, "Prune all commits older than the specified date", "DATE" },
   { "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" },
+  { "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" },
   { NULL }
 };
 
@@ -82,87 +84,53 @@ delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *can
 }
 
 static gboolean
-prune_commits_keep_younger_than_date (OstreeRepo *repo, const char *date, GCancellable *cancellable, GError **error)
+traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
+                            struct timespec *ts,
+                            GHashTable *reachable,
+                            GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GHashTable) refs = NULL;
-  g_autoptr(GHashTable) ref_heads = g_hash_table_new (g_str_hash, g_str_equal);
-  g_autoptr(GHashTable) objects = NULL;
-  GHashTableIter hash_iter;
-  gpointer key, value;
-  struct timespec ts;
-  gboolean ret = FALSE;
-
-  if (!parse_datetime (&ts, date, NULL))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Could not parse '%s'", date);
-      goto out;
-    }
-
-  if (!ot_enable_tombstone_commits (repo, error))
-    goto out;
-
-  if (!ostree_repo_list_refs (repo, NULL, &refs, cancellable, error))
-    goto out;
+  g_autofree char *next_checksum = g_strdup (checksum);
+  g_autoptr(GVariant) commit = NULL;
 
-  /* We used to prune the HEAD of a given ref by default, but that's
-   * broken for a few reasons.  One is that people may use branches as
-   * tags.  Second is that if we do it, we should be deleting the ref
-   * too, otherwise e.g. `summary -u` breaks trying to load it, etc.
+  /* This is the first commit in our loop, which has a ref pointing to it. We
+   * don't want to auto-prune it.
    */
-  g_hash_table_iter_init (&hash_iter, refs);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      /* Value is lifecycle bound to refs */
-      g_hash_table_add (ref_heads, (char*)value);
-    }
+  if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable,
+                                          cancellable, error))
+    return FALSE;
 
-  if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
-                                 cancellable, error))
-    goto out;
-
-  g_hash_table_iter_init (&hash_iter, objects);
-
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+  while (TRUE)
     {
-      GVariant *serialized_key = key;
-      const char *checksum;
-      OstreeObjectType objtype;
       guint64 commit_timestamp;
-      g_autoptr(GVariant) commit = NULL;
-
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-
-      if (objtype != OSTREE_OBJECT_TYPE_COMMIT)
-        continue;
 
-      if (g_hash_table_contains (ref_heads, checksum))
-        continue;
+      if (!ostree_repo_load_variant_if_exists (repo, OSTREE_OBJECT_TYPE_COMMIT, next_checksum,
+                                               &commit, error))
+        return FALSE;
 
-      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
-                                     &commit, error))
-        goto out;
+      if (!commit)
+        break; /* This commit was pruned, so we're done */
 
       commit_timestamp = ostree_commit_get_timestamp (commit);
-      if (commit_timestamp < ts.tv_sec)
+      /* Is this commit newer than our --keep-younger-than spec? */
+      if (commit_timestamp >= ts->tv_sec)
         {
-          if (opt_static_deltas_only)
-            {
-              if(!ostree_repo_prune_static_deltas (repo, checksum, cancellable, error))
-                goto out;
-            }
+          /* It's newer, traverse it */
+          if (!ostree_repo_traverse_commit_union (repo, next_checksum, 0, reachable,
+                                                  cancellable, error))
+            return FALSE;
+
+          g_free (next_checksum);
+          next_checksum = ostree_commit_get_parent (commit);
+          if (next_checksum)
+            g_clear_pointer (&commit, (GDestroyNotify)g_variant_unref);
           else
-            {
-              if (!ostree_repo_delete_object (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum, cancellable, error))
-                goto out;
-            }
+            break; /* No parent, we're done */
         }
+      else
+        break; /* It's older than our spec, we're done */
     }
 
-  ret = TRUE;
-
- out:
-  return ret;
+  return TRUE;
 }
 
 gboolean
@@ -185,6 +153,9 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError *
   if (!opt_no_prune && !ostree_ensure_repo_writable (repo, error))
     goto out;
 
+  /* Special handling for explicit commit deletion here - we do this
+   * first.
+   */
   if (opt_delete_commit)
     {
       if (opt_no_prune)
@@ -200,26 +171,133 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError *
         else if (!delete_commit (repo, opt_delete_commit, cancellable, error))
           goto out;
     }
-  if (opt_keep_younger_than)
-    {
-      if (opt_no_prune)
-        {
-          ot_util_usage_error (context, "Cannot specify both --keep-younger-than and --no-prune", error);
-          goto out;
-        }
-      if (!prune_commits_keep_younger_than_date (repo, opt_keep_younger_than, cancellable, error))
-        goto out;
-    }
 
   if (opt_refs_only)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
   if (opt_no_prune)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE;
 
-  if (!ostree_repo_prune (repo, pruneflags, opt_depth,
-                          &n_objects_total, &n_objects_pruned, &objsize_total,
-                          cancellable, error))
-    goto out;
+  /* If no newer more complex options are specified, drop down to the original
+   * prune API - both to avoid code duplication, and to keep it run from the
+   * test suite.
+   */
+  if (!(opt_retain_branch_depth || opt_keep_younger_than))
+    {
+      if (!ostree_repo_prune (repo, pruneflags, opt_depth,
+                              &n_objects_total, &n_objects_pruned, &objsize_total,
+                              cancellable, error))
+        goto out;
+    }
+  else
+    {
+      g_autoptr(GHashTable) all_refs = NULL;
+      g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable ();
+      g_autoptr(GHashTable) retain_branch_depth = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+      struct timespec keep_younger_than_ts;
+      GHashTableIter hash_iter;
+      gpointer key, value;
+
+      /* Otherwise, the default is --refs-only; we set this just as a note */
+      opt_refs_only = TRUE;
+
+      if (opt_keep_younger_than)
+        {
+          if (!parse_datetime (&keep_younger_than_ts, opt_keep_younger_than, NULL))
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Could not parse '%s'", opt_keep_younger_than);
+              goto out;
+            }
+        }
+
+      for (char **iter = opt_retain_branch_depth; iter && *iter; iter++)
+        {
+          /* bd should look like BRANCH=DEPTH where DEPTH is an int */
+          const char *bd = *iter;
+          const char *eq = strchr (bd, '=');
+          const char *depthstr;
+          gint64 depth;
+          char *endptr;
+
+          if (!eq)
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Invalid value %s, must specify BRANCH=DEPTH",
+                           bd);
+              goto out;
+            }
+          depthstr = eq + 1;
+          errno = EPERM;
+          depth = g_ascii_strtoll (depthstr, &endptr, 10);
+          if (depth == 0)
+            {
+              if (errno == EINVAL)
+                {
+                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Out of range depth %s", depthstr);
+                  goto out;
+                }
+              else if (endptr == depthstr)
+                {
+                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Invalid depth %s", depthstr);
+                  goto out;
+                }
+            }
+          g_hash_table_insert (retain_branch_depth, g_strndup (bd, eq - bd),
+                               GINT_TO_POINTER ((int)depth));
+        }
+
+      /* We start from the refs */
+      if (!ostree_repo_list_refs (repo, NULL, &all_refs,
+                                  cancellable, error))
+        return FALSE;
+
+      g_hash_table_iter_init (&hash_iter, all_refs);
+      while (g_hash_table_iter_next (&hash_iter, &key, &value))
+        {
+          const char *checksum = value;
+          gpointer depthp = g_hash_table_lookup (retain_branch_depth, key);
+          gint depth;
+
+          /* Here, we handle a spec like
+           * --retain-branch-depth=myos/x86_64/stable=-1
+           * --retain-branch-depth=myos/x86_64/dev=5
+           */
+          if (depthp)
+            depth = GPOINTER_TO_INT(depthp);
+          else if (opt_keep_younger_than)
+            {
+              if (!traverse_keep_younger_than (repo, checksum,
+                                               &keep_younger_than_ts,
+                                               reachable,
+                                               cancellable, error))
+                goto out;
+
+              /* Okay, we handled the younger-than case; the other
+               * two fall through to plain depth-based handling below.
+               */
+              continue;  /* Note again, we're skipping the below bit */
+            }
+          else
+            depth = opt_depth; /* No --retain-branch-depth for this branch, use
+                                  the global default */
+
+          g_debug ("Finding objects to keep for commit %s", checksum);
+          if (!ostree_repo_traverse_commit_union (repo, checksum, depth, reachable,
+                                                  cancellable, error))
+            return FALSE;
+        }
+
+      { OstreeRepoPruneOptions opts = { pruneflags, reachable };
+        if (!ostree_repo_prune_from_reachable (repo, &opts,
+                                               &n_objects_total,
+                                               &n_objects_pruned,
+                                               &objsize_total,
+                                               cancellable, error))
+          goto out;
+      }
+    }
 
   formatted_freed_size = g_format_size_full (objsize_total, 0);