builder: Change the way the cache is indexed

Instead of a single ref we use one ref per stage (init, build module X,
cleanup, finish). This means we don't have to iterate in the cache
lookup, and it also means we get clean refs for each stage so one
can easily inspect the differences between the stages.

4 files changed, 73 insertions, 12 deletions

diff --git a/builder/builder-cache.c b/builder/builder-cache.c
index 8dd1624..f919e42 100644
--- a/builder/builder-cache.c
+++ b/builder/builder-cache.c
@@ -42,6 +42,8 @@ struct BuilderCache {
   GFile *cache_dir;
   GFile *app_dir;
   char *branch;
+  char *stage;
+  GHashTable *unused_stages;
   char *last_parent;
   OstreeRepo *repo;
   gboolean disabled;
@@ -74,6 +76,8 @@ builder_cache_finalize (GObject *object)
   g_checksum_free (self->checksum);
   g_free (self->branch);
   g_free (self->last_parent);
+  if (self->unused_stages)
+    g_hash_table_unref (self->unused_stages);
 
   G_OBJECT_CLASS (builder_cache_parent_class)->finalize (object);
 }
@@ -189,10 +193,18 @@ builder_cache_get_checksum (BuilderCache *self)
   return self->checksum;
 }
 
+static char *
+get_ref (BuilderCache *self, const char *stage)
+{
+  return g_strdup_printf ("%s/%s", self->branch, stage);
+}
+
 gboolean
 builder_cache_open (BuilderCache *self,
                     GError **error)
 {
+  g_autoptr(GHashTable) all_refs = NULL;
+
   self->repo = ostree_repo_new (self->cache_dir);
 
   /* We don't need fsync on checkouts as they are transient, and we
@@ -208,6 +220,21 @@ builder_cache_open (BuilderCache *self,
   if (!ostree_repo_open (self->repo, NULL, error))
     return FALSE;
 
+  /* At one point we used just the branch name as a ref, make sure to
+   * remove this to handle using the branch as a subdir */
+  ostree_repo_set_ref_immediate (self->repo,
+                                 NULL,
+                                 self->branch,
+                                 NULL,
+                                 NULL, NULL);
+
+  /* List all stages first so we can purge unused ones at the end */
+  if (!ostree_repo_list_refs (self->repo,
+                              self->branch,
+                              &self->unused_stages,
+                              NULL, error))
+    return FALSE;
+
   return TRUE;
 }
 
@@ -273,21 +300,35 @@ builder_cache_ensure_checkout (BuilderCache *self)
   self->disabled = TRUE;
 }
 
+static char *
+builder_cache_get_current_ref (BuilderCache *self)
+{
+  return get_ref (self, self->stage);
+}
+
 gboolean
-builder_cache_lookup (BuilderCache *self)
+builder_cache_lookup (BuilderCache *self,
+                      const char *stage)
 {
   g_autofree char *current = NULL;
   g_autofree char *commit = NULL;
+  g_autofree char *ref = NULL;
+
+  g_free (self->stage);
+  self->stage = g_strdup (stage);
+
+  g_hash_table_remove (self->unused_stages, stage);
 
   if (self->disabled)
     return FALSE;
 
-  if (!ostree_repo_resolve_rev (self->repo, self->branch, TRUE, &commit, NULL))
+  ref = builder_cache_get_current_ref (self);
+  if (!ostree_repo_resolve_rev (self->repo, ref, TRUE, &commit, NULL))
     goto checkout;
 
   current = builder_cache_get_current (self);
 
-  while (commit != NULL)
+  if (commit != NULL)
     {
       g_autoptr(GVariant) variant = NULL;
       const gchar *subject;
@@ -306,9 +347,6 @@ builder_cache_lookup (BuilderCache *self)
 
           return TRUE;
         }
-
-      g_free (commit);
-      commit = ostree_commit_get_parent (variant);
     }
 
  checkout:
@@ -336,6 +374,7 @@ builder_cache_commit (BuilderCache  *self,
   g_autoptr(GFile) root = NULL;
   g_autofree char *commit_checksum = NULL;
   gboolean res = FALSE;
+  g_autofree char *ref = NULL;
 
   if (!ostree_repo_prepare_transaction (self->repo, NULL, NULL, error))
     return FALSE;
@@ -358,7 +397,8 @@ builder_cache_commit (BuilderCache  *self,
                                  &commit_checksum, NULL, error))
     goto out;
 
-  ostree_repo_transaction_set_ref (self->repo, NULL, self->branch, commit_checksum);
+  ref = builder_cache_get_current_ref (self);
+  ostree_repo_transaction_set_ref (self->repo, NULL, ref, commit_checksum);
 
   if (!ostree_repo_commit_transaction (self->repo, NULL, NULL, error))
     goto out;
@@ -511,7 +551,26 @@ builder_gc (BuilderCache  *self,
   gint objects_total;
   gint objects_pruned;
   guint64 pruned_object_size_total;
+  GHashTableIter iter;
+  gpointer key, value;
+
+  g_hash_table_iter_init (&iter, self->unused_stages);
+  while (g_hash_table_iter_next (&iter, &key, &value))
+    {
+      const char *unused_stage = (const char *)key;
+      g_autofree char *unused_ref = get_ref (self, unused_stage);
+
+      g_debug ("Removing unused ref %s", unused_ref);
+
+      if (!ostree_repo_set_ref_immediate (self->repo,
+                                          NULL,
+                                          unused_ref,
+                                          NULL,
+                                          NULL, error))
+        return FALSE;
+    }
 
+  g_print ("Pruning cache");
   return ostree_repo_prune (self->repo,
                             OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, -1,
                             &objects_total,
diff --git a/builder/builder-cache.h b/builder/builder-cache.h
index 6f2863a..5b44c08 100644
--- a/builder/builder-cache.h
+++ b/builder/builder-cache.h
@@ -41,7 +41,8 @@ void          builder_cache_disable_lookups (BuilderCache  *self);
 gboolean      builder_cache_open            (BuilderCache  *self,
                                              GError       **error);
 GChecksum *   builder_cache_get_checksum    (BuilderCache  *self);
-gboolean      builder_cache_lookup          (BuilderCache  *self);
+gboolean      builder_cache_lookup          (BuilderCache  *self,
+                                             const char    *stage);
 void          builder_cache_ensure_checkout (BuilderCache *self);
 gboolean      builder_cache_has_checkout    (BuilderCache *self);
 gboolean      builder_cache_commit          (BuilderCache  *self,
diff --git a/builder/builder-manifest.c b/builder/builder-manifest.c
index ccafe41..e4f46c4 100644
--- a/builder/builder-manifest.c
+++ b/builder/builder-manifest.c
@@ -709,10 +709,11 @@ builder_manifest_build (BuilderManifest *self,
     {
       BuilderModule *m = l->data;
       g_autoptr(GPtrArray) changes = NULL;
+      g_autofree char *stage = g_strdup_printf ("build-%s", builder_module_get_name (m));
 
       builder_module_checksum (m, cache, context);
 
-      if (!builder_cache_lookup (cache))
+      if (!builder_cache_lookup (cache, stage))
         {
           g_autofree char *body =
             g_strdup_printf ("Built %s\n", builder_module_get_name (m));
@@ -931,7 +932,7 @@ builder_manifest_cleanup (BuilderManifest *self,
   int i;
 
   builder_manifest_checksum_for_cleanup (self, cache, context);
-  if (!builder_cache_lookup (cache))
+  if (!builder_cache_lookup (cache, "cleanup"))
     {
       app_root = g_file_get_child (app_dir, "files");
 
@@ -1143,7 +1144,7 @@ builder_manifest_finish (BuilderManifest *self,
   JsonGenerator *generator;
 
   builder_manifest_checksum_for_finish (self, cache, context);
-  if (!builder_cache_lookup (cache))
+  if (!builder_cache_lookup (cache, "finish"))
     {
       g_print ("Finishing app\n");
 
diff --git a/builder/xdg-app-builder-main.c b/builder/xdg-app-builder-main.c
index c5e0493..ad7c7b3 100644
--- a/builder/xdg-app-builder-main.c
+++ b/builder/xdg-app-builder-main.c
@@ -251,7 +251,7 @@ main (int    argc,
 
   builder_manifest_checksum (manifest, cache, build_context);
 
-  if (!builder_cache_lookup (cache))
+  if (!builder_cache_lookup (cache, "init"))
     {
       g_autofree char *body =
         g_strdup_printf ("Initialized %s\n",