Merge pull request #3532 from alexlarsson/extra-data-size-in-summary

Extra data size in summary

5 files changed, 125 insertions, 32 deletions

diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h
index 65492321..1e89d1b3 100644
--- a/common/flatpak-dir-private.h
+++ b/common/flatpak-dir-private.h
@@ -91,6 +91,7 @@ GType flatpak_deploy_get_type (void);
 #define FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE "eol"
 #define FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE_REBASE "eolr"
 #define FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE "tokt"
+#define FLATPAK_SPARSE_CACHE_KEY_EXTRA_DATA_SIZE "eds"
 
 typedef struct
 {
diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c
index f8c126ea..2e2a710a 100644
--- a/common/flatpak-dir.c
+++ b/common/flatpak-dir.c
@@ -559,6 +559,21 @@ flatpak_remote_state_match_subrefs (FlatpakRemoteState *self,
   return flatpak_summary_match_subrefs (self->summary, NULL, ref);
 }
 
+/* 0 if not specified */
+static guint32
+flatpak_remote_state_get_cache_version (FlatpakRemoteState *self)
+{
+  VarMetadataRef meta;
+  VarSummaryRef summary;
+
+  if (!flatpak_remote_state_ensure_summary (self, NULL))
+    return 0;
+
+  summary = var_summary_from_gvariant (self->summary);
+  meta = var_summary_get_metadata (summary);
+
+  return GUINT32_FROM_LE (var_metadata_lookup_uint32 (meta, "xa.cache-version", 0));
+}
 
 static gboolean
 flatpak_remote_state_get_cache (FlatpakRemoteState *self,
@@ -4626,6 +4641,37 @@ extra_data_progress_report (guint64  downloaded_bytes,
   flatpak_progress_update_extra_data (progress, downloaded_bytes);
 }
 
+static void
+compute_extra_data_download_size (GVariant *commitv,
+                                  guint64 *out_n_extra_data,
+                                  guint64 *out_total_download_size)
+{
+  guint64 i;
+  guint64 n_extra_data = 0;
+  guint64 total_download_size = 0;
+  g_autoptr(GVariant) extra_data_sources = NULL;
+
+  extra_data_sources = flatpak_commit_get_extra_data_sources (commitv, NULL);
+  if (extra_data_sources != NULL)
+    {
+      n_extra_data = g_variant_n_children (extra_data_sources);
+      for (i = 0; i < n_extra_data; i++)
+        {
+          guint64 download_size;
+          flatpak_repo_parse_extra_data_sources (extra_data_sources, i,
+                                                 NULL,
+                                                 &download_size,
+                                                 NULL,
+                                                 NULL,
+                                                 NULL);
+          total_download_size += download_size;
+        }
+    }
+
+  *out_n_extra_data = n_extra_data;
+  *out_total_download_size = total_download_size;
+}
+
 static gboolean
 flatpak_dir_setup_extra_data (FlatpakDir                           *self,
                               FlatpakRemoteState                   *state,
@@ -4639,47 +4685,51 @@ flatpak_dir_setup_extra_data (FlatpakDir                           *self,
                               GCancellable                         *cancellable,
                               GError                              **error)
 {
-  g_autoptr(GVariant) extra_data_sources = NULL;
-  guint64 i;
-  guint64 n_extra_data;
-  guint64 total_download_size;
+  guint64 n_extra_data = 0;
+  guint64 total_download_size = 0;
 
   /* ostree-metadata and appstreams never have extra data, so ignore those */
   if (g_str_has_prefix (ref, "app/") || g_str_has_prefix (ref, "runtime/"))
     {
-      g_autoptr(GVariant) commitv = flatpak_remote_state_load_ref_commit (state, self, ref, rev, token, error);
-      if (commitv == NULL)
-        return FALSE;
+      g_autofree char *summary_checksum = NULL;
 
-      extra_data_sources = flatpak_commit_get_extra_data_sources (commitv, NULL);
-    }
-
-  n_extra_data = 0;
-  total_download_size = 0;
-
-  if (extra_data_sources != NULL)
-    n_extra_data = g_variant_n_children (extra_data_sources);
-
-  if (n_extra_data > 0)
-    {
-      if ((flatpak_flags & FLATPAK_PULL_FLAGS_DOWNLOAD_EXTRA_DATA) == 0)
-        return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Extra data not supported for non-gpg-verified local system installs"));
-
-      for (i = 0; i < n_extra_data; i++)
+      /* Version 1 added extra data details, so we can rely on it
+       * either being in the sparse cache or no extra data.  However,
+       * it only applies to the commit the summary contains, so verify
+       * that too.
+       */
+      if (state->summary &&
+          flatpak_summary_lookup_ref (state->summary, NULL, ref, &summary_checksum, NULL) &&
+          g_strcmp0 (rev, summary_checksum) == 0 &&
+          flatpak_remote_state_get_cache_version (state) >= 1)
         {
-          guint64 download_size;
+          VarMetadataRef metadata;
+          VarVariantRef res;
 
-          flatpak_repo_parse_extra_data_sources (extra_data_sources, i,
-                                                 NULL,
-                                                 &download_size,
-                                                 NULL,
-                                                 NULL,
-                                                 NULL);
+          if (flatpak_remote_state_lookup_sparse_cache (state, ref, &metadata, NULL) &&
+              var_metadata_lookup (metadata, FLATPAK_SPARSE_CACHE_KEY_EXTRA_DATA_SIZE, NULL, &res) &&
+              var_variant_is_type (res, VAR_EXTRA_DATA_SIZE_TYPEFORMAT))
+            {
+              VarExtraDataSizeRef eds = var_extra_data_size_from_variant (res);
+              n_extra_data = var_extra_data_size_get_n_extra_data (eds);
+              total_download_size = var_extra_data_size_get_total_size (eds);
+            }
+        }
+      else
+        {
+          /* No summary/cache or old cache version, download commit and get size from there */
+          g_autoptr(GVariant) commitv = flatpak_remote_state_load_ref_commit (state, self, ref, rev, token, error);
+          if (commitv == NULL)
+            return FALSE;
 
-          total_download_size += download_size;
+          compute_extra_data_download_size (commitv, &n_extra_data, &total_download_size);
         }
     }
 
+  if (n_extra_data > 0 &&
+      (flatpak_flags & FLATPAK_PULL_FLAGS_DOWNLOAD_EXTRA_DATA) == 0)
+    return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Extra data not supported for non-gpg-verified local system installs"));
+
   flatpak_progress_init_extra_data (progress, n_extra_data, total_download_size);
 
   return TRUE;
diff --git a/common/flatpak-utils-private.h b/common/flatpak-utils-private.h
index 7d101470..2004d26a 100644
--- a/common/flatpak-utils-private.h
+++ b/common/flatpak-utils-private.h
@@ -53,6 +53,9 @@
 #define FLATPAK_ANSI_ROW_N "\x1b[%d;1H"
 #define FLATPAK_ANSI_CLEAR "\x1b[0J"
 
+#define FLATPAK_XA_CACHE_VERSION 1
+/* version 1 added extra data download size */
+
 gboolean flatpak_set_tty_echo (gboolean echo);
 void flatpak_get_window_size (int *rows,
                               int *cols);
diff --git a/common/flatpak-utils.c b/common/flatpak-utils.c
index 6eaa4f0a..385467f9 100644
--- a/common/flatpak-utils.c
+++ b/common/flatpak-utils.c
@@ -3445,16 +3445,24 @@ populate_commit_data_cache (GVariant   *metadata,
   g_autoptr(GVariant) cache = NULL;
   g_autoptr(GVariant) sparse_cache = NULL;
   gsize n, i;
+  guint32 cache_version = 0;
   const char *old_collection_id;
 
   if (!g_variant_lookup (metadata, "ostree.summary.collection-id", "&s", &old_collection_id))
     old_collection_id = NULL;
 
   cache_v = g_variant_lookup_value (metadata, "xa.cache", NULL);
-
   if (cache_v == NULL)
     return;
 
+  if (g_variant_lookup (metadata, "xa.cache-version", "u", &cache_version))
+    cache_version = GUINT32_FROM_LE (cache_version);
+  else
+    cache_version = 0;
+
+  if (cache_version < FLATPAK_XA_CACHE_VERSION)
+    return; /* We need to rebuild the cache with the current version */
+
   cache = g_variant_get_child_value (cache_v, 0);
 
   sparse_cache = g_variant_lookup_value (metadata, "xa.sparse-cache", NULL);
@@ -3714,6 +3722,9 @@ flatpak_repo_update (OstreeRepo   *repo,
       const char *eol = NULL;
       const char *eol_rebase = NULL;
       int token_type = -1;
+      g_autoptr(GVariant) extra_data_sources = NULL;
+      guint32 n_extra_data = 0;
+      guint64 total_extra_data_download_size = 0;
 
       /* See if we already have the info on this revision */
       if (g_hash_table_lookup (commit_data_cache, rev))
@@ -3756,7 +3767,25 @@ flatpak_repo_update (OstreeRepo   *repo,
       g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "&s", &eol_rebase);
       if (g_variant_lookup (commit_metadata, "xa.token-type", "i", &token_type))
         token_type = GINT32_FROM_LE(token_type);
-      if (eol || eol_rebase || token_type >= 0)
+
+      extra_data_sources = flatpak_commit_get_extra_data_sources (commit_v, NULL);
+      if (extra_data_sources)
+        {
+          n_extra_data = g_variant_n_children (extra_data_sources);
+          for (int i = 0; i < n_extra_data; i++)
+            {
+              guint64 download_size;
+              flatpak_repo_parse_extra_data_sources (extra_data_sources, i,
+                                                     NULL,
+                                                     &download_size,
+                                                     NULL,
+                                                     NULL,
+                                                     NULL);
+              total_extra_data_download_size += download_size;
+            }
+        }
+
+      if (eol || eol_rebase || token_type >= 0 || n_extra_data > 0)
         {
           g_auto(GVariantBuilder) sparse_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER;
           g_variant_builder_init (&sparse_builder, G_VARIANT_TYPE_VARDICT);
@@ -3766,6 +3795,9 @@ flatpak_repo_update (OstreeRepo   *repo,
             g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE_REBASE, g_variant_new_string (eol_rebase));
           if (token_type >= 0)
             g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE, g_variant_new_int32 (GINT32_TO_LE(token_type)));
+          if (n_extra_data >= 0)
+            g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_EXTRA_DATA_SIZE,
+                                   g_variant_new ("(ut)", GUINT32_TO_LE(n_extra_data), GUINT64_TO_LE(total_extra_data_download_size)));
 
           rev_data->sparse_data = g_variant_ref_sink (g_variant_builder_end (&sparse_builder));
         }
@@ -3799,6 +3831,8 @@ flatpak_repo_update (OstreeRepo   *repo,
    * too old to care about collection IDs anyway. */
   g_variant_builder_add (builder, "{sv}", "xa.cache",
                          g_variant_new_variant (g_variant_builder_end (ref_data_builder)));
+  g_variant_builder_add (builder, "{sv}", "xa.cache-version",
+                         g_variant_new_uint32 (GUINT32_TO_LE (FLATPAK_XA_CACHE_VERSION)));
 
   g_variant_builder_add (builder, "{sv}", "xa.sparse-cache",
                          g_variant_builder_end (ref_sparse_data_builder));
diff --git a/data/flatpak-variants.gv b/data/flatpak-variants.gv
index f3434d29..c2f6fd4f 100644
--- a/data/flatpak-variants.gv
+++ b/data/flatpak-variants.gv
@@ -57,3 +57,8 @@ type ContentRating {
  rating_type: string;
  ratings: 'Ratings [string] string;
 };
+
+type ExtraDataSize {
+ n_extra_data: littleendian uint32;
+ total_size: littleendian uint64;
+};