Use the new metastore index API for OCI remotes

This means the url of the remote is a service supporting:
 https://github.com/owtaylor/metastore/blob/master/docs/protocol.md

And we use that to find all flatpak images and the repository url
itself.

This also add support for docker-v2 registries that support OCI
images.

Closes: #1171
Approved by: alexlarsson

8 files changed, 365 insertions, 160 deletions

diff --git a/app/flatpak-builtins-build-import-bundle.c b/app/flatpak-builtins-build-import-bundle.c
index 1dc5f96c..0efee51e 100644
--- a/app/flatpak-builtins-build-import-bundle.c
+++ b/app/flatpak-builtins-build-import-bundle.c
@@ -95,7 +95,7 @@ import_oci (OstreeRepo *repo, GFile *file,
 
   oci_digest = desc->parent.digest;
 
-  versioned = flatpak_oci_registry_load_versioned (registry,
+  versioned = flatpak_oci_registry_load_versioned (registry, NULL,
                                                    oci_digest, NULL,
                                                    cancellable, error);
   if (versioned == NULL)
@@ -115,7 +115,7 @@ import_oci (OstreeRepo *repo, GFile *file,
       return NULL;
     }
 
-  commit_checksum = flatpak_pull_from_oci (repo, registry, oci_digest, manifest,
+  commit_checksum = flatpak_pull_from_oci (repo, registry, NULL, oci_digest, manifest,
                                            NULL, target_ref, NULL, NULL, NULL, cancellable, error);
   if (commit_checksum == NULL)
     return NULL;
diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c
index d5f2a6ce..cc8ce0a4 100644
--- a/common/flatpak-dir.c
+++ b/common/flatpak-dir.c
@@ -2547,7 +2547,7 @@ flatpak_dir_pull_extra_data (FlatpakDir          *self,
       else
         {
           ensure_soup_session (self);
-          bytes = flatpak_load_http_uri (self->soup_session, extra_data_uri, NULL, NULL,
+          bytes = flatpak_load_http_uri (self->soup_session, extra_data_uri, 0, NULL, NULL,
                                          extra_data_progress_report, &extra_data_progress,
                                          cancellable, error);
         }
@@ -2624,6 +2624,22 @@ flatpak_dir_pull_extra_data (FlatpakDir          *self,
 }
 
 static char *
+lookup_oci_registry_uri_from_summary (GVariant *summary,
+                                      GError **error)
+{
+  g_autoptr(GVariant) extensions = g_variant_get_child_value (summary, 1);
+  g_autofree char *registry_uri = NULL;
+
+  if (!g_variant_lookup (extensions, "xa.oci-registry-uri", "s", &registry_uri))
+    {
+      flatpak_fail (error, _("Remote OCI index has no registry uri"));
+      return NULL;
+    }
+
+  return g_steal_pointer (&registry_uri);
+}
+
+static char *
 flatpak_dir_lookup_ref_from_summary (FlatpakDir          *self,
                                      const char          *remote,
                                      const          char *ref,
@@ -2806,6 +2822,7 @@ flatpak_dir_mirror_oci (FlatpakDir          *self,
 {
   g_autoptr(FlatpakOciRegistry) registry = NULL;
   g_autofree char *oci_uri = NULL;
+  g_autofree char *registry_uri = NULL;
   g_autofree char *oci_digest = NULL;
   g_autofree char *latest_rev = NULL;
   g_auto(GLnxConsoleRef) console = { 0, };
@@ -2840,7 +2857,11 @@ flatpak_dir_mirror_oci (FlatpakDir          *self,
 
   oci_digest = g_strconcat ("sha256:", latest_rev, NULL);
 
-  registry = flatpak_oci_registry_new (oci_uri, FALSE, -1, NULL, error);
+  registry_uri = lookup_oci_registry_uri_from_summary (summary, error);
+  if (registry_uri == NULL)
+    return FALSE;
+
+  registry = flatpak_oci_registry_new (registry_uri, FALSE, -1, NULL, error);
   if (registry == NULL)
     return FALSE;
 
@@ -2885,6 +2906,8 @@ flatpak_dir_pull_oci (FlatpakDir          *self,
   g_autoptr(FlatpakOciVersioned) versioned = NULL;
   g_autofree char *full_ref = NULL;
   g_autofree char *oci_uri = NULL;
+  g_autofree char *registry_uri = NULL;
+  g_autofree char *oci_repository = NULL;
   g_autofree char *oci_digest = NULL;
   g_autofree char *checksum = NULL;
   g_auto(GLnxConsoleRef) console = { 0, };
@@ -2915,6 +2938,7 @@ flatpak_dir_pull_oci (FlatpakDir          *self,
 
   metadata = g_variant_get_child_value (summary_element, 2);
   g_variant_lookup (metadata, "xa.oci-signature", "s", &signature_digest);
+  g_variant_lookup (metadata, "xa.oci-repository", "s", &oci_repository);
 
   oci_digest = g_strconcat ("sha256:", latest_rev, NULL);
 
@@ -2922,12 +2946,16 @@ flatpak_dir_pull_oci (FlatpakDir          *self,
   if (latest_alt_commit != NULL && strcmp (oci_digest + strlen ("sha256:"), latest_alt_commit) == 0)
     return TRUE;
 
-  registry = flatpak_oci_registry_new (oci_uri, FALSE, -1, NULL, error);
+  registry_uri = lookup_oci_registry_uri_from_summary (summary, error);
+  if (registry_uri == NULL)
+    return FALSE;
+
+  registry = flatpak_oci_registry_new (registry_uri, FALSE, -1, NULL, error);
   if (registry == NULL)
     return FALSE;
 
-  versioned = flatpak_oci_registry_load_versioned (registry, oci_digest, NULL,
-                                                   cancellable, error);
+  versioned = flatpak_oci_registry_load_versioned (registry, oci_repository, oci_digest,
+						   NULL, cancellable, error);
   if (versioned == NULL)
     return FALSE;
 
@@ -2957,7 +2985,7 @@ flatpak_dir_pull_oci (FlatpakDir          *self,
 
   g_debug ("Pulling OCI image %s", oci_digest);
 
-  checksum = flatpak_pull_from_oci (repo, registry, oci_digest, FLATPAK_OCI_MANIFEST (versioned),
+  checksum = flatpak_pull_from_oci (repo, registry, oci_repository, oci_digest, FLATPAK_OCI_MANIFEST (versioned),
                                     remote, ref, signature_digest, oci_pull_progress_cb, progress, cancellable, error);
 
   if (progress)
@@ -7413,18 +7441,6 @@ flatpak_dir_cache_summary (FlatpakDir  *self,
   G_UNLOCK (cache);
 }
 
-
-static int
-compare_mdp (const void *a, const void *b)
-{
-  FlatpakOciManifestDescriptor *aa = *(FlatpakOciManifestDescriptor **)a;
-  FlatpakOciManifestDescriptor *bb = *(FlatpakOciManifestDescriptor **)b;
-  const char *ref_a = flatpak_oci_manifest_descriptor_get_ref (aa);
-  const char *ref_b = flatpak_oci_manifest_descriptor_get_ref (bb);
-
-  return g_strcmp0 (ref_a, ref_b);
-}
-
 static gboolean
 flatpak_dir_remote_make_oci_summary (FlatpakDir   *self,
                                      const char   *remote,
@@ -7432,20 +7448,12 @@ flatpak_dir_remote_make_oci_summary (FlatpakDir   *self,
                                      GCancellable *cancellable,
                                      GError      **error)
 {
-  g_autoptr(FlatpakOciRegistry) registry = NULL;
   g_autofree char *oci_uri = NULL;
-  g_autoptr(GVariantBuilder) refs_builder = NULL;
-  g_autoptr(GVariantBuilder) additional_metadata_builder = NULL;
-  g_autoptr(GVariantBuilder) summary_builder = NULL;
   g_autoptr(GVariant) summary = NULL;
-  g_autoptr(GVariantBuilder) ref_data_builder = NULL;
-  g_autoptr(FlatpakOciIndex) index = NULL;
-  int i;
   g_autoptr(GFile) cache_dir = NULL;
   g_autoptr(GFile) summary_cache = NULL;
   g_autofree char *summary_name = NULL;
   g_autofree char *cache_etag = NULL;
-  g_autofree char *new_etag = NULL;
   g_autofree char *self_name = NULL;
   g_autoptr(GError) local_error = NULL;
   g_autoptr(GMappedFile) mfile = NULL;
@@ -7475,12 +7483,10 @@ flatpak_dir_remote_make_oci_summary (FlatpakDir   *self,
       g_variant_lookup (extensions, "xa.oci-etag", "s", &cache_etag);
     }
 
-  registry = flatpak_oci_registry_new (oci_uri, FALSE, -1, NULL, error);
-  if (registry == NULL)
-    return FALSE;
+  ensure_soup_session (self);
 
-  index = flatpak_oci_registry_load_index (registry, cache_etag, &new_etag, cancellable, &local_error);
-  if (index == NULL)
+  summary = flatpak_oci_index_fetch_summary (self->soup_session, oci_uri, cache_etag, cancellable, &local_error);
+  if (summary == NULL)
     {
       if (g_error_matches (local_error, FLATPAK_OCI_ERROR, FLATPAK_OCI_ERROR_NOT_CHANGED))
         {
@@ -7493,85 +7499,6 @@ flatpak_dir_remote_make_oci_summary (FlatpakDir   *self,
       return FALSE;
     }
 
-  refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))"));
-  ref_data_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{s(tts)}"));
-  additional_metadata_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
-
-  /* The summary has to be sorted by ref, so pre-sort the manifests */
-  if (index->manifests != NULL)
-    qsort (index->manifests, flatpak_oci_index_get_n_manifests (index), sizeof (FlatpakOciManifestDescriptor *), compare_mdp);
-
-  for (i = 0; index->manifests != NULL && index->manifests[i] != NULL; i++)
-    {
-      FlatpakOciManifestDescriptor *m = index->manifests[i];
-      FlatpakOciDescriptor *d = (FlatpakOciDescriptor *)m;
-      const char *ref;
-      const char *fake_commit;
-      guint64 installed_size = 0;
-      guint64 download_size = 0;
-      const char *installed_size_str;
-      const char *download_size_str;
-      const char *signature_digest;
-      const char *metadata_contents = NULL;
-      g_autoptr(GVariantBuilder) ref_metadata_builder = NULL;
-
-      ref = flatpak_oci_manifest_descriptor_get_ref (m);
-      if (ref == NULL)
-        continue;
-
-      metadata_contents = g_hash_table_lookup (d->annotations, "org.flatpak.metadata");
-      if (metadata_contents == NULL && !g_str_has_prefix (ref, "appstream/"))
-        continue; /* Not a flatpak, skip */
-
-      if (!g_str_has_prefix (d->digest, "sha256:"))
-        {
-          g_debug ("Ignoring digest type %s", d->digest);
-          continue;
-        }
-
-      fake_commit = d->digest + strlen ("sha256:");
-
-      installed_size_str = g_hash_table_lookup (d->annotations, "org.flatpak.installed-size");
-      if (installed_size_str)
-        installed_size = g_ascii_strtoull (installed_size_str, NULL, 10);
-
-      download_size_str = g_hash_table_lookup (d->annotations, "org.flatpak.download-size");
-      if (download_size_str)
-        download_size = g_ascii_strtoull (download_size_str, NULL, 10);
-
-      ref_metadata_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
-
-      signature_digest = g_hash_table_lookup (d->annotations, "org.flatpak.signature-digest");
-      if (signature_digest)
-        g_variant_builder_add (ref_metadata_builder, "{sv}", "xa.oci-signature",
-                               g_variant_new_string (signature_digest));
-
-      g_variant_builder_add_value (refs_builder,
-                                   g_variant_new ("(s(t@ay@a{sv}))", ref,
-                                                  0,
-                                                  ostree_checksum_to_bytes_v (fake_commit),
-                                                  g_variant_builder_end (ref_metadata_builder)));
-      g_variant_builder_add (ref_data_builder, "{s(tts)}",
-                             ref,
-                             GUINT64_TO_BE (installed_size),
-                             GUINT64_TO_BE (download_size),
-                             metadata_contents ? metadata_contents : "");
-
-    }
-
-  g_variant_builder_add (additional_metadata_builder, "{sv}", "xa.cache",
-                         g_variant_new_variant (g_variant_builder_end (ref_data_builder)));
-  if (new_etag)
-    g_variant_builder_add (additional_metadata_builder, "{sv}", "xa.oci-etag",
-                           g_variant_new_string (new_etag));
-
-  summary_builder = g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT);
-
-  g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder));
-  g_variant_builder_add_value (summary_builder, g_variant_builder_end (additional_metadata_builder));
-
-  summary = g_variant_ref_sink (g_variant_builder_end (summary_builder));
-
   *out_summary = g_variant_get_data_as_bytes (summary);
 
   if (!g_file_replace_contents (summary_cache,
diff --git a/common/flatpak-json-oci.c b/common/flatpak-json-oci.c
index 3c26c215..d1a41abc 100644
--- a/common/flatpak-json-oci.c
+++ b/common/flatpak-json-oci.c
@@ -750,7 +750,10 @@ flatpak_oci_add_annotations_for_commit (GHashTable *annotations,
                                        GVariant *commit_data)
 {
   if (ref)
-    add_annotation (annotations,"org.opencontainers.image.ref.name", ref);
+    {
+      add_annotation (annotations,"org.opencontainers.image.ref.name", ref);
+      add_annotation (annotations,"org.flatpak.ref", ref);
+    }
 
   if (commit)
     add_annotation (annotations,"org.flatpak.commit", commit);
diff --git a/common/flatpak-oci-registry.c b/common/flatpak-oci-registry.c
index c0e86ebc..a073d8ab 100644
--- a/common/flatpak-oci-registry.c
+++ b/common/flatpak-oci-registry.c
@@ -46,6 +46,7 @@ struct FlatpakOciRegistry
 
   gboolean for_write;
   gboolean valid;
+  gboolean is_docker;
   char *uri;
   int tmp_dfd;
 
@@ -122,9 +123,9 @@ flatpak_oci_registry_set_property (GObject         *object,
 
 static void
 flatpak_oci_registry_get_property (GObject         *object,
-                                  guint            prop_id,
-                                  GValue          *value,
-                                  GParamSpec      *pspec)
+				   guint            prop_id,
+				   GValue          *value,
+				   GParamSpec      *pspec)
 {
   FlatpakOciRegistry *self = FLATPAK_OCI_REGISTRY (object);
 
@@ -192,10 +193,10 @@ flatpak_oci_registry_get_uri (FlatpakOciRegistry   *self)
 
 FlatpakOciRegistry *
 flatpak_oci_registry_new (const char *uri,
-                         gboolean for_write,
-                         int tmp_dfd,
-                         GCancellable *cancellable,
-                         GError **error)
+			  gboolean for_write,
+			  int tmp_dfd,
+			  GCancellable *cancellable,
+			  GError **error)
 {
   FlatpakOciRegistry *oci_registry;
 
@@ -306,7 +307,7 @@ remote_load_file (SoupSession *soup_session,
 
   uri_s = soup_uri_to_string (uri, FALSE);
   bytes = flatpak_load_http_uri (soup_session,
-                                 uri_s, etag, etag_out,
+                                 uri_s, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, etag, etag_out,
                                  NULL, NULL,
                                  cancellable, error);
   if (bytes == NULL)
@@ -353,7 +354,7 @@ parse_json (GBytes *bytes, GCancellable *cancellable, GError **error)
 }
 
 static gboolean
-verify_oci_version (GBytes *oci_layout_bytes, GCancellable *cancellable, GError **error)
+verify_oci_version (GBytes *oci_layout_bytes, gboolean *not_json, GCancellable *cancellable, GError **error)
 {
   const char *version;
   g_autoptr(JsonNode) node = NULL;
@@ -361,8 +362,12 @@ verify_oci_version (GBytes *oci_layout_bytes, GCancellable *cancellable, GError
 
   node = parse_json (oci_layout_bytes, cancellable, error);
   if (node == NULL)
-    return FALSE;
+    {
+      *not_json = TRUE;
+      return FALSE;
+    }
 
+  *not_json = FALSE;
   oci_layout = json_node_get_object (node);
 
   version = json_object_get_string_member (oci_layout, "imageLayoutVersion");
@@ -384,15 +389,16 @@ verify_oci_version (GBytes *oci_layout_bytes, GCancellable *cancellable, GError
 
 static gboolean
 flatpak_oci_registry_ensure_local (FlatpakOciRegistry *self,
-                                  gboolean for_write,
-                                  GCancellable *cancellable,
-                                  GError **error)
+				   gboolean for_write,
+				   GCancellable *cancellable,
+				   GError **error)
 {
   g_autoptr(GFile) dir = g_file_new_for_uri (self->uri);
   glnx_autofd int local_dfd = -1;
   int dfd;
   g_autoptr(GError) local_error = NULL;
   g_autoptr(GBytes) oci_layout_bytes = NULL;
+  gboolean not_json;
 
   if (self->dfd != -1)
     dfd = self->dfd;
@@ -450,7 +456,7 @@ flatpak_oci_registry_ensure_local (FlatpakOciRegistry *self,
           return FALSE;
         }
     }
-  else if (!verify_oci_version (oci_layout_bytes, cancellable, error))
+  else if (!verify_oci_version (oci_layout_bytes, &not_json, cancellable, error))
     return FALSE;
 
   if (self->dfd == -1 && local_dfd != -1)
@@ -461,9 +467,9 @@ flatpak_oci_registry_ensure_local (FlatpakOciRegistry *self,
 
 static gboolean
 flatpak_oci_registry_ensure_remote (FlatpakOciRegistry *self,
-                                   gboolean for_write,
-                                   GCancellable *cancellable,
-                                   GError **error)
+				    gboolean for_write,
+				    GCancellable *cancellable,
+				    GError **error)
 {
   g_autoptr(SoupURI) baseuri = NULL;
   g_autoptr(GBytes) oci_layout_bytes = NULL;
@@ -484,12 +490,25 @@ flatpak_oci_registry_ensure_remote (FlatpakOciRegistry *self,
       return FALSE;
     }
 
-  oci_layout_bytes = remote_load_file (self->soup_session, baseuri, "oci-layout", NULL, NULL, cancellable, error);
-  if (oci_layout_bytes == NULL)
-    return FALSE;
-
-  if (!verify_oci_version (oci_layout_bytes, cancellable, error))
-    return FALSE;
+  oci_layout_bytes = remote_load_file (self->soup_session, baseuri, "oci-layout", NULL, NULL, cancellable, NULL);
+  if (oci_layout_bytes != NULL)
+    {
+      g_autoptr(GError) local_error = NULL;
+      gboolean not_json;
+
+      if (!verify_oci_version (oci_layout_bytes, &not_json, cancellable, &local_error))
+	{
+	  if (not_json)
+	    self->is_docker = TRUE;
+	  else
+	    {
+              g_propagate_error (error, g_steal_pointer (&local_error));
+	      return FALSE;
+	    }
+	}
+    }
+  else
+    self->is_docker = TRUE;
 
   self->base_uri = g_steal_pointer (&baseuri);
 
@@ -498,8 +517,8 @@ flatpak_oci_registry_ensure_remote (FlatpakOciRegistry *self,
 
 static gboolean
 flatpak_oci_registry_initable_init (GInitable     *initable,
-                                   GCancellable  *cancellable,
-                                   GError       **error)
+				    GCancellable  *cancellable,
+				    GError       **error)
 {
   FlatpakOciRegistry *self = FLATPAK_OCI_REGISTRY (initable);
   gboolean res;
@@ -632,9 +651,14 @@ splice_update_checksum (GOutputStream  *out,
 }
 
 static char *
-get_digest_subpath (const char *digest,
+get_digest_subpath (FlatpakOciRegistry *self,
+		    const char *repository,
+		    gboolean is_manifest,
+		    const char *digest,
                     GError **error)
 {
+  g_autoptr(GString) s = g_string_new ("");
+
   if (!g_str_has_prefix (digest, "sha256:"))
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
@@ -642,7 +666,30 @@ get_digest_subpath (const char *digest,
       return NULL;
     }
 
-  return g_strdup_printf ("blobs/sha256/%s", digest + strlen ("sha256:"));
+  if (self->is_docker)
+    g_string_append (s, "v2/");
+
+  if (repository)
+    {
+      g_string_append (s, repository);
+      g_string_append (s, "/");
+    }
+
+  if (self->is_docker)
+    {
+      if (is_manifest)
+	g_string_append (s, "manifests/");
+      else
+	g_string_append (s, "blobs/");
+      g_string_append (s, digest);
+    }
+  else
+    {
+      g_string_append (s, "blobs/sha256/");
+      g_string_append (s, digest + strlen ("sha256:"));
+    }
+
+  return g_string_free (g_steal_pointer (&s), FALSE);
 }
 
 static char *
@@ -661,6 +708,8 @@ checksum_fd (int fd, GCancellable *cancellable, GError **error)
 
 int
 flatpak_oci_registry_download_blob (FlatpakOciRegistry    *self,
+				    const char            *repository,
+				    gboolean               manifest,
                                     const char            *digest,
                                     FlatpakLoadUriProgress progress_cb,
                                     gpointer               user_data,
@@ -672,7 +721,7 @@ flatpak_oci_registry_download_blob (FlatpakOciRegistry    *self,
 
   g_assert (self->valid);
 
-  subpath = get_digest_subpath (digest, error);
+  subpath = get_digest_subpath (self, repository, manifest, digest, error);
   if (subpath == NULL)
     return -1;
 
@@ -713,7 +762,9 @@ flatpak_oci_registry_download_blob (FlatpakOciRegistry    *self,
       if (fd == -1)
         return -1;
 
-      if (!flatpak_download_http_uri (self->soup_session, uri_s, out_stream,
+      if (!flatpak_download_http_uri (self->soup_session, uri_s,
+				      FLATPAK_HTTP_FLAGS_ACCEPT_OCI,
+				      out_stream,
                                       progress_cb, user_data,
                                       cancellable, error))
         return -1;
@@ -762,7 +813,7 @@ flatpak_oci_registry_mirror_blob (FlatpakOciRegistry    *self,
       return FALSE;
     }
 
-  subpath = get_digest_subpath (digest, error);
+  subpath = get_digest_subpath (self, NULL, FALSE, digest, error);
   if (subpath == NULL)
     return FALSE;
 
@@ -802,7 +853,8 @@ flatpak_oci_registry_mirror_blob (FlatpakOciRegistry    *self,
       out_stream = g_unix_output_stream_new (tmpf.fd, FALSE);
 
       uri_s = soup_uri_to_string (uri, FALSE);
-      if (!flatpak_download_http_uri (source_registry->soup_session, uri_s, out_stream,
+      if (!flatpak_download_http_uri (source_registry->soup_session, uri_s,
+				      FLATPAK_HTTP_FLAGS_ACCEPT_OCI, out_stream,
                                       progress_cb, user_data,
                                       cancellable, error))
         return FALSE;
@@ -835,9 +887,11 @@ flatpak_oci_registry_mirror_blob (FlatpakOciRegistry    *self,
 
 GBytes *
 flatpak_oci_registry_load_blob (FlatpakOciRegistry  *self,
-                               const char         *digest,
-                               GCancellable       *cancellable,
-                               GError            **error)
+				const char          *repository,
+				gboolean             manifest,
+				const char          *digest,
+				GCancellable        *cancellable,
+				GError             **error)
 {
   g_autofree char *subpath = NULL;
   g_autoptr(GBytes) bytes = NULL;
@@ -845,11 +899,13 @@ flatpak_oci_registry_load_blob (FlatpakOciRegistry  *self,
 
   g_assert (self->valid);
 
-  subpath = get_digest_subpath (digest, error);
+  subpath = get_digest_subpath (self, repository, manifest, digest, error);
   if (subpath == NULL)
     return NULL;
 
   bytes = flatpak_oci_registry_load_file (self, subpath, NULL, NULL, cancellable, error);
+  if (bytes == NULL)
+    return NULL;
 
   json_checksum = g_compute_checksum_for_bytes  (G_CHECKSUM_SHA256, bytes);
 
@@ -902,6 +958,7 @@ flatpak_oci_registry_store_json (FlatpakOciRegistry    *self,
 
 FlatpakOciVersioned *
 flatpak_oci_registry_load_versioned (FlatpakOciRegistry  *self,
+				     const char          *repository,
                                      const char          *digest,
                                      gsize               *out_size,
                                      GCancellable        *cancellable,
@@ -911,7 +968,7 @@ flatpak_oci_registry_load_versioned (FlatpakOciRegistry  *self,
 
   g_assert (self->valid);
 
-  bytes = flatpak_oci_registry_load_blob (self, digest, cancellable, error);
+  bytes = flatpak_oci_registry_load_blob (self, repository, TRUE, digest, cancellable, error);
   if (bytes == NULL)
     return NULL;
 
@@ -1820,3 +1877,188 @@ flatpak_oci_verify_signature (OstreeRepo *repo,
 
   return g_steal_pointer (&json);
 }
+
+static const char *
+get_image_ref (FlatpakOciIndexImage *img)
+{
+  const char *ref;
+
+  ref = g_hash_table_lookup (img->annotations, "org.flatpak.ref");
+  if (ref == NULL)
+    ref = g_hash_table_lookup (img->annotations, "org.opencontainers.image.ref.name");
+  return ref;
+}
+
+typedef struct {
+  char *repository;
+  FlatpakOciIndexImage *image;
+} ImageInfo;
+
+static gint
+compare_image_by_ref (ImageInfo *a,
+                      ImageInfo *b)
+{
+  const char *a_ref = get_image_ref (a->image);
+  const char *b_ref = get_image_ref (b->image);
+
+  return g_strcmp0 (a_ref, b_ref);
+}
+
+GVariant *
+flatpak_oci_index_fetch_summary (SoupSession *soup_session,
+                                 const char *uri,
+                                 const char *etag,
+                                 GCancellable *cancellable,
+                                 GError **error)
+{
+  g_autoptr(GBytes) res = NULL;
+  g_autofree char *new_etag = NULL;
+  g_autoptr(FlatpakJson) json = NULL;
+  FlatpakOciIndexResponse *response;
+  g_autoptr(SoupURI) registry_uri = NULL;
+  g_autofree char *registry_uri_s = NULL;
+  g_autoptr(SoupURI) base_uri = NULL;
+  int i;
+  g_autoptr(GArray) images = g_array_new (FALSE, TRUE, sizeof (ImageInfo));
+  g_autoptr(GVariantBuilder) refs_builder = NULL;
+  g_autoptr(GVariantBuilder) additional_metadata_builder = NULL;
+  g_autoptr(GVariantBuilder) summary_builder = NULL;
+  g_autoptr(GVariant) summary = NULL;
+  g_autoptr(GVariantBuilder) ref_data_builder = NULL;
+  g_autoptr(GString) index_uri = g_string_new (uri);
+
+  if (!g_str_has_suffix (index_uri->str, "/"))
+    g_string_append_c (index_uri, '/');
+
+  if (!g_str_has_suffix (uri, "/index/"))
+    g_string_append (index_uri, "index/");
+
+  g_string_append (index_uri, "/static?os=linux&tag=latest");
+
+  res = flatpak_load_http_uri (soup_session,
+                               index_uri->str,
+                               0, etag,
+                               &new_etag, NULL, NULL,
+                               cancellable, error);
+  if (res == NULL)
+    return NULL;
+
+  json = flatpak_json_from_bytes (res, FLATPAK_TYPE_OCI_INDEX_RESPONSE, error);
+  if (json == NULL)
+    return NULL;
+
+  response = (FlatpakOciIndexResponse *)json;
+
+  base_uri = soup_uri_new (index_uri->str);
+  registry_uri = soup_uri_new_with_base (base_uri, response->registry);
+  registry_uri_s = soup_uri_to_string (registry_uri, FALSE);
+
+  for (i = 0; response->results != NULL && response->results[i] != NULL; i++)
+    {
+      FlatpakOciIndexRepository *r = response->results[i];
+      int j;
+      ImageInfo info = { r->name };
+
+      for (j = 0; r->images != NULL && r->images[j] != NULL; j++)
+	{
+	  info.image = r->images[j];
+	  g_array_append_val (images, info);
+	}
+
+      for (j = 0; r->lists != NULL && r->lists[j] != NULL; j++)
+        {
+          FlatpakOciIndexImageList *list =  r->lists[j];
+          int k;
+
+          for (k = 0; list->images != NULL && list->images[k] != NULL; k++)
+	    {
+	      info.image = list->images[k];
+	      g_array_append_val (images, info);
+	    }
+        }
+    }
+
+  refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))"));
+  ref_data_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{s(tts)}"));
+  additional_metadata_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
+
+  /* The summary has to be sorted by ref */
+  g_array_sort (images, (GCompareFunc)compare_image_by_ref);
+
+  for (i = 0; i < images->len; i++)
+    {
+      ImageInfo *info = &g_array_index (images, ImageInfo, i);
+      FlatpakOciIndexImage *image = info->image;
+      const char *ref = get_image_ref (image);
+      const char *fake_commit;
+      guint64 installed_size = 0;
+      guint64 download_size = 0;
+      const char *installed_size_str;
+      const char *download_size_str;
+      const char *signature_digest;
+      const char *metadata_contents = NULL;
+      g_autoptr(GVariantBuilder) ref_metadata_builder = NULL;
+
+      if (ref == NULL)
+        continue;
+
+      metadata_contents = g_hash_table_lookup (image->annotations, "org.flatpak.metadata");
+      if (metadata_contents == NULL && !g_str_has_prefix (ref, "appstream/"))
+        continue; /* Not a flatpak, skip */
+
+      if (!g_str_has_prefix (image->digest, "sha256:"))
+        {
+          g_debug ("Ignoring digest type %s", image->digest);
+          continue;
+        }
+
+      fake_commit = image->digest + strlen ("sha256:");
+
+      installed_size_str = g_hash_table_lookup (image->annotations, "org.flatpak.installed-size");
+      if (installed_size_str)
+        installed_size = g_ascii_strtoull (installed_size_str, NULL, 10);
+
+      download_size_str = g_hash_table_lookup (image->annotations, "org.flatpak.download-size");
+      if (download_size_str)
+        download_size = g_ascii_strtoull (download_size_str, NULL, 10);
+
+      ref_metadata_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
+
+      g_variant_builder_add (ref_metadata_builder, "{sv}", "xa.oci-repository",
+			     g_variant_new_string (info->repository));
+
+      signature_digest = g_hash_table_lookup (image->annotations, "org.flatpak.signature-digest");
+      if (signature_digest)
+        g_variant_builder_add (ref_metadata_builder, "{sv}", "xa.oci-signature",
+                               g_variant_new_string (signature_digest));
+
+      g_variant_builder_add_value (refs_builder,
+                                   g_variant_new ("(s(t@ay@a{sv}))", ref,
+                                                  0,
+                                                  ostree_checksum_to_bytes_v (fake_commit),
+                                                  g_variant_builder_end (ref_metadata_builder)));
+      g_variant_builder_add (ref_data_builder, "{s(tts)}",
+                             ref,
+                             GUINT64_TO_BE (installed_size),
+                             GUINT64_TO_BE (download_size),
+                             metadata_contents ? metadata_contents : "");
+    }
+
+  g_variant_builder_add (additional_metadata_builder, "{sv}", "xa.cache",
+                         g_variant_new_variant (g_variant_builder_end (ref_data_builder)));
+  if (new_etag)
+    g_variant_builder_add (additional_metadata_builder, "{sv}", "xa.oci-etag",
+                           g_variant_new_string (new_etag));
+
+  g_variant_builder_add (additional_metadata_builder, "{sv}", "xa.oci-registry-uri",
+                         g_variant_new_string (registry_uri_s));
+
+  summary_builder = g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT);
+
+  g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder));
+  g_variant_builder_add_value (summary_builder, g_variant_builder_end (additional_metadata_builder));
+
+  summary = g_variant_ref_sink (g_variant_builder_end (summary_builder));
+
+  return g_steal_pointer (&summary);
+}
diff --git a/common/flatpak-oci-registry.h b/common/flatpak-oci-registry.h
index 1458b9a1..c7f854dc 100644
--- a/common/flatpak-oci-registry.h
+++ b/common/flatpak-oci-registry.h
@@ -71,12 +71,16 @@ gboolean               flatpak_oci_registry_save_index           (FlatpakOciRegi
                                                                   GCancellable         *cancellable,
                                                                   GError              **error);
 int                    flatpak_oci_registry_download_blob        (FlatpakOciRegistry   *self,
+                                                                  const char           *repository,
+								  gboolean              manifest,
                                                                   const char           *digest,
                                                                   FlatpakLoadUriProgress progress_cb,
                                                                   gpointer               user_data,
                                                                   GCancellable         *cancellable,
                                                                   GError              **error);
 GBytes             *   flatpak_oci_registry_load_blob            (FlatpakOciRegistry   *self,
+                                                                  const char           *repository,
+								  gboolean              manifest,
                                                                   const char           *digest,
                                                                   GCancellable         *cancellable,
                                                                   GError              **error);
@@ -96,6 +100,7 @@ FlatpakOciDescriptor * flatpak_oci_registry_store_json           (FlatpakOciRegi
                                                                   GCancellable         *cancellable,
                                                                   GError              **error);
 FlatpakOciVersioned *  flatpak_oci_registry_load_versioned       (FlatpakOciRegistry   *self,
+                                                                  const char           *repository,
                                                                   const char           *digest,
                                                                   gsize                *out_size,
                                                                   GCancellable         *cancellable,
@@ -126,4 +131,10 @@ FlatpakOciSignature *flatpak_oci_verify_signature (OstreeRepo *repo,
                                                    GBytes *signature,
                                                    GError **error);
 
+GVariant *flatpak_oci_index_fetch_summary (SoupSession *soup_session,
+                                           const char *uri,
+                                           const char *etag,
+                                           GCancellable *cancellable,
+                                           GError **error);
+
 #endif /* __FLATPAK_OCI_REGISTRY_H__ */
diff --git a/common/flatpak-utils.c b/common/flatpak-utils.c
index c8f8719e..b90dbcaf 100644
--- a/common/flatpak-utils.c
+++ b/common/flatpak-utils.c
@@ -4976,7 +4976,7 @@ flatpak_mirror_image_from_oci (FlatpakOciRegistry *dst_registry,
       !flatpak_oci_registry_mirror_blob (dst_registry, registry, signature_digest, NULL, NULL, cancellable, error))
     return FALSE;
 
-  versioned = flatpak_oci_registry_load_versioned (dst_registry, digest, &versioned_size, cancellable, error);
+  versioned = flatpak_oci_registry_load_versioned (dst_registry, NULL, digest, &versioned_size, cancellable, error);
   if (versioned == NULL)
     return FALSE;
 
@@ -5046,6 +5046,7 @@ flatpak_mirror_image_from_oci (FlatpakOciRegistry *dst_registry,
 char *
 flatpak_pull_from_oci (OstreeRepo   *repo,
                        FlatpakOciRegistry *registry,
+                       const char *oci_repository,
                        const char *digest,
                        FlatpakOciManifest *manifest,
                        const char *remote,
@@ -5091,7 +5092,7 @@ flatpak_pull_from_oci (OstreeRepo   *repo,
           return NULL;
         }
 
-      signature_bytes = flatpak_oci_registry_load_blob (registry, signature_digest, cancellable, error);
+      signature_bytes = flatpak_oci_registry_load_blob (registry, oci_repository, FALSE, signature_digest, cancellable, error);
       if (signature_bytes == NULL)
         return NULL;
 
@@ -5173,7 +5174,8 @@ flatpak_pull_from_oci (OstreeRepo   *repo,
       opts.autocreate_parents = TRUE;
       opts.ignore_unsupported_content = TRUE;
 
-      layer_fd = flatpak_oci_registry_download_blob (registry, layer->digest,
+      layer_fd = flatpak_oci_registry_download_blob (registry, oci_repository, FALSE,
+						     layer->digest,
                                                      oci_layer_progress, &progress_data,
                                                      cancellable, error);
       if (layer_fd == -1)
@@ -5635,6 +5637,7 @@ flatpak_create_soup_session (const char *user_agent)
 GBytes *
 flatpak_load_http_uri (SoupSession *soup_session,
                        const char *uri,
+		       FlatpakHTTPFlags flags,
                        const char *etag,
                        char      **out_etag,
                        FlatpakLoadUriProgress progress,
@@ -5648,6 +5651,7 @@ flatpak_load_http_uri (SoupSession *soup_session,
   g_autoptr(GMainLoop) loop = NULL;
   g_autoptr(GString) content = g_string_new ("");
   LoadUriData data = { NULL };
+  SoupMessage *m;
 
   g_debug ("Loading %s using libsoup", uri);
 
@@ -5666,11 +5670,13 @@ flatpak_load_http_uri (SoupSession *soup_session,
   if (request == NULL)
     return NULL;
 
+  m = soup_request_http_get_message (request);
   if (etag)
-    {
-      SoupMessage *m = soup_request_http_get_message (request);
-      soup_message_headers_replace (m->request_headers, "If-None-Match", etag);
-    }
+    soup_message_headers_replace (m->request_headers, "If-None-Match", etag);
+
+  if (flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI)
+    soup_message_headers_replace (m->request_headers, "Accept",
+				  "application/vnd.oci.image.manifest.v1+json");
 
   soup_request_send_async (SOUP_REQUEST(request),
                            cancellable,
@@ -5699,6 +5705,7 @@ flatpak_load_http_uri (SoupSession *soup_session,
 gboolean
 flatpak_download_http_uri (SoupSession *soup_session,
                            const char   *uri,
+			   FlatpakHTTPFlags flags,
                            GOutputStream *out,
                            FlatpakLoadUriProgress progress,
                            gpointer      user_data,
@@ -5709,6 +5716,7 @@ flatpak_download_http_uri (SoupSession *soup_session,
   g_autoptr(GMainLoop) loop = NULL;
   g_autoptr(GMainContext) context = NULL;
   LoadUriData data = { NULL };
+  SoupMessage *m;
 
   g_debug ("Loading %s using libsoup", uri);
 
@@ -5727,6 +5735,11 @@ flatpak_download_http_uri (SoupSession *soup_session,
   if (request == NULL)
     return FALSE;
 
+  m = soup_request_http_get_message (request);
+  if (flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI)
+    soup_message_headers_replace (m->request_headers, "Accept",
+				  "application/vnd.oci.image.manifest.v1+json");
+
   soup_request_send_async (SOUP_REQUEST(request),
                            cancellable,
                            load_uri_callback, &data);
diff --git a/common/flatpak-utils.h b/common/flatpak-utils.h
index 38de0ee9..131b62e3 100644
--- a/common/flatpak-utils.h
+++ b/common/flatpak-utils.h
@@ -375,6 +375,7 @@ typedef void (*FlatpakOciPullProgress) (guint64 total_size, guint64 pulled_size,
 
 char * flatpak_pull_from_oci (OstreeRepo   *repo,
                               FlatpakOciRegistry *registry,
+			      const char *oci_repository,
                               const char *digest,
                               FlatpakOciManifest *manifest,
                               const char *remote,
@@ -643,8 +644,15 @@ gboolean flatpak_yes_no_prompt (const char *prompt, ...) G_GNUC_PRINTF(1, 2);
 long flatpak_number_prompt (int min, int max, const char *prompt, ...) G_GNUC_PRINTF(3, 4);
 
 SoupSession * flatpak_create_soup_session (const char *user_agent);
+
+typedef enum {
+  FLATPAK_HTTP_FLAGS_NONE = 0,
+  FLATPAK_HTTP_FLAGS_ACCEPT_OCI = 1<<0,
+} FlatpakHTTPFlags;
+
 GBytes * flatpak_load_http_uri (SoupSession *soup_session,
                                 const char   *uri,
+				FlatpakHTTPFlags flags,
                                 const char   *etag,
                                 char        **out_etag,
                                 FlatpakLoadUriProgress progress,
@@ -653,6 +661,7 @@ GBytes * flatpak_load_http_uri (SoupSession *soup_session,
                                 GError      **error);
 gboolean flatpak_download_http_uri (SoupSession *soup_session,
                                     const char   *uri,
+				    FlatpakHTTPFlags flags,
                                     GOutputStream *out,
                                     FlatpakLoadUriProgress progress,
                                     gpointer      user_data,
diff --git a/system-helper/flatpak-system-helper.c b/system-helper/flatpak-system-helper.c
index df0994aa..96ec7333 100644
--- a/system-helper/flatpak-system-helper.c
+++ b/system-helper/flatpak-system-helper.c
@@ -258,7 +258,7 @@ handle_deploy (FlatpakSystemHelper   *object,
 
       signature_digest = g_hash_table_lookup (desc->parent.annotations, "org.flatpak.signature-digest");
 
-      versioned = flatpak_oci_registry_load_versioned (registry, desc->parent.digest, NULL,
+      versioned = flatpak_oci_registry_load_versioned (registry, NULL, desc->parent.digest, NULL,
                                                        NULL, &error);
       if (versioned == NULL || !FLATPAK_IS_OCI_MANIFEST (versioned))
         {
@@ -267,7 +267,7 @@ handle_deploy (FlatpakSystemHelper   *object,
           return TRUE;
         }
 
-      checksum = flatpak_pull_from_oci (flatpak_dir_get_repo (system), registry, desc->parent.digest, FLATPAK_OCI_MANIFEST (versioned),
+      checksum = flatpak_pull_from_oci (flatpak_dir_get_repo (system), registry, NULL, desc->parent.digest, FLATPAK_OCI_MANIFEST (versioned),
                                         arg_origin, arg_ref, signature_digest, NULL, NULL, NULL, &error);
       if (checksum == NULL)
         {
@@ -443,7 +443,7 @@ handle_deploy_appstream (FlatpakSystemHelper   *object,
 
       signature_digest = g_hash_table_lookup (desc->parent.annotations, "org.flatpak.signature-digest");
 
-      versioned = flatpak_oci_registry_load_versioned (registry, desc->parent.digest, NULL,
+      versioned = flatpak_oci_registry_load_versioned (registry, NULL, desc->parent.digest, NULL,
                                                        NULL, &error);
       if (versioned == NULL || !FLATPAK_IS_OCI_MANIFEST (versioned))
         {
@@ -452,7 +452,7 @@ handle_deploy_appstream (FlatpakSystemHelper   *object,
           return TRUE;
         }
 
-      checksum = flatpak_pull_from_oci (flatpak_dir_get_repo (system), registry, desc->parent.digest, FLATPAK_OCI_MANIFEST (versioned),
+      checksum = flatpak_pull_from_oci (flatpak_dir_get_repo (system), registry, NULL, desc->parent.digest, FLATPAK_OCI_MANIFEST (versioned),
                                         arg_origin, branch, signature_digest, NULL, NULL, NULL, &error);
       if (checksum == NULL)
         {