summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOpenShift Merge Robot <openshift-merge-robot@users.noreply.github.com>2020-10-23 09:05:24 -0400
committerGitHub <noreply@github.com>2020-10-23 09:05:24 -0400
commit6d64477c8de9ecb6881afbd4c9881170f1d2b7a9 (patch)
treef1aa9ba629a58813607633d344876dce7c4291ed
parentf8f6464580574b40e0e3bccf7a60a5ec6cf8c105 (diff)
parent654f3d959a4563ccf7d36182f4ee7a25a70fa016 (diff)
downloadostree-6d64477c8de9ecb6881afbd4c9881170f1d2b7a9.tar.gz
Merge pull request #2188 from alexlarsson/delta-indexes
Add indexes for deltas outside of the summary
-rw-r--r--Makefile-libostree.am6
-rw-r--r--apidoc/ostree-sections.txt3
-rw-r--r--man/ostree.repo-config.xml14
-rw-r--r--src/libostree/libostree-devel.sym9
-rw-r--r--src/libostree/ostree-core-private.h3
-rw-r--r--src/libostree/ostree-core.c30
-rw-r--r--src/libostree/ostree-repo-private.h1
-rw-r--r--src/libostree/ostree-repo-pull-private.h5
-rw-r--r--src/libostree/ostree-repo-pull.c1205
-rw-r--r--src/libostree/ostree-repo-static-delta-core.c291
-rw-r--r--src/libostree/ostree-repo-static-delta-private.h5
-rw-r--r--src/libostree/ostree-repo.c121
-rw-r--r--src/libostree/ostree-repo.h33
-rw-r--r--src/ostree/ot-builtin-static-delta.c57
-rw-r--r--tests/pull-test.sh23
-rwxr-xr-xtests/test-delta.sh48
16 files changed, 1329 insertions, 525 deletions
diff --git a/Makefile-libostree.am b/Makefile-libostree.am
index bdc47122..eeb0b6c6 100644
--- a/Makefile-libostree.am
+++ b/Makefile-libostree.am
@@ -184,9 +184,9 @@ libostree_1_la_SOURCES += \
endif # USE_GPGME
symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
-#if BUILDOPT_IS_DEVEL_BUILD
-#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
-#endif
+if BUILDOPT_IS_DEVEL_BUILD
+symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
+endif
# http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
wl_versionscript_arg = -Wl,--version-script=
EXTRA_DIST += \
diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt
index c1d6b35e..64bc68d2 100644
--- a/apidoc/ostree-sections.txt
+++ b/apidoc/ostree-sections.txt
@@ -412,6 +412,8 @@ OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE
ostree_repo_list_objects
ostree_repo_list_commit_objects_starting_with
ostree_repo_list_static_delta_names
+ostree_repo_list_static_delta_indexes
+ostree_repo_static_delta_reindex
OstreeStaticDeltaGenerateOpt
ostree_repo_static_delta_generate
ostree_repo_static_delta_execute_offline_with_signature
@@ -445,6 +447,7 @@ ostree_repo_pull_default_console_progress_changed
ostree_repo_sign_commit
ostree_repo_append_gpg_signature
ostree_repo_add_gpg_signature_summary
+ostree_repo_gpg_sign_data
ostree_repo_gpg_verify_data
ostree_repo_verify_commit
ostree_repo_verify_commit_ext
diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml
index 7a01fc01..e4984430 100644
--- a/man/ostree.repo-config.xml
+++ b/man/ostree.repo-config.xml
@@ -249,6 +249,20 @@ Boston, MA 02111-1307, USA.
costly).
</para></listitem>
</varlistentry>
+
+ <varlistentry>
+ <term><varname>no-deltas-in-summary</varname></term>
+ <listitem><para>Boolean value controlling whether OSTree should skip
+ putting an index of available deltas in the summary file. Defaults to false.
+ </para>
+ <para>
+ Since 2020.7 OSTree can use delta indexes outside the summary file,
+ making the summary file smaller (especially for larger repositories). However
+ by default we still create the index in the summary file to make older clients
+ work. If you know all clients will be 2020.7 later you can enable this to
+ save network bandwidth.
+ </para></listitem>
+ </varlistentry>
</variablelist>
</refsect1>
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym
index 8e8d9c81..435be190 100644
--- a/src/libostree/libostree-devel.sym
+++ b/src/libostree/libostree-devel.sym
@@ -17,9 +17,12 @@
Boston, MA 02111-1307, USA.
***/
-/* Copy the bits below and uncomment the include in Makefile-libostree.am
- when adding a symbol.
-*/
+LIBOSTREE_2020.8 {
+global:
+ ostree_repo_list_static_delta_indexes;
+ ostree_repo_static_delta_reindex;
+ ostree_repo_gpg_sign_data;
+} LIBOSTREE_2020.7;
/* Stub section for the stable release *after* this development one; don't
* edit this other than to update the year. This is just a copy/paste
diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h
index 5d9d948a..89d95ad9 100644
--- a/src/libostree/ostree-core-private.h
+++ b/src/libostree/ostree-core-private.h
@@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char *from,
const char *to,
guint i);
+char *
+_ostree_get_relative_static_delta_index_path (const char *to);
+
static inline char * _ostree_get_commitpartial_path (const char *checksum)
{
return g_strconcat ("state/", checksum, ".commitpartial", NULL);
diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index 29528fa5..f822101a 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char *checksum,
return g_string_free (path, FALSE);
}
-char *
-_ostree_get_relative_static_delta_path (const char *from,
- const char *to,
- const char *target)
+static GString *
+static_delta_path_base (const char *dir,
+ const char *from,
+ const char *to)
{
guint8 csum_to[OSTREE_SHA256_DIGEST_LEN];
char to_b64[44];
guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN];
- GString *ret = g_string_new ("deltas/");
+ GString *ret = g_string_new (dir);
ostree_checksum_inplace_to_bytes (to, csum_to);
ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
@@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from,
g_string_append_c (ret, '/');
g_string_append (ret, to_b64 + 2);
+ return ret;
+}
+
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+ const char *to,
+ const char *target)
+{
+ GString *ret = static_delta_path_base ("deltas/", from, to);
+
if (target != NULL)
{
g_string_append_c (ret, '/');
@@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char *from,
return _ostree_get_relative_static_delta_path (from, to, partstr);
}
+char *
+_ostree_get_relative_static_delta_index_path (const char *to)
+{
+ GString *ret = static_delta_path_base ("delta-indexes/", NULL, to);
+
+ g_string_append (ret, ".index");
+
+ return g_string_free (ret, FALSE);
+}
+
gboolean
_ostree_parse_delta_name (const char *delta_name,
char **out_from,
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index a48feca4..cbbe6971 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -57,6 +57,7 @@ G_BEGIN_DECLS
#define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map"
#define OSTREE_SUMMARY_MODE "ostree.summary.mode"
#define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits"
+#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas"
#define _OSTREE_PAYLOAD_LINK_PREFIX "../"
#define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1)
diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h
index a7ea85cd..a827557a 100644
--- a/src/libostree/ostree-repo-pull-private.h
+++ b/src/libostree/ostree-repo-pull-private.h
@@ -78,7 +78,9 @@ typedef struct {
char *summary_sig_etag;
guint64 summary_sig_last_modified; /* seconds since the epoch */
GVariant *summary;
- GHashTable *summary_deltas_checksums;
+ GHashTable *summary_deltas_checksums; /* Filled from summary and delta indexes */
+ gboolean summary_has_deltas; /* True if the summary existed and had a delta index */
+ gboolean has_indexed_deltas;
GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
GHashTable *verified_commits; /* Set<checksum> of commits that have been verified */
GHashTable *signapi_verified_commits; /* Map<checksum,verification> of commits that have been signapi verified */
@@ -93,6 +95,7 @@ typedef struct {
GHashTable *requested_fallback_content; /* Maps checksum to itself */
GHashTable *pending_fetch_metadata; /* Map<ObjectName,FetchObjectData> */
GHashTable *pending_fetch_content; /* Map<checksum,FetchObjectData> */
+ GHashTable *pending_fetch_delta_indexes; /* Set<FetchDeltaIndexData> */
GHashTable *pending_fetch_delta_superblocks; /* Set<FetchDeltaSuperData> */
GHashTable *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
guint n_outstanding_metadata_fetches;
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index bb143136..da421db3 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -104,6 +104,14 @@ typedef struct {
guint n_retries_remaining;
} FetchDeltaSuperData;
+typedef struct {
+ OtPullData *pull_data;
+ char *from_revision;
+ char *to_revision;
+ OstreeCollectionRef *requested_ref; /* (nullable) */
+ guint n_retries_remaining;
+} FetchDeltaIndexData;
+
static void
variant_or_null_unref (gpointer data)
{
@@ -116,6 +124,8 @@ static void start_fetch_deltapart (OtPullData *pull_data,
FetchStaticDeltaData *fetch);
static void start_fetch_delta_superblock (OtPullData *pull_data,
FetchDeltaSuperData *fetch_data);
+static void start_fetch_delta_index (OtPullData *pull_data,
+ FetchDeltaIndexData *fetch_data);
static gboolean fetcher_queue_is_full (OtPullData *pull_data);
static void queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum,
@@ -135,6 +145,8 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da
static void enqueue_one_object_request_s (OtPullData *pull_data,
FetchObjectData *fetch_data);
+static void enqueue_one_static_delta_index_request_s (OtPullData *pull_data,
+ FetchDeltaIndexData *fetch_data);
static void enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data,
FetchDeltaSuperData *fetch_data);
static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data,
@@ -149,6 +161,11 @@ static gboolean scan_one_metadata_object (OtPullData *pull_data,
GCancellable *cancellable,
GError **error);
static void scan_object_queue_data_free (ScanObjectQueueData *scan_data);
+static gboolean initiate_delta_request (OtPullData *pull_data,
+ const OstreeCollectionRef *ref,
+ const char *to_revision,
+ const char *delta_from_revision,
+ GError **error);
static gboolean
update_progress (gpointer user_data)
@@ -287,6 +304,7 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);
g_queue_clear (&pull_data->scan_object_queue);
g_hash_table_remove_all (pull_data->pending_fetch_metadata);
+ g_hash_table_remove_all (pull_data->pending_fetch_delta_indexes);
g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks);
g_hash_table_remove_all (pull_data->pending_fetch_deltaparts);
g_hash_table_remove_all (pull_data->pending_fetch_content);
@@ -320,6 +338,16 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
g_variant_unref (objname);
}
+ /* Next, process delta index requests */
+ g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_indexes);
+ while (!fetcher_queue_is_full (pull_data) &&
+ g_hash_table_iter_next (&hiter, &key, &value))
+ {
+ FetchDeltaIndexData *fetch = key;
+ g_hash_table_iter_steal (&hiter);
+ start_fetch_delta_index (pull_data, g_steal_pointer (&fetch));
+ }
+
/* Next, process delta superblock requests */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks);
while (!fetcher_queue_is_full (pull_data) &&
@@ -2117,6 +2145,7 @@ start_fetch_deltapart (OtPullData *pull_data,
FetchStaticDeltaData *fetch)
{
g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i);
+ g_debug ("starting fetch of deltapart %s", deltapart_path);
pull_data->n_outstanding_deltapart_fetches++;
g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
@@ -2470,6 +2499,16 @@ fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data)
}
static void
+fetch_delta_index_data_free (FetchDeltaIndexData *fetch_data)
+{
+ g_free (fetch_data->from_revision);
+ g_free (fetch_data->to_revision);
+ if (fetch_data->requested_ref)
+ ostree_collection_ref_free (fetch_data->requested_ref);
+ g_free (fetch_data);
+}
+
+static void
set_required_deltas_error (GError **error,
const char *from_revision,
const char *to_revision)
@@ -2570,6 +2609,7 @@ start_fetch_delta_superblock (OtPullData *pull_data,
g_autofree char *delta_name =
_ostree_get_relative_static_delta_superblock_path (fetch_data->from_revision,
fetch_data->to_revision);
+ g_debug ("starting fetch of delta superblock %s", delta_name);
_ostree_fetcher_request_to_membuf (pull_data->fetcher,
pull_data->content_mirrorlist,
delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
@@ -2630,6 +2670,147 @@ validate_variant_is_csum (GVariant *csum,
}
static gboolean
+collect_available_deltas_for_pull (OtPullData *pull_data,
+ GVariant *deltas,
+ GError **error)
+{
+ gsize n;
+
+ n = deltas ? g_variant_n_children (deltas) : 0;
+ for (gsize i = 0; i < n; i++)
+ {
+ const char *delta;
+ g_autoptr(GVariant) csum_v = NULL;
+ g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i);
+
+ g_variant_get_child (ref, 0, "&s", &delta);
+ g_variant_get_child (ref, 1, "v", &csum_v);
+
+ if (!validate_variant_is_csum (csum_v, error))
+ return FALSE;
+
+ guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN);
+ memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32);
+ g_hash_table_insert (pull_data->summary_deltas_checksums,
+ g_strdup (delta),
+ csum_data);
+ }
+
+ return TRUE;
+}
+
+static void
+on_delta_index_fetched (GObject *src,
+ GAsyncResult *res,
+ gpointer data)
+
+{
+ FetchDeltaIndexData *fetch_data = data;
+ OtPullData *pull_data = fetch_data->pull_data;
+ g_autoptr(GError) local_error = NULL;
+ GError **error = &local_error;
+ g_autoptr(GBytes) delta_index_data = NULL;
+ const char *from_revision = fetch_data->from_revision;
+ const char *to_revision = fetch_data->to_revision;
+
+ if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
+ res,
+ &delta_index_data,
+ NULL, NULL, NULL,
+ error))
+ {
+ if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+ goto out;
+ g_clear_error (&local_error);
+
+ /* below call to initiate_delta_request() will fail finding the delta and fall back to commit */
+ }
+ else
+ {
+ g_autoptr(GVariant) delta_index = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, delta_index_data, FALSE));
+ g_autoptr(GVariant) deltas = g_variant_lookup_value (delta_index, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
+
+ if (!collect_available_deltas_for_pull (pull_data, deltas, error))
+ goto out;
+ }
+
+ if (!initiate_delta_request (pull_data,
+ fetch_data->requested_ref,
+ to_revision,
+ from_revision,
+ &local_error))
+ goto out;
+
+ out:
+ g_assert (pull_data->n_outstanding_metadata_fetches > 0);
+ pull_data->n_outstanding_metadata_fetches--;
+
+ if (local_error == NULL)
+ pull_data->n_fetched_metadata++;
+
+ if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+ enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fetch_data));
+ else
+ check_outstanding_requests_handle_error (pull_data, &local_error);
+
+ g_clear_pointer (&fetch_data, fetch_delta_index_data_free);
+}
+
+static void
+start_fetch_delta_index (OtPullData *pull_data,
+ FetchDeltaIndexData *fetch_data)
+{
+ g_autofree char *delta_name =
+ _ostree_get_relative_static_delta_index_path (fetch_data->to_revision);
+ g_debug ("starting fetch of delta index %s", delta_name);
+ _ostree_fetcher_request_to_membuf (pull_data->fetcher,
+ pull_data->content_mirrorlist,
+ delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ NULL, 0,
+ OSTREE_MAX_METADATA_SIZE,
+ 0, pull_data->cancellable,
+ on_delta_index_fetched,
+ g_steal_pointer (&fetch_data));
+ pull_data->n_outstanding_metadata_fetches++;
+ pull_data->n_requested_metadata++;
+}
+
+static void
+enqueue_one_static_delta_index_request_s (OtPullData *pull_data,
+ FetchDeltaIndexData *fetch_data)
+{
+ if (fetcher_queue_is_full (pull_data))
+ {
+ g_debug ("queuing fetch of static delta index to %s",
+ fetch_data->to_revision);
+
+ g_hash_table_add (pull_data->pending_fetch_delta_indexes,
+ g_steal_pointer (&fetch_data));
+ }
+ else
+ {
+ start_fetch_delta_index (pull_data, g_steal_pointer (&fetch_data));
+ }
+}
+
+/* Start a request for a static delta index */
+static void
+enqueue_one_static_delta_index_request (OtPullData *pull_data,
+ const char *to_revision,
+ const char *from_revision,
+ const OstreeCollectionRef *ref)
+{
+ FetchDeltaIndexData *fdata = g_new0(FetchDeltaIndexData, 1);
+ fdata->pull_data = pull_data;
+ fdata->from_revision = g_strdup (from_revision);
+ fdata->to_revision = g_strdup (to_revision);
+ fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
+ fdata->n_retries_remaining = pull_data->n_network_retries;
+
+ enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fdata));
+}
+
+static gboolean
_ostree_repo_verify_summary (OstreeRepo *self,
const char *name,
gboolean gpg_verify_summary,
@@ -3259,6 +3440,65 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name,
return TRUE;
}
+static gboolean
+initiate_delta_request (OtPullData *pull_data,
+ const OstreeCollectionRef *ref,
+ const char *to_revision,
+ const char *delta_from_revision,
+ GError **error)
+{
+ DeltaSearchResult deltares;
+
+ /* Look for a delta to @to_revision in the summary data */
+ if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares,
+ pull_data->cancellable, error))
+ return FALSE;
+
+ switch (deltares.result)
+ {
+ case DELTA_SEARCH_RESULT_NO_MATCH:
+ {
+ if (pull_data->require_static_deltas) /* No deltas found; are they required? */
+ {
+ set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision);
+ return FALSE;
+ }
+ else /* No deltas, fall back to object fetches. */
+ queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
+ }
+ break;
+ case DELTA_SEARCH_RESULT_FROM:
+ enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref);
+ break;
+ case DELTA_SEARCH_RESULT_SCRATCH:
+ {
+ /* If a from-scratch delta is available, we don’t want to use it if
+ * the ref already exists locally, since we are likely only a few
+ * commits out of date; so doing an object pull is likely more
+ * bandwidth efficient. */
+ if (delta_from_revision != NULL)
+ queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
+ else
+ enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref);
+ }
+ break;
+ case DELTA_SEARCH_RESULT_UNCHANGED:
+ {
+ /* If we already have the commit, here things get a little special; we've historically
+ * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas
+ * path, we don't, under the assumption the user wants as little network traffic as
+ * possible.
+ */
+ if (pull_data->require_static_deltas)
+ break;
+ else
+ queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
+ }
+ }
+
+ return TRUE;
+}
+
/*
* initiate_request:
* @ref: Optional ref name and collection ID
@@ -3298,57 +3538,17 @@ initiate_request (OtPullData *pull_data,
return FALSE;
}
- /* If we have a summary, we can use the newer logic */
- if (pull_data->summary)
+ /* If we have a summary or delta index, we can use the newer logic.
+ * We prefer the index as it might have more deltas than the summary
+ * (i.e. leave some deltas out of summary to make it smaller). */
+ if (pull_data->has_indexed_deltas)
{
- DeltaSearchResult deltares;
-
- /* Look for a delta to @to_revision in the summary data */
- if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares,
- pull_data->cancellable, error))
+ enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref);
+ }
+ else if (pull_data->summary_has_deltas)
+ {
+ if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error))
return FALSE;
-
- switch (deltares.result)
- {
- case DELTA_SEARCH_RESULT_NO_MATCH:
- {
- if (pull_data->require_static_deltas) /* No deltas found; are they required? */
- {
- set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision);
- return FALSE;
- }
- else /* No deltas, fall back to object fetches. */
- queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
- }
- break;
- case DELTA_SEARCH_RESULT_FROM:
- enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref);
- break;
- case DELTA_SEARCH_RESULT_SCRATCH:
- {
- /* If a from-scratch delta is available, we don’t want to use it if
- * the ref already exists locally, since we are likely only a few
- * commits out of date; so doing an object pull is likely more
- * bandwidth efficient. */
- if (delta_from_revision != NULL)
- queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
- else
- enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref);
- }
- break;
- case DELTA_SEARCH_RESULT_UNCHANGED:
- {
- /* If we already have the commit, here things get a little special; we've historically
- * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas
- * path, we don't, under the assumption the user wants as little network traffic as
- * possible.
- */
- if (pull_data->require_static_deltas)
- break;
- else
- queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
- }
- }
}
else if (ref != NULL)
{
@@ -3392,6 +3592,20 @@ initiate_request (OtPullData *pull_data,
return TRUE;
}
+static gboolean
+all_requested_refs_have_commit (GHashTable *requested_refs /* (element-type OstreeCollectionRef utf8) */)
+{
+ GLNX_HASH_TABLE_FOREACH_KV (requested_refs, const OstreeCollectionRef*, ref,
+ const char*, override_commitid)
+ {
+ /* Note: "" override means whatever is latest */
+ if (override_commitid == NULL || *override_commitid == 0)
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
/* ------------------------------------------------------------------------------------------
* Below is the libsoup-invariant API; these should match
* the stub functions in the #else clause
@@ -3497,9 +3711,11 @@ ostree_repo_pull_with_options (OstreeRepo *self,
gboolean opt_ref_keyring_map_set = FALSE;
gboolean disable_sign_verify = FALSE;
gboolean disable_sign_verify_summary = FALSE;
+ gboolean need_summary = FALSE;
const char *main_collection_id = NULL;
const char *url_override = NULL;
gboolean inherit_transaction = FALSE;
+ gboolean require_summary_for_mirror = FALSE;
g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */
gsize i;
g_autofree char **opt_localcache_repos = NULL;
@@ -3511,6 +3727,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
*/
const char *the_ref_to_fetch = NULL;
OstreeRepoTransactionStats tstats = { 0, };
+ gboolean remote_mode_loaded = FALSE;
/* Default */
pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE;
@@ -3657,6 +3874,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref,
(GDestroyNotify)fetch_object_data_free);
+ pull_data->pending_fetch_delta_indexes = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_index_data_free, NULL);
pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL);
pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
@@ -3933,424 +4151,509 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->is_commit_only)
pull_data->disable_static_deltas = TRUE;
- pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
-
- {
- g_autoptr(GBytes) bytes_sig = NULL;
- gboolean summary_sig_not_modified = FALSE;
- g_autofree char *summary_sig_etag = NULL;
- guint64 summary_sig_last_modified = 0;
- gsize n;
- g_autoptr(GVariant) refs = NULL;
- g_autoptr(GVariant) deltas = NULL;
- g_autoptr(GVariant) additional_metadata = NULL;
- gboolean summary_from_cache = FALSE;
- gboolean remote_mode_loaded = FALSE;
- gboolean tombstone_commits = FALSE;
-
- if (summary_sig_bytes_v)
- {
- /* Must both be specified */
- g_assert (summary_bytes_v);
+ /* Compute the set of collection-refs (and optional commit id) to fetch */
- bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v);
- bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v);
+ if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches)
+ {
+ require_summary_for_mirror = TRUE;
+ }
+ else if (opt_collection_refs_set)
+ {
+ const gchar *collection_id, *ref_name, *checksum;
- if (!bytes_sig || !bytes_summary)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "summary-bytes or summary-sig-bytes set to invalid value");
+ while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum))
+ {
+ if (!ostree_validate_rev (ref_name, error))
goto out;
- }
+ g_hash_table_insert (requested_refs_to_fetch,
+ ostree_collection_ref_new (collection_id, ref_name),
+ (*checksum != '\0') ? g_strdup (checksum) : NULL);
+ }
+ }
+ else if (refs_to_fetch != NULL)
+ {
+ char **strviter = refs_to_fetch;
+ char **commitid_strviter = override_commit_ids ?: NULL;
- g_debug ("Loaded %s summary from options", remote_name_or_baseurl);
- }
+ while (*strviter)
+ {
+ const char *branch = *strviter;
- if (!bytes_sig)
- {
- g_autofree char *summary_sig_if_none_match = NULL;
- guint64 summary_sig_if_modified_since = 0;
-
- /* Load the summary.sig from the network, but send its ETag and
- * Last-Modified from the on-disk cache (if it exists) to reduce the
- * download size if nothing’s changed. */
- _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig",
- &summary_sig_if_none_match, &summary_sig_if_modified_since);
-
- g_clear_pointer (&summary_sig_etag, g_free);
- summary_sig_last_modified = 0;
- if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
- pull_data->meta_mirrorlist,
- "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
- summary_sig_if_none_match, summary_sig_if_modified_since,
- pull_data->n_network_retries,
- &bytes_sig,
- &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error))
+ if (ostree_validate_checksum_string (branch, NULL))
+ {
+ char *key = g_strdup (branch);
+ g_hash_table_add (commits_to_fetch, key);
+ }
+ else
+ {
+ if (!ostree_validate_rev (branch, error))
+ goto out;
+ char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL;
+ g_hash_table_insert (requested_refs_to_fetch,
+ ostree_collection_ref_new (NULL, branch), commitid);
+ }
+
+ strviter++;
+ if (commitid_strviter)
+ commitid_strviter++;
+ }
+ }
+ else
+ {
+ char **branches_iter;
+
+ branches_iter = configured_branches;
+
+ if (!(branches_iter && *branches_iter))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No configured branches for remote %s", remote_name_or_baseurl);
goto out;
+ }
+ for (;branches_iter && *branches_iter; branches_iter++)
+ {
+ const char *branch = *branches_iter;
- /* The server returned HTTP status 304 Not Modified, so we’re clear to
- * load summary.sig from the cache. Also load summary, since
- * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
- if (summary_sig_not_modified)
- {
- g_clear_pointer (&bytes_sig, g_bytes_unref);
- g_clear_pointer (&bytes_summary, g_bytes_unref);
- if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig",
- &bytes_sig,
- cancellable, error))
- goto out;
+ g_hash_table_insert (requested_refs_to_fetch,
+ ostree_collection_ref_new (NULL, branch), NULL);
+ }
+ }
- if (!bytes_summary &&
- !pull_data->remote_repo_local &&
- !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
- &bytes_summary,
- cancellable, error))
- goto out;
- }
- }
+ /* Deltas are necessary when mirroring or resolving a requested ref to a commit.
+ * We try to avoid loading the potentially large summary if it is not needed. */
+ need_summary = require_summary_for_mirror || !all_requested_refs_have_commit (requested_refs_to_fetch) || summary_sig_bytes_v != NULL;
- if (bytes_sig &&
- !bytes_summary &&
- !pull_data->remote_repo_local &&
- !_ostree_repo_load_cache_summary_if_same_sig (self,
- remote_name_or_baseurl,
- bytes_sig,
- &bytes_summary,
- cancellable,
- error))
- goto out;
+ /* If we don't have indexed deltas, we need the summary for deltas, so check
+ * the config file for support.
+ * NOTE: Avoid download if we don't need deltas */
+ if (!need_summary && !pull_data->disable_static_deltas)
+ {
+ if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
+ goto out;
- if (bytes_summary && !summary_bytes_v)
- {
- g_debug ("Loaded %s summary from cache", remote_name_or_baseurl);
- summary_from_cache = TRUE;
- }
+ /* Check if remote has delta indexes outside summary */
+ if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "indexed-deltas", FALSE,
+ &pull_data->has_indexed_deltas, error))
+ goto out;
- if (!pull_data->summary && !bytes_summary)
- {
- g_autofree char *summary_if_none_match = NULL;
- guint64 summary_if_modified_since = 0;
+ if (!pull_data->has_indexed_deltas)
+ need_summary = TRUE;
+ }
- _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL,
- &summary_if_none_match, &summary_if_modified_since);
+ pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
- g_clear_pointer (&summary_etag, g_free);
- summary_last_modified = 0;
- if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
- pull_data->meta_mirrorlist,
- "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
- summary_if_none_match, summary_if_modified_since,
- pull_data->n_network_retries,
+ if (need_summary)
+ {
+ g_autoptr(GBytes) bytes_sig = NULL;
+ gboolean summary_sig_not_modified = FALSE;
+ g_autofree char *summary_sig_etag = NULL;
+ guint64 summary_sig_last_modified = 0;
+ gsize n;
+ g_autoptr(GVariant) refs = NULL;
+ g_autoptr(GVariant) deltas = NULL;
+ g_autoptr(GVariant) additional_metadata = NULL;
+ gboolean summary_from_cache = FALSE;
+ gboolean tombstone_commits = FALSE;
+
+ if (summary_sig_bytes_v)
+ {
+ /* Must both be specified */
+ g_assert (summary_bytes_v);
+
+ bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v);
+ bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v);
+
+ if (!bytes_sig || !bytes_summary)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "summary-bytes or summary-sig-bytes set to invalid value");
+ goto out;
+ }
+
+ g_debug ("Loaded %s summary from options", remote_name_or_baseurl);
+ }
+
+ if (!bytes_sig)
+ {
+ g_autofree char *summary_sig_if_none_match = NULL;
+ guint64 summary_sig_if_modified_since = 0;
+
+ /* Load the summary.sig from the network, but send its ETag and
+ * Last-Modified from the on-disk cache (if it exists) to reduce the
+ * download size if nothing’s changed. */
+ _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig",
+ &summary_sig_if_none_match, &summary_sig_if_modified_since);
+
+ g_clear_pointer (&summary_sig_etag, g_free);
+ summary_sig_last_modified = 0;
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ summary_sig_if_none_match, summary_sig_if_modified_since,
+ pull_data->n_network_retries,
+ &bytes_sig,
+ &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
+ goto out;
+
+ /* The server returned HTTP status 304 Not Modified, so we’re clear to
+ * load summary.sig from the cache. Also load summary, since
+ * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
+ if (summary_sig_not_modified)
+ {
+ g_clear_pointer (&bytes_sig, g_bytes_unref);
+ g_clear_pointer (&bytes_summary, g_bytes_unref);
+ if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig",
+ &bytes_sig,
+ cancellable, error))
+ goto out;
+
+ if (!bytes_summary &&
+ !pull_data->remote_repo_local &&
+ !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
- &summary_not_modified, &summary_etag, &summary_last_modified,
- OSTREE_MAX_METADATA_SIZE,
cancellable, error))
- goto out;
+ goto out;
+ }
+ }
- /* The server returned HTTP status 304 Not Modified, so we’re clear to
- * load summary from the cache. */
- if (summary_not_modified)
- {
- g_clear_pointer (&bytes_summary, g_bytes_unref);
- if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
- &bytes_summary,
- cancellable, error))
- goto out;
- }
- }
+ if (bytes_sig &&
+ !bytes_summary &&
+ !pull_data->remote_repo_local &&
+ !_ostree_repo_load_cache_summary_if_same_sig (self,
+ remote_name_or_baseurl,
+ bytes_sig,
+ &bytes_summary,
+ cancellable,
+ error))
+ goto out;
+
+ if (bytes_summary && !summary_bytes_v)
+ {
+ g_debug ("Loaded %s summary from cache", remote_name_or_baseurl);
+ summary_from_cache = TRUE;
+ }
+
+ if (!pull_data->summary && !bytes_summary)
+ {
+ g_autofree char *summary_if_none_match = NULL;
+ guint64 summary_if_modified_since = 0;
+
+ _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL,
+ &summary_if_none_match, &summary_if_modified_since);
+
+ g_clear_pointer (&summary_etag, g_free);
+ summary_last_modified = 0;
+
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ summary_if_none_match, summary_if_modified_since,
+ pull_data->n_network_retries,
+ &bytes_summary,
+ &summary_not_modified, &summary_etag, &summary_last_modified,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
+ goto out;
+
+ /* The server returned HTTP status 304 Not Modified, so we’re clear to
+ * load summary from the cache. */
+ if (summary_not_modified)
+ {
+ g_clear_pointer (&bytes_summary, g_bytes_unref);
+ if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
+ &bytes_summary,
+ cancellable, error))
+ goto out;
+ }
+ }
#ifndef OSTREE_DISABLE_GPGME
- if (!bytes_summary && pull_data->gpg_verify_summary)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
- "GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)");
- goto out;
- }
+ if (!bytes_summary && pull_data->gpg_verify_summary)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
+ "GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)");
+ goto out;
+ }
#endif /* OSTREE_DISABLE_GPGME */
- if (!bytes_summary && pull_data->require_static_deltas)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
- "Fetch configured to require static deltas, but no summary found");
- goto out;
- }
+ if (!bytes_summary && require_summary_for_mirror)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
+ goto out;
+ }
#ifndef OSTREE_DISABLE_GPGME
- if (!bytes_sig && pull_data->gpg_verify_summary)
- {
- g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
- "GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)");
- goto out;
- }
+ if (!bytes_sig && pull_data->gpg_verify_summary)
+ {
+ g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
+ "GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)");
+ goto out;
+ }
- if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
- {
- g_autoptr(OstreeGpgVerifyResult) result = NULL;
- g_autoptr(GError) temp_error = NULL;
+ if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
+ {
+ g_autoptr(OstreeGpgVerifyResult) result = NULL;
+ g_autoptr(GError) temp_error = NULL;
- result = ostree_repo_verify_summary (self, pull_data->remote_name,
- bytes_summary, bytes_sig,
- cancellable, &temp_error);
- if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error))
- {
- if (summary_from_cache)
- {
- /* The cached summary doesn't match, fetch a new one and verify again.
- * Don’t set the cache headers in the HTTP request, to force a
- * full download. */
- if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Remote %s cached summary invalid and "
- "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
- pull_data->remote_name);
+ result = ostree_repo_verify_summary (self, pull_data->remote_name,
+ bytes_summary, bytes_sig,
+ cancellable, &temp_error);
+ if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error))
+ {
+ if (summary_from_cache)
+ {
+ /* The cached summary doesn't match, fetch a new one and verify again.
+ * Don’t set the cache headers in the HTTP request, to force a
+ * full download. */
+ if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Remote %s cached summary invalid and "
+ "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
+ pull_data->remote_name);
+ goto out;
+ }
+ else
+ g_debug ("Remote %s cached summary invalid, pulling new version",
+ pull_data->remote_name);
+
+ summary_from_cache = FALSE;
+ g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
+ g_clear_pointer (&summary_etag, g_free);
+ summary_last_modified = 0;
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "summary",
+ OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ NULL, 0, /* no cache headers */
+ pull_data->n_network_retries,
+ &bytes_summary,
+ &summary_not_modified, &summary_etag, &summary_last_modified,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
goto out;
- }
- else
- g_debug ("Remote %s cached summary invalid, pulling new version",
- pull_data->remote_name);
-
- summary_from_cache = FALSE;
- g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
- g_clear_pointer (&summary_etag, g_free);
- summary_last_modified = 0;
- if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
- pull_data->meta_mirrorlist,
- "summary",
- OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
- NULL, 0, /* no cache headers */
- pull_data->n_network_retries,
- &bytes_summary,
- &summary_not_modified, &summary_etag, &summary_last_modified,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error))
- goto out;
- g_autoptr(OstreeGpgVerifyResult) retry =
- ostree_repo_verify_summary (self, pull_data->remote_name,
- bytes_summary, bytes_sig,
- cancellable, error);
- if (!ostree_gpg_verify_result_require_valid_signature (retry, error))
+ g_autoptr(OstreeGpgVerifyResult) retry =
+ ostree_repo_verify_summary (self, pull_data->remote_name,
+ bytes_summary, bytes_sig,
+ cancellable, error);
+ if (!ostree_gpg_verify_result_require_valid_signature (retry, error))
+ goto out;
+ }
+ else
+ {
+ g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
- }
- else
- {
- g_propagate_error (error, g_steal_pointer (&temp_error));
- goto out;
- }
- }
- }
+ }
+ }
+ }
#endif /* OSTREE_DISABLE_GPGME */
- if (pull_data->signapi_summary_verifiers)
- {
- if (!bytes_sig && pull_data->signapi_summary_verifiers)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)");
- goto out;
- }
- if (bytes_summary && bytes_sig)
- {
- g_autoptr(GVariant) signatures = NULL;
- g_autoptr(GError) temp_error = NULL;
-
- signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
- bytes_sig, FALSE);
-
-
- g_assert (pull_data->signapi_summary_verifiers);
- if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error))
- {
- if (summary_from_cache)
- {
- /* The cached summary doesn't match, fetch a new one and verify again.
- * Don’t set the cache headers in the HTTP request, to force a
- * full download. */
- if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Remote %s cached summary invalid and "
- "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
- pull_data->remote_name);
+ if (pull_data->signapi_summary_verifiers)
+ {
+ if (!bytes_sig && pull_data->signapi_summary_verifiers)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)");
+ goto out;
+ }
+ if (bytes_summary && bytes_sig)
+ {
+ g_autoptr(GVariant) signatures = NULL;
+ g_autoptr(GError) temp_error = NULL;
+
+ signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
+ bytes_sig, FALSE);
+
+ g_assert (pull_data->signapi_summary_verifiers);
+ if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error))
+ {
+ if (summary_from_cache)
+ {
+ /* The cached summary doesn't match, fetch a new one and verify again.
+ * Don’t set the cache headers in the HTTP request, to force a
+ * full download. */
+ if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Remote %s cached summary invalid and "
+ "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
+ pull_data->remote_name);
+ goto out;
+ }
+ else
+ g_debug ("Remote %s cached summary invalid, pulling new version",
+ pull_data->remote_name);
+
+ summary_from_cache = FALSE;
+ g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
+ g_clear_pointer (&summary_etag, g_free);
+ summary_last_modified = 0;
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "summary",
+ OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ NULL, 0, /* no cache headers */
+ pull_data->n_network_retries,
+ &bytes_summary,
+ &summary_not_modified, &summary_etag, &summary_last_modified,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
goto out;
- }
- else
- g_debug ("Remote %s cached summary invalid, pulling new version",
- pull_data->remote_name);
-
- summary_from_cache = FALSE;
- g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
- g_clear_pointer (&summary_etag, g_free);
- summary_last_modified = 0;
- if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
- pull_data->meta_mirrorlist,
- "summary",
- OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
- NULL, 0, /* no cache headers */
- pull_data->n_network_retries,
- &bytes_summary,
- &summary_not_modified, &summary_etag, &summary_last_modified,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error))
- goto out;
- if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error))
+ if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error))
goto out;
- }
- else
- {
- g_propagate_error (error, g_steal_pointer (&temp_error));
- goto out;
- }
- }
- }
- }
+ }
+ else
+ {
+ g_propagate_error (error, g_steal_pointer (&temp_error));
+ goto out;
+ }
+ }
+ }
+ }
- if (bytes_summary)
- {
- pull_data->summary_data = g_bytes_ref (bytes_summary);
- pull_data->summary_etag = g_strdup (summary_etag);
- pull_data->summary_last_modified = summary_last_modified;
- pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE);
+ if (bytes_summary)
+ {
+ pull_data->summary_data = g_bytes_ref (bytes_summary);
+ pull_data->summary_etag = g_strdup (summary_etag);
+ pull_data->summary_last_modified = summary_last_modified;
+ pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE);
- if (!g_variant_is_normal_form (pull_data->summary))
- {
- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Not normal form");
- goto out;
- }
- if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT))
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Doesn't match variant type '%s'",
- (char *)OSTREE_SUMMARY_GVARIANT_FORMAT);
- goto out;
- }
+ if (!g_variant_is_normal_form (pull_data->summary))
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Not normal form");
+ goto out;
+ }
+ if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Doesn't match variant type '%s'",
+ (char *)OSTREE_SUMMARY_GVARIANT_FORMAT);
+ goto out;
+ }
- if (bytes_sig)
- {
- pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
- pull_data->summary_sig_etag = g_strdup (summary_sig_etag);
- pull_data->summary_sig_last_modified = summary_sig_last_modified;
- }
- }
+ if (bytes_sig)
+ {
+ pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
+ pull_data->summary_sig_etag = g_strdup (summary_sig_etag);
+ pull_data->summary_sig_last_modified = summary_sig_last_modified;
+ }
+ }
- if (!summary_from_cache && bytes_summary && bytes_sig)
- {
- if (!pull_data->remote_repo_local &&
- !_ostree_repo_cache_summary (self,
- remote_name_or_baseurl,
- bytes_summary,
- summary_etag, summary_last_modified,
- bytes_sig,
- summary_sig_etag, summary_sig_last_modified,
- cancellable,
- error))
- goto out;
- }
+ if (!summary_from_cache && bytes_summary && bytes_sig)
+ {
+ if (!pull_data->remote_repo_local &&
+ !_ostree_repo_cache_summary (self,
+ remote_name_or_baseurl,
+ bytes_summary,
+ summary_etag, summary_last_modified,
+ bytes_sig,
+ summary_sig_etag, summary_sig_last_modified,
+ cancellable,
+ error))
+ goto out;
+ }
- if (pull_data->summary)
- {
- additional_metadata = g_variant_get_child_value (pull_data->summary, 1);
+ if (pull_data->summary)
+ {
+ additional_metadata = g_variant_get_child_value (pull_data->summary, 1);
- if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id))
- main_collection_id = NULL;
- else if (!ostree_validate_collection_id (main_collection_id, error))
- goto out;
+ if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id))
+ main_collection_id = NULL;
+ else if (!ostree_validate_collection_id (main_collection_id, error))
+ goto out;
- refs = g_variant_get_child_value (pull_data->summary, 0);
- for (i = 0, n = g_variant_n_children (refs); i < n; i++)
- {
- const char *refname;
- g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i);
+ refs = g_variant_get_child_value (pull_data->summary, 0);
+ for (i = 0, n = g_variant_n_children (refs); i < n; i++)
+ {
+ const char *refname;
+ g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i);
- g_variant_get_child (ref, 0, "&s", &refname);
+ g_variant_get_child (ref, 0, "&s", &refname);
- if (!ostree_validate_rev (refname, error))
- goto out;
+ if (!ostree_validate_rev (refname, error))
+ goto out;
- if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
- {
- g_hash_table_insert (requested_refs_to_fetch,
- ostree_collection_ref_new (main_collection_id, refname), NULL);
- }
- }
+ if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
+ {
+ g_hash_table_insert (requested_refs_to_fetch,
+ ostree_collection_ref_new (main_collection_id, refname), NULL);
+ }
+ }
- g_autoptr(GVariant) collection_map = NULL;
- collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}"));
- if (collection_map != NULL)
- {
- GVariantIter collection_map_iter;
- const char *collection_id;
- g_autoptr(GVariant) collection_refs = NULL;
+ g_autoptr(GVariant) collection_map = NULL;
+ collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}"));
+ if (collection_map != NULL)
+ {
+ GVariantIter collection_map_iter;
+ const char *collection_id;
+ g_autoptr(GVariant) collection_refs = NULL;
- g_variant_iter_init (&collection_map_iter, collection_map);
+ g_variant_iter_init (&collection_map_iter, collection_map);
- while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs))
- {
- if (!ostree_validate_collection_id (collection_id, error))
- goto out;
+ while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs))
+ {
+ if (!ostree_validate_collection_id (collection_id, error))
+ goto out;
- for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++)
- {
- const char *refname;
- g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i);
+ for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++)
+ {
+ const char *refname;
+ g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i);
- g_variant_get_child (ref, 0, "&s", &refname);
+ g_variant_get_child (ref, 0, "&s", &refname);
- if (!ostree_validate_rev (refname, error))
- goto out;
+ if (!ostree_validate_rev (refname, error))
+ goto out;
- if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
- {
- g_hash_table_insert (requested_refs_to_fetch,
- ostree_collection_ref_new (collection_id, refname), NULL);
- }
- }
- }
- }
+ if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
+ {
+ g_hash_table_insert (requested_refs_to_fetch,
+ ostree_collection_ref_new (collection_id, refname), NULL);
+ }
+ }
+ }
+ }
- deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
- n = deltas ? g_variant_n_children (deltas) : 0;
- for (i = 0; i < n; i++)
- {
- const char *delta;
- g_autoptr(GVariant) csum_v = NULL;
- g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i);
+ deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
+ pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0;
+ if (!collect_available_deltas_for_pull (pull_data, deltas, error))
+ goto out;
- g_variant_get_child (ref, 0, "&s", &delta);
- g_variant_get_child (ref, 1, "v", &csum_v);
+ g_variant_lookup (additional_metadata, OSTREE_SUMMARY_INDEXED_DELTAS, "b", &pull_data->has_indexed_deltas);
+ }
- if (!validate_variant_is_csum (csum_v, error))
- goto out;
+ if (pull_data->summary &&
+ g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) &&
+ g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits))
+ {
+ if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
+ goto out;
+ pull_data->has_tombstone_commits = tombstone_commits;
+ remote_mode_loaded = TRUE;
+ }
+ }
- guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN);
- memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32);
- g_hash_table_insert (pull_data->summary_deltas_checksums,
- g_strdup (delta),
- csum_data);
- }
- }
+ if (pull_data->require_static_deltas && !pull_data->has_indexed_deltas && !pull_data->summary_has_deltas)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
+ "Fetch configured to require static deltas, but no summary deltas or delta index found");
+ goto out;
+ }
- if (pull_data->summary &&
- g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) &&
- g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits))
- {
- if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
- goto out;
- pull_data->has_tombstone_commits = tombstone_commits;
- remote_mode_loaded = TRUE;
- }
- else if (pull_data->remote_repo_local == NULL)
+ if (remote_mode_loaded && pull_data->remote_repo_local == NULL)
{
/* Fall-back path which loads the necessary config from the remote’s
- * `config` file. Doing so is deprecated since it means an
+ * `config` file (unless we already read it above). Doing so is deprecated since it means an
* additional round trip to the remote for each pull. No need to do
* it for local pulls. */
- if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
+ if (remote_config == NULL &&
+ !load_remote_repo_config (pull_data, &remote_config, cancellable, error))
goto out;
if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
@@ -4367,85 +4670,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
remote_mode_loaded = TRUE;
}
- if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Can't pull from archives with mode \"%s\"",
- remote_mode_str);
- goto out;
- }
- }
-
- if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches)
- {
- if (!bytes_summary)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
- goto out;
- }
-
- }
- else if (opt_collection_refs_set)
- {
- const gchar *collection_id, *ref_name, *checksum;
-
- while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum))
- {
- if (!ostree_validate_rev (ref_name, error))
- goto out;
- g_hash_table_insert (requested_refs_to_fetch,
- ostree_collection_ref_new (collection_id, ref_name),
- (*checksum != '\0') ? g_strdup (checksum) : NULL);
- }
- }
- else if (refs_to_fetch != NULL)
- {
- char **strviter = refs_to_fetch;
- char **commitid_strviter = override_commit_ids ?: NULL;
-
- while (*strviter)
- {
- const char *branch = *strviter;
-
- if (ostree_validate_checksum_string (branch, NULL))
- {
- char *key = g_strdup (branch);
- g_hash_table_add (commits_to_fetch, key);
- }
- else
- {
- if (!ostree_validate_rev (branch, error))
- goto out;
- char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL;
- g_hash_table_insert (requested_refs_to_fetch,
- ostree_collection_ref_new (NULL, branch), commitid);
- }
-
- strviter++;
- if (commitid_strviter)
- commitid_strviter++;
- }
- }
- else
+ if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE)
{
- char **branches_iter;
-
- branches_iter = configured_branches;
-
- if (!(branches_iter && *branches_iter))
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "No configured branches for remote %s", remote_name_or_baseurl);
- goto out;
- }
- for (;branches_iter && *branches_iter; branches_iter++)
- {
- const char *branch = *branches_iter;
-
- g_hash_table_insert (requested_refs_to_fetch,
- ostree_collection_ref_new (NULL, branch), NULL);
- }
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Can't pull from archives with mode \"%s\"",
+ remote_mode_str);
+ goto out;
}
/* Resolve the checksum for each ref. This has to be done into a new hash table,
@@ -4900,6 +5130,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref);
+ g_clear_pointer (&pull_data->pending_fetch_delta_indexes, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref);
g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
index e263e928..c2536724 100644
--- a/src/libostree/ostree-repo-static-delta-core.c
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -168,6 +168,93 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
return TRUE;
}
+/**
+ * ostree_repo_list_static_delta_indexes:
+ * @self: Repo
+ * @out_indexes: (out) (element-type utf8) (transfer container): String name of delta indexes (checksum)
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * This function synchronously enumerates all static delta indexes in the
+ * repository, returning its result in @out_indexes.
+ *
+ * Since: 2020.7
+ */
+gboolean
+ostree_repo_list_static_delta_indexes (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_autoptr(GPtrArray) ret_indexes = g_ptr_array_new_with_free_func (g_free);
+
+ g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+ gboolean exists;
+ if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "delta-indexes", &dfd_iter,
+ &exists, error))
+ return FALSE;
+ if (!exists)
+ {
+ /* Note early return */
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ return TRUE;
+ }
+
+ while (TRUE)
+ {
+ g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, };
+ struct dirent *dent;
+
+ if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
+ return FALSE;
+ if (dent == NULL)
+ break;
+ if (dent->d_type != DT_DIR)
+ continue;
+ if (strlen (dent->d_name) != 2)
+ continue;
+
+ if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE,
+ &sub_dfd_iter, error))
+ return FALSE;
+
+ while (TRUE)
+ {
+ struct dirent *sub_dent;
+
+ if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent,
+ cancellable, error))
+ return FALSE;
+ if (sub_dent == NULL)
+ break;
+ if (sub_dent->d_type != DT_REG)
+ continue;
+
+ const char *name1 = dent->d_name;
+ const char *name2 = sub_dent->d_name;
+
+ /* base64 len is 43, but 2 chars are in the parent dir name */
+ if (strlen (name2) != 41 + strlen(".index") ||
+ !g_str_has_suffix (name2, ".index"))
+ continue;
+
+ g_autoptr(GString) out = g_string_new (name1);
+ g_string_append_len (out, name2, 41);
+
+ char checksum[OSTREE_SHA256_STRING_LEN+1];
+ guchar csum[OSTREE_SHA256_DIGEST_LEN];
+
+ ostree_checksum_b64_inplace_to_bytes (out->str, csum);
+ ostree_checksum_inplace_from_bytes (csum, checksum);
+
+ g_ptr_array_add (ret_indexes, g_strdup (checksum));
+ }
+ }
+
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ return TRUE;
+}
+
gboolean
_ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo,
GVariant *checksum_array,
@@ -1118,3 +1205,207 @@ ostree_repo_static_delta_verify_signature (OstreeRepo *self,
return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error);
}
+
+static void
+null_or_ptr_array_unref (GPtrArray *array)
+{
+ if (array != NULL)
+ g_ptr_array_unref (array);
+}
+
+static gboolean
+file_has_content (OstreeRepo *repo,
+ const char *subpath,
+ GBytes *data,
+ GCancellable *cancellable)
+{
+ struct stat stbuf;
+ glnx_autofd int existing_fd = -1;
+
+ if (!glnx_fstatat (repo->repo_dir_fd, subpath, &stbuf, 0, NULL))
+ return FALSE;
+
+ if (stbuf.st_size != g_bytes_get_size (data))
+ return FALSE;
+
+ if (!glnx_openat_rdonly (repo->repo_dir_fd, subpath, TRUE, &existing_fd, NULL))
+ return FALSE;
+
+ g_autoptr(GBytes) existing_data = glnx_fd_readall_bytes (existing_fd, cancellable, NULL);
+ if (existing_data == NULL)
+ return FALSE;
+
+ return g_bytes_equal (existing_data, data);
+}
+
+/**
+ * ostree_repo_static_delta_reindex:
+ * @repo: Repo
+ * @flags: Flags affecting the indexing operation
+ * @opt_to_commit: ASCII SHA256 checksum of target commit, or %NULL to index all targets
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * The delta index for a particular commit lists all the existing deltas that can be used
+ * when downloading that commit. This operation regenerates these indexes, either for
+ * a particular commit (if @opt_to_commit is non-%NULL), or for all commits that
+ * are reachable by an existing delta (if @opt_to_commit is %NULL).
+ *
+ * This is normally called automatically when the summary is updated in ostree_repo_regenerate_summary().
+ *
+ * Locking: shared
+ */
+gboolean
+ostree_repo_static_delta_reindex (OstreeRepo *repo,
+ OstreeStaticDeltaIndexFlags flags,
+ const char *opt_to_commit,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_autoptr(GPtrArray) all_deltas = NULL;
+ g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */
+ gboolean opt_indexed_deltas;
+
+ /* Protect against parallel prune operation */
+ g_autoptr(OstreeRepoAutoLock) lock =
+ _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_SHARED, cancellable, error);
+ if (!lock)
+ return FALSE;
+
+ /* Enusre that the "indexed-deltas" option is set on the config, so we know this when pulling */
+ if (!ot_keyfile_get_boolean_with_default (repo->config, "core",
+ "indexed-deltas", FALSE,
+ &opt_indexed_deltas, error))
+ return FALSE;
+
+ if (!opt_indexed_deltas)
+ {
+ g_autoptr(GKeyFile) config = ostree_repo_copy_config (repo);
+ g_key_file_set_boolean (config, "core", "indexed-deltas", TRUE);
+ if (!ostree_repo_write_config (repo, config, error))
+ return FALSE;
+ }
+
+ deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref);
+
+ if (opt_to_commit == NULL)
+ {
+ g_autoptr(GPtrArray) old_indexes = NULL;
+
+ /* To ensure all old index files either is regenerated, or
+ * removed, we initialize all existing indexes to NULL in the
+ * hashtable. */
+ if (!ostree_repo_list_static_delta_indexes (repo, &old_indexes, cancellable, error))
+ return FALSE;
+
+ for (int i = 0; i < old_indexes->len; i++)
+ {
+ const char *old_index = g_ptr_array_index (old_indexes, i);
+ g_hash_table_insert (deltas_to_commit_ht, g_strdup (old_index), NULL);
+ }
+ }
+ else
+ {
+ if (!ostree_validate_checksum_string (opt_to_commit, error))
+ return FALSE;
+
+ /* We ensure the specific old index either is regenerated, or removed */
+ g_hash_table_insert (deltas_to_commit_ht, g_strdup (opt_to_commit), NULL);
+ }
+
+ if (!ostree_repo_list_static_delta_names (repo, &all_deltas, cancellable, error))
+ return FALSE;
+
+ for (int i = 0; i < all_deltas->len; i++)
+ {
+ const char *delta_name = g_ptr_array_index (all_deltas, i);
+ g_autofree char *from = NULL;
+ g_autofree char *to = NULL;
+ GPtrArray *deltas_to_commit = NULL;
+
+ if (!_ostree_parse_delta_name (delta_name, &from, &to, error))
+ return FALSE;
+
+ if (opt_to_commit != NULL && strcmp (to, opt_to_commit) != 0)
+ continue;
+
+ deltas_to_commit = g_hash_table_lookup (deltas_to_commit_ht, to);
+ if (deltas_to_commit == NULL)
+ {
+ deltas_to_commit = g_ptr_array_new_with_free_func (g_free);
+ g_hash_table_insert (deltas_to_commit_ht, g_steal_pointer (&to), deltas_to_commit);
+ }
+
+ g_ptr_array_add (deltas_to_commit, g_steal_pointer (&from));
+ }
+
+ GLNX_HASH_TABLE_FOREACH_KV (deltas_to_commit_ht, const char*, to, GPtrArray*, froms)
+ {
+ g_autofree char *index_path = _ostree_get_relative_static_delta_index_path (to);
+
+ if (froms == NULL)
+ {
+ /* No index to this checksum seen, delete if it exists */
+
+ g_debug ("Removing delta index for %s", to);
+ if (!ot_ensure_unlinked_at (repo->repo_dir_fd, index_path, error))
+ return FALSE;
+ }
+ else
+ {
+ g_auto(GVariantDict) index_builder = OT_VARIANT_BUILDER_INITIALIZER;
+ g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+ g_autoptr(GVariant) index_variant = NULL;
+ g_autoptr(GBytes) index = NULL;
+
+ /* We sort on from here so that the index file is reproducible */
+ g_ptr_array_sort (froms, (GCompareFunc)g_strcmp0);
+
+ g_variant_dict_init (&deltas_builder, NULL);
+
+ for (int i = 0; i < froms->len; i++)
+ {
+ const char *from = g_ptr_array_index (froms, i);
+ g_autofree char *delta_name = NULL;
+ GVariant *digest;
+
+ digest = _ostree_repo_static_delta_superblock_digest (repo, from, to, cancellable, error);
+ if (digest == NULL)
+ return FALSE;
+
+ if (from != NULL)
+ delta_name = g_strconcat (from, "-", to, NULL);
+ else
+ delta_name = g_strdup (to);
+
+ g_variant_dict_insert_value (&deltas_builder, delta_name, digest);
+ }
+
+ /* The toplevel of the index is an a{sv} for extensibility, and we use same key name (and format) as when
+ * storing deltas in the summary. */
+ g_variant_dict_init (&index_builder, NULL);
+
+ g_variant_dict_insert_value (&index_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+
+ index_variant = g_variant_ref_sink (g_variant_dict_end (&index_builder));
+ index = g_variant_get_data_as_bytes (index_variant);
+
+ g_autofree char *index_dirname = g_path_get_dirname (index_path);
+ if (!glnx_shutil_mkdir_p_at (repo->repo_dir_fd, index_dirname, DEFAULT_DIRECTORY_MODE, cancellable, error))
+ return FALSE;
+
+ /* delta indexes are generally small and static, so reading it back and comparing is cheap, and it will
+ lower the write load (and particular sync-load) on the disk during reindexing (i.e. summary updates), */
+ if (file_has_content (repo, index_path, index, cancellable))
+ continue;
+
+ g_debug ("Updating delta index for %s", to);
+ if (!glnx_file_replace_contents_at (repo->repo_dir_fd, index_path,
+ g_bytes_get_data (index, NULL), g_bytes_get_size (index),
+ 0, cancellable, error))
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h
index 5a2e6879..d6c706da 100644
--- a/src/libostree/ostree-repo-static-delta-private.h
+++ b/src/libostree/ostree-repo-static-delta-private.h
@@ -227,6 +227,11 @@ _ostree_repo_static_delta_delete (OstreeRepo *repo,
const char *delta_id,
GCancellable *cancellable,
GError **error);
+gboolean
+_ostree_repo_static_delta_reindex (OstreeRepo *repo,
+ const char *opt_to_commit,
+ GCancellable *cancellable,
+ GError **error);
/* Used for static deltas which due to a historical mistake are
* inconsistent endian.
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index ba3e877f..3bbf5ea0 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -5222,6 +5222,67 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self,
#endif /* OSTREE_DISABLE_GPGME */
}
+
+/**
+ * ostree_repo_gpg_sign_data:
+ * @self: Self
+ * @data: Data as a #GBytes
+ * @old_signatures: Existing signatures to append to (or %NULL)
+ * @key_id: (array zero-terminated=1) (element-type utf8): NULL-terminated array of GPG keys.
+ * @homedir: (allow-none): GPG home directory, or %NULL
+ * @out_signature: (out): in case of success will contain signature
+ * @cancellable: A #GCancellable
+ * @error: a #GError
+ *
+ * Sign the given @data with the specified keys in @key_id. Similar to
+ * ostree_repo_add_gpg_signature_summary() but can be used on any
+ * data.
+ *
+ * You can use ostree_repo_gpg_verify_data() to verify the signatures.
+ *
+ * Returns: @TRUE if @data has been signed successfully,
+ * @FALSE in case of error (@error will contain the reason).
+ *
+ * Since: 2020.8
+ */
+gboolean
+ostree_repo_gpg_sign_data (OstreeRepo *self,
+ GBytes *data,
+ GBytes *old_signatures,
+ const gchar **key_id,
+ const gchar *homedir,
+ GBytes **out_signatures,
+ GCancellable *cancellable,
+ GError **error)
+{
+#ifndef OSTREE_DISABLE_GPGME
+ g_autoptr(GVariant) metadata = NULL;
+ g_autoptr(GVariant) res = NULL;
+
+ if (old_signatures)
+ metadata = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), old_signatures, FALSE));
+
+ for (guint i = 0; key_id[i]; i++)
+ {
+ g_autoptr(GBytes) signature_data = NULL;
+ if (!sign_data (self, data, key_id[i], homedir,
+ &signature_data,
+ cancellable, error))
+ return FALSE;
+
+ g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata);
+ metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data);
+ }
+
+ res = g_variant_get_normal_form (metadata);
+ *out_signatures = g_variant_get_data_as_bytes (res);
+ return TRUE;
+#else
+ return glnx_throw (error, "GPG feature is disabled in a build time");
+#endif /* OSTREE_DISABLE_GPGME */
+}
+
+
#ifndef OSTREE_DISABLE_GPGME
/* Special remote for _ostree_repo_gpg_verify_with_metadata() */
static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__";
@@ -5749,6 +5810,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self,
* commits from working.
*/
g_autoptr(OstreeRepoAutoLock) lock = NULL;
+ gboolean no_deltas_in_summary = FALSE;
+
lock = _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE,
cancellable, error);
if (!lock)
@@ -5781,35 +5844,41 @@ ostree_repo_regenerate_summary (OstreeRepo *self,
}
}
- {
- g_autoptr(GPtrArray) delta_names = NULL;
- g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+ if (!ot_keyfile_get_boolean_with_default (self->config, "core",
+ "no-deltas-in-summary", FALSE,
+ &no_deltas_in_summary, error))
+ return FALSE;
- if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error))
- return FALSE;
+ if (!no_deltas_in_summary)
+ {
+ g_autoptr(GPtrArray) delta_names = NULL;
+ g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
- g_variant_dict_init (&deltas_builder, NULL);
- for (guint i = 0; i < delta_names->len; i++)
- {
- g_autofree char *from = NULL;
- g_autofree char *to = NULL;
- GVariant *digest;
+ if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error))
+ return FALSE;
- if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
- return FALSE;
+ g_variant_dict_init (&deltas_builder, NULL);
+ for (guint i = 0; i < delta_names->len; i++)
+ {
+ g_autofree char *from = NULL;
+ g_autofree char *to = NULL;
+ GVariant *digest;
- digest = _ostree_repo_static_delta_superblock_digest (self,
- (from && from[0]) ? from : NULL,
- to, cancellable, error);
- if (digest == NULL)
- return FALSE;
+ if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
+ return FALSE;
- g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest);
- }
+ digest = _ostree_repo_static_delta_superblock_digest (self,
+ (from && from[0]) ? from : NULL,
+ to, cancellable, error);
+ if (digest == NULL)
+ return FALSE;
- if (delta_names->len > 0)
- g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
- }
+ g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest);
+ }
+
+ if (delta_names->len > 0)
+ g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+ }
{
g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_LAST_MODIFIED,
@@ -5834,6 +5903,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self,
g_variant_new_boolean (tombstone_commits));
}
+ g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_INDEXED_DELTAS,
+ g_variant_new_boolean (TRUE));
+
/* Add refs which have a collection specified, which could be in refs/mirrors,
* refs/heads, and/or refs/remotes. */
{
@@ -5927,6 +5999,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self,
g_variant_ref_sink (summary);
}
+ if (!ostree_repo_static_delta_reindex (self, 0, NULL, cancellable, error))
+ return FALSE;
+
if (!_ostree_repo_file_replace_contents (self,
self->repo_dir_fd,
"summary",
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index d52fc9c3..e64c3230 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -1046,6 +1046,12 @@ gboolean ostree_repo_list_static_delta_names (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+_OSTREE_PUBLIC
+gboolean ostree_repo_list_static_delta_indexes (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error);
+
/**
* OstreeStaticDeltaGenerateOpt:
* @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space
@@ -1068,6 +1074,23 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+/**
+ * OstreeStaticDeltaIndexFlags:
+ * @OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE: No special flags
+ *
+ * Flags controlling static delta index generation.
+ */
+typedef enum {
+ OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0,
+} OstreeStaticDeltaIndexFlags;
+
+_OSTREE_PUBLIC
+gboolean ostree_repo_static_delta_reindex (OstreeRepo *repo,
+ OstreeStaticDeltaIndexFlags flags,
+ const char *opt_to_commit,
+ GCancellable *cancellable,
+ GError **error);
+
_OSTREE_PUBLIC
gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self,
GFile *dir_or_file,
@@ -1394,6 +1417,16 @@ gboolean ostree_repo_append_gpg_signature (OstreeRepo *self,
GError **error);
_OSTREE_PUBLIC
+gboolean ostree_repo_gpg_sign_data (OstreeRepo *self,
+ GBytes *data,
+ GBytes *old_signatures,
+ const gchar **key_id,
+ const gchar *homedir,
+ GBytes **out_signatures,
+ GCancellable *cancellable,
+ GError **error);
+
+_OSTREE_PUBLIC
OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo *self,
const gchar *commit_checksum,
GFile *keyringdir,
diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c
index 3e0af5bd..ff31b574 100644
--- a/src/ostree/ot-builtin-static-delta.c
+++ b/src/ostree/ot-builtin-static-delta.c
@@ -53,6 +53,8 @@ BUILTINPROTO(delete);
BUILTINPROTO(generate);
BUILTINPROTO(apply_offline);
BUILTINPROTO(verify);
+BUILTINPROTO(indexes);
+BUILTINPROTO(reindex);
#undef BUILTINPROTO
@@ -75,6 +77,12 @@ static OstreeCommand static_delta_subcommands[] = {
{ "verify", OSTREE_BUILTIN_FLAG_NONE,
ot_static_delta_builtin_verify,
"Verify static delta signatures" },
+ { "indexes", OSTREE_BUILTIN_FLAG_NONE,
+ ot_static_delta_builtin_indexes,
+ "List static delta indexes" },
+ { "reindex", OSTREE_BUILTIN_FLAG_NONE,
+ ot_static_delta_builtin_reindex,
+ "Regenerate static delta indexes" },
{ NULL, 0, NULL, NULL }
};
@@ -126,6 +134,15 @@ static GOptionEntry verify_options[] = {
{ NULL }
};
+static GOptionEntry indexes_options[] = {
+ { NULL }
+};
+
+static GOptionEntry reindex_options[] = {
+ { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Only update delta index to revision REV", "REV" },
+ { NULL }
+};
+
static void
static_delta_usage (char **argv,
gboolean is_error)
@@ -177,6 +194,46 @@ ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *in
}
static gboolean
+ot_static_delta_builtin_indexes (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+ g_autoptr(OstreeRepo) repo = NULL;
+ g_autoptr(GOptionContext) context = g_option_context_new ("");
+ if (!ostree_option_context_parse (context, indexes_options, &argc, &argv,
+ invocation, &repo, cancellable, error))
+ return FALSE;
+
+ g_autoptr(GPtrArray) indexes = NULL;
+ if (!ostree_repo_list_static_delta_indexes (repo, &indexes, cancellable, error))
+ return FALSE;
+
+ if (indexes->len == 0)
+ g_print ("(No static deltas indexes)\n");
+ else
+ {
+ for (guint i = 0; i < indexes->len; i++)
+ g_print ("%s\n", (char*)indexes->pdata[i]);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+ot_static_delta_builtin_reindex (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+ g_autoptr(GOptionContext) context = g_option_context_new ("");
+
+ g_autoptr(OstreeRepo) repo = NULL;
+ if (!ostree_option_context_parse (context, reindex_options, &argc, &argv, invocation, &repo, cancellable, error))
+ return FALSE;
+
+ if (!ostree_repo_static_delta_reindex (repo, 0, opt_to_rev, cancellable, error))
+ return FALSE;
+
+ return TRUE;
+}
+
+
+static gboolean
ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
{
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index 082ed315..08921838 100644
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -55,10 +55,10 @@ function verify_initial_contents() {
}
if has_gpgme; then
- echo "1..35"
+ echo "1..36"
else
# 3 tests needs GPG support
- echo "1..32"
+ echo "1..33"
fi
# Try both syntaxes
@@ -381,6 +381,25 @@ assert_file_has_content err.txt "Upgrade.*is chronologically older"
${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${oldrev} origin main@${middlerev}
echo "ok pull timestamp checking"
+# test pull without override commit use summary, but with doesn't use summary
+# We temporarily replace summary with broken one to detect if it is used
+mv ostree-srv/gnomerepo/summary ostree-srv/gnomerepo/summary.backup
+echo "broken" > ostree-srv/gnomerepo/summary
+
+repo_init --no-sign-verify
+rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main)
+# This will need summary, so will fail
+if ${CMD_PREFIX} ostree --repo=repo -v pull origin main; then
+ assert_not_reached "Should have failed with broken summary"
+fi
+# This won't need summary so will not fail
+${CMD_PREFIX} ostree --repo=repo pull origin main@${rev}
+
+# Restore summary
+mv ostree-srv/gnomerepo/summary.backup ostree-srv/gnomerepo/summary
+
+echo "ok pull with override id doesn't use summary"
+
cd ${test_tmpdir}
repo_init --no-sign-verify
${CMD_PREFIX} ostree --repo=repo pull origin main
diff --git a/tests/test-delta.sh b/tests/test-delta.sh
index 557447bc..bfdec593 100755
--- a/tests/test-delta.sh
+++ b/tests/test-delta.sh
@@ -28,7 +28,7 @@ skip_without_user_xattrs
bindatafiles="bash true ostree"
morebindatafiles="false ls"
-echo '1..12'
+echo '1..13'
mkdir repo
ostree_repo_init repo --mode=archive
@@ -90,6 +90,11 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1
${CMD_PREFIX} ostree --repo=repo prune
${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount
+assert_file_has_content indexcount "^1$"
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1
+
permuteDirectory 1 files
${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files
@@ -119,6 +124,12 @@ ${CMD_PREFIX} ostree --repo=repo static-delta generate --max-bsdiff-size=10000 -
${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev}-${newrev} || exit 1
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount
+assert_file_has_content indexcount "^2$"
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${newrev} || exit 1
+
if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --empty 2>>err.txt; then
assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded"
fi
@@ -249,6 +260,41 @@ ${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null
echo 'ok pull empty delta part'
+rm -rf repo/delta-indexes
+${CMD_PREFIX} ostree --repo=repo summary -u
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_file_has_content summary.txt "ostree\.static\-deltas"
+
+${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary true
+${CMD_PREFIX} ostree --repo=repo summary -u
+
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_not_file_has_content summary.txt "ostree\.static\-deltas"
+
+rm -rf repo2
+mkdir repo2 && ostree_repo_init repo2 --mode=bare-user
+${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
+
+rm -rf repo/delta-indexes
+if ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} &> no-delta.txt; then
+ assert_not_reached "failing pull with --require-static-deltas unexpectedly succeeded"
+fi
+assert_file_has_content no-delta.txt "Static deltas required, but none found for"
+
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev}
+
+${CMD_PREFIX} ostree --repo=repo2 fsck
+${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null
+
+${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary false
+${CMD_PREFIX} ostree --repo=repo summary -u
+
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_file_has_content summary.txt "ostree\.static\-deltas"
+
+echo 'ok pull delta part with delta index'
+
# Make a new branch to test "rebase deltas"
echo otherbranch-content > files/otherbranch-content
${CMD_PREFIX} ostree --repo=repo commit -b otherbranch --tree=dir=files