diff options
author | OpenShift Merge Robot <openshift-merge-robot@users.noreply.github.com> | 2020-10-22 18:20:23 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-10-22 18:20:23 -0400 |
commit | f8f6464580574b40e0e3bccf7a60a5ec6cf8c105 (patch) | |
tree | 866f6efaab08236cdb3e6767d13cf048be40f083 /src/libostree/ostree-repo-pull.c | |
parent | 95a6d151457842d4a130f7ea7131659eebe674f5 (diff) | |
parent | 0974a7faf174988c93832e0b4693a11e3a42821e (diff) | |
download | ostree-f8f6464580574b40e0e3bccf7a60a5ec6cf8c105.tar.gz |
Merge pull request #2205 from pwithnall/etags-and-last-modified
Add support for ETag and Last-Modified headers for summary and summary.sig
Diffstat (limited to 'src/libostree/ostree-repo-pull.c')
-rw-r--r-- | src/libostree/ostree-repo-pull.c | 279 |
1 files changed, 264 insertions, 15 deletions
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 58c80543..bb143136 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -46,6 +46,7 @@ #include <gio/gunixinputstream.h> #include <sys/statvfs.h> +#include <sys/time.h> #ifdef HAVE_LIBSYSTEMD #include <systemd/sd-journal.h> #endif @@ -458,8 +459,9 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, g_autoptr(GBytes) bytes = NULL; if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION, + NULL, 0, n_network_retries, - &bytes, + &bytes, NULL, NULL, NULL, OSTREE_MAX_METADATA_SIZE, cancellable, error)) return FALSE; @@ -965,7 +967,7 @@ content_fetch_on_complete (GObject *object, OstreeObjectType objtype; gboolean free_fetch_data = TRUE; - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error)) goto out; ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); @@ -1105,7 +1107,7 @@ meta_fetch_on_complete (GObject *object, g_debug ("fetch of %s%s complete", checksum_obj, fetch_data->is_detached_meta ? " (detached)" : ""); - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error)) { if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { @@ -1282,7 +1284,7 @@ static_deltapart_fetch_on_complete (GObject *object, g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum); - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error)) goto out; /* Transfer ownership of the fd */ @@ -1994,7 +1996,7 @@ start_fetch (OtPullData *pull_data, if (!is_meta && pull_data->trusted_http_direct) flags |= OSTREE_FETCHER_REQUEST_LINKABLE; _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist, - obj_subpath, flags, expected_max_size, + obj_subpath, flags, NULL, 0, expected_max_size, is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY : OSTREE_REPO_PULL_CONTENT_PRIORITY, pull_data->cancellable, @@ -2119,7 +2121,7 @@ start_fetch_deltapart (OtPullData *pull_data, g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, pull_data->content_mirrorlist, - deltapart_path, 0, fetch->size, + deltapart_path, 0, NULL, 0, fetch->size, OSTREE_FETCHER_DEFAULT_PRIORITY, pull_data->cancellable, static_deltapart_fetch_on_complete, @@ -2494,6 +2496,7 @@ on_superblock_fetched (GObject *src, if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src, res, &delta_superblock_data, + NULL, NULL, NULL, error)) { if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) @@ -2570,6 +2573,7 @@ start_fetch_delta_superblock (OtPullData *pull_data, _ostree_fetcher_request_to_membuf (pull_data->fetcher, pull_data->content_mirrorlist, delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, OSTREE_MAX_METADATA_SIZE, 0, pull_data->cancellable, on_superblock_fetched, @@ -2699,6 +2703,64 @@ _ostree_repo_verify_summary (OstreeRepo *self, return TRUE; } +static void +_ostree_repo_load_cache_summary_properties (OstreeRepo *self, + const char *filename, + const char *extension, + char **out_etag, + guint64 *out_last_modified) +{ + const char *file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", filename, extension); + glnx_autofd int fd = -1; + + if (self->cache_dir_fd == -1) + return; + + if (!glnx_openat_rdonly (self->cache_dir_fd, file, TRUE, &fd, NULL)) + return; + + if (out_etag != NULL) + { + g_autoptr(GBytes) etag_bytes = glnx_fgetxattr_bytes (fd, "user.etag", NULL); + if (etag_bytes != NULL) + { + const guint8 *buf; + gsize buf_len; + + buf = g_bytes_get_data (etag_bytes, &buf_len); + + /* Loosely validate against https://tools.ietf.org/html/rfc7232#section-2.3 + * by checking there are no embedded nuls. */ + for (gsize i = 0; i < buf_len; i++) + { + if (buf[i] == 0) + { + buf_len = 0; + break; + } + } + + /* Nul-terminate and return */ + if (buf_len > 0) + *out_etag = g_strndup ((const char *) buf, buf_len); + else + *out_etag = NULL; + } + else + *out_etag = NULL; + } + + if (out_last_modified != NULL) + { + struct stat statbuf; + + if (glnx_fstatat (fd, "", &statbuf, AT_EMPTY_PATH, NULL)) + *out_last_modified = statbuf.st_mtim.tv_sec; + else + *out_last_modified = 0; + } +} + static gboolean _ostree_repo_load_cache_summary_file (OstreeRepo *self, const char *filename, @@ -2774,11 +2836,38 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, return TRUE; } +static void +store_file_cache_properties (int dir_fd, + const char *filename, + const char *etag, + guint64 last_modified) +{ + glnx_autofd int fd = -1; + struct timespec time_vals[] = + { + { .tv_sec = last_modified, .tv_nsec = UTIME_OMIT }, /* access, leave unchanged */ + { .tv_sec = last_modified, .tv_nsec = 0 }, /* modification */ + }; + + if (!glnx_openat_rdonly (dir_fd, filename, TRUE, &fd, NULL)) + return; + + if (etag != NULL) + TEMP_FAILURE_RETRY (fsetxattr (fd, "user.etag", etag, strlen (etag), 0)); + else + TEMP_FAILURE_RETRY (fremovexattr (fd, "user.etag")); + + if (last_modified > 0) + TEMP_FAILURE_RETRY (futimens (fd, time_vals)); +} + static gboolean _ostree_repo_save_cache_summary_file (OstreeRepo *self, const char *filename, const char *extension, GBytes *data, + const char *etag, + guint64 last_modified, GCancellable *cancellable, GError **error) { @@ -2799,6 +2888,9 @@ _ostree_repo_save_cache_summary_file (OstreeRepo *self, cancellable, error)) return FALSE; + /* Store the caching properties. This is non-fatal on failure. */ + store_file_cache_properties (self->cache_dir_fd, file, etag, last_modified); + return TRUE; } @@ -2807,16 +2899,24 @@ static gboolean _ostree_repo_cache_summary (OstreeRepo *self, const char *remote, GBytes *summary, + const char *summary_etag, + guint64 summary_last_modified, GBytes *summary_sig, + const char *summary_sig_etag, + guint64 summary_sig_last_modified, GCancellable *cancellable, GError **error) { if (!_ostree_repo_save_cache_summary_file (self, remote, NULL, - summary, cancellable, error)) + summary, + summary_etag, summary_last_modified, + cancellable, error)) return FALSE; if (!_ostree_repo_save_cache_summary_file (self, remote, ".sig", - summary_sig, cancellable, error)) + summary_sig, + summary_sig_etag, summary_sig_last_modified, + cancellable, error)) return FALSE; return TRUE; @@ -2964,8 +3064,13 @@ _ostree_preload_metadata_file (OstreeRepo *self, GPtrArray *mirrorlist, const char *filename, gboolean is_metalink, + const char *if_none_match, + guint64 if_modified_since, guint n_network_retries, GBytes **out_bytes, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GCancellable *cancellable, GError **error) { @@ -3000,8 +3105,10 @@ _ostree_preload_metadata_file (OstreeRepo *self, { return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + if_none_match, if_modified_since, n_network_retries, - out_bytes, OSTREE_MAX_METADATA_SIZE, + out_bytes, out_not_modified, out_etag, out_last_modified, + OSTREE_MAX_METADATA_SIZE, cancellable, error); } } @@ -3360,6 +3467,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, { gboolean ret = FALSE; g_autoptr(GBytes) bytes_summary = NULL; + gboolean summary_not_modified = FALSE; + g_autofree char *summary_etag = NULL; + guint64 summary_last_modified = 0; g_autofree char *metalink_url_str = NULL; g_autoptr(GHashTable) requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ g_autoptr(GHashTable) commits_to_fetch = NULL; @@ -3827,6 +3937,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, { g_autoptr(GBytes) bytes_sig = NULL; + gboolean summary_sig_not_modified = FALSE; + g_autofree char *summary_sig_etag = NULL; + guint64 summary_sig_last_modified = 0; gsize n; g_autoptr(GVariant) refs = NULL; g_autoptr(GVariant) deltas = NULL; @@ -3855,14 +3968,47 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!bytes_sig) { + g_autofree char *summary_sig_if_none_match = NULL; + guint64 summary_sig_if_modified_since = 0; + + /* Load the summary.sig from the network, but send its ETag and + * Last-Modified from the on-disk cache (if it exists) to reduce the + * download size if nothing’s changed. */ + _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig", + &summary_sig_if_none_match, &summary_sig_if_modified_since); + + g_clear_pointer (&summary_sig_etag, g_free); + summary_sig_last_modified = 0; if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + summary_sig_if_none_match, summary_sig_if_modified_since, pull_data->n_network_retries, &bytes_sig, + &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary.sig from the cache. Also load summary, since + * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ + if (summary_sig_not_modified) + { + g_clear_pointer (&bytes_sig, g_bytes_unref); + g_clear_pointer (&bytes_summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig", + &bytes_sig, + cancellable, error)) + goto out; + + if (!bytes_summary && + !pull_data->remote_repo_local && + !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, + &bytes_summary, + cancellable, error)) + goto out; + } } if (bytes_sig && @@ -3884,14 +4030,35 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!pull_data->summary && !bytes_summary) { + g_autofree char *summary_if_none_match = NULL; + guint64 summary_if_modified_since = 0; + + _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL, + &summary_if_none_match, &summary_if_modified_since); + + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + summary_if_none_match, summary_if_modified_since, pull_data->n_network_retries, &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary from the cache. */ + if (summary_not_modified) + { + g_clear_pointer (&bytes_summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, + &bytes_summary, + cancellable, error)) + goto out; + } } #ifndef OSTREE_DISABLE_GPGME @@ -3930,7 +4097,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, { if (summary_from_cache) { - /* The cached summary doesn't match, fetch a new one and verify again */ + /* The cached summary doesn't match, fetch a new one and verify again. + * Don’t set the cache headers in the HTTP request, to force a + * full download. */ if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -3945,12 +4114,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, summary_from_cache = FALSE; g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, /* no cache headers */ pull_data->n_network_retries, &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; @@ -3993,7 +4166,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, { if (summary_from_cache) { - /* The cached summary doesn't match, fetch a new one and verify again */ + /* The cached summary doesn't match, fetch a new one and verify again. + * Don’t set the cache headers in the HTTP request, to force a + * full download. */ if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -4008,12 +4183,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, summary_from_cache = FALSE; g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, /* no cache headers */ pull_data->n_network_retries, &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; @@ -4033,6 +4212,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (bytes_summary) { pull_data->summary_data = g_bytes_ref (bytes_summary); + pull_data->summary_etag = g_strdup (summary_etag); + pull_data->summary_last_modified = summary_last_modified; pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); if (!g_variant_is_normal_form (pull_data->summary)) @@ -4050,7 +4231,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, } if (bytes_sig) - pull_data->summary_data_sig = g_bytes_ref (bytes_sig); + { + pull_data->summary_data_sig = g_bytes_ref (bytes_sig); + pull_data->summary_sig_etag = g_strdup (summary_sig_etag); + pull_data->summary_sig_last_modified = summary_sig_last_modified; + } } if (!summary_from_cache && bytes_summary && bytes_sig) @@ -4059,7 +4244,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, !_ostree_repo_cache_summary (self, remote_name_or_baseurl, bytes_summary, + summary_etag, summary_last_modified, bytes_sig, + summary_sig_etag, summary_sig_last_modified, cancellable, error)) goto out; @@ -4494,6 +4681,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; + store_file_cache_properties (pull_data->repo->repo_dir_fd, "summary", + pull_data->summary_etag, pull_data->summary_last_modified); + if (pull_data->summary_data_sig) { buf = g_bytes_get_data (pull_data->summary_data_sig, &len); @@ -4501,6 +4691,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, buf, len, replaceflag, cancellable, error)) goto out; + + store_file_cache_properties (pull_data->repo->repo_dir_fd, "summary.sig", + pull_data->summary_sig_etag, pull_data->summary_sig_last_modified); } } @@ -4687,7 +4880,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref); + g_clear_pointer (&pull_data->summary_etag, g_free); g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref); + g_clear_pointer (&pull_data->summary_sig_etag, g_free); g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref); g_clear_pointer (&pull_data->static_delta_superblocks, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->commit_to_depth, (GDestroyNotify) g_hash_table_unref); @@ -5499,8 +5694,10 @@ find_remotes_cb (GObject *obj, mirrorlist, commit_filename, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, data->n_network_retries, &commit_bytes, + NULL, NULL, NULL, 0, /* no maximum size */ cancellable, &error)) @@ -6114,6 +6311,16 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autoptr(GPtrArray) mirrorlist = NULL; const char *append_user_agent = NULL; guint n_network_retries = DEFAULT_N_NETWORK_RETRIES; + gboolean summary_sig_not_modified = FALSE; + g_autofree char *summary_sig_if_none_match = NULL; + g_autofree char *summary_sig_etag = NULL; + gboolean summary_not_modified = FALSE; + g_autofree char *summary_if_none_match = NULL; + g_autofree char *summary_etag = NULL; + guint64 summary_sig_if_modified_since = 0; + guint64 summary_sig_last_modified = 0; + guint64 summary_if_modified_since = 0; + guint64 summary_last_modified = 0; g_return_val_if_fail (OSTREE_REPO (self), FALSE); g_return_val_if_fail (name != NULL, FALSE); @@ -6159,22 +6366,48 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, &mirrorlist, cancellable, error)) return FALSE; - /* FIXME: Send the ETag from the cache with the request for summary.sig to + /* Send the ETag from the cache with the request for summary.sig to * avoid downloading summary.sig unnecessarily. This won’t normally provide - * any benefits (but won’t do any harm) since summary.sig is typically 500B - * in size. But if a repository has multiple keys, the signature file will + * much benefit since summary.sig is typically 590B in size (vs a 0B HTTP 304 + * response). But if a repository has multiple keys, the signature file will * grow and this optimisation may be useful. */ + _ostree_repo_load_cache_summary_properties (self, name, ".sig", + &summary_sig_if_none_match, &summary_sig_if_modified_since); + _ostree_repo_load_cache_summary_properties (self, name, NULL, + &summary_if_none_match, &summary_if_modified_since); + if (!_ostree_preload_metadata_file (self, fetcher, mirrorlist, "summary.sig", metalink_url_string ? TRUE : FALSE, + summary_sig_if_none_match, summary_sig_if_modified_since, n_network_retries, &signatures, + &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified, cancellable, error)) return FALSE; + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary.sig from the cache. Also load summary, since + * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ + if (summary_sig_not_modified) + { + g_clear_pointer (&signatures, g_bytes_unref); + g_clear_pointer (&summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, name, ".sig", + &signatures, + cancellable, error)) + return FALSE; + + if (!summary && + !_ostree_repo_load_cache_summary_file (self, name, NULL, + &summary, + cancellable, error)) + return FALSE; + } + if (signatures) { if (!_ostree_repo_load_cache_summary_if_same_sig (self, @@ -6195,11 +6428,25 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, mirrorlist, "summary", metalink_url_string ? TRUE : FALSE, + summary_if_none_match, summary_if_modified_since, n_network_retries, &summary, + &summary_not_modified, &summary_etag, &summary_last_modified, cancellable, error)) return FALSE; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary.sig from the cache. Also load summary, since + * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ + if (summary_not_modified) + { + g_clear_pointer (&summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, name, NULL, + &summary, + cancellable, error)) + return FALSE; + } } if (!_ostree_repo_verify_summary (self, name, @@ -6215,7 +6462,9 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (!_ostree_repo_cache_summary (self, name, summary, + summary_etag, summary_last_modified, signatures, + summary_sig_etag, summary_sig_last_modified, cancellable, &temp_error)) { |