lib/repo-pull: Support retrying requests on transient network errors

Allow network requests to be re-queued if they failed with a transient
error, such as a socket timeout. Retry each request up to a limit
(default: 5), and only then fail the entire pull and propagate the error
to the caller.

Add a new ostree_repo_pull_with_options() option, n-network-retries, to
control the number of retries (including setting it back to the old
default of 0, if the caller wants).

Currently, retries are not supported for FetchDeltaSuperData requests,
as they are not queued. Once they are queued, adding support for retries
should be trivial. A FIXME comment has been left for this.

Signed-off-by: Philip Withnall <withnall@endlessm.com>

Closes: #1594
Approved by: jlebon

5 files changed, 202 insertions, 38 deletions

diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c
index aca31358..9cdb82c6 100644
--- a/src/libostree/ostree-fetcher-util.c
+++ b/src/libostree/ostree-fetcher-util.c
@@ -52,15 +52,15 @@ fetch_uri_sync_on_complete (GObject        *object,
   data->done = TRUE;
 }
 
-gboolean
-_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
-                                            GPtrArray     *mirrorlist,
-                                            const char     *filename,
-                                            OstreeFetcherRequestFlags flags,
-                                            GBytes         **out_contents,
-                                            guint64        max_size,
-                                            GCancellable   *cancellable,
-                                            GError         **error)
+static gboolean
+_ostree_fetcher_mirrored_request_to_membuf_once  (OstreeFetcher              *fetcher,
+                                                  GPtrArray                  *mirrorlist,
+                                                  const char                 *filename,
+                                                  OstreeFetcherRequestFlags   flags,
+                                                  GBytes                    **out_contents,
+                                                  guint64                     max_size,
+                                                  GCancellable               *cancellable,
+                                                  GError                    **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GMainContext) mainctx = NULL;
@@ -108,11 +108,42 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
   return ret;
 }
 
+gboolean
+_ostree_fetcher_mirrored_request_to_membuf  (OstreeFetcher              *fetcher,
+                                             GPtrArray                  *mirrorlist,
+                                             const char                 *filename,
+                                             OstreeFetcherRequestFlags   flags,
+                                             guint                       n_network_retries,
+                                             GBytes                    **out_contents,
+                                             guint64                     max_size,
+                                             GCancellable               *cancellable,
+                                             GError                    **error)
+{
+  g_autoptr(GError) local_error = NULL;
+  guint n_retries_remaining = n_network_retries;
+
+  do
+    {
+      g_clear_error (&local_error);
+      if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist,
+                                                           filename, flags,
+                                                           out_contents, max_size,
+                                                           cancellable, &local_error))
+        return TRUE;
+    }
+  while (_ostree_fetcher_should_retry_request (local_error, n_retries_remaining--));
+
+  g_assert (local_error != NULL);
+  g_propagate_error (error, g_steal_pointer (&local_error));
+  return FALSE;
+}
+
 /* Helper for callers who just want to fetch single one-off URIs */
 gboolean
 _ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
                                        OstreeFetcherURI *uri,
                                        OstreeFetcherRequestFlags flags,
+                                       guint          n_network_retries,
                                        GBytes         **out_contents,
                                        guint64        max_size,
                                        GCancellable   *cancellable,
@@ -121,7 +152,7 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
   g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
   g_ptr_array_add (mirrorlist, uri); /* no transfer */
   return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags,
-                                                     out_contents, max_size,
+                                                     n_network_retries, out_contents, max_size,
                                                      cancellable, error);
 }
 
@@ -144,3 +175,46 @@ _ostree_fetcher_journal_failure (const char *remote_name,
                    NULL);
 #endif
 }
+
+/* Check whether a particular operation should be retried. This is entirely
+ * based on how it failed (if at all) last time, and whether the operation has
+ * some retries left. The retry count is set when the operation is first
+ * created, and must be decremented by the caller. (@n_retries_remaining == 0)
+ * will always return %FALSE from this function.
+ *
+ * FIXME: In future, we may decide to use transient failures like this as a hint
+ * to prioritise other mirrors for a particular pull operation (for example). */
+gboolean
+_ostree_fetcher_should_retry_request (const GError *error,
+                                      guint         n_retries_remaining)
+{
+  if (error == NULL)
+    g_debug ("%s: error: unset, n_retries_remaining: %u",
+             G_STRFUNC, n_retries_remaining);
+  else
+    g_debug ("%s: error: %u:%u %s, n_retries_remaining: %u",
+             G_STRFUNC, error->domain, error->code, error->message,
+             n_retries_remaining);
+
+  if (error == NULL || n_retries_remaining == 0)
+    return FALSE;
+
+  /* Return TRUE for transient errors. */
+  if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_TIMED_OUT) ||
+      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) ||
+      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) ||
+#if !GLIB_CHECK_VERSION(2, 44, 0)
+      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BROKEN_PIPE) ||
+#else
+      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CONNECTION_CLOSED) ||
+#endif
+      g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_NOT_FOUND) ||
+      g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_TEMPORARY_FAILURE))
+    {
+      g_debug ("Should retry request (remaining: %u retries), due to transient error: %s",
+               n_retries_remaining, error->message);
+      return TRUE;
+    }
+
+  return FALSE;
+}
diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h
index 1e2dabe5..5f62ad45 100644
--- a/src/libostree/ostree-fetcher-util.h
+++ b/src/libostree/ostree-fetcher-util.h
@@ -56,6 +56,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
                                                      GPtrArray     *mirrorlist,
                                                      const char    *filename,
                                                      OstreeFetcherRequestFlags flags,
+                                                     guint          n_network_retries,
                                                      GBytes         **out_contents,
                                                      guint64        max_size,
                                                      GCancellable   *cancellable,
@@ -64,6 +65,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
 gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
                                                 OstreeFetcherURI *uri,
                                                 OstreeFetcherRequestFlags flags,
+                                                guint          n_network_retries,
                                                 GBytes         **out_contents,
                                                 guint64        max_size,
                                                 GCancellable   *cancellable,
@@ -73,6 +75,8 @@ void _ostree_fetcher_journal_failure (const char *remote_name,
                                       const char *url,
                                       const char *msg);
 
+gboolean _ostree_fetcher_should_retry_request (const GError *error,
+                                               guint         n_retries_remaining);
 
 G_END_DECLS
 
diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c
index 7cb879f7..cb8a50e3 100644
--- a/src/libostree/ostree-metalink.c
+++ b/src/libostree/ostree-metalink.c
@@ -50,6 +50,7 @@ struct OstreeMetalink
   OstreeFetcher *fetcher;
   char *requested_file;
   guint64 max_size;
+  guint n_network_retries;
 };
 
 G_DEFINE_TYPE (OstreeMetalink, _ostree_metalink, G_TYPE_OBJECT)
@@ -401,7 +402,8 @@ OstreeMetalink *
 _ostree_metalink_new (OstreeFetcher  *fetcher,
                       const char     *requested_file,
                       guint64         max_size,
-                      OstreeFetcherURI *uri)
+                      OstreeFetcherURI *uri,
+                      guint             n_network_retries)
 {
   OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL);
 
@@ -409,6 +411,7 @@ _ostree_metalink_new (OstreeFetcher  *fetcher,
   self->requested_file = g_strdup (requested_file);
   self->max_size = max_size;
   self->uri = _ostree_fetcher_uri_clone (uri);
+  self->n_network_retries = n_network_retries;
 
   return self;
 }
@@ -432,7 +435,9 @@ try_one_url (OstreeMetalinkRequest *self,
   gssize n_bytes;
 
   if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher,
-                                              uri, 0, &bytes,
+                                              uri, 0,
+                                              self->metalink->n_network_retries,
+                                              &bytes,
                                               self->metalink->max_size,
                                               self->cancellable,
                                               error))
@@ -613,6 +618,7 @@ _ostree_metalink_request_sync (OstreeMetalink        *self,
   request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL);
 
   if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0,
+                                              self->n_network_retries,
                                               &contents, self->max_size,
                                               cancellable, error))
     goto out;
diff --git a/src/libostree/ostree-metalink.h b/src/libostree/ostree-metalink.h
index a9a090b8..1a48945f 100644
--- a/src/libostree/ostree-metalink.h
+++ b/src/libostree/ostree-metalink.h
@@ -48,7 +48,8 @@ GType   _ostree_metalink_get_type (void) G_GNUC_CONST;
 OstreeMetalink *_ostree_metalink_new (OstreeFetcher  *fetcher,
                                       const char     *requested_file,
                                       guint64         max_size,
-                                      OstreeFetcherURI *uri);
+                                      OstreeFetcherURI *uri,
+                                      guint             n_network_retries);
 
 gboolean _ostree_metalink_request_sync (OstreeMetalink        *self,
                                         OstreeFetcherURI      **out_target_uri,
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 16e9efda..668968e6 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -61,6 +61,12 @@
 #define OSTREE_REPO_PULL_CONTENT_PRIORITY  (OSTREE_FETCHER_DEFAULT_PRIORITY)
 #define OSTREE_REPO_PULL_METADATA_PRIORITY (OSTREE_REPO_PULL_CONTENT_PRIORITY - 100)
 
+/* Arbitrarily chosen number of retries for all download operations when they
+ * receive a transient network error (such as a socket timeout) — see
+ * _ostree_fetcher_should_retry_request(). This is the default value for the
+ * `n-network-retries` pull option. */
+#define DEFAULT_N_NETWORK_RETRIES 5
+
 typedef enum {
   OSTREE_FETCHER_SECURITY_STATE_CA_PINNED,
   OSTREE_FETCHER_SECURITY_STATE_TLS,
@@ -92,6 +98,7 @@ typedef struct {
   gboolean      dry_run;
   gboolean      dry_run_emitted_progress;
   gboolean      legacy_transaction_resuming;
+  guint         n_network_retries;
   enum {
     OSTREE_PULL_PHASE_FETCHING_REFS,
     OSTREE_PULL_PHASE_FETCHING_OBJECTS
@@ -177,6 +184,7 @@ typedef struct {
   gboolean     object_is_stored;
 
   OstreeCollectionRef *requested_ref;  /* (nullable) */
+  guint n_retries_remaining;
 } FetchObjectData;
 
 typedef struct {
@@ -187,6 +195,7 @@ typedef struct {
   char *to_revision;
   guint i;
   guint64 size;
+  guint n_retries_remaining;
 } FetchStaticDeltaData;
 
 typedef struct {
@@ -502,6 +511,8 @@ idle_worker (gpointer user_data)
   scan_one_metadata_object (pull_data, checksum, scan_data->objtype,
                             scan_data->path, scan_data->recursion_depth,
                             scan_data->requested_ref, pull_data->cancellable, &error);
+
+  /* No need to retry scan tasks, since they’re local. */
   check_outstanding_requests_handle_error (pull_data, &error);
   scan_object_queue_data_free (scan_data);
 
@@ -532,6 +543,7 @@ static gboolean
 fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
                                        GPtrArray      *mirrorlist,
                                        const char     *filename,
+                                       guint           n_network_retries,
                                        char          **out_contents,
                                        GCancellable   *cancellable,
                                        GError        **error)
@@ -539,6 +551,7 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
   g_autoptr(GBytes) bytes = NULL;
   if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
                                                    filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION,
+                                                   n_network_retries,
                                                    &bytes,
                                                    OSTREE_MAX_METADATA_SIZE,
                                                    cancellable, error))
@@ -557,6 +570,7 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
 static gboolean
 fetch_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
                               OstreeFetcherURI *uri,
+                              guint           n_network_retries,
                               char          **out_contents,
                               GCancellable   *cancellable,
                               GError        **error)
@@ -564,7 +578,8 @@ fetch_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
   g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
   g_ptr_array_add (mirrorlist, uri); /* no transfer */
   return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist,
-                                                NULL, out_contents,
+                                                NULL, n_network_retries,
+                                                out_contents,
                                                 cancellable, error);
 }
 
@@ -718,6 +733,7 @@ on_local_object_imported (GObject        *object,
   pull_data->n_imported_content++;
   g_assert_cmpint (pull_data->n_outstanding_content_write_requests, >, 0);
   pull_data->n_outstanding_content_write_requests--;
+  /* No retries for local reads. */
   check_outstanding_requests_handle_error (pull_data, &local_error);
 }
 
@@ -893,7 +909,8 @@ fetch_ref_contents (OtPullData                 *pull_data,
 
       if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
                                                   pull_data->meta_mirrorlist,
-                                                  filename, &ret_contents,
+                                                  filename, pull_data->n_network_retries,
+                                                  &ret_contents,
                                                   cancellable, error))
         return FALSE;
 
@@ -1017,6 +1034,7 @@ content_fetch_on_write_complete (GObject        *object,
     pull_data->n_fetched_deltapart_fallbacks++;
  out:
   pull_data->n_outstanding_content_write_requests--;
+  /* No retries for local writes. */
   check_outstanding_requests_handle_error (pull_data, &local_error);
   fetch_object_data_free (fetch_data);
 }
@@ -1102,9 +1120,14 @@ content_fetch_on_complete (GObject        *object,
 
  out:
   pull_data->n_outstanding_content_fetches--;
-  check_outstanding_requests_handle_error (pull_data, &local_error);
+
+  if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+    enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
+  else
+    check_outstanding_requests_handle_error (pull_data, &local_error);
+
   if (free_fetch_data)
-    fetch_object_data_free (fetch_data);
+    g_clear_pointer (&fetch_data, fetch_object_data_free);
 }
 
 static void
@@ -1148,6 +1171,7 @@ on_metadata_written (GObject           *object,
   pull_data->n_outstanding_metadata_write_requests--;
   fetch_object_data_free (fetch_data);
 
+  /* No need to retry local write operations. */
   check_outstanding_requests_handle_error (pull_data, &local_error);
 }
 
@@ -1285,10 +1309,17 @@ meta_fetch_on_complete (GObject           *object,
  out:
   g_assert (pull_data->n_outstanding_metadata_fetches > 0);
   pull_data->n_outstanding_metadata_fetches--;
-  pull_data->n_fetched_metadata++;
-  check_outstanding_requests_handle_error (pull_data, &local_error);
+
+  if (local_error == NULL)
+    pull_data->n_fetched_metadata++;
+
+  if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+    enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
+  else
+    check_outstanding_requests_handle_error (pull_data, &local_error);
+
   if (free_fetch_data)
-    fetch_object_data_free (fetch_data);
+    g_clear_pointer (&fetch_data, fetch_object_data_free);
 }
 
 static void
@@ -1320,6 +1351,7 @@ on_static_delta_written (GObject           *object,
  out:
   g_assert (pull_data->n_outstanding_deltapart_write_requests > 0);
   pull_data->n_outstanding_deltapart_write_requests--;
+  /* No need to retry on failure to write locally. */
   check_outstanding_requests_handle_error (pull_data, &local_error);
   /* Always free state */
   fetch_static_delta_data_free (fetch_data);
@@ -1365,10 +1397,17 @@ static_deltapart_fetch_on_complete (GObject           *object,
  out:
   g_assert (pull_data->n_outstanding_deltapart_fetches > 0);
   pull_data->n_outstanding_deltapart_fetches--;
-  pull_data->n_fetched_deltaparts++;
-  check_outstanding_requests_handle_error (pull_data, &local_error);
+
+  if (local_error == NULL)
+    pull_data->n_fetched_deltaparts++;
+
+  if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+    enqueue_one_static_delta_part_request_s (pull_data, g_steal_pointer (&fetch_data));
+  else
+    check_outstanding_requests_handle_error (pull_data, &local_error);
+
   if (free_fetch_data)
-    fetch_static_delta_data_free (fetch_data);
+    g_clear_pointer (&fetch_data, fetch_static_delta_data_free);
 }
 
 static gboolean
@@ -2084,10 +2123,9 @@ enqueue_one_object_request (OtPullData                *pull_data,
   fetch_data->is_detached_meta = is_detached_meta;
   fetch_data->object_is_stored = object_is_stored;
   fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
+  fetch_data->n_retries_remaining = pull_data->n_network_retries;
 
-  gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
-
-  if (is_meta)
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
     pull_data->n_requested_metadata++;
   else
     pull_data->n_requested_content++;
@@ -2167,8 +2205,8 @@ load_remote_repo_config (OtPullData    *pull_data,
 
   if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
                                               pull_data->meta_mirrorlist,
-                                              "config", &contents,
-                                              cancellable, error))
+                                              "config", pull_data->n_network_retries,
+                                              &contents, cancellable, error))
     return FALSE;
 
   g_autoptr(GKeyFile) ret_keyfile = g_key_file_new ();
@@ -2350,6 +2388,7 @@ process_one_static_delta (OtPullData                 *pull_data,
           fetch_data->is_detached_meta = FALSE;
           fetch_data->object_is_stored = FALSE;
           fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
+          fetch_data->n_retries_remaining = pull_data->n_network_retries;
 
           ostree_repo_write_metadata_async (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
                                             to_commit,
@@ -2423,6 +2462,7 @@ process_one_static_delta (OtPullData                 *pull_data,
       fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
       fetch_data->size = size;
       fetch_data->i = i;
+      fetch_data->n_retries_remaining = pull_data->n_network_retries;
 
       if (inline_part_bytes != NULL)
         {
@@ -2705,7 +2745,11 @@ on_superblock_fetched (GObject   *src,
  out:
   g_assert (pull_data->n_outstanding_metadata_fetches > 0);
   pull_data->n_outstanding_metadata_fetches--;
-  pull_data->n_fetched_metadata++;
+
+  if (local_error == NULL)
+    pull_data->n_fetched_metadata++;
+
+  /* FIXME: This should check _ostree_fetcher_should_retry_request(). */
   check_outstanding_requests_handle_error (pull_data, &local_error);
 
   g_clear_pointer (&fetch_data, fetch_delta_super_data_free);
@@ -2938,6 +2982,7 @@ _ostree_preload_metadata_file (OstreeRepo    *self,
                                GPtrArray     *mirrorlist,
                                const char    *filename,
                                gboolean      is_metalink,
+                               guint         n_network_retries,
                                GBytes        **out_bytes,
                                GCancellable  *cancellable,
                                GError        **error)
@@ -2951,7 +2996,7 @@ _ostree_preload_metadata_file (OstreeRepo    *self,
       g_autoptr(OstreeMetalink) metalink =
         _ostree_metalink_new (fetcher, filename,
                               OSTREE_MAX_METADATA_SIZE,
-                              mirrorlist->pdata[0]);
+                              mirrorlist->pdata[0], n_network_retries);
 
       _ostree_metalink_request_sync (metalink, NULL, out_bytes,
                                      cancellable, &local_error);
@@ -2973,6 +3018,7 @@ _ostree_preload_metadata_file (OstreeRepo    *self,
     {
       return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename,
                                                          OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+                                                         n_network_retries,
                                                          out_bytes, OSTREE_MAX_METADATA_SIZE,
                                                          cancellable, error);
     }
@@ -2981,6 +3027,7 @@ _ostree_preload_metadata_file (OstreeRepo    *self,
 static gboolean
 fetch_mirrorlist (OstreeFetcher  *fetcher,
                   const char     *mirrorlist_url,
+                  guint           n_network_retries,
                   GPtrArray     **out_mirrorlist,
                   GCancellable   *cancellable,
                   GError        **error)
@@ -2993,8 +3040,8 @@ fetch_mirrorlist (OstreeFetcher  *fetcher,
     return FALSE;
 
   g_autofree char *contents = NULL;
-  if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents,
-                                     cancellable, error))
+  if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, n_network_retries,
+                                     &contents, cancellable, error))
     return glnx_prefix_error (error, "While fetching mirrorlist '%s'",
                               mirrorlist_url);
 
@@ -3040,8 +3087,8 @@ fetch_mirrorlist (OstreeFetcher  *fetcher,
           GError *local_error = NULL;
           g_autoptr(OstreeFetcherURI) config_uri = _ostree_fetcher_uri_new_subpath (mirror_uri, "config");
 
-          if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL,
-                                            cancellable, &local_error))
+          if (fetch_uri_contents_utf8_sync (fetcher, config_uri, n_network_retries,
+                                            NULL, cancellable, &local_error))
             g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
           else
             {
@@ -3083,12 +3130,14 @@ repo_remote_fetch_summary (OstreeRepo    *self,
   g_autoptr(GVariant) extra_headers = NULL;
   g_autoptr(GPtrArray) mirrorlist = NULL;
   const char *append_user_agent = NULL;
+  guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
 
   if (options)
     {
       (void) g_variant_lookup (options, "override-url", "&s", &url_override);
       (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers);
       (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent);
+      (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries);
     }
 
   mainctx = g_main_context_new ();
@@ -3117,7 +3166,7 @@ repo_remote_fetch_summary (OstreeRepo    *self,
         g_str_has_prefix (url_string, "mirrorlist="))
       {
         if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="),
-                               &mirrorlist, cancellable, error))
+                               n_network_retries, &mirrorlist, cancellable, error))
           goto out;
       }
     else
@@ -3143,6 +3192,7 @@ repo_remote_fetch_summary (OstreeRepo    *self,
                                       mirrorlist,
                                       "summary.sig",
                                       metalink_url_string ? TRUE : FALSE,
+                                      n_network_retries,
                                       out_signatures,
                                       cancellable,
                                       error))
@@ -3168,6 +3218,7 @@ repo_remote_fetch_summary (OstreeRepo    *self,
                                           mirrorlist,
                                           "summary",
                                           metalink_url_string ? TRUE : FALSE,
+                                          n_network_retries,
                                           out_summary,
                                           cancellable,
                                           error))
@@ -3390,6 +3441,9 @@ initiate_request (OtPullData                 *pull_data,
  *   * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
  *   * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches
  *   * append-user-agent (s): Additional string to append to the user agent
+ *   * n-network-retries (u): Number of times to retry each download on receiving
+ *     a transient network error, such as a socket timeout; default is 5, 0
+ *     means return errors without retrying
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -3423,6 +3477,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   gboolean opt_gpg_verify_set = FALSE;
   gboolean opt_gpg_verify_summary_set = FALSE;
   gboolean opt_collection_refs_set = FALSE;
+  gboolean opt_n_network_retries_set = FALSE;
   const char *main_collection_id = NULL;
   const char *url_override = NULL;
   gboolean inherit_transaction = FALSE;
@@ -3462,6 +3517,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
       (void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check);
       (void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent);
+      opt_n_network_retries_set =
+        g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries);
 
       if (pull_data->remote_refspec_name != NULL)
         pull_data->remote_name = g_strdup (pull_data->remote_refspec_name);
@@ -3502,6 +3559,9 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   pull_data->main_context = g_main_context_ref_thread_default ();
   pull_data->flags = flags;
 
+  if (!opt_n_network_retries_set)
+    pull_data->n_network_retries = DEFAULT_N_NETWORK_RETRIES;
+
   pull_data->repo = self;
   pull_data->progress = progress;
 
@@ -3647,6 +3707,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         {
           if (!fetch_mirrorlist (pull_data->fetcher,
                                  baseurl + strlen ("mirrorlist="),
+                                 pull_data->n_network_retries,
                                  &pull_data->meta_mirrorlist,
                                  cancellable, error))
             goto out;
@@ -3673,7 +3734,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         goto out;
 
       metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
-                                       OSTREE_MAX_METADATA_SIZE, metalink_uri);
+                                       OSTREE_MAX_METADATA_SIZE, metalink_uri,
+                                       pull_data->n_network_retries);
 
       if (! _ostree_metalink_request_sync (metalink,
                                            &target_uri,
@@ -3719,6 +3781,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
           {
             if (!fetch_mirrorlist (pull_data->fetcher,
                                    contenturl + strlen ("mirrorlist="),
+                                   pull_data->n_network_retries,
                                    &pull_data->content_mirrorlist,
                                    cancellable, error))
               goto out;
@@ -3865,6 +3928,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
                                                          pull_data->meta_mirrorlist,
                                                          "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+                                                         pull_data->n_network_retries,
                                                          &bytes_sig,
                                                          OSTREE_MAX_METADATA_SIZE,
                                                          cancellable, error))
@@ -3889,6 +3953,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
                                                          pull_data->meta_mirrorlist,
                                                          "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+                                                         pull_data->n_network_retries,
                                                          &bytes_summary,
                                                          OSTREE_MAX_METADATA_SIZE,
                                                          cancellable, error))
@@ -4663,6 +4728,7 @@ typedef struct
   GVariant *options;
   OstreeAsyncProgress *progress;
   OstreeRepoFinder *default_finder_avahi;
+  guint n_network_retries;
 } FindRemotesData;
 
 static void
@@ -4682,7 +4748,8 @@ static FindRemotesData *
 find_remotes_data_new (const OstreeCollectionRef * const *refs,
                        GVariant                      *options,
                        OstreeAsyncProgress           *progress,
-                       OstreeRepoFinder              *default_finder_avahi)
+                       OstreeRepoFinder              *default_finder_avahi,
+                       guint                          n_network_retries)
 {
   g_autoptr(FindRemotesData) data = NULL;
 
@@ -4691,6 +4758,7 @@ find_remotes_data_new (const OstreeCollectionRef * const *refs,
   data->options = (options != NULL) ? g_variant_ref (options) : NULL;
   data->progress = (progress != NULL) ? g_object_ref (progress) : NULL;
   data->default_finder_avahi = (default_finder_avahi != NULL) ? g_object_ref (default_finder_avahi) : NULL;
+  data->n_network_retries = n_network_retries;
 
   return g_steal_pointer (&data);
 }
@@ -4770,6 +4838,9 @@ static void find_remotes_cb (GObject      *obj,
  *   * `override-commit-ids` (`as`): Array of specific commit IDs to fetch. The nth
  *   commit ID applies to the nth ref, so this must be the same length as @refs, if
  *   provided.
+ *   * `n-network-retries` (`u`): Number of times to retry each download on
+ *   receiving a transient network error, such as a socket timeout; default is
+ *   5, 0 means return errors without retrying.
  *
  * @finders must be a non-empty %NULL-terminated array of the #OstreeRepoFinder
  * instances to use, or %NULL to use the system default set of finders, which
@@ -4799,6 +4870,7 @@ ostree_repo_find_remotes_async (OstreeRepo                     *self,
   g_autoptr(OstreeRepoFinder) finder_mount = NULL;
   g_autoptr(OstreeRepoFinder) finder_avahi = NULL;
   g_autofree char **override_commit_ids = NULL;
+  guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
 
   g_return_if_fail (OSTREE_IS_REPO (self));
   g_return_if_fail (is_valid_collection_ref_array (refs));
@@ -4812,6 +4884,8 @@ ostree_repo_find_remotes_async (OstreeRepo                     *self,
     {
       (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
       g_return_if_fail (override_commit_ids == NULL || g_strv_length ((gchar **) refs) == g_strv_length (override_commit_ids));
+
+      (void) g_variant_lookup (options, "n-network-retries", "u", &n_network_retries);
     }
 
   /* Set up a task for the whole operation. */
@@ -4867,7 +4941,7 @@ ostree_repo_find_remotes_async (OstreeRepo                     *self,
   /* We need to keep a pointer to the default Avahi finder so we can stop it
    * again after the operation, which happens implicitly by dropping the final
    * ref. */
-  data = find_remotes_data_new (refs, options, progress, finder_avahi);
+  data = find_remotes_data_new (refs, options, progress, finder_avahi, n_network_retries);
   g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free);
 
   /* Asynchronously resolve all possible remotes for the given refs. */
@@ -5260,6 +5334,7 @@ find_remotes_cb (GObject      *obj,
                                                                mirrorlist,
                                                                commit_filename,
                                                                OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+                                                               data->n_network_retries,
                                                                &commit_bytes,
                                                                0,  /* no maximum size */
                                                                cancellable,
@@ -5710,6 +5785,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo                           *self,
       copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as"));
       copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u"));
       copy_option (&options_dict, &local_options_dict, "append-user-agent", G_VARIANT_TYPE ("s"));
+      copy_option (&options_dict, &local_options_dict, "n-network-retries", G_VARIANT_TYPE ("u"));
 
       local_options = g_variant_dict_end (&local_options_dict);
 
@@ -5827,6 +5903,9 @@ ostree_repo_pull_from_remotes_finish (OstreeRepo    *self,
  * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options
  * - http-headers (a(ss)): Additional headers to add to all HTTP requests
  * - append-user-agent (s): Additional string to append to the user agent
+ * - n-network-retries (u): Number of times to retry each download on receiving
+ *   a transient network error, such as a socket timeout; default is 5, 0
+ *   means return errors without retrying
  *
  * Returns: %TRUE on success, %FALSE on failure
  */