From 3ec7b5db18cc6249048e46049be537ed135451b3 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 5 Apr 2023 16:26:10 -0600 Subject: tests: Ensure real GIO backends aren't used Set a few environment variables during tests to ensure fake GIO backends are used. This is particularly important with the soup fetcher backend as it can cause strange test errors in containerized test environments. Upstream soup has been setting these 3 environment variables for their tests since 2015. --- Makefile-tests.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile-tests.am b/Makefile-tests.am index a30b86dd..429dafb9 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -36,6 +36,9 @@ AM_TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ PATH=$$(cd $(top_builddir)/tests && pwd):$${PATH} \ OSTREE_FEATURES="$(OSTREE_FEATURES)" \ PYTHONUNBUFFERED=1 \ + GSETTINGS_BACKEND=memory \ + GIO_USE_PROXY_RESOLVER=dummy \ + GIO_USE_VFS=local \ $(NULL) if BUILDOPT_ASAN AM_TESTS_ENVIRONMENT += OT_SKIP_READDIR_RAND=1 G_SLICE=always-malloc -- cgit v1.2.1 From d0ea2db4300eb7871b59d0b997a7f06869120297 Mon Sep 17 00:00:00 2001 From: Daniel Kolesa Date: Thu, 17 Feb 2022 20:12:18 +0100 Subject: fetcher: add libsoup3 backend The default is still soup2, you can use --with-soup3 to enable the soup3 backend instead. --- Makefile-libostree.am | 7 + Makefile-man.am | 2 +- Makefile-ostree.am | 2 +- Makefile-tests.am | 4 +- Makefile.am | 12 +- configure.ac | 35 +- src/libostree/ostree-fetcher-soup3.c | 1325 ++++++++++++++++++++++++++++++++++ src/ostree/main.c | 2 +- src/ostree/ostree-trivial-httpd.c | 228 ++++-- 9 files changed, 1544 insertions(+), 73 deletions(-) create mode 100644 src/libostree/ostree-fetcher-soup3.c diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 52ea49d4..d18714ae 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -221,18 +221,25 @@ libostree_1_la_SOURCES += \ $(NULL) endif +# Only enable one fetcher backend. if USE_CURL libostree_1_la_SOURCES += src/libostree/ostree-fetcher-curl.c \ $(NULL) libostree_1_la_CFLAGS += $(OT_DEP_CURL_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_CURL_LIBS) else +if USE_LIBSOUP3 +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup3.c +libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) +libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) +else if USE_LIBSOUP libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup.c libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) endif endif +endif if USE_LIBMOUNT libostree_1_la_CFLAGS += $(OT_DEP_LIBMOUNT_CFLAGS) diff --git a/Makefile-man.am b/Makefile-man.am index 5c7f2413..41c59327 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -34,7 +34,7 @@ ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 \ ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 \ ostree-rev-parse.1 ostree-show.1 ostree-sign.1 ostree-summary.1 \ ostree-static-delta.1 -if USE_LIBSOUP +if USE_LIBSOUP_OR_LIBSOUP3 man1_files += ostree-trivial-httpd.1 else # We still want to distribute the source, even if we are not building it diff --git a/Makefile-ostree.am b/Makefile-ostree.am index fb377075..05d58a20 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -141,7 +141,7 @@ if USE_CURL_OR_SOUP ostree_SOURCES += src/ostree/ot-builtin-pull.c endif -if USE_LIBSOUP +if USE_LIBSOUP_OR_LIBSOUP3 # Eventually once we stop things from using this, we should support disabling this ostree_SOURCES += src/ostree/ot-builtin-trivial-httpd.c pkglibexec_PROGRAMS += ostree-trivial-httpd diff --git a/Makefile-tests.am b/Makefile-tests.am index 429dafb9..32817704 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -185,7 +185,7 @@ else EXTRA_DIST += tests/test-rofiles-fuse.sh endif -if USE_LIBSOUP +if USE_LIBSOUP_OR_LIBSOUP3 _installed_or_uninstalled_test_scripts += tests/test-remote-cookies.sh endif @@ -438,7 +438,7 @@ dist_test_scripts += $(_installed_or_uninstalled_test_scripts) test_programs += $(_installed_or_uninstalled_test_programs) endif -if !USE_LIBSOUP +if !USE_LIBSOUP_OR_LIBSOUP3 no-soup-for-you-warning: @echo "WARNING: $(PACKAGE) was built without libsoup, which is currently" 1>&2 @echo "WARNING: required for many unit tests." 1>&2 diff --git a/Makefile.am b/Makefile.am index 4e669164..50282156 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,8 +29,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DOSTREE_COMPILATION \ -DG_LOG_DOMAIN=\"OSTree\" \ -DOSTREE_GITREV='"$(OSTREE_GITREV)"' \ - -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_66 '-DGLIB_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,70)' \ - -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 '-DSOUP_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,48)' + -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_66 '-DGLIB_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,70)' # For strict aliasing, see https://bugzilla.gnome.org/show_bug.cgi?id=791622 AM_CFLAGS += -std=gnu99 -fno-strict-aliasing $(WARN_CFLAGS) AM_DISTCHECK_CONFIGURE_FLAGS += \ @@ -39,6 +38,10 @@ AM_DISTCHECK_CONFIGURE_FLAGS += \ --disable-maintainer-mode \ $(NULL) +if USE_LIBSOUP +AM_CPPFLAGS += -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 '-DSOUP_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,48)' +endif + GITIGNOREFILES = aclocal.m4 build-aux/ buildutil/*.m4 config.h.in gtk-doc.make # Generated by ci/gh-build.sh @@ -69,8 +72,13 @@ EXTRA_DIST += autogen.sh COPYING README.md OT_INTERNAL_GIO_UNIX_CFLAGS = $(OT_DEP_GIO_UNIX_CFLAGS) OT_INTERNAL_GIO_UNIX_LIBS = $(OT_DEP_GIO_UNIX_LIBS) +if USE_LIBSOUP3 +OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP3_CFLAGS) +OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP3_LIBS) +else OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP_CFLAGS) OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP_LIBS) +endif # This canonicalizes the PKG_CHECK_MODULES or AM_PATH_GPGME results if USE_GPGME diff --git a/configure.ac b/configure.ac index 21f204f3..115096ef 100644 --- a/configure.ac +++ b/configure.ac @@ -149,6 +149,21 @@ AS_IF([test x$enable_http2 != xno ], [ OSTREE_FEATURES="$OSTREE_FEATURES no-http2" ]) +SOUP3_DEPENDENCY="libsoup-3.0 >= 3.0.0" +AC_ARG_WITH(soup3, + AS_HELP_STRING([--with-soup3], [Use libsoup3 @<:@default=no@:>@]), + [], [with_soup3=no]) +AS_IF([test x$with_soup3 != xno], [ + PKG_CHECK_MODULES(OT_DEP_SOUP3, $SOUP3_DEPENDENCY) + with_soup3=yes + AC_DEFINE([HAVE_LIBSOUP3], 1, [Define if we have libsoup3]) + OSTREE_FEATURES="$OSTREE_FEATURES libsoup3" + with_soup_default=no + dnl soup3 always supports client certs + have_libsoup_client_certs=yes +], [with_soup_default=check]) +AM_CONDITIONAL(USE_LIBSOUP3, test x$with_soup3 != xno) + dnl When bumping the libsoup-2.4 dependency, remember to bump dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in dnl Makefile.am @@ -180,7 +195,7 @@ AS_IF([test x$with_soup != xno], [ ], [], [#include ]) AS_IF([test x$enable_libsoup_client_certs = xyes && test x$have_libsoup_client_certs != xyes], [ AC_MSG_ERROR([libsoup client certs explicitly requested but not found]) - ]) + ]) CFLAGS=$save_CFLAGS ], [ with_soup=no @@ -190,6 +205,13 @@ if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES libsoup"; fi AM_CONDITIONAL(USE_LIBSOUP, test x$with_soup != xno) AM_CONDITIONAL(HAVE_LIBSOUP_CLIENT_CERTS, test x$have_libsoup_client_certs = xyes) +dnl Some components use either soup2 or soup3. +AM_CONDITIONAL([USE_LIBSOUP_OR_LIBSOUP3], + [test x$with_soup = xyes || test x$with_soup3 = xyes]) +AS_IF([test x$with_soup = xyes || test x$with_soup3 = xyes], [ + AC_DEFINE([HAVE_LIBSOUP_OR_LIBSOUP3], 1, [Define if we have libsoup.pc or libsoup3.pc]) +]) + AC_ARG_ENABLE(trivial-httpd-cmdline, [AS_HELP_STRING([--enable-trivial-httpd-cmdline], [Continue to support "ostree trivial-httpd" [default=no]])],, @@ -198,13 +220,16 @@ AS_IF([test x$enable_trivial_httpd_cmdline = xyes], [AC_DEFINE([BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE], 1, [Define if we are enabling ostree trivial-httpd entrypoint])] ) -AS_IF([test x$with_curl = xyes && test x$with_soup = xno], [ +AS_IF([test x$with_curl = xyes && test x$with_soup = xno && test x$with_soup3 = xno], [ AC_MSG_WARN([Curl enabled, but libsoup is not; libsoup is needed for tests (make check, etc.)]) ]) -AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno) -AS_IF([test x$with_curl != xno || test x$with_soup != xno], +AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno || test x$with_soup3 != xno) +AS_IF([test x$with_curl != xno || test x$with_soup != xno || test x$with_soup3 != xno], [AC_DEFINE([HAVE_LIBCURL_OR_LIBSOUP], 1, [Define if we have soup or curl])]) -AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], [test x$with_soup = xyes], [fetcher_backend=libsoup], [fetcher_backend=none]) +AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], + [test x$with_soup = xyes], [fetcher_backend=libsoup], + [test x$with_soup3 = xyes], [fetcher_backend=libsoup3], + [fetcher_backend=none]) m4_ifdef([GOBJECT_INTROSPECTION_CHECK], [ GOBJECT_INTROSPECTION_CHECK([1.51.5]) diff --git a/src/libostree/ostree-fetcher-soup3.c b/src/libostree/ostree-fetcher-soup3.c new file mode 100644 index 00000000..9f495a72 --- /dev/null +++ b/src/libostree/ostree-fetcher-soup3.c @@ -0,0 +1,1325 @@ +/* + * Copyright (C) 2011 Colin Walters + * Copyright (C) 2022 Igalia S.L. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + * Author: Colin Walters + * Author: Daniel Kolesa + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "libglnx.h" +#include "ostree-fetcher.h" +#include "ostree-fetcher-util.h" +#include "ostree-tls-cert-interaction-private.h" +#include "ostree-enumtypes.h" +#include "ostree.h" +#include "ostree-repo-private.h" +#include "otutil.h" + +typedef enum { + OSTREE_FETCHER_STATE_PENDING, + OSTREE_FETCHER_STATE_DOWNLOADING, + OSTREE_FETCHER_STATE_COMPLETE +} OstreeFetcherState; + +typedef struct { + int ref_count; /* atomic */ + + SoupSession *session; /* not referenced */ + GMainContext *main_context; + volatile gint running; + GError *initialization_error; /* Any failure to load the db */ + + char *remote_name; + int base_tmpdir_dfd; + + GVariant *extra_headers; + gboolean transfer_gzip; + + /* Our active HTTP requests */ + GHashTable *outstanding; + + /* Shared across threads; be sure to lock. */ + GHashTable *output_stream_set; /* set */ + GMutex output_stream_set_lock; + + /* Also protected by output_stream_set_lock. */ + guint64 total_downloaded; + + GError *oob_error; +} ThreadClosure; + +typedef struct { + int ref_count; /* atomic */ + + ThreadClosure *thread_closure; + GPtrArray *mirrorlist; /* list of base URIs */ + char *filename; /* relative name to fetch or NULL */ + guint mirrorlist_idx; + + OstreeFetcherState state; + + SoupMessage *message; + GFile *file; + struct OstreeFetcher *fetcher; + + gboolean is_membuf; + OstreeFetcherRequestFlags flags; + char *if_none_match; /* request ETag */ + guint64 if_modified_since; /* seconds since the epoch */ + GInputStream *request_body; + GLnxTmpfile tmpf; + GOutputStream *out_stream; + gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */ + char *out_etag; /* response ETag */ + guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */ + + guint64 max_size; + guint64 current_size; + guint64 content_length; +} OstreeFetcherPendingURI; + +/* Used by session_thread_idle_add() */ +typedef void (*SessionThreadFunc) (ThreadClosure *thread_closure, + gpointer data); + +/* Used by session_thread_idle_add() */ +typedef struct { + ThreadClosure *thread_closure; + SessionThreadFunc function; + gpointer data; + GDestroyNotify notify; +} IdleClosure; + +struct OstreeFetcher +{ + GObject parent_instance; + + OstreeFetcherConfigFlags config_flags; + + GThread *session_thread; + ThreadClosure *thread_closure; +}; + +enum { + PROP_0, + PROP_CONFIG_FLAGS +}; + +G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) + +static ThreadClosure * +thread_closure_ref (ThreadClosure *thread_closure) +{ + int refcount; + g_return_val_if_fail (thread_closure != NULL, NULL); + refcount = g_atomic_int_add (&thread_closure->ref_count, 1); + g_assert (refcount > 0); + return thread_closure; +} + +static void +thread_closure_unref (ThreadClosure *thread_closure) +{ + g_return_if_fail (thread_closure != NULL); + + if (g_atomic_int_dec_and_test (&thread_closure->ref_count)) + { + /* The session thread should have cleared this by now. */ + g_assert (thread_closure->session == NULL); + + g_clear_pointer (&thread_closure->main_context, g_main_context_unref); + + g_clear_pointer (&thread_closure->extra_headers, g_variant_unref); + + g_clear_pointer (&thread_closure->output_stream_set, g_hash_table_unref); + g_mutex_clear (&thread_closure->output_stream_set_lock); + + g_clear_pointer (&thread_closure->oob_error, g_error_free); + + g_free (thread_closure->remote_name); + + g_slice_free (ThreadClosure, thread_closure); + } +} + +static void +idle_closure_free (IdleClosure *idle_closure) +{ + g_clear_pointer (&idle_closure->thread_closure, thread_closure_unref); + + if (idle_closure->notify != NULL) + idle_closure->notify (idle_closure->data); + + g_slice_free (IdleClosure, idle_closure); +} + +static OstreeFetcherPendingURI * +pending_uri_ref (OstreeFetcherPendingURI *pending) +{ + gint refcount; + g_assert (pending); + refcount = g_atomic_int_add (&pending->ref_count, 1); + g_assert (refcount > 0); + return pending; +} + +static void +pending_uri_unref (OstreeFetcherPendingURI *pending) +{ + if (!g_atomic_int_dec_and_test (&pending->ref_count)) + return; + + g_clear_pointer (&pending->thread_closure, thread_closure_unref); + + g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref); + g_free (pending->filename); + g_clear_object (&pending->message); + g_clear_object (&pending->file); + g_clear_object (&pending->request_body); + g_free (pending->if_none_match); + glnx_tmpfile_clear (&pending->tmpf); + g_clear_object (&pending->out_stream); + g_free (pending->out_etag); + g_free (pending); +} + +static gboolean +session_thread_idle_dispatch (gpointer data) +{ + IdleClosure *idle_closure = data; + + idle_closure->function (idle_closure->thread_closure, + idle_closure->data); + + return G_SOURCE_REMOVE; +} + +static void +session_thread_idle_add (ThreadClosure *thread_closure, + SessionThreadFunc function, + gpointer data, + GDestroyNotify notify) +{ + IdleClosure *idle_closure; + + g_return_if_fail (thread_closure != NULL); + g_return_if_fail (function != NULL); + + idle_closure = g_slice_new (IdleClosure); + idle_closure->thread_closure = thread_closure_ref (thread_closure); + idle_closure->function = function; + idle_closure->data = data; + idle_closure->notify = notify; + + g_main_context_invoke_full (thread_closure->main_context, + G_PRIORITY_DEFAULT, + session_thread_idle_dispatch, + idle_closure, /* takes ownership */ + (GDestroyNotify) idle_closure_free); +} + +static void +session_thread_add_logger (ThreadClosure *thread_closure, + gpointer data) +{ + glnx_unref_object SoupLogger *logger = NULL; + + logger = soup_logger_new (SOUP_LOGGER_LOG_BODY); + soup_logger_set_max_body_size (logger, 500); + soup_session_add_feature (thread_closure->session, + SOUP_SESSION_FEATURE (logger)); +} + +static void +session_thread_set_proxy_cb (ThreadClosure *thread_closure, + gpointer data) +{ + GProxyResolver *resolver = data; + + g_object_set (thread_closure->session, + "proxy-resolver", + g_object_ref (resolver), NULL); +} + +static void +session_thread_set_cookie_jar_cb (ThreadClosure *thread_closure, + gpointer data) +{ + SoupCookieJar *jar = data; + + soup_session_add_feature (thread_closure->session, + SOUP_SESSION_FEATURE (jar)); +} + +static void +session_thread_set_headers_cb (ThreadClosure *thread_closure, + gpointer data) +{ + GVariant *headers = data; + + g_clear_pointer (&thread_closure->extra_headers, g_variant_unref); + thread_closure->extra_headers = g_variant_ref (headers); +} + +static void +session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *cert_and_key_path = data; /* str\0str\0 in one malloc buf */ + const char *cert_path = cert_and_key_path; + const char *key_path = cert_and_key_path + strlen (cert_and_key_path) + 1; + g_autoptr(OstreeTlsCertInteraction) interaction = NULL; + + /* The GTlsInteraction instance must be created in the + * session thread so it uses the correct GMainContext. */ + interaction = _ostree_tls_cert_interaction_new (cert_path, key_path); + + g_object_set (thread_closure->session, + "tls-interaction", + interaction, NULL); +} + +static void +session_thread_set_tls_database_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *db_path = data; + + if (db_path != NULL) + { + glnx_unref_object GTlsDatabase *tlsdb = NULL; + + g_clear_error (&thread_closure->initialization_error); + tlsdb = g_tls_file_database_new (db_path, &thread_closure->initialization_error); + + if (tlsdb) + g_object_set (thread_closure->session, + "tls-database", + tlsdb, NULL); + } +} + +static void +session_thread_set_extra_user_agent_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *extra_user_agent = data; + if (extra_user_agent != NULL) + { + g_autofree char *ua = + g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent); + g_object_set (thread_closure->session, "user-agent", ua, NULL); + } + else + { + g_object_set (thread_closure->session, "user-agent", + OSTREE_FETCHER_USERAGENT_STRING, NULL); + } +} + +static void +on_request_sent (GObject *object, GAsyncResult *result, gpointer user_data); + +static void +request_send_async (ThreadClosure *thread_closure, + GTask *task) +{ + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; + + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); + + if (pending->file) + g_file_read_async (pending->file, g_task_get_priority (task), + cancellable, on_request_sent, task); + else + { + soup_session_send_async (thread_closure->session, pending->message, + g_task_get_priority (task), + cancellable, on_request_sent, task); + } +} + +static void +start_pending_request (ThreadClosure *thread_closure, + GTask *task) +{ + + OstreeFetcherPendingURI *pending; + + pending = g_task_get_task_data (task); + + g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending)); + request_send_async (thread_closure, g_object_ref (task)); +} + +static gboolean +_message_accept_cert_loose (SoupMessage *msg, + GTlsCertificate *tls_peer_certificate, + GTlsCertificateFlags tls_peer_errors, + gpointer data) +{ + OstreeFetcherConfigFlags config_flags; + + config_flags = GPOINTER_TO_UINT (data); + + return ((config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0); +} + +static void +create_pending_soup_request (OstreeFetcherPendingURI *pending) +{ + OstreeFetcherURI *next_mirror = NULL; + g_autoptr(OstreeFetcherURI) uri = NULL; + GUri *guri = NULL; + + g_assert (pending->mirrorlist); + g_assert (pending->mirrorlist_idx < pending->mirrorlist->len); + + next_mirror = g_ptr_array_index (pending->mirrorlist, pending->mirrorlist_idx); + if (pending->filename) + uri = _ostree_fetcher_uri_new_subpath (next_mirror, pending->filename); + if (!uri) + uri = (OstreeFetcherURI*)g_uri_ref ((GUri*)next_mirror); + + g_clear_object (&pending->message); + g_clear_object (&pending->file); + + guri = (GUri*)(uri ? uri : next_mirror); + + /* file:// URI is handle via GFile */ + if (!strcmp (g_uri_get_scheme (guri), "file")) + { + char *str = g_uri_to_string (guri); + pending->file = g_file_new_for_uri (str); + g_free (str); + return; + } + + pending->message = soup_message_new_from_uri ("GET", guri); + + if (pending->if_none_match != NULL) + { + soup_message_headers_append (soup_message_get_request_headers (pending->message), + "If-None-Match", pending->if_none_match); + } + + if (pending->if_modified_since > 0) + { + g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (pending->if_modified_since); + g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z"); + soup_message_headers_append (soup_message_get_request_headers (pending->message), + "If-Modified-Since", mod_date); + } + + if (pending->message && pending->fetcher->config_flags != 0) + { + g_signal_connect (pending->message, "accept-certificate", + G_CALLBACK (_message_accept_cert_loose), + GUINT_TO_POINTER (pending->fetcher->config_flags)); + } +} + +static void +session_thread_request_uri (ThreadClosure *thread_closure, + gpointer data) +{ + GTask *task = G_TASK (data); + OstreeFetcherPendingURI *pending; + + pending = g_task_get_task_data (task); + + /* If we caught an error in init, re-throw it for every request */ + if (thread_closure->initialization_error) + { + g_task_return_error (task, g_error_copy (thread_closure->initialization_error)); + return; + } + + create_pending_soup_request (pending); + + if (pending->message && thread_closure->extra_headers) + { + g_autoptr(GVariantIter) viter = g_variant_iter_new (thread_closure->extra_headers); + const char *key; + const char *value; + + while (g_variant_iter_next (viter, "(&s&s)", &key, &value)) + soup_message_headers_append (soup_message_get_request_headers (pending->message), key, value); + } + + if (pending->is_membuf) + request_send_async (thread_closure, g_object_ref (task)); + else + start_pending_request (thread_closure, task); +} + +static gpointer +ostree_fetcher_session_thread (gpointer data) +{ + ThreadClosure *closure = data; + g_autoptr(GMainContext) mainctx = g_main_context_ref (closure->main_context); + + /* This becomes the GMainContext that SoupSession schedules async + * callbacks and emits signals from. Make it the thread-default + * context for this thread before creating the session. */ + g_main_context_push_thread_default (mainctx); + + /* We retain ownership of the SoupSession reference. */ + closure->session = soup_session_new_with_options ("user-agent", OSTREE_FETCHER_USERAGENT_STRING, + "timeout", 60, + "idle-timeout", 60, + "max-conns-per-host", _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS, + NULL); + + /* SoupContentDecoder is included in the session by default. Remove it + * if gzip compression isn't in use. + */ + if (!closure->transfer_gzip) + soup_session_remove_feature_by_type (closure->session, SOUP_TYPE_CONTENT_DECODER); + + /* This model ensures we don't hit a race using g_main_loop_quit(); + * see also what pull_termination_condition() in ostree-repo-pull.c + * is doing. + */ + while (g_atomic_int_get (&closure->running)) + g_main_context_iteration (closure->main_context, TRUE); + + /* Since the ThreadClosure may be finalized from any thread we + * unreference all data related to the SoupSession ourself to ensure + * it's freed in the same thread where it was created. */ + g_clear_pointer (&closure->outstanding, g_hash_table_unref); + g_clear_pointer (&closure->session, g_object_unref); + + thread_closure_unref (closure); + + /* Do this last, since libsoup uses g_main_current_source() which + * relies on it. + */ + g_main_context_pop_thread_default (mainctx); + + return NULL; +} + +static void +_ostree_fetcher_set_property (GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + self->config_flags = g_value_get_flags (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_get_property (GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + g_value_set_flags (value, self->config_flags); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_finalize (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + /* Terminate the session thread. */ + g_atomic_int_set (&self->thread_closure->running, 0); + g_main_context_wakeup (self->thread_closure->main_context); + if (self->session_thread) + { + /* We need to explicitly synchronize to clean up TLS */ + if (self->session_thread != g_thread_self ()) + g_thread_join (self->session_thread); + else + g_clear_pointer (&self->session_thread, g_thread_unref); + } + g_clear_pointer (&self->thread_closure, thread_closure_unref); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); +} + +static void +_ostree_fetcher_constructed (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + g_autoptr(GMainContext) main_context = NULL; + const char *http_proxy; + + main_context = g_main_context_new (); + + self->thread_closure = g_slice_new0 (ThreadClosure); + self->thread_closure->ref_count = 1; + self->thread_closure->main_context = g_main_context_ref (main_context); + self->thread_closure->running = 1; + self->thread_closure->transfer_gzip = (self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) != 0; + + self->thread_closure->outstanding = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)pending_uri_unref); + self->thread_closure->output_stream_set = g_hash_table_new_full (NULL, NULL, + (GDestroyNotify) NULL, + (GDestroyNotify) g_object_unref); + g_mutex_init (&self->thread_closure->output_stream_set_lock); + + if (g_getenv ("OSTREE_DEBUG_HTTP")) + { + session_thread_idle_add (self->thread_closure, + session_thread_add_logger, + NULL, (GDestroyNotify) NULL); + } + + http_proxy = g_getenv ("http_proxy"); + if (http_proxy != NULL && http_proxy[0] != '\0') + _ostree_fetcher_set_proxy (self, http_proxy); + + /* FIXME Maybe implement GInitableIface and use g_thread_try_new() + * so we can try to handle thread creation errors gracefully? */ + self->session_thread = g_thread_new ("fetcher-session-thread", + ostree_fetcher_session_thread, + thread_closure_ref (self->thread_closure)); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); +} + +static void +_ostree_fetcher_class_init (OstreeFetcherClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = _ostree_fetcher_set_property; + gobject_class->get_property = _ostree_fetcher_get_property; + gobject_class->finalize = _ostree_fetcher_finalize; + gobject_class->constructed = _ostree_fetcher_constructed; + + g_object_class_install_property (gobject_class, + PROP_CONFIG_FLAGS, + g_param_spec_flags ("config-flags", + "", + "", + OSTREE_TYPE_FETCHER_CONFIG_FLAGS, + OSTREE_FETCHER_FLAGS_NONE, + G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS)); +} + +static void +_ostree_fetcher_init (OstreeFetcher *self) +{ +} + +OstreeFetcher * +_ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, + OstreeFetcherConfigFlags flags) +{ + OstreeFetcher *self; + + self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + self->thread_closure->remote_name = g_strdup (remote_name); + self->thread_closure->base_tmpdir_dfd = tmpdir_dfd; + + return self; +} + +int +_ostree_fetcher_get_dfd (OstreeFetcher *fetcher) +{ + return fetcher->thread_closure->base_tmpdir_dfd; +} + +void +_ostree_fetcher_set_proxy (OstreeFetcher *self, + const char *http_proxy) +{ + GProxyResolver *resolver; + GUri *guri; + + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (http_proxy != NULL && http_proxy[0] != '\0'); + + /* validate first */ + guri = g_uri_parse (http_proxy, SOUP_HTTP_URI_FLAGS, NULL); + + if (!guri) + { + g_warning ("Invalid proxy URI '%s'", http_proxy); + return; + } + + g_uri_unref (guri); + + resolver = g_simple_proxy_resolver_new (http_proxy, NULL); + + session_thread_idle_add (self->thread_closure, + session_thread_set_proxy_cb, + resolver, /* takes ownership */ + (GDestroyNotify) g_object_unref); + } + +void +_ostree_fetcher_set_cookie_jar (OstreeFetcher *self, + const char *jar_path) +{ + SoupCookieJar *jar; + + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (jar_path != NULL); + + jar = soup_cookie_jar_text_new (jar_path, TRUE); + + session_thread_idle_add (self->thread_closure, + session_thread_set_cookie_jar_cb, + jar, /* takes ownership */ + (GDestroyNotify) g_object_unref); +} + +void +_ostree_fetcher_set_client_cert (OstreeFetcher *self, + const char *cert_path, + const char *key_path) +{ + g_autoptr(GString) buf = NULL; + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + if (cert_path) + { + buf = g_string_new (cert_path); + g_string_append_c (buf, '\0'); + g_string_append (buf, key_path); + } + + session_thread_idle_add (self->thread_closure, + session_thread_set_tls_interaction_cb, + g_string_free (g_steal_pointer (&buf), FALSE), + (GDestroyNotify) g_free); +} + +void +_ostree_fetcher_set_tls_database (OstreeFetcher *self, + const char *tlsdb_path) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + session_thread_idle_add (self->thread_closure, + session_thread_set_tls_database_cb, + g_strdup (tlsdb_path), + (GDestroyNotify) g_free); +} + +void +_ostree_fetcher_set_extra_headers (OstreeFetcher *self, + GVariant *extra_headers) +{ + session_thread_idle_add (self->thread_closure, + session_thread_set_headers_cb, + g_variant_ref (extra_headers), + (GDestroyNotify) g_variant_unref); +} + +void +_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, + const char *extra_user_agent) +{ + session_thread_idle_add (self->thread_closure, + session_thread_set_extra_user_agent_cb, + g_strdup (extra_user_agent), + (GDestroyNotify) g_free); +} + +static gboolean +finish_stream (OstreeFetcherPendingURI *pending, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + struct stat stbuf; + + /* Close it here since we do an async fstat(), where we don't want + * to hit a bad fd. + */ + if (pending->out_stream) + { + if ((pending->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + { + const guint8 nulchar = 0; + gsize bytes_written; + + if (!g_output_stream_write_all (pending->out_stream, &nulchar, 1, &bytes_written, + cancellable, error)) + goto out; + } + + if (!g_output_stream_close (pending->out_stream, cancellable, error)) + goto out; + + g_mutex_lock (&pending->thread_closure->output_stream_set_lock); + g_hash_table_remove (pending->thread_closure->output_stream_set, + pending->out_stream); + g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + } + + if (!pending->is_membuf) + { + if (!glnx_fstat (pending->tmpf.fd, &stbuf, error)) + goto out; + } + + pending->state = OSTREE_FETCHER_STATE_COMPLETE; + + if (!pending->is_membuf) + { + if (stbuf.st_size < pending->content_length) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete"); + goto out; + } + else + { + g_mutex_lock (&pending->thread_closure->output_stream_set_lock); + pending->thread_closure->total_downloaded += stbuf.st_size; + g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + } + } + + ret = TRUE; + out: + (void) g_input_stream_close (pending->request_body, NULL, NULL); + return ret; +} + +static void +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data); + +static void +remove_pending (OstreeFetcherPendingURI *pending) +{ + /* Hold a temporary ref to ensure the reference to + * pending->thread_closure is valid. + */ + pending_uri_ref (pending); + g_hash_table_remove (pending->thread_closure->outstanding, pending); + pending_uri_unref (pending); +} + +static void +on_out_splice_complete (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + GTask *task = G_TASK (user_data); + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; + gssize bytes_written; + GError *local_error = NULL; + + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); + + bytes_written = g_output_stream_splice_finish ((GOutputStream *)object, + result, + &local_error); + if (bytes_written < 0) + goto out; + + g_input_stream_read_bytes_async (pending->request_body, + 8192, G_PRIORITY_DEFAULT, + cancellable, + on_stream_read, + g_object_ref (task)); + + out: + if (local_error) + { + g_task_return_error (task, local_error); + remove_pending (pending); + } + + g_object_unref (task); +} + +static void +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + GTask *task = G_TASK (user_data); + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; + g_autoptr(GBytes) bytes = NULL; + gsize bytes_read; + GError *local_error = NULL; + + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); + + /* Only open the output stream on demand to ensure we use as + * few file descriptors as possible. + */ + if (!pending->out_stream) + { + if (!pending->is_membuf) + { + if (!_ostree_fetcher_tmpf_from_flags (pending->flags, pending->thread_closure->base_tmpdir_dfd, + &pending->tmpf, &local_error)) + goto out; + pending->out_stream = g_unix_output_stream_new (pending->tmpf.fd, FALSE); + } + else + { + pending->out_stream = g_memory_output_stream_new_resizable (); + } + + g_mutex_lock (&pending->thread_closure->output_stream_set_lock); + g_hash_table_add (pending->thread_closure->output_stream_set, + g_object_ref (pending->out_stream)); + g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + } + + /* Get a GBytes buffer */ + bytes = g_input_stream_read_bytes_finish ((GInputStream*)object, result, &local_error); + if (!bytes) + goto out; + bytes_read = g_bytes_get_size (bytes); + + /* Was this the end of the stream? */ + if (bytes_read == 0) + { + if (!finish_stream (pending, cancellable, &local_error)) + goto out; + if (pending->is_membuf) + { + g_task_return_pointer (task, + g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream*)pending->out_stream), + (GDestroyNotify) g_bytes_unref); + } + else + { + if (lseek (pending->tmpf.fd, 0, SEEK_SET) < 0) + { + glnx_set_error_from_errno (&local_error); + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else + g_task_return_boolean (task, TRUE); + } + remove_pending (pending); + } + else + { + /* Verify max size */ + if (pending->max_size > 0) + { + if (bytes_read > pending->max_size || + (bytes_read + pending->current_size) > pending->max_size) + { + g_autofree char *uristr = NULL; + + if (pending->file) + uristr = g_file_get_uri (pending->file); + else + uristr = g_uri_to_string (soup_message_get_uri (pending->message)); + + local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, + "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", + uristr, pending->max_size); + goto out; + } + } + + pending->current_size += bytes_read; + + /* We do this instead of _write_bytes_async() as that's not + * guaranteed to do a complete write. + */ + { + g_autoptr(GInputStream) membuf = + g_memory_input_stream_new_from_bytes (bytes); + g_output_stream_splice_async (pending->out_stream, membuf, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, + G_PRIORITY_DEFAULT, + cancellable, + on_out_splice_complete, + g_object_ref (task)); + } + } + + out: + if (local_error) + { + g_task_return_error (task, local_error); + remove_pending (pending); + } + + g_object_unref (task); +} + +static void +on_request_sent (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + GTask *task = G_TASK (user_data); + /* Hold a ref to the pending across this function, since we remove + * it from the hash early in some cases, not in others. */ + OstreeFetcherPendingURI *pending = pending_uri_ref (g_task_get_task_data (task)); + GCancellable *cancellable = g_task_get_cancellable (task); + GError *local_error = NULL; + glnx_unref_object SoupMessage *msg = NULL; + SoupStatus status; + + pending->state = OSTREE_FETCHER_STATE_COMPLETE; + if (pending->file) + pending->request_body = (GInputStream *)g_file_read_finish ((GFile *) object, + result, + &local_error); + else + pending->request_body = soup_session_send_finish ((SoupSession*) object, + result, &local_error); + + if (!pending->request_body) + goto out; + + if (pending->message) + { + status = soup_message_get_status (pending->message); + if (status == SOUP_STATUS_NOT_MODIFIED && + (pending->if_none_match != NULL || pending->if_modified_since > 0)) + { + /* Version on the server is unchanged from the version we have cached locally; + * report this as an out-argument, a zero-length response buffer, and no error */ + pending->out_not_modified = TRUE; + } + else if (!SOUP_STATUS_IS_SUCCESSFUL (status)) + { + /* is there another mirror we can try? */ + if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len) + { + pending->mirrorlist_idx++; + create_pending_soup_request (pending); + + (void) g_input_stream_close (pending->request_body, NULL, NULL); + + start_pending_request (pending->thread_closure, task); + } + else + { + g_autofree char *uristring = g_uri_to_string (soup_message_get_uri (pending->message)); + GIOErrorEnum code = _ostree_fetcher_http_status_code_to_io_error (status); + { + g_autofree char *errmsg = + g_strdup_printf ("Server returned status %u: %s", + status, + soup_status_get_phrase (status)); + + /* Let's make OOB errors be the final one since they're probably + * the cause for the error here. */ + if (pending->thread_closure->oob_error) + { + local_error = + g_error_copy (pending->thread_closure->oob_error); + g_prefix_error (&local_error, "%s: ", errmsg); + } + else + local_error = g_error_new_literal (G_IO_ERROR, code, errmsg); + } + + if (pending->mirrorlist->len > 1) + g_prefix_error (&local_error, + "All %u mirrors failed. Last error was: ", + pending->mirrorlist->len); + if (pending->thread_closure->remote_name && + !((pending->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && + code == G_IO_ERROR_NOT_FOUND)) + _ostree_fetcher_journal_failure (pending->thread_closure->remote_name, + uristring, local_error->message); + + } + goto out; + } + + /* Grab cache properties from the response */ + pending->out_etag = g_strdup (soup_message_headers_get_one (soup_message_get_response_headers (pending->message), "ETag")); + pending->out_last_modified = 0; + + const char *last_modified_str = soup_message_headers_get_one (soup_message_get_response_headers (pending->message), "Last-Modified"); + if (last_modified_str != NULL) + { + GDateTime *soup_date = soup_date_time_new_from_http_string (last_modified_str); + if (soup_date != NULL) + { + pending->out_last_modified = g_date_time_to_unix (soup_date); + g_date_time_unref (soup_date); + } + } + } + + pending->state = OSTREE_FETCHER_STATE_DOWNLOADING; + + if (pending->file) + { + GFileInfo *info = g_file_query_info (pending->file, + G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE "," + G_FILE_ATTRIBUTE_STANDARD_SIZE, + 0, NULL, NULL); + if (info) + { + pending->content_length = g_file_info_get_size (info); + g_object_unref (info); + } + else + pending->content_length = -1; + } + else + pending->content_length = soup_message_headers_get_content_length (soup_message_get_response_headers (pending->message)); + + g_input_stream_read_bytes_async (pending->request_body, + 8192, G_PRIORITY_DEFAULT, + cancellable, + on_stream_read, + g_object_ref (task)); + + out: + if (local_error) + { + if (pending->request_body) + (void) g_input_stream_close (pending->request_body, NULL, NULL); + g_task_return_error (task, local_error); + remove_pending (pending); + } + + pending_uri_unref (pending); + g_object_unref (task); +} + +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + OstreeFetcherPendingURI *pending; + + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (mirrorlist != NULL); + g_return_if_fail (mirrorlist->len > 0); + + /* SoupRequest is created in session thread. */ + pending = g_new0 (OstreeFetcherPendingURI, 1); + pending->ref_count = 1; + pending->thread_closure = thread_closure_ref (self->thread_closure); + pending->mirrorlist = g_ptr_array_ref (mirrorlist); + pending->filename = g_strdup (filename); + pending->flags = flags; + pending->if_none_match = g_strdup (if_none_match); + pending->if_modified_since = if_modified_since; + pending->max_size = max_size; + pending->is_membuf = is_membuf; + pending->fetcher = self; + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, _ostree_fetcher_request_async); + g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_unref); + + /* We'll use the GTask priority for our own priority queue. */ + g_task_set_priority (task, priority); + + session_thread_idle_add (self->thread_closure, + session_thread_request_uri, + g_object_ref (task), + (GDestroyNotify) g_object_unref); +} + +void +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, FALSE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) +{ + GTask *task; + OstreeFetcherPendingURI *pending; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + pending = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (!pending->is_membuf); + *out_tmpf = pending->tmpf; + pending->tmpf.initialized = FALSE; /* Transfer ownership */ + + if (out_not_modified != NULL) + *out_not_modified = pending->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&pending->out_etag); + if (out_last_modified != NULL) + *out_last_modified = pending->out_last_modified; + + return TRUE; +} + +void +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, TRUE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) +{ + GTask *task; + OstreeFetcherPendingURI *pending; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + pending = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (pending->is_membuf); + g_assert (out_buf); + *out_buf = ret; + + if (out_not_modified != NULL) + *out_not_modified = pending->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&pending->out_etag); + if (out_last_modified != NULL) + *out_last_modified = pending->out_last_modified; + + return TRUE; +} + + +guint64 +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +{ + g_return_val_if_fail (OSTREE_IS_FETCHER (self), 0); + + g_mutex_lock (&self->thread_closure->output_stream_set_lock); + + guint64 ret = self->thread_closure->total_downloaded; + + GLNX_HASH_TABLE_FOREACH (self->thread_closure->output_stream_set, + GFileOutputStream*, stream) + { + if (G_IS_FILE_DESCRIPTOR_BASED (stream)) + { + int fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)stream); + struct stat stbuf; + + if (glnx_fstat (fd, &stbuf, NULL)) + ret += stbuf.st_size; + } + } + + g_mutex_unlock (&self->thread_closure->output_stream_set_lock); + + return ret; +} diff --git a/src/ostree/main.c b/src/ostree/main.c index 7d17080c..badfa6df 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -118,7 +118,7 @@ static OstreeCommand commands[] = { { "summary", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_summary, "Manage summary metadata" }, -#if defined(HAVE_LIBSOUP) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE) +#if defined(HAVE_LIBSOUP_OR_LIBSOUP3) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE) { "trivial-httpd", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_trivial_httpd, NULL }, diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 6b16737d..855d2cea 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -34,6 +34,19 @@ #include #include +#if ! SOUP_CHECK_VERSION (3, 0, 0) +# define SoupServerMessage SoupMessage +# define soup_server_message_get_method(msg) ((msg)->method) +# define soup_server_message_get_request_headers(msg) ((msg)->request_headers) +# define soup_server_message_get_response_headers(msg) ((msg)->response_headers) +# define soup_server_message_get_response_body(msg) ((msg)->response_body) +# define soup_server_message_set_status(msg, status) soup_message_set_status(msg, status) +# define soup_server_message_set_redirect(msg, status, uri) soup_message_set_redirect(msg, status, uri) +# define soup_server_message_set_response(msg, ct, ru, rb, rl) soup_message_set_response(msg, ct, ru, rb, rl) +#else +# define soup_server_message_set_status(msg, status) soup_server_message_set_status(msg, status, NULL) +#endif + static char *opt_port_file = NULL; static char *opt_log = NULL; static gboolean opt_daemonize; @@ -188,15 +201,12 @@ is_safe_to_access (struct stat *stbuf) } static void -close_socket (SoupMessage *msg, gpointer user_data) +close_socket (SoupServerMessage *msg, gpointer user_data) { - SoupSocket *sock = user_data; + GSocket *sock = user_data; int sockfd; - /* Actually calling soup_socket_disconnect() here would cause - * us to leak memory, so just shutdown the socket instead. - */ - sockfd = soup_socket_get_fd (sock); + sockfd = g_socket_get_fd (sock); #ifdef G_OS_WIN32 shutdown (sockfd, SD_SEND); #else @@ -213,12 +223,55 @@ calculate_etag (GMappedFile *mapping) return g_strconcat ("\"", checksum, "\"", NULL); } +static GSList * +_server_cookies_from_request (SoupServerMessage *msg) +{ + SoupCookie *cookie; + GSList *cookies = NULL; + GHashTable *params; + GHashTableIter iter; + gpointer name, value; + const char *header; + const char *host; + + header = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), + "Cookie"); + if (!header) + return NULL; + +#if ! SOUP_CHECK_VERSION (3, 0, 0) + host = soup_uri_get_host (soup_message_get_uri (msg)); +#else + host = g_uri_get_host (soup_server_message_get_uri (msg)); +#endif + params = soup_header_parse_semi_param_list (header); + g_hash_table_iter_init (&iter, params); + + while (g_hash_table_iter_next (&iter, &name, &value)) + { + if (!name || !value) continue; + cookie = soup_cookie_new (name, value, host, NULL, 0); + cookies = g_slist_prepend (cookies, cookie); + } + + soup_header_free_param_list (params); + + return g_slist_reverse (cookies); +} + static void +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (OtTrivialHttpd *self, SoupServer *server, - SoupMessage *msg, + SoupServerMessage *msg, const char *path, SoupClientContext *context) +#else +do_get (OtTrivialHttpd *self, + SoupServer *server, + SoupServerMessage *msg, + const char *path) +#endif { char *slash; int ret; @@ -228,7 +281,7 @@ do_get (OtTrivialHttpd *self, if (opt_expected_cookies) { - GSList *cookies = soup_cookies_from_request (msg); + GSList *cookies = _server_cookies_from_request (msg); GSList *l; int i; @@ -253,12 +306,12 @@ do_get (OtTrivialHttpd *self, if (!found) { httpd_log (self, "Expected cookie not found %s\n", k); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); - soup_cookies_free (cookies); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + g_slist_free_full (cookies, (GDestroyNotify)soup_cookie_free); goto out; } } - soup_cookies_free (cookies); + g_slist_free_full (cookies, (GDestroyNotify)soup_cookie_free); } if (opt_expected_headers) @@ -273,18 +326,18 @@ do_get (OtTrivialHttpd *self, { g_autofree char *k = g_strndup (kv, eq - kv); const gchar *expected_v = eq + 1; - const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k); + const gchar *found_v = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), k); if (!found_v) { httpd_log (self, "Expected header not found %s\n", k); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } if (strcmp (found_v, expected_v) != 0) { httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } } @@ -293,7 +346,7 @@ do_get (OtTrivialHttpd *self, if (strstr (path, "../") != NULL) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } @@ -302,7 +355,7 @@ do_get (OtTrivialHttpd *self, g_random_int_range (0, 100) < opt_random_500s_percentage) { emitted_random_500s_count++; - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } else if (opt_random_408s_percentage > 0 && @@ -310,7 +363,7 @@ do_get (OtTrivialHttpd *self, g_random_int_range (0, 100) < opt_random_408s_percentage) { emitted_random_408s_count++; - soup_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); + soup_server_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); goto out; } @@ -323,17 +376,17 @@ do_get (OtTrivialHttpd *self, if (ret == -1) { if (errno == EPERM) - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); else if (errno == ENOENT) - soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_FOUND); else - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } if (!is_safe_to_access (&stbuf)) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } @@ -344,9 +397,13 @@ do_get (OtTrivialHttpd *self, { g_autofree char *redir_uri = NULL; - redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path); - soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY, - redir_uri); +#if ! SOUP_CHECK_VERSION (3, 0, 0) + redir_uri = g_strdup_printf ("%s/", soup_uri_get_path (soup_message_get_uri (msg))); +#else + redir_uri = g_strdup_printf ("%s/", g_uri_get_path (soup_server_message_get_uri (msg))); +#endif + soup_server_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY, + redir_uri); } else { @@ -354,15 +411,19 @@ do_get (OtTrivialHttpd *self, if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1) { g_autofree char *index_path = g_strconcat (path, "/index.html", NULL); +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (self, server, msg, index_path, context); +#else + do_get (self, server, msg, index_path); +#endif } else { GString *listing = get_directory_listing (self->root_dfd, path); - soup_message_set_response (msg, "text/html", - SOUP_MEMORY_TAKE, - listing->str, listing->len); - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_response (msg, "text/html", + SOUP_MEMORY_TAKE, + listing->str, listing->len); + soup_server_message_set_status (msg, SOUP_STATUS_OK); g_string_free (listing, FALSE); } } @@ -371,21 +432,21 @@ do_get (OtTrivialHttpd *self, { if (!S_ISREG (stbuf.st_mode)) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } glnx_autofd int fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC); if (fd < 0) { - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } g_autoptr(GMappedFile) mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL); if (!mapping) { - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } (void) close (fd); fd = -1; @@ -395,14 +456,14 @@ do_get (OtTrivialHttpd *self, if (last_modified != NULL) { g_autofree gchar *formatted = g_date_time_format (last_modified, "%a, %d %b %Y %H:%M:%S GMT"); - soup_message_headers_append (msg->response_headers, "Last-Modified", formatted); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Last-Modified", formatted); } g_autofree gchar *etag = calculate_etag (mapping); if (etag != NULL) - soup_message_headers_append (msg->response_headers, "ETag", etag); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "ETag", etag); - if (msg->method == SOUP_METHOD_GET) + if (!strcmp (soup_server_message_get_method (msg), "GET")) { gsize buffer_length, file_size; SoupRange *ranges; @@ -410,13 +471,13 @@ do_get (OtTrivialHttpd *self, gboolean have_ranges; file_size = g_mapped_file_get_length (mapping); - have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length); + have_ranges = soup_message_headers_get_ranges(soup_server_message_get_request_headers (msg), file_size, &ranges, &ranges_length); if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL) { - SoupSocket *sock; + GSocket *sock; buffer_length = file_size/2; - soup_message_headers_set_content_length (msg->response_headers, file_size); - soup_message_headers_append (msg->response_headers, + soup_message_headers_set_content_length (soup_server_message_get_response_headers (msg), file_size); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Connection", "close"); /* soup-message-io will wait for us to add @@ -424,7 +485,11 @@ do_get (OtTrivialHttpd *self, * the declared Content-Length. Instead, we * forcibly close the socket at that point. */ - sock = soup_client_context_get_socket (context); +#if ! SOUP_CHECK_VERSION (3, 0, 0) + sock = soup_client_context_get_gsocket (context); +#else + sock = soup_server_message_get_socket (msg); +#endif g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock); } else @@ -434,12 +499,13 @@ do_get (OtTrivialHttpd *self, { if (ranges_length > 0 && ranges[0].start >= file_size) { - soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); - soup_message_headers_free_ranges (msg->request_headers, ranges); + soup_server_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + soup_message_headers_free_ranges (soup_server_message_get_request_headers (msg), ranges); goto out; } - soup_message_headers_free_ranges (msg->request_headers, ranges); + soup_message_headers_free_ranges (soup_server_message_get_request_headers (msg), ranges); } +#if ! SOUP_CHECK_VERSION (3, 0, 0) if (buffer_length > 0) { SoupBuffer *buffer; @@ -451,8 +517,22 @@ do_get (OtTrivialHttpd *self, soup_message_body_append_buffer (msg->response_body, buffer); soup_buffer_free (buffer); } +#else + if (buffer_length > 0 && buffer_length == file_size) + { + GBytes *bytes = g_mapped_file_get_bytes (mapping); + soup_message_body_append_bytes (soup_server_message_get_response_body (msg), bytes); + g_bytes_unref (bytes); + } + else if (buffer_length > 0) + { + gchar *contents = g_mapped_file_get_contents (mapping); + soup_message_body_append (soup_server_message_get_response_body (msg), + SOUP_MEMORY_COPY, contents, buffer_length); + } +#endif } - else /* msg->method == SOUP_METHOD_HEAD */ + else /* method == HEAD */ { g_autofree char *length = NULL; @@ -461,56 +541,59 @@ do_get (OtTrivialHttpd *self, * But we'll optimize and avoid the extra I/O. */ length = g_strdup_printf ("%lu", (gulong)stbuf.st_size); - soup_message_headers_append (msg->response_headers, + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Content-Length", length); } /* Check client’s caching headers. */ - const gchar *if_modified_since = soup_message_headers_get_one (msg->request_headers, + const gchar *if_modified_since = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), "If-Modified-Since"); - const gchar *if_none_match = soup_message_headers_get_one (msg->request_headers, + const gchar *if_none_match = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), "If-None-Match"); if (if_none_match != NULL && etag != NULL) { if (g_strcmp0 (etag, if_none_match) == 0) { - soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); - soup_message_body_truncate (msg->response_body); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); + soup_message_body_truncate (soup_server_message_get_response_body (msg)); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } } else if (if_modified_since != NULL && last_modified != NULL) { - SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since); g_autoptr(GDateTime) if_modified_since_dt = NULL; +#if ! SOUP_CHECK_VERSION (3, 0, 0) + SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since); if (if_modified_since_sd != NULL) if_modified_since_dt = g_date_time_new_from_unix_utc (soup_date_to_time_t (if_modified_since_sd)); +#else + if_modified_since_dt = soup_date_time_new_from_http_string (if_modified_since); +#endif if (if_modified_since_dt != NULL && g_date_time_compare (last_modified, if_modified_since_dt) <= 0) { - soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); - soup_message_body_truncate (msg->response_body); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); + soup_message_body_truncate (soup_server_message_get_response_body (msg)); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } - - g_clear_pointer (&if_modified_since_sd, soup_date_free); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } } out: { +#if ! SOUP_CHECK_VERSION (3, 0, 0) guint status = 0; g_autofree gchar *reason = NULL; @@ -518,26 +601,41 @@ do_get (OtTrivialHttpd *self, "status-code", &status, "reason-phrase", &reason, NULL); +#else + guint status = soup_server_message_get_status (msg); + const char *reason = soup_server_message_get_reason_phrase (msg); +#endif + httpd_log (self, " status: %s (%u)\n", reason, status); } return; } static void -httpd_callback (SoupServer *server, SoupMessage *msg, +#if ! SOUP_CHECK_VERSION (3, 0, 0) +httpd_callback (SoupServer *server, SoupServerMessage *msg, const char *path, GHashTable *query, SoupClientContext *context, gpointer data) +#else +httpd_callback (SoupServer *server, SoupServerMessage *msg, + const char *path, GHashTable *query, gpointer data) +#endif { OtTrivialHttpd *self = data; + const char *meth = soup_server_message_get_method (msg); - if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD) + if (!strcmp (meth, "GET") || !strcmp(meth, "HEAD")) +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (self, server, msg, path, context); +#else + do_get (self, server, msg, path); +#endif else - soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); } static gboolean -basic_auth_callback (SoupAuthDomain *auth_domain, SoupMessage *msg, +basic_auth_callback (SoupAuthDomain *auth_domain, SoupServerMessage *msg, const char *username, const char *password, gpointer data) { return g_str_equal (username, "foouser") && g_str_equal (password, "barpw"); @@ -703,7 +801,7 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) } #if SOUP_CHECK_VERSION(2, 48, 0) - server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); + server = soup_server_new ("server-header", "ostree-httpd ", NULL); if (!soup_server_listen_all (server, opt_port, 0, error)) goto out; #else @@ -711,13 +809,21 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); #endif + if (opt_require_basic_auth) { +#if ! SOUP_CHECK_VERSION (3, 0, 0) glnx_unref_object SoupAuthDomain *auth_domain = soup_auth_domain_basic_new (SOUP_AUTH_DOMAIN_REALM, "auth-test", SOUP_AUTH_DOMAIN_ADD_PATH, "/", SOUP_AUTH_DOMAIN_BASIC_AUTH_CALLBACK, basic_auth_callback, NULL); +#else + glnx_unref_object SoupAuthDomain *auth_domain = + soup_auth_domain_basic_new ("realm", "auth-test", NULL); + soup_auth_domain_add_path (auth_domain, "/"); + soup_auth_domain_basic_set_auth_callback (auth_domain, basic_auth_callback, NULL, NULL); +#endif soup_server_add_auth_domain (server, auth_domain); } -- cgit v1.2.1 From 945f0a860299e93985be02719c0c427162de551c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 12 Apr 2023 22:34:23 -0600 Subject: ci: Add test configuration with soup3 This needs to be on Debian testing for now since bullseye doesn't have soup3. --- .github/workflows/tests.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b74a34ac..966f6416 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -194,6 +194,19 @@ jobs: apt-get update apt-get install -y git + # A build using libsoup3. After bookworm is released, this can + # be switched to Debian Stable. + - name: Debian Testing with libsoup3 + image: debian:testing-slim + container-options: --security-opt seccomp=unconfined + pre-checkout-setup: | + apt-get update + apt-get install -y git + extra-packages: >- + libsoup-3.0-dev + configure-options: >- + --with-soup3 + # Ubuntu builds. Unfortunately, when the latest release is # also the latest LTS, latest and rolling are the same. Other # options would be to test the previous LTS by name or to test -- cgit v1.2.1 From a0407225f15415ddf207c25201c48445418ab964 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 7 Apr 2023 09:47:14 -0600 Subject: fetcher/soup3: Rewrite without threads soup3 works best using only the async API from a single thread[1]. Rework the fetcher to stop using worker threads. In order to maximize session usage across requests, sessions will be reused for each main context. 1. https://libsoup.org/libsoup-3.0/client-thread-safety.html --- src/libostree/ostree-fetcher-soup3.c | 1239 ++++++++++++---------------------- 1 file changed, 413 insertions(+), 826 deletions(-) diff --git a/src/libostree/ostree-fetcher-soup3.c b/src/libostree/ostree-fetcher-soup3.c index 9f495a72..b46ce34f 100644 --- a/src/libostree/ostree-fetcher-soup3.c +++ b/src/libostree/ostree-fetcher-soup3.c @@ -1,6 +1,7 @@ /* * Copyright (C) 2011 Colin Walters * Copyright (C) 2022 Igalia S.L. + * Copyright (C) 2023 Endless OS Foundation, LLC * * SPDX-License-Identifier: LGPL-2.0+ * @@ -19,12 +20,12 @@ * * Author: Colin Walters * Author: Daniel Kolesa + * Author: Dan Nicholson */ #include "config.h" #include -#include #include #include @@ -33,62 +34,24 @@ #include "ostree-fetcher-util.h" #include "ostree-tls-cert-interaction-private.h" #include "ostree-enumtypes.h" -#include "ostree.h" #include "ostree-repo-private.h" -#include "otutil.h" - -typedef enum { - OSTREE_FETCHER_STATE_PENDING, - OSTREE_FETCHER_STATE_DOWNLOADING, - OSTREE_FETCHER_STATE_COMPLETE -} OstreeFetcherState; - -typedef struct { - int ref_count; /* atomic */ - - SoupSession *session; /* not referenced */ - GMainContext *main_context; - volatile gint running; - GError *initialization_error; /* Any failure to load the db */ - - char *remote_name; - int base_tmpdir_dfd; - - GVariant *extra_headers; - gboolean transfer_gzip; - - /* Our active HTTP requests */ - GHashTable *outstanding; - - /* Shared across threads; be sure to lock. */ - GHashTable *output_stream_set; /* set */ - GMutex output_stream_set_lock; - - /* Also protected by output_stream_set_lock. */ - guint64 total_downloaded; - - GError *oob_error; -} ThreadClosure; typedef struct { - int ref_count; /* atomic */ - - ThreadClosure *thread_closure; GPtrArray *mirrorlist; /* list of base URIs */ char *filename; /* relative name to fetch or NULL */ guint mirrorlist_idx; - OstreeFetcherState state; - SoupMessage *message; - GFile *file; struct OstreeFetcher *fetcher; + GMainContext *mainctx; + SoupSession *session; + GFile *file; gboolean is_membuf; OstreeFetcherRequestFlags flags; char *if_none_match; /* request ETag */ guint64 if_modified_since; /* seconds since the epoch */ - GInputStream *request_body; + GInputStream *response_body; GLnxTmpfile tmpf; GOutputStream *out_stream; gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */ @@ -97,29 +60,26 @@ typedef struct { guint64 max_size; guint64 current_size; - guint64 content_length; -} OstreeFetcherPendingURI; - -/* Used by session_thread_idle_add() */ -typedef void (*SessionThreadFunc) (ThreadClosure *thread_closure, - gpointer data); - -/* Used by session_thread_idle_add() */ -typedef struct { - ThreadClosure *thread_closure; - SessionThreadFunc function; - gpointer data; - GDestroyNotify notify; -} IdleClosure; + goffset content_length; +} FetcherRequest; struct OstreeFetcher { GObject parent_instance; OstreeFetcherConfigFlags config_flags; + char *remote_name; + int tmpdir_dfd; - GThread *session_thread; - ThreadClosure *thread_closure; + GHashTable *sessions; /* (element-type GMainContext SoupSession ) */ + GProxyResolver *proxy_resolver; + SoupCookieJar *cookie_jar; + OstreeTlsCertInteraction *client_cert; + GTlsDatabase *tls_database; + GVariant *extra_headers; + char *user_agent; + + guint64 bytes_transferred; }; enum { @@ -129,399 +89,113 @@ enum { G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) -static ThreadClosure * -thread_closure_ref (ThreadClosure *thread_closure) -{ - int refcount; - g_return_val_if_fail (thread_closure != NULL, NULL); - refcount = g_atomic_int_add (&thread_closure->ref_count, 1); - g_assert (refcount > 0); - return thread_closure; -} - -static void -thread_closure_unref (ThreadClosure *thread_closure) -{ - g_return_if_fail (thread_closure != NULL); - - if (g_atomic_int_dec_and_test (&thread_closure->ref_count)) - { - /* The session thread should have cleared this by now. */ - g_assert (thread_closure->session == NULL); - - g_clear_pointer (&thread_closure->main_context, g_main_context_unref); - - g_clear_pointer (&thread_closure->extra_headers, g_variant_unref); - - g_clear_pointer (&thread_closure->output_stream_set, g_hash_table_unref); - g_mutex_clear (&thread_closure->output_stream_set_lock); - - g_clear_pointer (&thread_closure->oob_error, g_error_free); - - g_free (thread_closure->remote_name); - - g_slice_free (ThreadClosure, thread_closure); - } -} - static void -idle_closure_free (IdleClosure *idle_closure) -{ - g_clear_pointer (&idle_closure->thread_closure, thread_closure_unref); - - if (idle_closure->notify != NULL) - idle_closure->notify (idle_closure->data); - - g_slice_free (IdleClosure, idle_closure); -} - -static OstreeFetcherPendingURI * -pending_uri_ref (OstreeFetcherPendingURI *pending) -{ - gint refcount; - g_assert (pending); - refcount = g_atomic_int_add (&pending->ref_count, 1); - g_assert (refcount > 0); - return pending; +fetcher_request_free (FetcherRequest *request) +{ + g_debug ("Freeing request for %s", request->filename); + g_clear_pointer (&request->mirrorlist, g_ptr_array_unref); + g_clear_pointer (&request->filename, g_free); + g_clear_object (&request->message); + g_clear_pointer (&request->mainctx, g_main_context_unref); + g_clear_object (&request->session); + g_clear_object (&request->file); + g_clear_pointer (&request->if_none_match, g_free); + g_clear_object (&request->response_body); + glnx_tmpfile_clear (&request->tmpf); + g_clear_object (&request->out_stream); + g_clear_pointer (&request->out_etag, g_free); + g_free (request); } static void -pending_uri_unref (OstreeFetcherPendingURI *pending) -{ - if (!g_atomic_int_dec_and_test (&pending->ref_count)) - return; - - g_clear_pointer (&pending->thread_closure, thread_closure_unref); - - g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref); - g_free (pending->filename); - g_clear_object (&pending->message); - g_clear_object (&pending->file); - g_clear_object (&pending->request_body); - g_free (pending->if_none_match); - glnx_tmpfile_clear (&pending->tmpf); - g_clear_object (&pending->out_stream); - g_free (pending->out_etag); - g_free (pending); -} - -static gboolean -session_thread_idle_dispatch (gpointer data) -{ - IdleClosure *idle_closure = data; - - idle_closure->function (idle_closure->thread_closure, - idle_closure->data); - - return G_SOURCE_REMOVE; -} - -static void -session_thread_idle_add (ThreadClosure *thread_closure, - SessionThreadFunc function, - gpointer data, - GDestroyNotify notify) -{ - IdleClosure *idle_closure; - - g_return_if_fail (thread_closure != NULL); - g_return_if_fail (function != NULL); - - idle_closure = g_slice_new (IdleClosure); - idle_closure->thread_closure = thread_closure_ref (thread_closure); - idle_closure->function = function; - idle_closure->data = data; - idle_closure->notify = notify; - - g_main_context_invoke_full (thread_closure->main_context, - G_PRIORITY_DEFAULT, - session_thread_idle_dispatch, - idle_closure, /* takes ownership */ - (GDestroyNotify) idle_closure_free); -} - -static void -session_thread_add_logger (ThreadClosure *thread_closure, - gpointer data) -{ - glnx_unref_object SoupLogger *logger = NULL; - - logger = soup_logger_new (SOUP_LOGGER_LOG_BODY); - soup_logger_set_max_body_size (logger, 500); - soup_session_add_feature (thread_closure->session, - SOUP_SESSION_FEATURE (logger)); -} - -static void -session_thread_set_proxy_cb (ThreadClosure *thread_closure, - gpointer data) -{ - GProxyResolver *resolver = data; - - g_object_set (thread_closure->session, - "proxy-resolver", - g_object_ref (resolver), NULL); -} - -static void -session_thread_set_cookie_jar_cb (ThreadClosure *thread_closure, - gpointer data) -{ - SoupCookieJar *jar = data; - - soup_session_add_feature (thread_closure->session, - SOUP_SESSION_FEATURE (jar)); -} - -static void -session_thread_set_headers_cb (ThreadClosure *thread_closure, - gpointer data) -{ - GVariant *headers = data; - - g_clear_pointer (&thread_closure->extra_headers, g_variant_unref); - thread_closure->extra_headers = g_variant_ref (headers); -} - -static void -session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, - gpointer data) -{ - const char *cert_and_key_path = data; /* str\0str\0 in one malloc buf */ - const char *cert_path = cert_and_key_path; - const char *key_path = cert_and_key_path + strlen (cert_and_key_path) + 1; - g_autoptr(OstreeTlsCertInteraction) interaction = NULL; - - /* The GTlsInteraction instance must be created in the - * session thread so it uses the correct GMainContext. */ - interaction = _ostree_tls_cert_interaction_new (cert_path, key_path); - - g_object_set (thread_closure->session, - "tls-interaction", - interaction, NULL); -} - -static void -session_thread_set_tls_database_cb (ThreadClosure *thread_closure, - gpointer data) -{ - const char *db_path = data; - - if (db_path != NULL) - { - glnx_unref_object GTlsDatabase *tlsdb = NULL; - - g_clear_error (&thread_closure->initialization_error); - tlsdb = g_tls_file_database_new (db_path, &thread_closure->initialization_error); - - if (tlsdb) - g_object_set (thread_closure->session, - "tls-database", - tlsdb, NULL); - } -} - -static void -session_thread_set_extra_user_agent_cb (ThreadClosure *thread_closure, - gpointer data) -{ - const char *extra_user_agent = data; - if (extra_user_agent != NULL) - { - g_autofree char *ua = - g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent); - g_object_set (thread_closure->session, "user-agent", ua, NULL); - } - else - { - g_object_set (thread_closure->session, "user-agent", - OSTREE_FETCHER_USERAGENT_STRING, NULL); - } -} - -static void -on_request_sent (GObject *object, GAsyncResult *result, gpointer user_data); - -static void -request_send_async (ThreadClosure *thread_closure, - GTask *task) -{ - OstreeFetcherPendingURI *pending; - GCancellable *cancellable; - - pending = g_task_get_task_data (task); - cancellable = g_task_get_cancellable (task); - - if (pending->file) - g_file_read_async (pending->file, g_task_get_priority (task), - cancellable, on_request_sent, task); - else - { - soup_session_send_async (thread_closure->session, pending->message, - g_task_get_priority (task), - cancellable, on_request_sent, task); - } -} - -static void -start_pending_request (ThreadClosure *thread_closure, - GTask *task) -{ - - OstreeFetcherPendingURI *pending; - - pending = g_task_get_task_data (task); - - g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending)); - request_send_async (thread_closure, g_object_ref (task)); -} +on_request_sent (GObject *object, + GAsyncResult *result, + gpointer user_data); static gboolean _message_accept_cert_loose (SoupMessage *msg, GTlsCertificate *tls_peer_certificate, - GTlsCertificateFlags tls_peer_errors, + GTlsCertificateFlags tls_peer_errors, gpointer data) { - OstreeFetcherConfigFlags config_flags; - - config_flags = GPOINTER_TO_UINT (data); - - return ((config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0); + return TRUE; } static void -create_pending_soup_request (OstreeFetcherPendingURI *pending) +create_request_message (FetcherRequest *request) { - OstreeFetcherURI *next_mirror = NULL; - g_autoptr(OstreeFetcherURI) uri = NULL; - GUri *guri = NULL; + g_assert (request->mirrorlist); + g_assert (request->mirrorlist_idx < request->mirrorlist->len); - g_assert (pending->mirrorlist); - g_assert (pending->mirrorlist_idx < pending->mirrorlist->len); - - next_mirror = g_ptr_array_index (pending->mirrorlist, pending->mirrorlist_idx); - if (pending->filename) - uri = _ostree_fetcher_uri_new_subpath (next_mirror, pending->filename); + OstreeFetcherURI *next_mirror = g_ptr_array_index (request->mirrorlist, request->mirrorlist_idx); + g_autoptr(OstreeFetcherURI) uri = NULL; + if (request->filename) + uri = _ostree_fetcher_uri_new_subpath (next_mirror, request->filename); if (!uri) - uri = (OstreeFetcherURI*)g_uri_ref ((GUri*)next_mirror); + uri = (OstreeFetcherURI *) g_uri_ref ((GUri *) next_mirror); - g_clear_object (&pending->message); - g_clear_object (&pending->file); + g_clear_object (&request->message); + g_clear_object (&request->file); + g_clear_object (&request->response_body); - guri = (GUri*)(uri ? uri : next_mirror); + GUri *guri = (GUri *) uri; /* file:// URI is handle via GFile */ if (!strcmp (g_uri_get_scheme (guri), "file")) { - char *str = g_uri_to_string (guri); - pending->file = g_file_new_for_uri (str); - g_free (str); + g_autofree char *str = g_uri_to_string (guri); + request->file = g_file_new_for_uri (str); return; } - pending->message = soup_message_new_from_uri ("GET", guri); + request->message = soup_message_new_from_uri ("GET", guri); - if (pending->if_none_match != NULL) - { - soup_message_headers_append (soup_message_get_request_headers (pending->message), - "If-None-Match", pending->if_none_match); - } + if (request->if_none_match != NULL) + soup_message_headers_append (soup_message_get_request_headers (request->message), + "If-None-Match", request->if_none_match); - if (pending->if_modified_since > 0) + if (request->if_modified_since > 0) { - g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (pending->if_modified_since); + g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (request->if_modified_since); g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z"); - soup_message_headers_append (soup_message_get_request_headers (pending->message), + soup_message_headers_append (soup_message_get_request_headers (request->message), "If-Modified-Since", mod_date); } - if (pending->message && pending->fetcher->config_flags != 0) - { - g_signal_connect (pending->message, "accept-certificate", - G_CALLBACK (_message_accept_cert_loose), - GUINT_TO_POINTER (pending->fetcher->config_flags)); - } -} + if ((request->fetcher->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) != 0) + g_signal_connect (request->message, "accept-certificate", + G_CALLBACK (_message_accept_cert_loose), NULL); -static void -session_thread_request_uri (ThreadClosure *thread_closure, - gpointer data) -{ - GTask *task = G_TASK (data); - OstreeFetcherPendingURI *pending; - - pending = g_task_get_task_data (task); - - /* If we caught an error in init, re-throw it for every request */ - if (thread_closure->initialization_error) + if (request->fetcher->extra_headers) { - g_task_return_error (task, g_error_copy (thread_closure->initialization_error)); - return; - } - - create_pending_soup_request (pending); - - if (pending->message && thread_closure->extra_headers) - { - g_autoptr(GVariantIter) viter = g_variant_iter_new (thread_closure->extra_headers); + g_autoptr(GVariantIter) viter = g_variant_iter_new (request->fetcher->extra_headers); const char *key; const char *value; while (g_variant_iter_next (viter, "(&s&s)", &key, &value)) - soup_message_headers_append (soup_message_get_request_headers (pending->message), key, value); + soup_message_headers_append (soup_message_get_request_headers (request->message), key, value); } - - if (pending->is_membuf) - request_send_async (thread_closure, g_object_ref (task)); - else - start_pending_request (thread_closure, task); } -static gpointer -ostree_fetcher_session_thread (gpointer data) +static void +initiate_task_request (GTask *task) { - ThreadClosure *closure = data; - g_autoptr(GMainContext) mainctx = g_main_context_ref (closure->main_context); - - /* This becomes the GMainContext that SoupSession schedules async - * callbacks and emits signals from. Make it the thread-default - * context for this thread before creating the session. */ - g_main_context_push_thread_default (mainctx); - - /* We retain ownership of the SoupSession reference. */ - closure->session = soup_session_new_with_options ("user-agent", OSTREE_FETCHER_USERAGENT_STRING, - "timeout", 60, - "idle-timeout", 60, - "max-conns-per-host", _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS, - NULL); + FetcherRequest *request = g_task_get_task_data (task); + create_request_message (request); - /* SoupContentDecoder is included in the session by default. Remove it - * if gzip compression isn't in use. - */ - if (!closure->transfer_gzip) - soup_session_remove_feature_by_type (closure->session, SOUP_TYPE_CONTENT_DECODER); - - /* This model ensures we don't hit a race using g_main_loop_quit(); - * see also what pull_termination_condition() in ostree-repo-pull.c - * is doing. - */ - while (g_atomic_int_get (&closure->running)) - g_main_context_iteration (closure->main_context, TRUE); - - /* Since the ThreadClosure may be finalized from any thread we - * unreference all data related to the SoupSession ourself to ensure - * it's freed in the same thread where it was created. */ - g_clear_pointer (&closure->outstanding, g_hash_table_unref); - g_clear_pointer (&closure->session, g_object_unref); - - thread_closure_unref (closure); - - /* Do this last, since libsoup uses g_main_current_source() which - * relies on it. - */ - g_main_context_pop_thread_default (mainctx); - - return NULL; + GCancellable *cancellable = g_task_get_cancellable (task); + int priority = g_task_get_priority (task); + if (request->file) + g_file_read_async (request->file, priority, cancellable, on_request_sent, task); + else + { + g_autofree char *uri = g_uri_to_string (soup_message_get_uri (request->message)); + const char *dest = request->is_membuf ? "memory" : "tmpfile"; + g_debug ("Requesting %s to %s for session %p in main context %p", + uri, dest, request->session, request->mainctx); + soup_session_send_async (request->session, request->message, + priority, cancellable, on_request_sent, task); + } } static void @@ -567,18 +241,14 @@ _ostree_fetcher_finalize (GObject *object) { OstreeFetcher *self = OSTREE_FETCHER (object); - /* Terminate the session thread. */ - g_atomic_int_set (&self->thread_closure->running, 0); - g_main_context_wakeup (self->thread_closure->main_context); - if (self->session_thread) - { - /* We need to explicitly synchronize to clean up TLS */ - if (self->session_thread != g_thread_self ()) - g_thread_join (self->session_thread); - else - g_clear_pointer (&self->session_thread, g_thread_unref); - } - g_clear_pointer (&self->thread_closure, thread_closure_unref); + g_clear_pointer (&self->remote_name, g_free); + g_clear_pointer (&self->sessions, g_hash_table_unref); + g_clear_object (&self->proxy_resolver); + g_clear_object (&self->cookie_jar); + g_clear_object (&self->client_cert); + g_clear_object (&self->tls_database); + g_clear_pointer (&self->extra_headers, g_variant_unref); + g_clear_pointer (&self->user_agent, g_free); G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); } @@ -587,40 +257,11 @@ static void _ostree_fetcher_constructed (GObject *object) { OstreeFetcher *self = OSTREE_FETCHER (object); - g_autoptr(GMainContext) main_context = NULL; - const char *http_proxy; - - main_context = g_main_context_new (); - self->thread_closure = g_slice_new0 (ThreadClosure); - self->thread_closure->ref_count = 1; - self->thread_closure->main_context = g_main_context_ref (main_context); - self->thread_closure->running = 1; - self->thread_closure->transfer_gzip = (self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) != 0; - - self->thread_closure->outstanding = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)pending_uri_unref); - self->thread_closure->output_stream_set = g_hash_table_new_full (NULL, NULL, - (GDestroyNotify) NULL, - (GDestroyNotify) g_object_unref); - g_mutex_init (&self->thread_closure->output_stream_set_lock); - - if (g_getenv ("OSTREE_DEBUG_HTTP")) - { - session_thread_idle_add (self->thread_closure, - session_thread_add_logger, - NULL, (GDestroyNotify) NULL); - } - - http_proxy = g_getenv ("http_proxy"); + const char *http_proxy = g_getenv ("http_proxy"); if (http_proxy != NULL && http_proxy[0] != '\0') _ostree_fetcher_set_proxy (self, http_proxy); - /* FIXME Maybe implement GInitableIface and use g_thread_try_new() - * so we can try to handle thread creation errors gracefully? */ - self->session_thread = g_thread_new ("fetcher-session-thread", - ostree_fetcher_session_thread, - thread_closure_ref (self->thread_closure)); - G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); } @@ -649,72 +290,56 @@ _ostree_fetcher_class_init (OstreeFetcherClass *klass) static void _ostree_fetcher_init (OstreeFetcher *self) { + self->sessions = g_hash_table_new (g_direct_hash, g_direct_equal); } OstreeFetcher * -_ostree_fetcher_new (int tmpdir_dfd, - const char *remote_name, - OstreeFetcherConfigFlags flags) +_ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, + OstreeFetcherConfigFlags flags) { - OstreeFetcher *self; - - self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); - self->thread_closure->remote_name = g_strdup (remote_name); - self->thread_closure->base_tmpdir_dfd = tmpdir_dfd; + OstreeFetcher *self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + self->remote_name = g_strdup (remote_name); + self->tmpdir_dfd = tmpdir_dfd; return self; } int -_ostree_fetcher_get_dfd (OstreeFetcher *fetcher) +_ostree_fetcher_get_dfd (OstreeFetcher *self) { - return fetcher->thread_closure->base_tmpdir_dfd; + return self->tmpdir_dfd; } void _ostree_fetcher_set_proxy (OstreeFetcher *self, const char *http_proxy) { - GProxyResolver *resolver; - GUri *guri; - g_return_if_fail (OSTREE_IS_FETCHER (self)); g_return_if_fail (http_proxy != NULL && http_proxy[0] != '\0'); /* validate first */ - guri = g_uri_parse (http_proxy, SOUP_HTTP_URI_FLAGS, NULL); - - if (!guri) + g_autoptr(GError) local_error = NULL; + g_autoptr(GUri) guri = g_uri_parse (http_proxy, G_URI_FLAGS_NONE, &local_error); + if (guri == NULL) { - g_warning ("Invalid proxy URI '%s'", http_proxy); + g_warning ("Invalid proxy URI '%s': %s", http_proxy, local_error->message); return; } - g_uri_unref (guri); - - resolver = g_simple_proxy_resolver_new (http_proxy, NULL); - - session_thread_idle_add (self->thread_closure, - session_thread_set_proxy_cb, - resolver, /* takes ownership */ - (GDestroyNotify) g_object_unref); - } + g_clear_object (&self->proxy_resolver); + self->proxy_resolver = g_simple_proxy_resolver_new (http_proxy, NULL); +} void _ostree_fetcher_set_cookie_jar (OstreeFetcher *self, const char *jar_path) { - SoupCookieJar *jar; - g_return_if_fail (OSTREE_IS_FETCHER (self)); g_return_if_fail (jar_path != NULL); - jar = soup_cookie_jar_text_new (jar_path, TRUE); - - session_thread_idle_add (self->thread_closure, - session_thread_set_cookie_jar_cb, - jar, /* takes ownership */ - (GDestroyNotify) g_object_unref); + g_clear_object (&self->cookie_jar); + self->cookie_jar = soup_cookie_jar_text_new (jar_path, TRUE); } void @@ -722,20 +347,10 @@ _ostree_fetcher_set_client_cert (OstreeFetcher *self, const char *cert_path, const char *key_path) { - g_autoptr(GString) buf = NULL; g_return_if_fail (OSTREE_IS_FETCHER (self)); - if (cert_path) - { - buf = g_string_new (cert_path); - g_string_append_c (buf, '\0'); - g_string_append (buf, key_path); - } - - session_thread_idle_add (self->thread_closure, - session_thread_set_tls_interaction_cb, - g_string_free (g_steal_pointer (&buf), FALSE), - (GDestroyNotify) g_free); + g_clear_object (&self->client_cert); + self->client_cert = _ostree_tls_cert_interaction_new (cert_path, key_path); } void @@ -744,235 +359,201 @@ _ostree_fetcher_set_tls_database (OstreeFetcher *self, { g_return_if_fail (OSTREE_IS_FETCHER (self)); - session_thread_idle_add (self->thread_closure, - session_thread_set_tls_database_cb, - g_strdup (tlsdb_path), - (GDestroyNotify) g_free); + g_autoptr(GError) local_error = NULL; + GTlsDatabase *tlsdb = g_tls_file_database_new (tlsdb_path, &local_error); + if (tlsdb == NULL) + { + g_warning ("Invalid TLS database '%s': %s", tlsdb_path, local_error->message); + return; + } + + g_clear_object (&self->tls_database); + self->tls_database = tlsdb; } void _ostree_fetcher_set_extra_headers (OstreeFetcher *self, GVariant *extra_headers) { - session_thread_idle_add (self->thread_closure, - session_thread_set_headers_cb, - g_variant_ref (extra_headers), - (GDestroyNotify) g_variant_unref); + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + g_clear_pointer (&self->extra_headers, g_variant_unref); + self->extra_headers = g_variant_ref (extra_headers); } void _ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, const char *extra_user_agent) { - session_thread_idle_add (self->thread_closure, - session_thread_set_extra_user_agent_cb, - g_strdup (extra_user_agent), - (GDestroyNotify) g_free); + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + g_clear_pointer (&self->user_agent, g_free); + if (extra_user_agent != NULL) + self->user_agent = g_strdup_printf ("%s %s", + OSTREE_FETCHER_USERAGENT_STRING, + extra_user_agent); } static gboolean -finish_stream (OstreeFetcherPendingURI *pending, - GCancellable *cancellable, - GError **error) +finish_stream (FetcherRequest *request, + GCancellable *cancellable, + GError **error) { - gboolean ret = FALSE; - struct stat stbuf; - /* Close it here since we do an async fstat(), where we don't want * to hit a bad fd. */ - if (pending->out_stream) + if (request->out_stream) { - if ((pending->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + if ((request->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) { const guint8 nulchar = 0; - gsize bytes_written; - if (!g_output_stream_write_all (pending->out_stream, &nulchar, 1, &bytes_written, + if (!g_output_stream_write_all (request->out_stream, &nulchar, 1, NULL, cancellable, error)) - goto out; + return FALSE; } - if (!g_output_stream_close (pending->out_stream, cancellable, error)) - goto out; - - g_mutex_lock (&pending->thread_closure->output_stream_set_lock); - g_hash_table_remove (pending->thread_closure->output_stream_set, - pending->out_stream); - g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + if (!g_output_stream_close (request->out_stream, cancellable, error)) + return FALSE; } - if (!pending->is_membuf) + if (!request->is_membuf) { - if (!glnx_fstat (pending->tmpf.fd, &stbuf, error)) - goto out; - } + struct stat stbuf; - pending->state = OSTREE_FETCHER_STATE_COMPLETE; + if (!glnx_fstat (request->tmpf.fd, &stbuf, error)) + return FALSE; - if (!pending->is_membuf) - { - if (stbuf.st_size < pending->content_length) + if (request->content_length >= 0 && stbuf.st_size < request->content_length) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete"); - goto out; - } - else - { - g_mutex_lock (&pending->thread_closure->output_stream_set_lock); - pending->thread_closure->total_downloaded += stbuf.st_size; - g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + return FALSE; } } - ret = TRUE; - out: - (void) g_input_stream_close (pending->request_body, NULL, NULL); - return ret; + return TRUE; } static void -on_stream_read (GObject *object, - GAsyncResult *result, - gpointer user_data); +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data); static void -remove_pending (OstreeFetcherPendingURI *pending) +on_out_splice_complete (GObject *object, + GAsyncResult *result, + gpointer user_data) { - /* Hold a temporary ref to ensure the reference to - * pending->thread_closure is valid. - */ - pending_uri_ref (pending); - g_hash_table_remove (pending->thread_closure->outstanding, pending); - pending_uri_unref (pending); -} - -static void -on_out_splice_complete (GObject *object, - GAsyncResult *result, - gpointer user_data) -{ - GTask *task = G_TASK (user_data); - OstreeFetcherPendingURI *pending; - GCancellable *cancellable; - gssize bytes_written; + g_autoptr(GTask) task = G_TASK (user_data); GError *local_error = NULL; - pending = g_task_get_task_data (task); - cancellable = g_task_get_cancellable (task); - - bytes_written = g_output_stream_splice_finish ((GOutputStream *)object, - result, - &local_error); + gssize bytes_written = g_output_stream_splice_finish ((GOutputStream *) object, result, &local_error); if (bytes_written < 0) - goto out; + { + g_task_return_error (task, local_error); + return; + } + + FetcherRequest *request = g_task_get_task_data (task); + request->fetcher->bytes_transferred += bytes_written; - g_input_stream_read_bytes_async (pending->request_body, + GCancellable *cancellable = g_task_get_cancellable (task); + g_input_stream_read_bytes_async (request->response_body, 8192, G_PRIORITY_DEFAULT, cancellable, on_stream_read, g_object_ref (task)); - - out: - if (local_error) - { - g_task_return_error (task, local_error); - remove_pending (pending); - } - - g_object_unref (task); } static void -on_stream_read (GObject *object, - GAsyncResult *result, - gpointer user_data) +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data) { - GTask *task = G_TASK (user_data); - OstreeFetcherPendingURI *pending; - GCancellable *cancellable; - g_autoptr(GBytes) bytes = NULL; - gsize bytes_read; + g_autoptr(GTask) task = G_TASK (user_data); + FetcherRequest *request = g_task_get_task_data (task); GError *local_error = NULL; - pending = g_task_get_task_data (task); - cancellable = g_task_get_cancellable (task); - /* Only open the output stream on demand to ensure we use as * few file descriptors as possible. */ - if (!pending->out_stream) + if (!request->out_stream) { - if (!pending->is_membuf) + if (!request->is_membuf) { - if (!_ostree_fetcher_tmpf_from_flags (pending->flags, pending->thread_closure->base_tmpdir_dfd, - &pending->tmpf, &local_error)) - goto out; - pending->out_stream = g_unix_output_stream_new (pending->tmpf.fd, FALSE); + if (!_ostree_fetcher_tmpf_from_flags (request->flags, request->fetcher->tmpdir_dfd, + &request->tmpf, &local_error)) + { + g_task_return_error (task, local_error); + return; + } + request->out_stream = g_unix_output_stream_new (request->tmpf.fd, FALSE); } else { - pending->out_stream = g_memory_output_stream_new_resizable (); + request->out_stream = g_memory_output_stream_new_resizable (); } - - g_mutex_lock (&pending->thread_closure->output_stream_set_lock); - g_hash_table_add (pending->thread_closure->output_stream_set, - g_object_ref (pending->out_stream)); - g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); } /* Get a GBytes buffer */ - bytes = g_input_stream_read_bytes_finish ((GInputStream*)object, result, &local_error); + g_autoptr(GBytes) bytes = g_input_stream_read_bytes_finish ((GInputStream *) object, result, &local_error); if (!bytes) - goto out; - bytes_read = g_bytes_get_size (bytes); + { + g_task_return_error (task, local_error); + return; + } /* Was this the end of the stream? */ + GCancellable *cancellable = g_task_get_cancellable (task); + gsize bytes_read = g_bytes_get_size (bytes); if (bytes_read == 0) { - if (!finish_stream (pending, cancellable, &local_error)) - goto out; - if (pending->is_membuf) + if (!finish_stream (request, cancellable, &local_error)) + { + g_task_return_error (task, local_error); + return; + } + if (request->is_membuf) { - g_task_return_pointer (task, - g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream*)pending->out_stream), - (GDestroyNotify) g_bytes_unref); + GBytes *mem_bytes = g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream *) request->out_stream); + g_task_return_pointer (task, mem_bytes, (GDestroyNotify) g_bytes_unref); } else { - if (lseek (pending->tmpf.fd, 0, SEEK_SET) < 0) + if (lseek (request->tmpf.fd, 0, SEEK_SET) < 0) { glnx_set_error_from_errno (&local_error); g_task_return_error (task, g_steal_pointer (&local_error)); + return; } - else - g_task_return_boolean (task, TRUE); + + g_task_return_boolean (task, TRUE); } - remove_pending (pending); } else { /* Verify max size */ - if (pending->max_size > 0) + if (request->max_size > 0) { - if (bytes_read > pending->max_size || - (bytes_read + pending->current_size) > pending->max_size) + if (bytes_read > request->max_size || + (bytes_read + request->current_size) > request->max_size) { g_autofree char *uristr = NULL; - if (pending->file) - uristr = g_file_get_uri (pending->file); + if (request->file) + uristr = g_file_get_uri (request->file); else - uristr = g_uri_to_string (soup_message_get_uri (pending->message)); + uristr = g_uri_to_string (soup_message_get_uri (request->message)); local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", - uristr, pending->max_size); - goto out; + uristr, request->max_size); + g_task_return_error (task, local_error); + return; } } - pending->current_size += bytes_read; + request->current_size += bytes_read; /* We do this instead of _write_bytes_async() as that's not * guaranteed to do a complete write. @@ -980,7 +561,7 @@ on_stream_read (GObject *object, { g_autoptr(GInputStream) membuf = g_memory_input_stream_new_from_bytes (bytes); - g_output_stream_splice_async (pending->out_stream, membuf, + g_output_stream_splice_async (request->out_stream, membuf, G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, G_PRIORITY_DEFAULT, cancellable, @@ -988,214 +569,251 @@ on_stream_read (GObject *object, g_object_ref (task)); } } - - out: - if (local_error) - { - g_task_return_error (task, local_error); - remove_pending (pending); - } - - g_object_unref (task); } static void -on_request_sent (GObject *object, - GAsyncResult *result, - gpointer user_data) +on_request_sent (GObject *object, + GAsyncResult *result, + gpointer user_data) { - GTask *task = G_TASK (user_data); - /* Hold a ref to the pending across this function, since we remove - * it from the hash early in some cases, not in others. */ - OstreeFetcherPendingURI *pending = pending_uri_ref (g_task_get_task_data (task)); - GCancellable *cancellable = g_task_get_cancellable (task); + g_autoptr(GTask) task = G_TASK (user_data); + FetcherRequest *request = g_task_get_task_data (task); GError *local_error = NULL; - glnx_unref_object SoupMessage *msg = NULL; - SoupStatus status; - - pending->state = OSTREE_FETCHER_STATE_COMPLETE; - if (pending->file) - pending->request_body = (GInputStream *)g_file_read_finish ((GFile *) object, - result, - &local_error); + + if (request->file) + request->response_body = (GInputStream *) g_file_read_finish ((GFile *) object, + result, + &local_error); else - pending->request_body = soup_session_send_finish ((SoupSession*) object, - result, &local_error); + request->response_body = soup_session_send_finish ((SoupSession *) object, + result, &local_error); - if (!pending->request_body) - goto out; + if (!request->response_body) + { + g_task_return_error (task, local_error); + return; + } - if (pending->message) + if (request->message) { - status = soup_message_get_status (pending->message); + SoupStatus status = soup_message_get_status (request->message); if (status == SOUP_STATUS_NOT_MODIFIED && - (pending->if_none_match != NULL || pending->if_modified_since > 0)) + (request->if_none_match != NULL || request->if_modified_since > 0)) { /* Version on the server is unchanged from the version we have cached locally; * report this as an out-argument, a zero-length response buffer, and no error */ - pending->out_not_modified = TRUE; + request->out_not_modified = TRUE; } else if (!SOUP_STATUS_IS_SUCCESSFUL (status)) { /* is there another mirror we can try? */ - if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len) + if (request->mirrorlist_idx + 1 < request->mirrorlist->len) { - pending->mirrorlist_idx++; - create_pending_soup_request (pending); - - (void) g_input_stream_close (pending->request_body, NULL, NULL); - - start_pending_request (pending->thread_closure, task); + request->mirrorlist_idx++; + initiate_task_request (g_object_ref (task)); + return; } else { - g_autofree char *uristring = g_uri_to_string (soup_message_get_uri (pending->message)); + g_autofree char *uristring = g_uri_to_string (soup_message_get_uri (request->message)); GIOErrorEnum code = _ostree_fetcher_http_status_code_to_io_error (status); { g_autofree char *errmsg = g_strdup_printf ("Server returned status %u: %s", status, soup_status_get_phrase (status)); - - /* Let's make OOB errors be the final one since they're probably - * the cause for the error here. */ - if (pending->thread_closure->oob_error) - { - local_error = - g_error_copy (pending->thread_closure->oob_error); - g_prefix_error (&local_error, "%s: ", errmsg); - } - else - local_error = g_error_new_literal (G_IO_ERROR, code, errmsg); + local_error = g_error_new_literal (G_IO_ERROR, code, errmsg); } - if (pending->mirrorlist->len > 1) + if (request->mirrorlist->len > 1) g_prefix_error (&local_error, "All %u mirrors failed. Last error was: ", - pending->mirrorlist->len); - if (pending->thread_closure->remote_name && - !((pending->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && + request->mirrorlist->len); + if (request->fetcher->remote_name && + !((request->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && code == G_IO_ERROR_NOT_FOUND)) - _ostree_fetcher_journal_failure (pending->thread_closure->remote_name, + _ostree_fetcher_journal_failure (request->fetcher->remote_name, uristring, local_error->message); + g_task_return_error (task, local_error); + return; } - goto out; } /* Grab cache properties from the response */ - pending->out_etag = g_strdup (soup_message_headers_get_one (soup_message_get_response_headers (pending->message), "ETag")); - pending->out_last_modified = 0; + request->out_etag = g_strdup (soup_message_headers_get_one (soup_message_get_response_headers (request->message), "ETag")); + request->out_last_modified = 0; - const char *last_modified_str = soup_message_headers_get_one (soup_message_get_response_headers (pending->message), "Last-Modified"); + const char *last_modified_str = soup_message_headers_get_one (soup_message_get_response_headers (request->message), "Last-Modified"); if (last_modified_str != NULL) { GDateTime *soup_date = soup_date_time_new_from_http_string (last_modified_str); if (soup_date != NULL) { - pending->out_last_modified = g_date_time_to_unix (soup_date); + request->out_last_modified = g_date_time_to_unix (soup_date); g_date_time_unref (soup_date); } } } - pending->state = OSTREE_FETCHER_STATE_DOWNLOADING; - - if (pending->file) + if (request->file) { - GFileInfo *info = g_file_query_info (pending->file, + GFileInfo *info = g_file_query_info (request->file, G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE "," G_FILE_ATTRIBUTE_STANDARD_SIZE, 0, NULL, NULL); if (info) { - pending->content_length = g_file_info_get_size (info); + request->content_length = g_file_info_get_size (info); g_object_unref (info); } else - pending->content_length = -1; + request->content_length = -1; } else - pending->content_length = soup_message_headers_get_content_length (soup_message_get_response_headers (pending->message)); + { + SoupMessageHeaders *headers = soup_message_get_response_headers (request->message); + if (soup_message_headers_get_list (headers, "Content-Encoding") == NULL) + request->content_length = soup_message_headers_get_content_length (headers); + else + request->content_length = -1; + } - g_input_stream_read_bytes_async (pending->request_body, + GCancellable *cancellable = g_task_get_cancellable (task); + g_input_stream_read_bytes_async (request->response_body, 8192, G_PRIORITY_DEFAULT, cancellable, on_stream_read, g_object_ref (task)); +} - out: - if (local_error) +static SoupSession * +create_soup_session (OstreeFetcher *self) +{ + const char *user_agent = self->user_agent ?: OSTREE_FETCHER_USERAGENT_STRING; + SoupSession *session = + soup_session_new_with_options ("user-agent", user_agent, + "timeout", 60, + "idle-timeout", 60, + "max-conns-per-host", _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS, + NULL); + + /* SoupContentDecoder is included in the session by default. Remove it + * if gzip compression isn't in use. + */ + if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) == 0) + soup_session_remove_feature_by_type (session, SOUP_TYPE_CONTENT_DECODER); + + if (g_getenv ("OSTREE_DEBUG_HTTP")) { - if (pending->request_body) - (void) g_input_stream_close (pending->request_body, NULL, NULL); - g_task_return_error (task, local_error); - remove_pending (pending); + glnx_unref_object SoupLogger *logger = soup_logger_new (SOUP_LOGGER_LOG_BODY); + soup_session_add_feature (session, SOUP_SESSION_FEATURE (logger)); } - pending_uri_unref (pending); - g_object_unref (task); + if (self->proxy_resolver != NULL) + soup_session_set_proxy_resolver (session, self->proxy_resolver); + if (self->cookie_jar != NULL) + soup_session_add_feature (session, SOUP_SESSION_FEATURE (self->cookie_jar)); + if (self->client_cert != NULL) + soup_session_set_tls_interaction (session, (GTlsInteraction *) self->client_cert); + if (self->tls_database != NULL) + soup_session_set_tls_database (session, self->tls_database); + + return session; +} + +static gboolean +match_value (gpointer key, + gpointer value, + gpointer user_data) +{ + return value == user_data; } static void -_ostree_fetcher_request_async (OstreeFetcher *self, - GPtrArray *mirrorlist, - const char *filename, - OstreeFetcherRequestFlags flags, - const char *if_none_match, - guint64 if_modified_since, - gboolean is_membuf, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data) +on_session_finalized (gpointer data, + GObject *object) { - g_autoptr(GTask) task = NULL; - OstreeFetcherPendingURI *pending; + GHashTable *sessions = data; + g_debug ("Removing session %p from sessions hash table", object); + (void) g_hash_table_foreach_remove (sessions, match_value, object); +} +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ g_return_if_fail (OSTREE_IS_FETCHER (self)); g_return_if_fail (mirrorlist != NULL); g_return_if_fail (mirrorlist->len > 0); - /* SoupRequest is created in session thread. */ - pending = g_new0 (OstreeFetcherPendingURI, 1); - pending->ref_count = 1; - pending->thread_closure = thread_closure_ref (self->thread_closure); - pending->mirrorlist = g_ptr_array_ref (mirrorlist); - pending->filename = g_strdup (filename); - pending->flags = flags; - pending->if_none_match = g_strdup (if_none_match); - pending->if_modified_since = if_modified_since; - pending->max_size = max_size; - pending->is_membuf = is_membuf; - pending->fetcher = self; - - task = g_task_new (self, cancellable, callback, user_data); + FetcherRequest *request = g_new0 (FetcherRequest, 1); + request->mirrorlist = g_ptr_array_ref (mirrorlist); + request->filename = g_strdup (filename); + request->flags = flags; + request->if_none_match = g_strdup (if_none_match); + request->if_modified_since = if_modified_since; + request->max_size = max_size; + request->is_membuf = is_membuf; + request->fetcher = self; + request->mainctx = g_main_context_ref_thread_default (); + + /* Ideally each fetcher would have a single soup session. However, each + * session needs to be used from a single main context and the fetcher + * doesn't have that limitation. Instead, a mapping from main context to + * session is kept in the fetcher. + */ + g_debug ("Looking up session for main context %p", request->mainctx); + request->session = (SoupSession *) g_hash_table_lookup (self->sessions, request->mainctx); + if (request->session != NULL) + { + g_debug ("Using existing session %p", request->session); + g_object_ref (request->session); + } + else + { + request->session = create_soup_session (self); + g_debug ("Created new session %p", request->session); + g_hash_table_insert (self->sessions, request->mainctx, request->session); + + /* Add a weak ref to the session so that when it's finalized it can be + * removed from the hash table. + */ + g_object_weak_ref (G_OBJECT (request->session), on_session_finalized, self->sessions); + } + + g_autoptr(GTask) task = g_task_new (self, cancellable, callback, user_data); g_task_set_source_tag (task, _ostree_fetcher_request_async); - g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_unref); + g_task_set_task_data (task, request, (GDestroyNotify) fetcher_request_free); /* We'll use the GTask priority for our own priority queue. */ g_task_set_priority (task, priority); - session_thread_idle_add (self->thread_closure, - session_thread_request_uri, - g_object_ref (task), - (GDestroyNotify) g_object_unref); + initiate_task_request (g_object_ref (task)); } void -_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, - GPtrArray *mirrorlist, - const char *filename, - OstreeFetcherRequestFlags flags, - const char *if_none_match, - guint64 if_modified_since, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data) +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) { _ostree_fetcher_request_async (self, mirrorlist, filename, flags, if_none_match, if_modified_since, FALSE, @@ -1204,54 +822,49 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, } gboolean -_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, - GAsyncResult *result, - GLnxTmpfile *out_tmpf, - gboolean *out_not_modified, - char **out_etag, - guint64 *out_last_modified, - GError **error) +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) { - GTask *task; - OstreeFetcherPendingURI *pending; - gpointer ret; - g_return_val_if_fail (g_task_is_valid (result, self), FALSE); g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); - task = (GTask*)result; - pending = g_task_get_task_data (task); - - ret = g_task_propagate_pointer (task, error); + GTask *task = (GTask *) result; + gpointer ret = g_task_propagate_pointer (task, error); if (!ret) return FALSE; - g_assert (!pending->is_membuf); - *out_tmpf = pending->tmpf; - pending->tmpf.initialized = FALSE; /* Transfer ownership */ + FetcherRequest *request = g_task_get_task_data (task); + g_assert (!request->is_membuf); + *out_tmpf = request->tmpf; + request->tmpf.initialized = FALSE; /* Transfer ownership */ if (out_not_modified != NULL) - *out_not_modified = pending->out_not_modified; + *out_not_modified = request->out_not_modified; if (out_etag != NULL) - *out_etag = g_steal_pointer (&pending->out_etag); + *out_etag = g_steal_pointer (&request->out_etag); if (out_last_modified != NULL) - *out_last_modified = pending->out_last_modified; + *out_last_modified = request->out_last_modified; return TRUE; } void -_ostree_fetcher_request_to_membuf (OstreeFetcher *self, - GPtrArray *mirrorlist, - const char *filename, - OstreeFetcherRequestFlags flags, - const char *if_none_match, - guint64 if_modified_since, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data) +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) { _ostree_fetcher_request_async (self, mirrorlist, filename, flags, if_none_match, if_modified_since, TRUE, @@ -1260,66 +873,40 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher *self, } gboolean -_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, - GAsyncResult *result, - GBytes **out_buf, - gboolean *out_not_modified, - char **out_etag, - guint64 *out_last_modified, - GError **error) +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) { - GTask *task; - OstreeFetcherPendingURI *pending; - gpointer ret; - g_return_val_if_fail (g_task_is_valid (result, self), FALSE); g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); - task = (GTask*)result; - pending = g_task_get_task_data (task); - - ret = g_task_propagate_pointer (task, error); + GTask *task = (GTask *) result; + gpointer ret = g_task_propagate_pointer (task, error); if (!ret) return FALSE; - g_assert (pending->is_membuf); + FetcherRequest *request = g_task_get_task_data (task); + g_assert (request->is_membuf); g_assert (out_buf); *out_buf = ret; if (out_not_modified != NULL) - *out_not_modified = pending->out_not_modified; + *out_not_modified = request->out_not_modified; if (out_etag != NULL) - *out_etag = g_steal_pointer (&pending->out_etag); + *out_etag = g_steal_pointer (&request->out_etag); if (out_last_modified != NULL) - *out_last_modified = pending->out_last_modified; + *out_last_modified = request->out_last_modified; return TRUE; } guint64 -_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) { - g_return_val_if_fail (OSTREE_IS_FETCHER (self), 0); - - g_mutex_lock (&self->thread_closure->output_stream_set_lock); - - guint64 ret = self->thread_closure->total_downloaded; - - GLNX_HASH_TABLE_FOREACH (self->thread_closure->output_stream_set, - GFileOutputStream*, stream) - { - if (G_IS_FILE_DESCRIPTOR_BASED (stream)) - { - int fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)stream); - struct stat stbuf; - - if (glnx_fstat (fd, &stbuf, NULL)) - ret += stbuf.st_size; - } - } - - g_mutex_unlock (&self->thread_closure->output_stream_set_lock); - - return ret; + return self->bytes_transferred; } -- cgit v1.2.1