From d0ea2db4300eb7871b59d0b997a7f06869120297 Mon Sep 17 00:00:00 2001 From: Daniel Kolesa Date: Thu, 17 Feb 2022 20:12:18 +0100 Subject: fetcher: add libsoup3 backend The default is still soup2, you can use --with-soup3 to enable the soup3 backend instead. --- Makefile-libostree.am | 7 + Makefile-man.am | 2 +- Makefile-ostree.am | 2 +- Makefile-tests.am | 4 +- Makefile.am | 12 +- configure.ac | 35 +- src/libostree/ostree-fetcher-soup3.c | 1325 ++++++++++++++++++++++++++++++++++ src/ostree/main.c | 2 +- src/ostree/ostree-trivial-httpd.c | 228 ++++-- 9 files changed, 1544 insertions(+), 73 deletions(-) create mode 100644 src/libostree/ostree-fetcher-soup3.c diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 52ea49d4..d18714ae 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -221,18 +221,25 @@ libostree_1_la_SOURCES += \ $(NULL) endif +# Only enable one fetcher backend. if USE_CURL libostree_1_la_SOURCES += src/libostree/ostree-fetcher-curl.c \ $(NULL) libostree_1_la_CFLAGS += $(OT_DEP_CURL_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_CURL_LIBS) else +if USE_LIBSOUP3 +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup3.c +libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) +libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) +else if USE_LIBSOUP libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup.c libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) endif endif +endif if USE_LIBMOUNT libostree_1_la_CFLAGS += $(OT_DEP_LIBMOUNT_CFLAGS) diff --git a/Makefile-man.am b/Makefile-man.am index 5c7f2413..41c59327 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -34,7 +34,7 @@ ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 \ ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 \ ostree-rev-parse.1 ostree-show.1 ostree-sign.1 ostree-summary.1 \ ostree-static-delta.1 -if USE_LIBSOUP +if USE_LIBSOUP_OR_LIBSOUP3 man1_files += ostree-trivial-httpd.1 else # We still want to distribute the source, even if we are not building it diff --git a/Makefile-ostree.am b/Makefile-ostree.am index fb377075..05d58a20 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -141,7 +141,7 @@ if USE_CURL_OR_SOUP ostree_SOURCES += src/ostree/ot-builtin-pull.c endif -if USE_LIBSOUP +if USE_LIBSOUP_OR_LIBSOUP3 # Eventually once we stop things from using this, we should support disabling this ostree_SOURCES += src/ostree/ot-builtin-trivial-httpd.c pkglibexec_PROGRAMS += ostree-trivial-httpd diff --git a/Makefile-tests.am b/Makefile-tests.am index 429dafb9..32817704 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -185,7 +185,7 @@ else EXTRA_DIST += tests/test-rofiles-fuse.sh endif -if USE_LIBSOUP +if USE_LIBSOUP_OR_LIBSOUP3 _installed_or_uninstalled_test_scripts += tests/test-remote-cookies.sh endif @@ -438,7 +438,7 @@ dist_test_scripts += $(_installed_or_uninstalled_test_scripts) test_programs += $(_installed_or_uninstalled_test_programs) endif -if !USE_LIBSOUP +if !USE_LIBSOUP_OR_LIBSOUP3 no-soup-for-you-warning: @echo "WARNING: $(PACKAGE) was built without libsoup, which is currently" 1>&2 @echo "WARNING: required for many unit tests." 1>&2 diff --git a/Makefile.am b/Makefile.am index 4e669164..50282156 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,8 +29,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DOSTREE_COMPILATION \ -DG_LOG_DOMAIN=\"OSTree\" \ -DOSTREE_GITREV='"$(OSTREE_GITREV)"' \ - -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_66 '-DGLIB_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,70)' \ - -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 '-DSOUP_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,48)' + -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_66 '-DGLIB_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,70)' # For strict aliasing, see https://bugzilla.gnome.org/show_bug.cgi?id=791622 AM_CFLAGS += -std=gnu99 -fno-strict-aliasing $(WARN_CFLAGS) AM_DISTCHECK_CONFIGURE_FLAGS += \ @@ -39,6 +38,10 @@ AM_DISTCHECK_CONFIGURE_FLAGS += \ --disable-maintainer-mode \ $(NULL) +if USE_LIBSOUP +AM_CPPFLAGS += -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 '-DSOUP_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,48)' +endif + GITIGNOREFILES = aclocal.m4 build-aux/ buildutil/*.m4 config.h.in gtk-doc.make # Generated by ci/gh-build.sh @@ -69,8 +72,13 @@ EXTRA_DIST += autogen.sh COPYING README.md OT_INTERNAL_GIO_UNIX_CFLAGS = $(OT_DEP_GIO_UNIX_CFLAGS) OT_INTERNAL_GIO_UNIX_LIBS = $(OT_DEP_GIO_UNIX_LIBS) +if USE_LIBSOUP3 +OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP3_CFLAGS) +OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP3_LIBS) +else OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP_CFLAGS) OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP_LIBS) +endif # This canonicalizes the PKG_CHECK_MODULES or AM_PATH_GPGME results if USE_GPGME diff --git a/configure.ac b/configure.ac index 21f204f3..115096ef 100644 --- a/configure.ac +++ b/configure.ac @@ -149,6 +149,21 @@ AS_IF([test x$enable_http2 != xno ], [ OSTREE_FEATURES="$OSTREE_FEATURES no-http2" ]) +SOUP3_DEPENDENCY="libsoup-3.0 >= 3.0.0" +AC_ARG_WITH(soup3, + AS_HELP_STRING([--with-soup3], [Use libsoup3 @<:@default=no@:>@]), + [], [with_soup3=no]) +AS_IF([test x$with_soup3 != xno], [ + PKG_CHECK_MODULES(OT_DEP_SOUP3, $SOUP3_DEPENDENCY) + with_soup3=yes + AC_DEFINE([HAVE_LIBSOUP3], 1, [Define if we have libsoup3]) + OSTREE_FEATURES="$OSTREE_FEATURES libsoup3" + with_soup_default=no + dnl soup3 always supports client certs + have_libsoup_client_certs=yes +], [with_soup_default=check]) +AM_CONDITIONAL(USE_LIBSOUP3, test x$with_soup3 != xno) + dnl When bumping the libsoup-2.4 dependency, remember to bump dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in dnl Makefile.am @@ -180,7 +195,7 @@ AS_IF([test x$with_soup != xno], [ ], [], [#include ]) AS_IF([test x$enable_libsoup_client_certs = xyes && test x$have_libsoup_client_certs != xyes], [ AC_MSG_ERROR([libsoup client certs explicitly requested but not found]) - ]) + ]) CFLAGS=$save_CFLAGS ], [ with_soup=no @@ -190,6 +205,13 @@ if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES libsoup"; fi AM_CONDITIONAL(USE_LIBSOUP, test x$with_soup != xno) AM_CONDITIONAL(HAVE_LIBSOUP_CLIENT_CERTS, test x$have_libsoup_client_certs = xyes) +dnl Some components use either soup2 or soup3. +AM_CONDITIONAL([USE_LIBSOUP_OR_LIBSOUP3], + [test x$with_soup = xyes || test x$with_soup3 = xyes]) +AS_IF([test x$with_soup = xyes || test x$with_soup3 = xyes], [ + AC_DEFINE([HAVE_LIBSOUP_OR_LIBSOUP3], 1, [Define if we have libsoup.pc or libsoup3.pc]) +]) + AC_ARG_ENABLE(trivial-httpd-cmdline, [AS_HELP_STRING([--enable-trivial-httpd-cmdline], [Continue to support "ostree trivial-httpd" [default=no]])],, @@ -198,13 +220,16 @@ AS_IF([test x$enable_trivial_httpd_cmdline = xyes], [AC_DEFINE([BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE], 1, [Define if we are enabling ostree trivial-httpd entrypoint])] ) -AS_IF([test x$with_curl = xyes && test x$with_soup = xno], [ +AS_IF([test x$with_curl = xyes && test x$with_soup = xno && test x$with_soup3 = xno], [ AC_MSG_WARN([Curl enabled, but libsoup is not; libsoup is needed for tests (make check, etc.)]) ]) -AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno) -AS_IF([test x$with_curl != xno || test x$with_soup != xno], +AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno || test x$with_soup3 != xno) +AS_IF([test x$with_curl != xno || test x$with_soup != xno || test x$with_soup3 != xno], [AC_DEFINE([HAVE_LIBCURL_OR_LIBSOUP], 1, [Define if we have soup or curl])]) -AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], [test x$with_soup = xyes], [fetcher_backend=libsoup], [fetcher_backend=none]) +AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], + [test x$with_soup = xyes], [fetcher_backend=libsoup], + [test x$with_soup3 = xyes], [fetcher_backend=libsoup3], + [fetcher_backend=none]) m4_ifdef([GOBJECT_INTROSPECTION_CHECK], [ GOBJECT_INTROSPECTION_CHECK([1.51.5]) diff --git a/src/libostree/ostree-fetcher-soup3.c b/src/libostree/ostree-fetcher-soup3.c new file mode 100644 index 00000000..9f495a72 --- /dev/null +++ b/src/libostree/ostree-fetcher-soup3.c @@ -0,0 +1,1325 @@ +/* + * Copyright (C) 2011 Colin Walters + * Copyright (C) 2022 Igalia S.L. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + * Author: Colin Walters + * Author: Daniel Kolesa + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "libglnx.h" +#include "ostree-fetcher.h" +#include "ostree-fetcher-util.h" +#include "ostree-tls-cert-interaction-private.h" +#include "ostree-enumtypes.h" +#include "ostree.h" +#include "ostree-repo-private.h" +#include "otutil.h" + +typedef enum { + OSTREE_FETCHER_STATE_PENDING, + OSTREE_FETCHER_STATE_DOWNLOADING, + OSTREE_FETCHER_STATE_COMPLETE +} OstreeFetcherState; + +typedef struct { + int ref_count; /* atomic */ + + SoupSession *session; /* not referenced */ + GMainContext *main_context; + volatile gint running; + GError *initialization_error; /* Any failure to load the db */ + + char *remote_name; + int base_tmpdir_dfd; + + GVariant *extra_headers; + gboolean transfer_gzip; + + /* Our active HTTP requests */ + GHashTable *outstanding; + + /* Shared across threads; be sure to lock. */ + GHashTable *output_stream_set; /* set */ + GMutex output_stream_set_lock; + + /* Also protected by output_stream_set_lock. */ + guint64 total_downloaded; + + GError *oob_error; +} ThreadClosure; + +typedef struct { + int ref_count; /* atomic */ + + ThreadClosure *thread_closure; + GPtrArray *mirrorlist; /* list of base URIs */ + char *filename; /* relative name to fetch or NULL */ + guint mirrorlist_idx; + + OstreeFetcherState state; + + SoupMessage *message; + GFile *file; + struct OstreeFetcher *fetcher; + + gboolean is_membuf; + OstreeFetcherRequestFlags flags; + char *if_none_match; /* request ETag */ + guint64 if_modified_since; /* seconds since the epoch */ + GInputStream *request_body; + GLnxTmpfile tmpf; + GOutputStream *out_stream; + gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */ + char *out_etag; /* response ETag */ + guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */ + + guint64 max_size; + guint64 current_size; + guint64 content_length; +} OstreeFetcherPendingURI; + +/* Used by session_thread_idle_add() */ +typedef void (*SessionThreadFunc) (ThreadClosure *thread_closure, + gpointer data); + +/* Used by session_thread_idle_add() */ +typedef struct { + ThreadClosure *thread_closure; + SessionThreadFunc function; + gpointer data; + GDestroyNotify notify; +} IdleClosure; + +struct OstreeFetcher +{ + GObject parent_instance; + + OstreeFetcherConfigFlags config_flags; + + GThread *session_thread; + ThreadClosure *thread_closure; +}; + +enum { + PROP_0, + PROP_CONFIG_FLAGS +}; + +G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) + +static ThreadClosure * +thread_closure_ref (ThreadClosure *thread_closure) +{ + int refcount; + g_return_val_if_fail (thread_closure != NULL, NULL); + refcount = g_atomic_int_add (&thread_closure->ref_count, 1); + g_assert (refcount > 0); + return thread_closure; +} + +static void +thread_closure_unref (ThreadClosure *thread_closure) +{ + g_return_if_fail (thread_closure != NULL); + + if (g_atomic_int_dec_and_test (&thread_closure->ref_count)) + { + /* The session thread should have cleared this by now. */ + g_assert (thread_closure->session == NULL); + + g_clear_pointer (&thread_closure->main_context, g_main_context_unref); + + g_clear_pointer (&thread_closure->extra_headers, g_variant_unref); + + g_clear_pointer (&thread_closure->output_stream_set, g_hash_table_unref); + g_mutex_clear (&thread_closure->output_stream_set_lock); + + g_clear_pointer (&thread_closure->oob_error, g_error_free); + + g_free (thread_closure->remote_name); + + g_slice_free (ThreadClosure, thread_closure); + } +} + +static void +idle_closure_free (IdleClosure *idle_closure) +{ + g_clear_pointer (&idle_closure->thread_closure, thread_closure_unref); + + if (idle_closure->notify != NULL) + idle_closure->notify (idle_closure->data); + + g_slice_free (IdleClosure, idle_closure); +} + +static OstreeFetcherPendingURI * +pending_uri_ref (OstreeFetcherPendingURI *pending) +{ + gint refcount; + g_assert (pending); + refcount = g_atomic_int_add (&pending->ref_count, 1); + g_assert (refcount > 0); + return pending; +} + +static void +pending_uri_unref (OstreeFetcherPendingURI *pending) +{ + if (!g_atomic_int_dec_and_test (&pending->ref_count)) + return; + + g_clear_pointer (&pending->thread_closure, thread_closure_unref); + + g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref); + g_free (pending->filename); + g_clear_object (&pending->message); + g_clear_object (&pending->file); + g_clear_object (&pending->request_body); + g_free (pending->if_none_match); + glnx_tmpfile_clear (&pending->tmpf); + g_clear_object (&pending->out_stream); + g_free (pending->out_etag); + g_free (pending); +} + +static gboolean +session_thread_idle_dispatch (gpointer data) +{ + IdleClosure *idle_closure = data; + + idle_closure->function (idle_closure->thread_closure, + idle_closure->data); + + return G_SOURCE_REMOVE; +} + +static void +session_thread_idle_add (ThreadClosure *thread_closure, + SessionThreadFunc function, + gpointer data, + GDestroyNotify notify) +{ + IdleClosure *idle_closure; + + g_return_if_fail (thread_closure != NULL); + g_return_if_fail (function != NULL); + + idle_closure = g_slice_new (IdleClosure); + idle_closure->thread_closure = thread_closure_ref (thread_closure); + idle_closure->function = function; + idle_closure->data = data; + idle_closure->notify = notify; + + g_main_context_invoke_full (thread_closure->main_context, + G_PRIORITY_DEFAULT, + session_thread_idle_dispatch, + idle_closure, /* takes ownership */ + (GDestroyNotify) idle_closure_free); +} + +static void +session_thread_add_logger (ThreadClosure *thread_closure, + gpointer data) +{ + glnx_unref_object SoupLogger *logger = NULL; + + logger = soup_logger_new (SOUP_LOGGER_LOG_BODY); + soup_logger_set_max_body_size (logger, 500); + soup_session_add_feature (thread_closure->session, + SOUP_SESSION_FEATURE (logger)); +} + +static void +session_thread_set_proxy_cb (ThreadClosure *thread_closure, + gpointer data) +{ + GProxyResolver *resolver = data; + + g_object_set (thread_closure->session, + "proxy-resolver", + g_object_ref (resolver), NULL); +} + +static void +session_thread_set_cookie_jar_cb (ThreadClosure *thread_closure, + gpointer data) +{ + SoupCookieJar *jar = data; + + soup_session_add_feature (thread_closure->session, + SOUP_SESSION_FEATURE (jar)); +} + +static void +session_thread_set_headers_cb (ThreadClosure *thread_closure, + gpointer data) +{ + GVariant *headers = data; + + g_clear_pointer (&thread_closure->extra_headers, g_variant_unref); + thread_closure->extra_headers = g_variant_ref (headers); +} + +static void +session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *cert_and_key_path = data; /* str\0str\0 in one malloc buf */ + const char *cert_path = cert_and_key_path; + const char *key_path = cert_and_key_path + strlen (cert_and_key_path) + 1; + g_autoptr(OstreeTlsCertInteraction) interaction = NULL; + + /* The GTlsInteraction instance must be created in the + * session thread so it uses the correct GMainContext. */ + interaction = _ostree_tls_cert_interaction_new (cert_path, key_path); + + g_object_set (thread_closure->session, + "tls-interaction", + interaction, NULL); +} + +static void +session_thread_set_tls_database_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *db_path = data; + + if (db_path != NULL) + { + glnx_unref_object GTlsDatabase *tlsdb = NULL; + + g_clear_error (&thread_closure->initialization_error); + tlsdb = g_tls_file_database_new (db_path, &thread_closure->initialization_error); + + if (tlsdb) + g_object_set (thread_closure->session, + "tls-database", + tlsdb, NULL); + } +} + +static void +session_thread_set_extra_user_agent_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *extra_user_agent = data; + if (extra_user_agent != NULL) + { + g_autofree char *ua = + g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent); + g_object_set (thread_closure->session, "user-agent", ua, NULL); + } + else + { + g_object_set (thread_closure->session, "user-agent", + OSTREE_FETCHER_USERAGENT_STRING, NULL); + } +} + +static void +on_request_sent (GObject *object, GAsyncResult *result, gpointer user_data); + +static void +request_send_async (ThreadClosure *thread_closure, + GTask *task) +{ + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; + + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); + + if (pending->file) + g_file_read_async (pending->file, g_task_get_priority (task), + cancellable, on_request_sent, task); + else + { + soup_session_send_async (thread_closure->session, pending->message, + g_task_get_priority (task), + cancellable, on_request_sent, task); + } +} + +static void +start_pending_request (ThreadClosure *thread_closure, + GTask *task) +{ + + OstreeFetcherPendingURI *pending; + + pending = g_task_get_task_data (task); + + g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending)); + request_send_async (thread_closure, g_object_ref (task)); +} + +static gboolean +_message_accept_cert_loose (SoupMessage *msg, + GTlsCertificate *tls_peer_certificate, + GTlsCertificateFlags tls_peer_errors, + gpointer data) +{ + OstreeFetcherConfigFlags config_flags; + + config_flags = GPOINTER_TO_UINT (data); + + return ((config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0); +} + +static void +create_pending_soup_request (OstreeFetcherPendingURI *pending) +{ + OstreeFetcherURI *next_mirror = NULL; + g_autoptr(OstreeFetcherURI) uri = NULL; + GUri *guri = NULL; + + g_assert (pending->mirrorlist); + g_assert (pending->mirrorlist_idx < pending->mirrorlist->len); + + next_mirror = g_ptr_array_index (pending->mirrorlist, pending->mirrorlist_idx); + if (pending->filename) + uri = _ostree_fetcher_uri_new_subpath (next_mirror, pending->filename); + if (!uri) + uri = (OstreeFetcherURI*)g_uri_ref ((GUri*)next_mirror); + + g_clear_object (&pending->message); + g_clear_object (&pending->file); + + guri = (GUri*)(uri ? uri : next_mirror); + + /* file:// URI is handle via GFile */ + if (!strcmp (g_uri_get_scheme (guri), "file")) + { + char *str = g_uri_to_string (guri); + pending->file = g_file_new_for_uri (str); + g_free (str); + return; + } + + pending->message = soup_message_new_from_uri ("GET", guri); + + if (pending->if_none_match != NULL) + { + soup_message_headers_append (soup_message_get_request_headers (pending->message), + "If-None-Match", pending->if_none_match); + } + + if (pending->if_modified_since > 0) + { + g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (pending->if_modified_since); + g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z"); + soup_message_headers_append (soup_message_get_request_headers (pending->message), + "If-Modified-Since", mod_date); + } + + if (pending->message && pending->fetcher->config_flags != 0) + { + g_signal_connect (pending->message, "accept-certificate", + G_CALLBACK (_message_accept_cert_loose), + GUINT_TO_POINTER (pending->fetcher->config_flags)); + } +} + +static void +session_thread_request_uri (ThreadClosure *thread_closure, + gpointer data) +{ + GTask *task = G_TASK (data); + OstreeFetcherPendingURI *pending; + + pending = g_task_get_task_data (task); + + /* If we caught an error in init, re-throw it for every request */ + if (thread_closure->initialization_error) + { + g_task_return_error (task, g_error_copy (thread_closure->initialization_error)); + return; + } + + create_pending_soup_request (pending); + + if (pending->message && thread_closure->extra_headers) + { + g_autoptr(GVariantIter) viter = g_variant_iter_new (thread_closure->extra_headers); + const char *key; + const char *value; + + while (g_variant_iter_next (viter, "(&s&s)", &key, &value)) + soup_message_headers_append (soup_message_get_request_headers (pending->message), key, value); + } + + if (pending->is_membuf) + request_send_async (thread_closure, g_object_ref (task)); + else + start_pending_request (thread_closure, task); +} + +static gpointer +ostree_fetcher_session_thread (gpointer data) +{ + ThreadClosure *closure = data; + g_autoptr(GMainContext) mainctx = g_main_context_ref (closure->main_context); + + /* This becomes the GMainContext that SoupSession schedules async + * callbacks and emits signals from. Make it the thread-default + * context for this thread before creating the session. */ + g_main_context_push_thread_default (mainctx); + + /* We retain ownership of the SoupSession reference. */ + closure->session = soup_session_new_with_options ("user-agent", OSTREE_FETCHER_USERAGENT_STRING, + "timeout", 60, + "idle-timeout", 60, + "max-conns-per-host", _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS, + NULL); + + /* SoupContentDecoder is included in the session by default. Remove it + * if gzip compression isn't in use. + */ + if (!closure->transfer_gzip) + soup_session_remove_feature_by_type (closure->session, SOUP_TYPE_CONTENT_DECODER); + + /* This model ensures we don't hit a race using g_main_loop_quit(); + * see also what pull_termination_condition() in ostree-repo-pull.c + * is doing. + */ + while (g_atomic_int_get (&closure->running)) + g_main_context_iteration (closure->main_context, TRUE); + + /* Since the ThreadClosure may be finalized from any thread we + * unreference all data related to the SoupSession ourself to ensure + * it's freed in the same thread where it was created. */ + g_clear_pointer (&closure->outstanding, g_hash_table_unref); + g_clear_pointer (&closure->session, g_object_unref); + + thread_closure_unref (closure); + + /* Do this last, since libsoup uses g_main_current_source() which + * relies on it. + */ + g_main_context_pop_thread_default (mainctx); + + return NULL; +} + +static void +_ostree_fetcher_set_property (GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + self->config_flags = g_value_get_flags (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_get_property (GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + g_value_set_flags (value, self->config_flags); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_finalize (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + /* Terminate the session thread. */ + g_atomic_int_set (&self->thread_closure->running, 0); + g_main_context_wakeup (self->thread_closure->main_context); + if (self->session_thread) + { + /* We need to explicitly synchronize to clean up TLS */ + if (self->session_thread != g_thread_self ()) + g_thread_join (self->session_thread); + else + g_clear_pointer (&self->session_thread, g_thread_unref); + } + g_clear_pointer (&self->thread_closure, thread_closure_unref); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); +} + +static void +_ostree_fetcher_constructed (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + g_autoptr(GMainContext) main_context = NULL; + const char *http_proxy; + + main_context = g_main_context_new (); + + self->thread_closure = g_slice_new0 (ThreadClosure); + self->thread_closure->ref_count = 1; + self->thread_closure->main_context = g_main_context_ref (main_context); + self->thread_closure->running = 1; + self->thread_closure->transfer_gzip = (self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) != 0; + + self->thread_closure->outstanding = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)pending_uri_unref); + self->thread_closure->output_stream_set = g_hash_table_new_full (NULL, NULL, + (GDestroyNotify) NULL, + (GDestroyNotify) g_object_unref); + g_mutex_init (&self->thread_closure->output_stream_set_lock); + + if (g_getenv ("OSTREE_DEBUG_HTTP")) + { + session_thread_idle_add (self->thread_closure, + session_thread_add_logger, + NULL, (GDestroyNotify) NULL); + } + + http_proxy = g_getenv ("http_proxy"); + if (http_proxy != NULL && http_proxy[0] != '\0') + _ostree_fetcher_set_proxy (self, http_proxy); + + /* FIXME Maybe implement GInitableIface and use g_thread_try_new() + * so we can try to handle thread creation errors gracefully? */ + self->session_thread = g_thread_new ("fetcher-session-thread", + ostree_fetcher_session_thread, + thread_closure_ref (self->thread_closure)); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); +} + +static void +_ostree_fetcher_class_init (OstreeFetcherClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = _ostree_fetcher_set_property; + gobject_class->get_property = _ostree_fetcher_get_property; + gobject_class->finalize = _ostree_fetcher_finalize; + gobject_class->constructed = _ostree_fetcher_constructed; + + g_object_class_install_property (gobject_class, + PROP_CONFIG_FLAGS, + g_param_spec_flags ("config-flags", + "", + "", + OSTREE_TYPE_FETCHER_CONFIG_FLAGS, + OSTREE_FETCHER_FLAGS_NONE, + G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS)); +} + +static void +_ostree_fetcher_init (OstreeFetcher *self) +{ +} + +OstreeFetcher * +_ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, + OstreeFetcherConfigFlags flags) +{ + OstreeFetcher *self; + + self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + self->thread_closure->remote_name = g_strdup (remote_name); + self->thread_closure->base_tmpdir_dfd = tmpdir_dfd; + + return self; +} + +int +_ostree_fetcher_get_dfd (OstreeFetcher *fetcher) +{ + return fetcher->thread_closure->base_tmpdir_dfd; +} + +void +_ostree_fetcher_set_proxy (OstreeFetcher *self, + const char *http_proxy) +{ + GProxyResolver *resolver; + GUri *guri; + + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (http_proxy != NULL && http_proxy[0] != '\0'); + + /* validate first */ + guri = g_uri_parse (http_proxy, SOUP_HTTP_URI_FLAGS, NULL); + + if (!guri) + { + g_warning ("Invalid proxy URI '%s'", http_proxy); + return; + } + + g_uri_unref (guri); + + resolver = g_simple_proxy_resolver_new (http_proxy, NULL); + + session_thread_idle_add (self->thread_closure, + session_thread_set_proxy_cb, + resolver, /* takes ownership */ + (GDestroyNotify) g_object_unref); + } + +void +_ostree_fetcher_set_cookie_jar (OstreeFetcher *self, + const char *jar_path) +{ + SoupCookieJar *jar; + + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (jar_path != NULL); + + jar = soup_cookie_jar_text_new (jar_path, TRUE); + + session_thread_idle_add (self->thread_closure, + session_thread_set_cookie_jar_cb, + jar, /* takes ownership */ + (GDestroyNotify) g_object_unref); +} + +void +_ostree_fetcher_set_client_cert (OstreeFetcher *self, + const char *cert_path, + const char *key_path) +{ + g_autoptr(GString) buf = NULL; + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + if (cert_path) + { + buf = g_string_new (cert_path); + g_string_append_c (buf, '\0'); + g_string_append (buf, key_path); + } + + session_thread_idle_add (self->thread_closure, + session_thread_set_tls_interaction_cb, + g_string_free (g_steal_pointer (&buf), FALSE), + (GDestroyNotify) g_free); +} + +void +_ostree_fetcher_set_tls_database (OstreeFetcher *self, + const char *tlsdb_path) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + session_thread_idle_add (self->thread_closure, + session_thread_set_tls_database_cb, + g_strdup (tlsdb_path), + (GDestroyNotify) g_free); +} + +void +_ostree_fetcher_set_extra_headers (OstreeFetcher *self, + GVariant *extra_headers) +{ + session_thread_idle_add (self->thread_closure, + session_thread_set_headers_cb, + g_variant_ref (extra_headers), + (GDestroyNotify) g_variant_unref); +} + +void +_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, + const char *extra_user_agent) +{ + session_thread_idle_add (self->thread_closure, + session_thread_set_extra_user_agent_cb, + g_strdup (extra_user_agent), + (GDestroyNotify) g_free); +} + +static gboolean +finish_stream (OstreeFetcherPendingURI *pending, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + struct stat stbuf; + + /* Close it here since we do an async fstat(), where we don't want + * to hit a bad fd. + */ + if (pending->out_stream) + { + if ((pending->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + { + const guint8 nulchar = 0; + gsize bytes_written; + + if (!g_output_stream_write_all (pending->out_stream, &nulchar, 1, &bytes_written, + cancellable, error)) + goto out; + } + + if (!g_output_stream_close (pending->out_stream, cancellable, error)) + goto out; + + g_mutex_lock (&pending->thread_closure->output_stream_set_lock); + g_hash_table_remove (pending->thread_closure->output_stream_set, + pending->out_stream); + g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + } + + if (!pending->is_membuf) + { + if (!glnx_fstat (pending->tmpf.fd, &stbuf, error)) + goto out; + } + + pending->state = OSTREE_FETCHER_STATE_COMPLETE; + + if (!pending->is_membuf) + { + if (stbuf.st_size < pending->content_length) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete"); + goto out; + } + else + { + g_mutex_lock (&pending->thread_closure->output_stream_set_lock); + pending->thread_closure->total_downloaded += stbuf.st_size; + g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + } + } + + ret = TRUE; + out: + (void) g_input_stream_close (pending->request_body, NULL, NULL); + return ret; +} + +static void +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data); + +static void +remove_pending (OstreeFetcherPendingURI *pending) +{ + /* Hold a temporary ref to ensure the reference to + * pending->thread_closure is valid. + */ + pending_uri_ref (pending); + g_hash_table_remove (pending->thread_closure->outstanding, pending); + pending_uri_unref (pending); +} + +static void +on_out_splice_complete (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + GTask *task = G_TASK (user_data); + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; + gssize bytes_written; + GError *local_error = NULL; + + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); + + bytes_written = g_output_stream_splice_finish ((GOutputStream *)object, + result, + &local_error); + if (bytes_written < 0) + goto out; + + g_input_stream_read_bytes_async (pending->request_body, + 8192, G_PRIORITY_DEFAULT, + cancellable, + on_stream_read, + g_object_ref (task)); + + out: + if (local_error) + { + g_task_return_error (task, local_error); + remove_pending (pending); + } + + g_object_unref (task); +} + +static void +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + GTask *task = G_TASK (user_data); + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; + g_autoptr(GBytes) bytes = NULL; + gsize bytes_read; + GError *local_error = NULL; + + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); + + /* Only open the output stream on demand to ensure we use as + * few file descriptors as possible. + */ + if (!pending->out_stream) + { + if (!pending->is_membuf) + { + if (!_ostree_fetcher_tmpf_from_flags (pending->flags, pending->thread_closure->base_tmpdir_dfd, + &pending->tmpf, &local_error)) + goto out; + pending->out_stream = g_unix_output_stream_new (pending->tmpf.fd, FALSE); + } + else + { + pending->out_stream = g_memory_output_stream_new_resizable (); + } + + g_mutex_lock (&pending->thread_closure->output_stream_set_lock); + g_hash_table_add (pending->thread_closure->output_stream_set, + g_object_ref (pending->out_stream)); + g_mutex_unlock (&pending->thread_closure->output_stream_set_lock); + } + + /* Get a GBytes buffer */ + bytes = g_input_stream_read_bytes_finish ((GInputStream*)object, result, &local_error); + if (!bytes) + goto out; + bytes_read = g_bytes_get_size (bytes); + + /* Was this the end of the stream? */ + if (bytes_read == 0) + { + if (!finish_stream (pending, cancellable, &local_error)) + goto out; + if (pending->is_membuf) + { + g_task_return_pointer (task, + g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream*)pending->out_stream), + (GDestroyNotify) g_bytes_unref); + } + else + { + if (lseek (pending->tmpf.fd, 0, SEEK_SET) < 0) + { + glnx_set_error_from_errno (&local_error); + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else + g_task_return_boolean (task, TRUE); + } + remove_pending (pending); + } + else + { + /* Verify max size */ + if (pending->max_size > 0) + { + if (bytes_read > pending->max_size || + (bytes_read + pending->current_size) > pending->max_size) + { + g_autofree char *uristr = NULL; + + if (pending->file) + uristr = g_file_get_uri (pending->file); + else + uristr = g_uri_to_string (soup_message_get_uri (pending->message)); + + local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, + "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", + uristr, pending->max_size); + goto out; + } + } + + pending->current_size += bytes_read; + + /* We do this instead of _write_bytes_async() as that's not + * guaranteed to do a complete write. + */ + { + g_autoptr(GInputStream) membuf = + g_memory_input_stream_new_from_bytes (bytes); + g_output_stream_splice_async (pending->out_stream, membuf, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, + G_PRIORITY_DEFAULT, + cancellable, + on_out_splice_complete, + g_object_ref (task)); + } + } + + out: + if (local_error) + { + g_task_return_error (task, local_error); + remove_pending (pending); + } + + g_object_unref (task); +} + +static void +on_request_sent (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + GTask *task = G_TASK (user_data); + /* Hold a ref to the pending across this function, since we remove + * it from the hash early in some cases, not in others. */ + OstreeFetcherPendingURI *pending = pending_uri_ref (g_task_get_task_data (task)); + GCancellable *cancellable = g_task_get_cancellable (task); + GError *local_error = NULL; + glnx_unref_object SoupMessage *msg = NULL; + SoupStatus status; + + pending->state = OSTREE_FETCHER_STATE_COMPLETE; + if (pending->file) + pending->request_body = (GInputStream *)g_file_read_finish ((GFile *) object, + result, + &local_error); + else + pending->request_body = soup_session_send_finish ((SoupSession*) object, + result, &local_error); + + if (!pending->request_body) + goto out; + + if (pending->message) + { + status = soup_message_get_status (pending->message); + if (status == SOUP_STATUS_NOT_MODIFIED && + (pending->if_none_match != NULL || pending->if_modified_since > 0)) + { + /* Version on the server is unchanged from the version we have cached locally; + * report this as an out-argument, a zero-length response buffer, and no error */ + pending->out_not_modified = TRUE; + } + else if (!SOUP_STATUS_IS_SUCCESSFUL (status)) + { + /* is there another mirror we can try? */ + if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len) + { + pending->mirrorlist_idx++; + create_pending_soup_request (pending); + + (void) g_input_stream_close (pending->request_body, NULL, NULL); + + start_pending_request (pending->thread_closure, task); + } + else + { + g_autofree char *uristring = g_uri_to_string (soup_message_get_uri (pending->message)); + GIOErrorEnum code = _ostree_fetcher_http_status_code_to_io_error (status); + { + g_autofree char *errmsg = + g_strdup_printf ("Server returned status %u: %s", + status, + soup_status_get_phrase (status)); + + /* Let's make OOB errors be the final one since they're probably + * the cause for the error here. */ + if (pending->thread_closure->oob_error) + { + local_error = + g_error_copy (pending->thread_closure->oob_error); + g_prefix_error (&local_error, "%s: ", errmsg); + } + else + local_error = g_error_new_literal (G_IO_ERROR, code, errmsg); + } + + if (pending->mirrorlist->len > 1) + g_prefix_error (&local_error, + "All %u mirrors failed. Last error was: ", + pending->mirrorlist->len); + if (pending->thread_closure->remote_name && + !((pending->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && + code == G_IO_ERROR_NOT_FOUND)) + _ostree_fetcher_journal_failure (pending->thread_closure->remote_name, + uristring, local_error->message); + + } + goto out; + } + + /* Grab cache properties from the response */ + pending->out_etag = g_strdup (soup_message_headers_get_one (soup_message_get_response_headers (pending->message), "ETag")); + pending->out_last_modified = 0; + + const char *last_modified_str = soup_message_headers_get_one (soup_message_get_response_headers (pending->message), "Last-Modified"); + if (last_modified_str != NULL) + { + GDateTime *soup_date = soup_date_time_new_from_http_string (last_modified_str); + if (soup_date != NULL) + { + pending->out_last_modified = g_date_time_to_unix (soup_date); + g_date_time_unref (soup_date); + } + } + } + + pending->state = OSTREE_FETCHER_STATE_DOWNLOADING; + + if (pending->file) + { + GFileInfo *info = g_file_query_info (pending->file, + G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE "," + G_FILE_ATTRIBUTE_STANDARD_SIZE, + 0, NULL, NULL); + if (info) + { + pending->content_length = g_file_info_get_size (info); + g_object_unref (info); + } + else + pending->content_length = -1; + } + else + pending->content_length = soup_message_headers_get_content_length (soup_message_get_response_headers (pending->message)); + + g_input_stream_read_bytes_async (pending->request_body, + 8192, G_PRIORITY_DEFAULT, + cancellable, + on_stream_read, + g_object_ref (task)); + + out: + if (local_error) + { + if (pending->request_body) + (void) g_input_stream_close (pending->request_body, NULL, NULL); + g_task_return_error (task, local_error); + remove_pending (pending); + } + + pending_uri_unref (pending); + g_object_unref (task); +} + +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + OstreeFetcherPendingURI *pending; + + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (mirrorlist != NULL); + g_return_if_fail (mirrorlist->len > 0); + + /* SoupRequest is created in session thread. */ + pending = g_new0 (OstreeFetcherPendingURI, 1); + pending->ref_count = 1; + pending->thread_closure = thread_closure_ref (self->thread_closure); + pending->mirrorlist = g_ptr_array_ref (mirrorlist); + pending->filename = g_strdup (filename); + pending->flags = flags; + pending->if_none_match = g_strdup (if_none_match); + pending->if_modified_since = if_modified_since; + pending->max_size = max_size; + pending->is_membuf = is_membuf; + pending->fetcher = self; + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, _ostree_fetcher_request_async); + g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_unref); + + /* We'll use the GTask priority for our own priority queue. */ + g_task_set_priority (task, priority); + + session_thread_idle_add (self->thread_closure, + session_thread_request_uri, + g_object_ref (task), + (GDestroyNotify) g_object_unref); +} + +void +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, FALSE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) +{ + GTask *task; + OstreeFetcherPendingURI *pending; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + pending = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (!pending->is_membuf); + *out_tmpf = pending->tmpf; + pending->tmpf.initialized = FALSE; /* Transfer ownership */ + + if (out_not_modified != NULL) + *out_not_modified = pending->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&pending->out_etag); + if (out_last_modified != NULL) + *out_last_modified = pending->out_last_modified; + + return TRUE; +} + +void +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, TRUE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) +{ + GTask *task; + OstreeFetcherPendingURI *pending; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + pending = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (pending->is_membuf); + g_assert (out_buf); + *out_buf = ret; + + if (out_not_modified != NULL) + *out_not_modified = pending->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&pending->out_etag); + if (out_last_modified != NULL) + *out_last_modified = pending->out_last_modified; + + return TRUE; +} + + +guint64 +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +{ + g_return_val_if_fail (OSTREE_IS_FETCHER (self), 0); + + g_mutex_lock (&self->thread_closure->output_stream_set_lock); + + guint64 ret = self->thread_closure->total_downloaded; + + GLNX_HASH_TABLE_FOREACH (self->thread_closure->output_stream_set, + GFileOutputStream*, stream) + { + if (G_IS_FILE_DESCRIPTOR_BASED (stream)) + { + int fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)stream); + struct stat stbuf; + + if (glnx_fstat (fd, &stbuf, NULL)) + ret += stbuf.st_size; + } + } + + g_mutex_unlock (&self->thread_closure->output_stream_set_lock); + + return ret; +} diff --git a/src/ostree/main.c b/src/ostree/main.c index 7d17080c..badfa6df 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -118,7 +118,7 @@ static OstreeCommand commands[] = { { "summary", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_summary, "Manage summary metadata" }, -#if defined(HAVE_LIBSOUP) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE) +#if defined(HAVE_LIBSOUP_OR_LIBSOUP3) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE) { "trivial-httpd", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_trivial_httpd, NULL }, diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 6b16737d..855d2cea 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -34,6 +34,19 @@ #include #include +#if ! SOUP_CHECK_VERSION (3, 0, 0) +# define SoupServerMessage SoupMessage +# define soup_server_message_get_method(msg) ((msg)->method) +# define soup_server_message_get_request_headers(msg) ((msg)->request_headers) +# define soup_server_message_get_response_headers(msg) ((msg)->response_headers) +# define soup_server_message_get_response_body(msg) ((msg)->response_body) +# define soup_server_message_set_status(msg, status) soup_message_set_status(msg, status) +# define soup_server_message_set_redirect(msg, status, uri) soup_message_set_redirect(msg, status, uri) +# define soup_server_message_set_response(msg, ct, ru, rb, rl) soup_message_set_response(msg, ct, ru, rb, rl) +#else +# define soup_server_message_set_status(msg, status) soup_server_message_set_status(msg, status, NULL) +#endif + static char *opt_port_file = NULL; static char *opt_log = NULL; static gboolean opt_daemonize; @@ -188,15 +201,12 @@ is_safe_to_access (struct stat *stbuf) } static void -close_socket (SoupMessage *msg, gpointer user_data) +close_socket (SoupServerMessage *msg, gpointer user_data) { - SoupSocket *sock = user_data; + GSocket *sock = user_data; int sockfd; - /* Actually calling soup_socket_disconnect() here would cause - * us to leak memory, so just shutdown the socket instead. - */ - sockfd = soup_socket_get_fd (sock); + sockfd = g_socket_get_fd (sock); #ifdef G_OS_WIN32 shutdown (sockfd, SD_SEND); #else @@ -213,12 +223,55 @@ calculate_etag (GMappedFile *mapping) return g_strconcat ("\"", checksum, "\"", NULL); } +static GSList * +_server_cookies_from_request (SoupServerMessage *msg) +{ + SoupCookie *cookie; + GSList *cookies = NULL; + GHashTable *params; + GHashTableIter iter; + gpointer name, value; + const char *header; + const char *host; + + header = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), + "Cookie"); + if (!header) + return NULL; + +#if ! SOUP_CHECK_VERSION (3, 0, 0) + host = soup_uri_get_host (soup_message_get_uri (msg)); +#else + host = g_uri_get_host (soup_server_message_get_uri (msg)); +#endif + params = soup_header_parse_semi_param_list (header); + g_hash_table_iter_init (&iter, params); + + while (g_hash_table_iter_next (&iter, &name, &value)) + { + if (!name || !value) continue; + cookie = soup_cookie_new (name, value, host, NULL, 0); + cookies = g_slist_prepend (cookies, cookie); + } + + soup_header_free_param_list (params); + + return g_slist_reverse (cookies); +} + static void +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (OtTrivialHttpd *self, SoupServer *server, - SoupMessage *msg, + SoupServerMessage *msg, const char *path, SoupClientContext *context) +#else +do_get (OtTrivialHttpd *self, + SoupServer *server, + SoupServerMessage *msg, + const char *path) +#endif { char *slash; int ret; @@ -228,7 +281,7 @@ do_get (OtTrivialHttpd *self, if (opt_expected_cookies) { - GSList *cookies = soup_cookies_from_request (msg); + GSList *cookies = _server_cookies_from_request (msg); GSList *l; int i; @@ -253,12 +306,12 @@ do_get (OtTrivialHttpd *self, if (!found) { httpd_log (self, "Expected cookie not found %s\n", k); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); - soup_cookies_free (cookies); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + g_slist_free_full (cookies, (GDestroyNotify)soup_cookie_free); goto out; } } - soup_cookies_free (cookies); + g_slist_free_full (cookies, (GDestroyNotify)soup_cookie_free); } if (opt_expected_headers) @@ -273,18 +326,18 @@ do_get (OtTrivialHttpd *self, { g_autofree char *k = g_strndup (kv, eq - kv); const gchar *expected_v = eq + 1; - const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k); + const gchar *found_v = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), k); if (!found_v) { httpd_log (self, "Expected header not found %s\n", k); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } if (strcmp (found_v, expected_v) != 0) { httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } } @@ -293,7 +346,7 @@ do_get (OtTrivialHttpd *self, if (strstr (path, "../") != NULL) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } @@ -302,7 +355,7 @@ do_get (OtTrivialHttpd *self, g_random_int_range (0, 100) < opt_random_500s_percentage) { emitted_random_500s_count++; - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } else if (opt_random_408s_percentage > 0 && @@ -310,7 +363,7 @@ do_get (OtTrivialHttpd *self, g_random_int_range (0, 100) < opt_random_408s_percentage) { emitted_random_408s_count++; - soup_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); + soup_server_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); goto out; } @@ -323,17 +376,17 @@ do_get (OtTrivialHttpd *self, if (ret == -1) { if (errno == EPERM) - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); else if (errno == ENOENT) - soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_FOUND); else - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } if (!is_safe_to_access (&stbuf)) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } @@ -344,9 +397,13 @@ do_get (OtTrivialHttpd *self, { g_autofree char *redir_uri = NULL; - redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path); - soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY, - redir_uri); +#if ! SOUP_CHECK_VERSION (3, 0, 0) + redir_uri = g_strdup_printf ("%s/", soup_uri_get_path (soup_message_get_uri (msg))); +#else + redir_uri = g_strdup_printf ("%s/", g_uri_get_path (soup_server_message_get_uri (msg))); +#endif + soup_server_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY, + redir_uri); } else { @@ -354,15 +411,19 @@ do_get (OtTrivialHttpd *self, if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1) { g_autofree char *index_path = g_strconcat (path, "/index.html", NULL); +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (self, server, msg, index_path, context); +#else + do_get (self, server, msg, index_path); +#endif } else { GString *listing = get_directory_listing (self->root_dfd, path); - soup_message_set_response (msg, "text/html", - SOUP_MEMORY_TAKE, - listing->str, listing->len); - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_response (msg, "text/html", + SOUP_MEMORY_TAKE, + listing->str, listing->len); + soup_server_message_set_status (msg, SOUP_STATUS_OK); g_string_free (listing, FALSE); } } @@ -371,21 +432,21 @@ do_get (OtTrivialHttpd *self, { if (!S_ISREG (stbuf.st_mode)) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } glnx_autofd int fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC); if (fd < 0) { - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } g_autoptr(GMappedFile) mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL); if (!mapping) { - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } (void) close (fd); fd = -1; @@ -395,14 +456,14 @@ do_get (OtTrivialHttpd *self, if (last_modified != NULL) { g_autofree gchar *formatted = g_date_time_format (last_modified, "%a, %d %b %Y %H:%M:%S GMT"); - soup_message_headers_append (msg->response_headers, "Last-Modified", formatted); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Last-Modified", formatted); } g_autofree gchar *etag = calculate_etag (mapping); if (etag != NULL) - soup_message_headers_append (msg->response_headers, "ETag", etag); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "ETag", etag); - if (msg->method == SOUP_METHOD_GET) + if (!strcmp (soup_server_message_get_method (msg), "GET")) { gsize buffer_length, file_size; SoupRange *ranges; @@ -410,13 +471,13 @@ do_get (OtTrivialHttpd *self, gboolean have_ranges; file_size = g_mapped_file_get_length (mapping); - have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length); + have_ranges = soup_message_headers_get_ranges(soup_server_message_get_request_headers (msg), file_size, &ranges, &ranges_length); if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL) { - SoupSocket *sock; + GSocket *sock; buffer_length = file_size/2; - soup_message_headers_set_content_length (msg->response_headers, file_size); - soup_message_headers_append (msg->response_headers, + soup_message_headers_set_content_length (soup_server_message_get_response_headers (msg), file_size); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Connection", "close"); /* soup-message-io will wait for us to add @@ -424,7 +485,11 @@ do_get (OtTrivialHttpd *self, * the declared Content-Length. Instead, we * forcibly close the socket at that point. */ - sock = soup_client_context_get_socket (context); +#if ! SOUP_CHECK_VERSION (3, 0, 0) + sock = soup_client_context_get_gsocket (context); +#else + sock = soup_server_message_get_socket (msg); +#endif g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock); } else @@ -434,12 +499,13 @@ do_get (OtTrivialHttpd *self, { if (ranges_length > 0 && ranges[0].start >= file_size) { - soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); - soup_message_headers_free_ranges (msg->request_headers, ranges); + soup_server_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + soup_message_headers_free_ranges (soup_server_message_get_request_headers (msg), ranges); goto out; } - soup_message_headers_free_ranges (msg->request_headers, ranges); + soup_message_headers_free_ranges (soup_server_message_get_request_headers (msg), ranges); } +#if ! SOUP_CHECK_VERSION (3, 0, 0) if (buffer_length > 0) { SoupBuffer *buffer; @@ -451,8 +517,22 @@ do_get (OtTrivialHttpd *self, soup_message_body_append_buffer (msg->response_body, buffer); soup_buffer_free (buffer); } +#else + if (buffer_length > 0 && buffer_length == file_size) + { + GBytes *bytes = g_mapped_file_get_bytes (mapping); + soup_message_body_append_bytes (soup_server_message_get_response_body (msg), bytes); + g_bytes_unref (bytes); + } + else if (buffer_length > 0) + { + gchar *contents = g_mapped_file_get_contents (mapping); + soup_message_body_append (soup_server_message_get_response_body (msg), + SOUP_MEMORY_COPY, contents, buffer_length); + } +#endif } - else /* msg->method == SOUP_METHOD_HEAD */ + else /* method == HEAD */ { g_autofree char *length = NULL; @@ -461,56 +541,59 @@ do_get (OtTrivialHttpd *self, * But we'll optimize and avoid the extra I/O. */ length = g_strdup_printf ("%lu", (gulong)stbuf.st_size); - soup_message_headers_append (msg->response_headers, + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Content-Length", length); } /* Check client’s caching headers. */ - const gchar *if_modified_since = soup_message_headers_get_one (msg->request_headers, + const gchar *if_modified_since = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), "If-Modified-Since"); - const gchar *if_none_match = soup_message_headers_get_one (msg->request_headers, + const gchar *if_none_match = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), "If-None-Match"); if (if_none_match != NULL && etag != NULL) { if (g_strcmp0 (etag, if_none_match) == 0) { - soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); - soup_message_body_truncate (msg->response_body); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); + soup_message_body_truncate (soup_server_message_get_response_body (msg)); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } } else if (if_modified_since != NULL && last_modified != NULL) { - SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since); g_autoptr(GDateTime) if_modified_since_dt = NULL; +#if ! SOUP_CHECK_VERSION (3, 0, 0) + SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since); if (if_modified_since_sd != NULL) if_modified_since_dt = g_date_time_new_from_unix_utc (soup_date_to_time_t (if_modified_since_sd)); +#else + if_modified_since_dt = soup_date_time_new_from_http_string (if_modified_since); +#endif if (if_modified_since_dt != NULL && g_date_time_compare (last_modified, if_modified_since_dt) <= 0) { - soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); - soup_message_body_truncate (msg->response_body); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); + soup_message_body_truncate (soup_server_message_get_response_body (msg)); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } - - g_clear_pointer (&if_modified_since_sd, soup_date_free); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } } out: { +#if ! SOUP_CHECK_VERSION (3, 0, 0) guint status = 0; g_autofree gchar *reason = NULL; @@ -518,26 +601,41 @@ do_get (OtTrivialHttpd *self, "status-code", &status, "reason-phrase", &reason, NULL); +#else + guint status = soup_server_message_get_status (msg); + const char *reason = soup_server_message_get_reason_phrase (msg); +#endif + httpd_log (self, " status: %s (%u)\n", reason, status); } return; } static void -httpd_callback (SoupServer *server, SoupMessage *msg, +#if ! SOUP_CHECK_VERSION (3, 0, 0) +httpd_callback (SoupServer *server, SoupServerMessage *msg, const char *path, GHashTable *query, SoupClientContext *context, gpointer data) +#else +httpd_callback (SoupServer *server, SoupServerMessage *msg, + const char *path, GHashTable *query, gpointer data) +#endif { OtTrivialHttpd *self = data; + const char *meth = soup_server_message_get_method (msg); - if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD) + if (!strcmp (meth, "GET") || !strcmp(meth, "HEAD")) +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (self, server, msg, path, context); +#else + do_get (self, server, msg, path); +#endif else - soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); } static gboolean -basic_auth_callback (SoupAuthDomain *auth_domain, SoupMessage *msg, +basic_auth_callback (SoupAuthDomain *auth_domain, SoupServerMessage *msg, const char *username, const char *password, gpointer data) { return g_str_equal (username, "foouser") && g_str_equal (password, "barpw"); @@ -703,7 +801,7 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) } #if SOUP_CHECK_VERSION(2, 48, 0) - server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); + server = soup_server_new ("server-header", "ostree-httpd ", NULL); if (!soup_server_listen_all (server, opt_port, 0, error)) goto out; #else @@ -711,13 +809,21 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); #endif + if (opt_require_basic_auth) { +#if ! SOUP_CHECK_VERSION (3, 0, 0) glnx_unref_object SoupAuthDomain *auth_domain = soup_auth_domain_basic_new (SOUP_AUTH_DOMAIN_REALM, "auth-test", SOUP_AUTH_DOMAIN_ADD_PATH, "/", SOUP_AUTH_DOMAIN_BASIC_AUTH_CALLBACK, basic_auth_callback, NULL); +#else + glnx_unref_object SoupAuthDomain *auth_domain = + soup_auth_domain_basic_new ("realm", "auth-test", NULL); + soup_auth_domain_add_path (auth_domain, "/"); + soup_auth_domain_basic_set_auth_callback (auth_domain, basic_auth_callback, NULL, NULL); +#endif soup_server_add_auth_domain (server, auth_domain); } -- cgit v1.2.1