diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/libostree/ostree-fetcher-soup3.c | 912 | ||||
-rw-r--r-- | src/ostree/main.c | 2 | ||||
-rw-r--r-- | src/ostree/ostree-trivial-httpd.c | 228 |
3 files changed, 1080 insertions, 62 deletions
diff --git a/src/libostree/ostree-fetcher-soup3.c b/src/libostree/ostree-fetcher-soup3.c new file mode 100644 index 00000000..b46ce34f --- /dev/null +++ b/src/libostree/ostree-fetcher-soup3.c @@ -0,0 +1,912 @@ +/* + * Copyright (C) 2011 Colin Walters <walters@verbum.org> + * Copyright (C) 2022 Igalia S.L. + * Copyright (C) 2023 Endless OS Foundation, LLC + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see <https://www.gnu.org/licenses/>. + * + * Author: Colin Walters <walters@verbum.org> + * Author: Daniel Kolesa <dkolesa@igalia.com> + * Author: Dan Nicholson <dbn@endlessos.org> + */ + +#include "config.h" + +#include <gio/gio.h> +#include <gio/gunixoutputstream.h> +#include <libsoup/soup.h> + +#include "libglnx.h" +#include "ostree-fetcher.h" +#include "ostree-fetcher-util.h" +#include "ostree-tls-cert-interaction-private.h" +#include "ostree-enumtypes.h" +#include "ostree-repo-private.h" + +typedef struct { + GPtrArray *mirrorlist; /* list of base URIs */ + char *filename; /* relative name to fetch or NULL */ + guint mirrorlist_idx; + + SoupMessage *message; + struct OstreeFetcher *fetcher; + GMainContext *mainctx; + SoupSession *session; + GFile *file; + + gboolean is_membuf; + OstreeFetcherRequestFlags flags; + char *if_none_match; /* request ETag */ + guint64 if_modified_since; /* seconds since the epoch */ + GInputStream *response_body; + GLnxTmpfile tmpf; + GOutputStream *out_stream; + gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */ + char *out_etag; /* response ETag */ + guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */ + + guint64 max_size; + guint64 current_size; + goffset content_length; +} FetcherRequest; + +struct OstreeFetcher +{ + GObject parent_instance; + + OstreeFetcherConfigFlags config_flags; + char *remote_name; + int tmpdir_dfd; + + GHashTable *sessions; /* (element-type GMainContext SoupSession ) */ + GProxyResolver *proxy_resolver; + SoupCookieJar *cookie_jar; + OstreeTlsCertInteraction *client_cert; + GTlsDatabase *tls_database; + GVariant *extra_headers; + char *user_agent; + + guint64 bytes_transferred; +}; + +enum { + PROP_0, + PROP_CONFIG_FLAGS +}; + +G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) + +static void +fetcher_request_free (FetcherRequest *request) +{ + g_debug ("Freeing request for %s", request->filename); + g_clear_pointer (&request->mirrorlist, g_ptr_array_unref); + g_clear_pointer (&request->filename, g_free); + g_clear_object (&request->message); + g_clear_pointer (&request->mainctx, g_main_context_unref); + g_clear_object (&request->session); + g_clear_object (&request->file); + g_clear_pointer (&request->if_none_match, g_free); + g_clear_object (&request->response_body); + glnx_tmpfile_clear (&request->tmpf); + g_clear_object (&request->out_stream); + g_clear_pointer (&request->out_etag, g_free); + g_free (request); +} + +static void +on_request_sent (GObject *object, + GAsyncResult *result, + gpointer user_data); + +static gboolean +_message_accept_cert_loose (SoupMessage *msg, + GTlsCertificate *tls_peer_certificate, + GTlsCertificateFlags tls_peer_errors, + gpointer data) +{ + return TRUE; +} + +static void +create_request_message (FetcherRequest *request) +{ + g_assert (request->mirrorlist); + g_assert (request->mirrorlist_idx < request->mirrorlist->len); + + OstreeFetcherURI *next_mirror = g_ptr_array_index (request->mirrorlist, request->mirrorlist_idx); + g_autoptr(OstreeFetcherURI) uri = NULL; + if (request->filename) + uri = _ostree_fetcher_uri_new_subpath (next_mirror, request->filename); + if (!uri) + uri = (OstreeFetcherURI *) g_uri_ref ((GUri *) next_mirror); + + g_clear_object (&request->message); + g_clear_object (&request->file); + g_clear_object (&request->response_body); + + GUri *guri = (GUri *) uri; + + /* file:// URI is handle via GFile */ + if (!strcmp (g_uri_get_scheme (guri), "file")) + { + g_autofree char *str = g_uri_to_string (guri); + request->file = g_file_new_for_uri (str); + return; + } + + request->message = soup_message_new_from_uri ("GET", guri); + + if (request->if_none_match != NULL) + soup_message_headers_append (soup_message_get_request_headers (request->message), + "If-None-Match", request->if_none_match); + + if (request->if_modified_since > 0) + { + g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (request->if_modified_since); + g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z"); + soup_message_headers_append (soup_message_get_request_headers (request->message), + "If-Modified-Since", mod_date); + } + + if ((request->fetcher->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) != 0) + g_signal_connect (request->message, "accept-certificate", + G_CALLBACK (_message_accept_cert_loose), NULL); + + if (request->fetcher->extra_headers) + { + g_autoptr(GVariantIter) viter = g_variant_iter_new (request->fetcher->extra_headers); + const char *key; + const char *value; + + while (g_variant_iter_next (viter, "(&s&s)", &key, &value)) + soup_message_headers_append (soup_message_get_request_headers (request->message), key, value); + } +} + +static void +initiate_task_request (GTask *task) +{ + FetcherRequest *request = g_task_get_task_data (task); + create_request_message (request); + + GCancellable *cancellable = g_task_get_cancellable (task); + int priority = g_task_get_priority (task); + if (request->file) + g_file_read_async (request->file, priority, cancellable, on_request_sent, task); + else + { + g_autofree char *uri = g_uri_to_string (soup_message_get_uri (request->message)); + const char *dest = request->is_membuf ? "memory" : "tmpfile"; + g_debug ("Requesting %s to %s for session %p in main context %p", + uri, dest, request->session, request->mainctx); + soup_session_send_async (request->session, request->message, + priority, cancellable, on_request_sent, task); + } +} + +static void +_ostree_fetcher_set_property (GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + self->config_flags = g_value_get_flags (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_get_property (GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + g_value_set_flags (value, self->config_flags); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_finalize (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + g_clear_pointer (&self->remote_name, g_free); + g_clear_pointer (&self->sessions, g_hash_table_unref); + g_clear_object (&self->proxy_resolver); + g_clear_object (&self->cookie_jar); + g_clear_object (&self->client_cert); + g_clear_object (&self->tls_database); + g_clear_pointer (&self->extra_headers, g_variant_unref); + g_clear_pointer (&self->user_agent, g_free); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); +} + +static void +_ostree_fetcher_constructed (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + const char *http_proxy = g_getenv ("http_proxy"); + if (http_proxy != NULL && http_proxy[0] != '\0') + _ostree_fetcher_set_proxy (self, http_proxy); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); +} + +static void +_ostree_fetcher_class_init (OstreeFetcherClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = _ostree_fetcher_set_property; + gobject_class->get_property = _ostree_fetcher_get_property; + gobject_class->finalize = _ostree_fetcher_finalize; + gobject_class->constructed = _ostree_fetcher_constructed; + + g_object_class_install_property (gobject_class, + PROP_CONFIG_FLAGS, + g_param_spec_flags ("config-flags", + "", + "", + OSTREE_TYPE_FETCHER_CONFIG_FLAGS, + OSTREE_FETCHER_FLAGS_NONE, + G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS)); +} + +static void +_ostree_fetcher_init (OstreeFetcher *self) +{ + self->sessions = g_hash_table_new (g_direct_hash, g_direct_equal); +} + +OstreeFetcher * +_ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, + OstreeFetcherConfigFlags flags) +{ + OstreeFetcher *self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + self->remote_name = g_strdup (remote_name); + self->tmpdir_dfd = tmpdir_dfd; + + return self; +} + +int +_ostree_fetcher_get_dfd (OstreeFetcher *self) +{ + return self->tmpdir_dfd; +} + +void +_ostree_fetcher_set_proxy (OstreeFetcher *self, + const char *http_proxy) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (http_proxy != NULL && http_proxy[0] != '\0'); + + /* validate first */ + g_autoptr(GError) local_error = NULL; + g_autoptr(GUri) guri = g_uri_parse (http_proxy, G_URI_FLAGS_NONE, &local_error); + if (guri == NULL) + { + g_warning ("Invalid proxy URI '%s': %s", http_proxy, local_error->message); + return; + } + + g_clear_object (&self->proxy_resolver); + self->proxy_resolver = g_simple_proxy_resolver_new (http_proxy, NULL); +} + +void +_ostree_fetcher_set_cookie_jar (OstreeFetcher *self, + const char *jar_path) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (jar_path != NULL); + + g_clear_object (&self->cookie_jar); + self->cookie_jar = soup_cookie_jar_text_new (jar_path, TRUE); +} + +void +_ostree_fetcher_set_client_cert (OstreeFetcher *self, + const char *cert_path, + const char *key_path) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + g_clear_object (&self->client_cert); + self->client_cert = _ostree_tls_cert_interaction_new (cert_path, key_path); +} + +void +_ostree_fetcher_set_tls_database (OstreeFetcher *self, + const char *tlsdb_path) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + g_autoptr(GError) local_error = NULL; + GTlsDatabase *tlsdb = g_tls_file_database_new (tlsdb_path, &local_error); + if (tlsdb == NULL) + { + g_warning ("Invalid TLS database '%s': %s", tlsdb_path, local_error->message); + return; + } + + g_clear_object (&self->tls_database); + self->tls_database = tlsdb; +} + +void +_ostree_fetcher_set_extra_headers (OstreeFetcher *self, + GVariant *extra_headers) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + g_clear_pointer (&self->extra_headers, g_variant_unref); + self->extra_headers = g_variant_ref (extra_headers); +} + +void +_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, + const char *extra_user_agent) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + + g_clear_pointer (&self->user_agent, g_free); + if (extra_user_agent != NULL) + self->user_agent = g_strdup_printf ("%s %s", + OSTREE_FETCHER_USERAGENT_STRING, + extra_user_agent); +} + +static gboolean +finish_stream (FetcherRequest *request, + GCancellable *cancellable, + GError **error) +{ + /* Close it here since we do an async fstat(), where we don't want + * to hit a bad fd. + */ + if (request->out_stream) + { + if ((request->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + { + const guint8 nulchar = 0; + + if (!g_output_stream_write_all (request->out_stream, &nulchar, 1, NULL, + cancellable, error)) + return FALSE; + } + + if (!g_output_stream_close (request->out_stream, cancellable, error)) + return FALSE; + } + + if (!request->is_membuf) + { + struct stat stbuf; + + if (!glnx_fstat (request->tmpf.fd, &stbuf, error)) + return FALSE; + + if (request->content_length >= 0 && stbuf.st_size < request->content_length) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete"); + return FALSE; + } + } + + return TRUE; +} + +static void +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data); + +static void +on_out_splice_complete (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + g_autoptr(GTask) task = G_TASK (user_data); + GError *local_error = NULL; + + gssize bytes_written = g_output_stream_splice_finish ((GOutputStream *) object, result, &local_error); + if (bytes_written < 0) + { + g_task_return_error (task, local_error); + return; + } + + FetcherRequest *request = g_task_get_task_data (task); + request->fetcher->bytes_transferred += bytes_written; + + GCancellable *cancellable = g_task_get_cancellable (task); + g_input_stream_read_bytes_async (request->response_body, + 8192, G_PRIORITY_DEFAULT, + cancellable, + on_stream_read, + g_object_ref (task)); +} + +static void +on_stream_read (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + g_autoptr(GTask) task = G_TASK (user_data); + FetcherRequest *request = g_task_get_task_data (task); + GError *local_error = NULL; + + /* Only open the output stream on demand to ensure we use as + * few file descriptors as possible. + */ + if (!request->out_stream) + { + if (!request->is_membuf) + { + if (!_ostree_fetcher_tmpf_from_flags (request->flags, request->fetcher->tmpdir_dfd, + &request->tmpf, &local_error)) + { + g_task_return_error (task, local_error); + return; + } + request->out_stream = g_unix_output_stream_new (request->tmpf.fd, FALSE); + } + else + { + request->out_stream = g_memory_output_stream_new_resizable (); + } + } + + /* Get a GBytes buffer */ + g_autoptr(GBytes) bytes = g_input_stream_read_bytes_finish ((GInputStream *) object, result, &local_error); + if (!bytes) + { + g_task_return_error (task, local_error); + return; + } + + /* Was this the end of the stream? */ + GCancellable *cancellable = g_task_get_cancellable (task); + gsize bytes_read = g_bytes_get_size (bytes); + if (bytes_read == 0) + { + if (!finish_stream (request, cancellable, &local_error)) + { + g_task_return_error (task, local_error); + return; + } + if (request->is_membuf) + { + GBytes *mem_bytes = g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream *) request->out_stream); + g_task_return_pointer (task, mem_bytes, (GDestroyNotify) g_bytes_unref); + } + else + { + if (lseek (request->tmpf.fd, 0, SEEK_SET) < 0) + { + glnx_set_error_from_errno (&local_error); + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + g_task_return_boolean (task, TRUE); + } + } + else + { + /* Verify max size */ + if (request->max_size > 0) + { + if (bytes_read > request->max_size || + (bytes_read + request->current_size) > request->max_size) + { + g_autofree char *uristr = NULL; + + if (request->file) + uristr = g_file_get_uri (request->file); + else + uristr = g_uri_to_string (soup_message_get_uri (request->message)); + + local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, + "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", + uristr, request->max_size); + g_task_return_error (task, local_error); + return; + } + } + + request->current_size += bytes_read; + + /* We do this instead of _write_bytes_async() as that's not + * guaranteed to do a complete write. + */ + { + g_autoptr(GInputStream) membuf = + g_memory_input_stream_new_from_bytes (bytes); + g_output_stream_splice_async (request->out_stream, membuf, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, + G_PRIORITY_DEFAULT, + cancellable, + on_out_splice_complete, + g_object_ref (task)); + } + } +} + +static void +on_request_sent (GObject *object, + GAsyncResult *result, + gpointer user_data) +{ + g_autoptr(GTask) task = G_TASK (user_data); + FetcherRequest *request = g_task_get_task_data (task); + GError *local_error = NULL; + + if (request->file) + request->response_body = (GInputStream *) g_file_read_finish ((GFile *) object, + result, + &local_error); + else + request->response_body = soup_session_send_finish ((SoupSession *) object, + result, &local_error); + + if (!request->response_body) + { + g_task_return_error (task, local_error); + return; + } + + if (request->message) + { + SoupStatus status = soup_message_get_status (request->message); + if (status == SOUP_STATUS_NOT_MODIFIED && + (request->if_none_match != NULL || request->if_modified_since > 0)) + { + /* Version on the server is unchanged from the version we have cached locally; + * report this as an out-argument, a zero-length response buffer, and no error */ + request->out_not_modified = TRUE; + } + else if (!SOUP_STATUS_IS_SUCCESSFUL (status)) + { + /* is there another mirror we can try? */ + if (request->mirrorlist_idx + 1 < request->mirrorlist->len) + { + request->mirrorlist_idx++; + initiate_task_request (g_object_ref (task)); + return; + } + else + { + g_autofree char *uristring = g_uri_to_string (soup_message_get_uri (request->message)); + GIOErrorEnum code = _ostree_fetcher_http_status_code_to_io_error (status); + { + g_autofree char *errmsg = + g_strdup_printf ("Server returned status %u: %s", + status, + soup_status_get_phrase (status)); + local_error = g_error_new_literal (G_IO_ERROR, code, errmsg); + } + + if (request->mirrorlist->len > 1) + g_prefix_error (&local_error, + "All %u mirrors failed. Last error was: ", + request->mirrorlist->len); + if (request->fetcher->remote_name && + !((request->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && + code == G_IO_ERROR_NOT_FOUND)) + _ostree_fetcher_journal_failure (request->fetcher->remote_name, + uristring, local_error->message); + + g_task_return_error (task, local_error); + return; + } + } + + /* Grab cache properties from the response */ + request->out_etag = g_strdup (soup_message_headers_get_one (soup_message_get_response_headers (request->message), "ETag")); + request->out_last_modified = 0; + + const char *last_modified_str = soup_message_headers_get_one (soup_message_get_response_headers (request->message), "Last-Modified"); + if (last_modified_str != NULL) + { + GDateTime *soup_date = soup_date_time_new_from_http_string (last_modified_str); + if (soup_date != NULL) + { + request->out_last_modified = g_date_time_to_unix (soup_date); + g_date_time_unref (soup_date); + } + } + } + + if (request->file) + { + GFileInfo *info = g_file_query_info (request->file, + G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE "," + G_FILE_ATTRIBUTE_STANDARD_SIZE, + 0, NULL, NULL); + if (info) + { + request->content_length = g_file_info_get_size (info); + g_object_unref (info); + } + else + request->content_length = -1; + } + else + { + SoupMessageHeaders *headers = soup_message_get_response_headers (request->message); + if (soup_message_headers_get_list (headers, "Content-Encoding") == NULL) + request->content_length = soup_message_headers_get_content_length (headers); + else + request->content_length = -1; + } + + GCancellable *cancellable = g_task_get_cancellable (task); + g_input_stream_read_bytes_async (request->response_body, + 8192, G_PRIORITY_DEFAULT, + cancellable, + on_stream_read, + g_object_ref (task)); +} + +static SoupSession * +create_soup_session (OstreeFetcher *self) +{ + const char *user_agent = self->user_agent ?: OSTREE_FETCHER_USERAGENT_STRING; + SoupSession *session = + soup_session_new_with_options ("user-agent", user_agent, + "timeout", 60, + "idle-timeout", 60, + "max-conns-per-host", _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS, + NULL); + + /* SoupContentDecoder is included in the session by default. Remove it + * if gzip compression isn't in use. + */ + if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) == 0) + soup_session_remove_feature_by_type (session, SOUP_TYPE_CONTENT_DECODER); + + if (g_getenv ("OSTREE_DEBUG_HTTP")) + { + glnx_unref_object SoupLogger *logger = soup_logger_new (SOUP_LOGGER_LOG_BODY); + soup_session_add_feature (session, SOUP_SESSION_FEATURE (logger)); + } + + if (self->proxy_resolver != NULL) + soup_session_set_proxy_resolver (session, self->proxy_resolver); + if (self->cookie_jar != NULL) + soup_session_add_feature (session, SOUP_SESSION_FEATURE (self->cookie_jar)); + if (self->client_cert != NULL) + soup_session_set_tls_interaction (session, (GTlsInteraction *) self->client_cert); + if (self->tls_database != NULL) + soup_session_set_tls_database (session, self->tls_database); + + return session; +} + +static gboolean +match_value (gpointer key, + gpointer value, + gpointer user_data) +{ + return value == user_data; +} + +static void +on_session_finalized (gpointer data, + GObject *object) +{ + GHashTable *sessions = data; + g_debug ("Removing session %p from sessions hash table", object); + (void) g_hash_table_foreach_remove (sessions, match_value, object); +} + +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_return_if_fail (OSTREE_IS_FETCHER (self)); + g_return_if_fail (mirrorlist != NULL); + g_return_if_fail (mirrorlist->len > 0); + + FetcherRequest *request = g_new0 (FetcherRequest, 1); + request->mirrorlist = g_ptr_array_ref (mirrorlist); + request->filename = g_strdup (filename); + request->flags = flags; + request->if_none_match = g_strdup (if_none_match); + request->if_modified_since = if_modified_since; + request->max_size = max_size; + request->is_membuf = is_membuf; + request->fetcher = self; + request->mainctx = g_main_context_ref_thread_default (); + + /* Ideally each fetcher would have a single soup session. However, each + * session needs to be used from a single main context and the fetcher + * doesn't have that limitation. Instead, a mapping from main context to + * session is kept in the fetcher. + */ + g_debug ("Looking up session for main context %p", request->mainctx); + request->session = (SoupSession *) g_hash_table_lookup (self->sessions, request->mainctx); + if (request->session != NULL) + { + g_debug ("Using existing session %p", request->session); + g_object_ref (request->session); + } + else + { + request->session = create_soup_session (self); + g_debug ("Created new session %p", request->session); + g_hash_table_insert (self->sessions, request->mainctx, request->session); + + /* Add a weak ref to the session so that when it's finalized it can be + * removed from the hash table. + */ + g_object_weak_ref (G_OBJECT (request->session), on_session_finalized, self->sessions); + } + + g_autoptr(GTask) task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, _ostree_fetcher_request_async); + g_task_set_task_data (task, request, (GDestroyNotify) fetcher_request_free); + + /* We'll use the GTask priority for our own priority queue. */ + g_task_set_priority (task, priority); + + initiate_task_request (g_object_ref (task)); +} + +void +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, FALSE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + GTask *task = (GTask *) result; + gpointer ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + FetcherRequest *request = g_task_get_task_data (task); + g_assert (!request->is_membuf); + *out_tmpf = request->tmpf; + request->tmpf.initialized = FALSE; /* Transfer ownership */ + + if (out_not_modified != NULL) + *out_not_modified = request->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&request->out_etag); + if (out_last_modified != NULL) + *out_last_modified = request->out_last_modified; + + return TRUE; +} + +void +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, TRUE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + GTask *task = (GTask *) result; + gpointer ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + FetcherRequest *request = g_task_get_task_data (task); + g_assert (request->is_membuf); + g_assert (out_buf); + *out_buf = ret; + + if (out_not_modified != NULL) + *out_not_modified = request->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&request->out_etag); + if (out_last_modified != NULL) + *out_last_modified = request->out_last_modified; + + return TRUE; +} + + +guint64 +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +{ + return self->bytes_transferred; +} diff --git a/src/ostree/main.c b/src/ostree/main.c index 7d17080c..badfa6df 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -118,7 +118,7 @@ static OstreeCommand commands[] = { { "summary", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_summary, "Manage summary metadata" }, -#if defined(HAVE_LIBSOUP) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE) +#if defined(HAVE_LIBSOUP_OR_LIBSOUP3) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE) { "trivial-httpd", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_trivial_httpd, NULL }, diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 6b16737d..855d2cea 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -34,6 +34,19 @@ #include <sys/prctl.h> #include <signal.h> +#if ! SOUP_CHECK_VERSION (3, 0, 0) +# define SoupServerMessage SoupMessage +# define soup_server_message_get_method(msg) ((msg)->method) +# define soup_server_message_get_request_headers(msg) ((msg)->request_headers) +# define soup_server_message_get_response_headers(msg) ((msg)->response_headers) +# define soup_server_message_get_response_body(msg) ((msg)->response_body) +# define soup_server_message_set_status(msg, status) soup_message_set_status(msg, status) +# define soup_server_message_set_redirect(msg, status, uri) soup_message_set_redirect(msg, status, uri) +# define soup_server_message_set_response(msg, ct, ru, rb, rl) soup_message_set_response(msg, ct, ru, rb, rl) +#else +# define soup_server_message_set_status(msg, status) soup_server_message_set_status(msg, status, NULL) +#endif + static char *opt_port_file = NULL; static char *opt_log = NULL; static gboolean opt_daemonize; @@ -188,15 +201,12 @@ is_safe_to_access (struct stat *stbuf) } static void -close_socket (SoupMessage *msg, gpointer user_data) +close_socket (SoupServerMessage *msg, gpointer user_data) { - SoupSocket *sock = user_data; + GSocket *sock = user_data; int sockfd; - /* Actually calling soup_socket_disconnect() here would cause - * us to leak memory, so just shutdown the socket instead. - */ - sockfd = soup_socket_get_fd (sock); + sockfd = g_socket_get_fd (sock); #ifdef G_OS_WIN32 shutdown (sockfd, SD_SEND); #else @@ -213,12 +223,55 @@ calculate_etag (GMappedFile *mapping) return g_strconcat ("\"", checksum, "\"", NULL); } +static GSList * +_server_cookies_from_request (SoupServerMessage *msg) +{ + SoupCookie *cookie; + GSList *cookies = NULL; + GHashTable *params; + GHashTableIter iter; + gpointer name, value; + const char *header; + const char *host; + + header = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), + "Cookie"); + if (!header) + return NULL; + +#if ! SOUP_CHECK_VERSION (3, 0, 0) + host = soup_uri_get_host (soup_message_get_uri (msg)); +#else + host = g_uri_get_host (soup_server_message_get_uri (msg)); +#endif + params = soup_header_parse_semi_param_list (header); + g_hash_table_iter_init (&iter, params); + + while (g_hash_table_iter_next (&iter, &name, &value)) + { + if (!name || !value) continue; + cookie = soup_cookie_new (name, value, host, NULL, 0); + cookies = g_slist_prepend (cookies, cookie); + } + + soup_header_free_param_list (params); + + return g_slist_reverse (cookies); +} + static void +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (OtTrivialHttpd *self, SoupServer *server, - SoupMessage *msg, + SoupServerMessage *msg, const char *path, SoupClientContext *context) +#else +do_get (OtTrivialHttpd *self, + SoupServer *server, + SoupServerMessage *msg, + const char *path) +#endif { char *slash; int ret; @@ -228,7 +281,7 @@ do_get (OtTrivialHttpd *self, if (opt_expected_cookies) { - GSList *cookies = soup_cookies_from_request (msg); + GSList *cookies = _server_cookies_from_request (msg); GSList *l; int i; @@ -253,12 +306,12 @@ do_get (OtTrivialHttpd *self, if (!found) { httpd_log (self, "Expected cookie not found %s\n", k); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); - soup_cookies_free (cookies); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + g_slist_free_full (cookies, (GDestroyNotify)soup_cookie_free); goto out; } } - soup_cookies_free (cookies); + g_slist_free_full (cookies, (GDestroyNotify)soup_cookie_free); } if (opt_expected_headers) @@ -273,18 +326,18 @@ do_get (OtTrivialHttpd *self, { g_autofree char *k = g_strndup (kv, eq - kv); const gchar *expected_v = eq + 1; - const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k); + const gchar *found_v = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), k); if (!found_v) { httpd_log (self, "Expected header not found %s\n", k); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } if (strcmp (found_v, expected_v) != 0) { httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v); - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } } @@ -293,7 +346,7 @@ do_get (OtTrivialHttpd *self, if (strstr (path, "../") != NULL) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } @@ -302,7 +355,7 @@ do_get (OtTrivialHttpd *self, g_random_int_range (0, 100) < opt_random_500s_percentage) { emitted_random_500s_count++; - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } else if (opt_random_408s_percentage > 0 && @@ -310,7 +363,7 @@ do_get (OtTrivialHttpd *self, g_random_int_range (0, 100) < opt_random_408s_percentage) { emitted_random_408s_count++; - soup_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); + soup_server_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); goto out; } @@ -323,17 +376,17 @@ do_get (OtTrivialHttpd *self, if (ret == -1) { if (errno == EPERM) - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); else if (errno == ENOENT) - soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_FOUND); else - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } if (!is_safe_to_access (&stbuf)) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } @@ -344,9 +397,13 @@ do_get (OtTrivialHttpd *self, { g_autofree char *redir_uri = NULL; - redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path); - soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY, - redir_uri); +#if ! SOUP_CHECK_VERSION (3, 0, 0) + redir_uri = g_strdup_printf ("%s/", soup_uri_get_path (soup_message_get_uri (msg))); +#else + redir_uri = g_strdup_printf ("%s/", g_uri_get_path (soup_server_message_get_uri (msg))); +#endif + soup_server_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY, + redir_uri); } else { @@ -354,15 +411,19 @@ do_get (OtTrivialHttpd *self, if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1) { g_autofree char *index_path = g_strconcat (path, "/index.html", NULL); +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (self, server, msg, index_path, context); +#else + do_get (self, server, msg, index_path); +#endif } else { GString *listing = get_directory_listing (self->root_dfd, path); - soup_message_set_response (msg, "text/html", - SOUP_MEMORY_TAKE, - listing->str, listing->len); - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_response (msg, "text/html", + SOUP_MEMORY_TAKE, + listing->str, listing->len); + soup_server_message_set_status (msg, SOUP_STATUS_OK); g_string_free (listing, FALSE); } } @@ -371,21 +432,21 @@ do_get (OtTrivialHttpd *self, { if (!S_ISREG (stbuf.st_mode)) { - soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); + soup_server_message_set_status (msg, SOUP_STATUS_FORBIDDEN); goto out; } glnx_autofd int fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC); if (fd < 0) { - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } g_autoptr(GMappedFile) mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL); if (!mapping) { - soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); + soup_server_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } (void) close (fd); fd = -1; @@ -395,14 +456,14 @@ do_get (OtTrivialHttpd *self, if (last_modified != NULL) { g_autofree gchar *formatted = g_date_time_format (last_modified, "%a, %d %b %Y %H:%M:%S GMT"); - soup_message_headers_append (msg->response_headers, "Last-Modified", formatted); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Last-Modified", formatted); } g_autofree gchar *etag = calculate_etag (mapping); if (etag != NULL) - soup_message_headers_append (msg->response_headers, "ETag", etag); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "ETag", etag); - if (msg->method == SOUP_METHOD_GET) + if (!strcmp (soup_server_message_get_method (msg), "GET")) { gsize buffer_length, file_size; SoupRange *ranges; @@ -410,13 +471,13 @@ do_get (OtTrivialHttpd *self, gboolean have_ranges; file_size = g_mapped_file_get_length (mapping); - have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length); + have_ranges = soup_message_headers_get_ranges(soup_server_message_get_request_headers (msg), file_size, &ranges, &ranges_length); if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL) { - SoupSocket *sock; + GSocket *sock; buffer_length = file_size/2; - soup_message_headers_set_content_length (msg->response_headers, file_size); - soup_message_headers_append (msg->response_headers, + soup_message_headers_set_content_length (soup_server_message_get_response_headers (msg), file_size); + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Connection", "close"); /* soup-message-io will wait for us to add @@ -424,7 +485,11 @@ do_get (OtTrivialHttpd *self, * the declared Content-Length. Instead, we * forcibly close the socket at that point. */ - sock = soup_client_context_get_socket (context); +#if ! SOUP_CHECK_VERSION (3, 0, 0) + sock = soup_client_context_get_gsocket (context); +#else + sock = soup_server_message_get_socket (msg); +#endif g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock); } else @@ -434,12 +499,13 @@ do_get (OtTrivialHttpd *self, { if (ranges_length > 0 && ranges[0].start >= file_size) { - soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); - soup_message_headers_free_ranges (msg->request_headers, ranges); + soup_server_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + soup_message_headers_free_ranges (soup_server_message_get_request_headers (msg), ranges); goto out; } - soup_message_headers_free_ranges (msg->request_headers, ranges); + soup_message_headers_free_ranges (soup_server_message_get_request_headers (msg), ranges); } +#if ! SOUP_CHECK_VERSION (3, 0, 0) if (buffer_length > 0) { SoupBuffer *buffer; @@ -451,8 +517,22 @@ do_get (OtTrivialHttpd *self, soup_message_body_append_buffer (msg->response_body, buffer); soup_buffer_free (buffer); } +#else + if (buffer_length > 0 && buffer_length == file_size) + { + GBytes *bytes = g_mapped_file_get_bytes (mapping); + soup_message_body_append_bytes (soup_server_message_get_response_body (msg), bytes); + g_bytes_unref (bytes); + } + else if (buffer_length > 0) + { + gchar *contents = g_mapped_file_get_contents (mapping); + soup_message_body_append (soup_server_message_get_response_body (msg), + SOUP_MEMORY_COPY, contents, buffer_length); + } +#endif } - else /* msg->method == SOUP_METHOD_HEAD */ + else /* method == HEAD */ { g_autofree char *length = NULL; @@ -461,56 +541,59 @@ do_get (OtTrivialHttpd *self, * But we'll optimize and avoid the extra I/O. */ length = g_strdup_printf ("%lu", (gulong)stbuf.st_size); - soup_message_headers_append (msg->response_headers, + soup_message_headers_append (soup_server_message_get_response_headers (msg), "Content-Length", length); } /* Check client’s caching headers. */ - const gchar *if_modified_since = soup_message_headers_get_one (msg->request_headers, + const gchar *if_modified_since = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), "If-Modified-Since"); - const gchar *if_none_match = soup_message_headers_get_one (msg->request_headers, + const gchar *if_none_match = soup_message_headers_get_one (soup_server_message_get_request_headers (msg), "If-None-Match"); if (if_none_match != NULL && etag != NULL) { if (g_strcmp0 (etag, if_none_match) == 0) { - soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); - soup_message_body_truncate (msg->response_body); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); + soup_message_body_truncate (soup_server_message_get_response_body (msg)); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } } else if (if_modified_since != NULL && last_modified != NULL) { - SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since); g_autoptr(GDateTime) if_modified_since_dt = NULL; +#if ! SOUP_CHECK_VERSION (3, 0, 0) + SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since); if (if_modified_since_sd != NULL) if_modified_since_dt = g_date_time_new_from_unix_utc (soup_date_to_time_t (if_modified_since_sd)); +#else + if_modified_since_dt = soup_date_time_new_from_http_string (if_modified_since); +#endif if (if_modified_since_dt != NULL && g_date_time_compare (last_modified, if_modified_since_dt) <= 0) { - soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); - soup_message_body_truncate (msg->response_body); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED); + soup_message_body_truncate (soup_server_message_get_response_body (msg)); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } - - g_clear_pointer (&if_modified_since_sd, soup_date_free); } else { - soup_message_set_status (msg, SOUP_STATUS_OK); + soup_server_message_set_status (msg, SOUP_STATUS_OK); } } out: { +#if ! SOUP_CHECK_VERSION (3, 0, 0) guint status = 0; g_autofree gchar *reason = NULL; @@ -518,26 +601,41 @@ do_get (OtTrivialHttpd *self, "status-code", &status, "reason-phrase", &reason, NULL); +#else + guint status = soup_server_message_get_status (msg); + const char *reason = soup_server_message_get_reason_phrase (msg); +#endif + httpd_log (self, " status: %s (%u)\n", reason, status); } return; } static void -httpd_callback (SoupServer *server, SoupMessage *msg, +#if ! SOUP_CHECK_VERSION (3, 0, 0) +httpd_callback (SoupServer *server, SoupServerMessage *msg, const char *path, GHashTable *query, SoupClientContext *context, gpointer data) +#else +httpd_callback (SoupServer *server, SoupServerMessage *msg, + const char *path, GHashTable *query, gpointer data) +#endif { OtTrivialHttpd *self = data; + const char *meth = soup_server_message_get_method (msg); - if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD) + if (!strcmp (meth, "GET") || !strcmp(meth, "HEAD")) +#if ! SOUP_CHECK_VERSION (3, 0, 0) do_get (self, server, msg, path, context); +#else + do_get (self, server, msg, path); +#endif else - soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); + soup_server_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); } static gboolean -basic_auth_callback (SoupAuthDomain *auth_domain, SoupMessage *msg, +basic_auth_callback (SoupAuthDomain *auth_domain, SoupServerMessage *msg, const char *username, const char *password, gpointer data) { return g_str_equal (username, "foouser") && g_str_equal (password, "barpw"); @@ -703,7 +801,7 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) } #if SOUP_CHECK_VERSION(2, 48, 0) - server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); + server = soup_server_new ("server-header", "ostree-httpd ", NULL); if (!soup_server_listen_all (server, opt_port, 0, error)) goto out; #else @@ -711,13 +809,21 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); #endif + if (opt_require_basic_auth) { +#if ! SOUP_CHECK_VERSION (3, 0, 0) glnx_unref_object SoupAuthDomain *auth_domain = soup_auth_domain_basic_new (SOUP_AUTH_DOMAIN_REALM, "auth-test", SOUP_AUTH_DOMAIN_ADD_PATH, "/", SOUP_AUTH_DOMAIN_BASIC_AUTH_CALLBACK, basic_auth_callback, NULL); +#else + glnx_unref_object SoupAuthDomain *auth_domain = + soup_auth_domain_basic_new ("realm", "auth-test", NULL); + soup_auth_domain_add_path (auth_domain, "/"); + soup_auth_domain_basic_set_auth_callback (auth_domain, basic_auth_callback, NULL, NULL); +#endif soup_server_add_auth_domain (server, auth_domain); } |