diff options
author | Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl> | 2021-12-07 07:35:34 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-12-07 07:35:34 +0100 |
commit | f66d548c1aa54bb6e9514e7e281c83bd2a8128b6 (patch) | |
tree | f0891701edc06d849ee18918c1c11de2e3458c11 | |
parent | 45aa0e841ba084ef259d5970e340702130cd548c (diff) | |
parent | d71ece3f0b85c7a3decc50143b68ac07fc5831ae (diff) | |
download | systemd-f66d548c1aa54bb6e9514e7e281c83bd2a8128b6.tar.gz |
Merge pull request #21598 from DaanDeMeyer/journald-file
journal: Fix re-enabling COW for archived journal files on BTRFS
22 files changed, 793 insertions, 676 deletions
diff --git a/meson.build b/meson.build index 43f5bf2c5a..519de53345 100644 --- a/meson.build +++ b/meson.build @@ -2814,7 +2814,7 @@ if conf.get('ENABLE_REMOTE') == 1 and conf.get('HAVE_MICROHTTPD') == 1 public_programs += executable( 'systemd-journal-remote', systemd_journal_remote_sources, - include_directories : includes, + include_directories : journal_includes, link_with : [libshared, libsystemd_journal_remote], dependencies : [threads, @@ -2830,7 +2830,7 @@ if conf.get('ENABLE_REMOTE') == 1 and conf.get('HAVE_MICROHTTPD') == 1 public_programs += executable( 'systemd-journal-gatewayd', systemd_journal_gatewayd_sources, - include_directories : includes, + include_directories : journal_includes, link_with : [libshared], dependencies : [threads, libmicrohttpd, diff --git a/mkosi.default.d/fedora/10-mkosi.fedora b/mkosi.default.d/fedora/10-mkosi.fedora index c348d66c34..78bc0ec342 100644 --- a/mkosi.default.d/fedora/10-mkosi.fedora +++ b/mkosi.default.d/fedora/10-mkosi.fedora @@ -73,3 +73,5 @@ Packages= tpm2-tss less netcat + e2fsprogs + compsize diff --git a/src/basic/chattr-util.c b/src/basic/chattr-util.c index 59707a79c0..807288a649 100644 --- a/src/basic/chattr-util.c +++ b/src/basic/chattr-util.c @@ -59,11 +59,25 @@ int chattr_full(const char *path, } if (ioctl(fd, FS_IOC_SETFLAGS, &new_attr) >= 0) { - if (ret_previous) - *ret_previous = old_attr; - if (ret_final) - *ret_final = new_attr; - return 1; + unsigned attr; + + /* Some filesystems (BTRFS) silently fail when a flag cannot be set. Let's make sure our + * changes actually went through by querying the flags again and verifying they're equal to + * the flags we tried to configure. */ + + if (ioctl(fd, FS_IOC_GETFLAGS, &attr) < 0) + return -errno; + + if (new_attr == attr) { + if (ret_previous) + *ret_previous = old_attr; + if (ret_final) + *ret_final = new_attr; + return 1; + } + + /* Trigger the fallback logic. */ + errno = EINVAL; } if ((errno != EINVAL && !ERRNO_IS_NOT_SUPPORTED(errno)) || diff --git a/src/journal-remote/journal-remote-write.c b/src/journal-remote/journal-remote-write.c index c953a5e93e..fd7cb91f2c 100644 --- a/src/journal-remote/journal-remote-write.c +++ b/src/journal-remote/journal-remote-write.c @@ -3,11 +3,11 @@ #include "alloc-util.h" #include "journal-remote.h" -static int do_rotate(JournalFile **f, bool compress, bool seal) { - int r = journal_file_rotate(f, compress, UINT64_MAX, seal, NULL); +static int do_rotate(JournaldFile **f, bool compress, bool seal) { + int r = journald_file_rotate(f, compress, UINT64_MAX, seal, NULL); if (r < 0) { if (*f) - log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); + log_error_errno(r, "Failed to rotate %s: %m", (*f)->file->path); else log_error_errno(r, "Failed to create rotated journal: %m"); } @@ -39,8 +39,8 @@ static Writer* writer_free(Writer *w) { return NULL; if (w->journal) { - log_debug("Closing journal file %s.", w->journal->path); - journal_file_close(w->journal); + log_debug("Closing journal file %s.", w->journal->file->path); + journald_file_close(w->journal); } if (w->server && w->hashmap_key) @@ -68,15 +68,15 @@ int writer_write(Writer *w, assert(iovw); assert(iovw->count > 0); - if (journal_file_rotate_suggested(w->journal, 0, LOG_DEBUG)) { + if (journal_file_rotate_suggested(w->journal->file, 0, LOG_DEBUG)) { log_info("%s: Journal header limits reached or header out-of-date, rotating", - w->journal->path); + w->journal->file->path); r = do_rotate(&w->journal, compress, seal); if (r < 0) return r; } - r = journal_file_append_entry(w->journal, ts, boot_id, + r = journal_file_append_entry(w->journal->file, ts, boot_id, iovw->iovec, iovw->count, &w->seqnum, NULL, NULL); if (r >= 0) { @@ -86,15 +86,15 @@ int writer_write(Writer *w, } else if (r == -EBADMSG) return r; - log_debug_errno(r, "%s: Write failed, rotating: %m", w->journal->path); + log_debug_errno(r, "%s: Write failed, rotating: %m", w->journal->file->path); r = do_rotate(&w->journal, compress, seal); if (r < 0) return r; else - log_debug("%s: Successfully rotated journal", w->journal->path); + log_debug("%s: Successfully rotated journal", w->journal->file->path); log_debug("Retrying write."); - r = journal_file_append_entry(w->journal, ts, boot_id, + r = journal_file_append_entry(w->journal->file, ts, boot_id, iovw->iovec, iovw->count, &w->seqnum, NULL, NULL); if (r < 0) diff --git a/src/journal-remote/journal-remote-write.h b/src/journal-remote/journal-remote-write.h index 123015b844..d97f6c674b 100644 --- a/src/journal-remote/journal-remote-write.h +++ b/src/journal-remote/journal-remote-write.h @@ -1,13 +1,13 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #pragma once -#include "journal-file.h" +#include "journald-file.h" #include "journal-importer.h" typedef struct RemoteServer RemoteServer; typedef struct Writer { - JournalFile *journal; + JournaldFile *journal; JournalMetrics metrics; MMapCache *mmap; diff --git a/src/journal-remote/journal-remote.c b/src/journal-remote/journal-remote.c index 9fd6a23076..e91b46473b 100644 --- a/src/journal-remote/journal-remote.c +++ b/src/journal-remote/journal-remote.c @@ -14,7 +14,7 @@ #include "errno-util.h" #include "escape.h" #include "fd-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-remote-write.h" #include "journal-remote.h" #include "journald-native.h" @@ -61,16 +61,16 @@ static int open_output(RemoteServer *s, Writer *w, const char* host) { assert_not_reached(); } - r = journal_file_open_reliably(filename, - O_RDWR|O_CREAT, 0640, - s->compress, UINT64_MAX, s->seal, - &w->metrics, - w->mmap, NULL, - NULL, &w->journal); + r = journald_file_open_reliably(filename, + O_RDWR|O_CREAT, 0640, + s->compress, UINT64_MAX, s->seal, + &w->metrics, + w->mmap, NULL, + NULL, &w->journal); if (r < 0) return log_error_errno(r, "Failed to open output journal %s: %m", filename); - log_debug("Opened output file %s", w->journal->path); + log_debug("Opened output file %s", w->journal->file->path); return 0; } diff --git a/src/journal-remote/meson.build b/src/journal-remote/meson.build index 54b314552b..b9f8b472cb 100644 --- a/src/journal-remote/meson.build +++ b/src/journal-remote/meson.build @@ -25,6 +25,7 @@ libsystemd_journal_remote = static_library( 'systemd-journal-remote', libsystemd_journal_remote_sources, include_directories : journal_includes, + link_with : libjournal_core, dependencies : [threads, libmicrohttpd, libgnutls, @@ -77,5 +78,7 @@ endif fuzzers += [ [['src/journal-remote/fuzz-journal-remote.c'], [libsystemd_journal_remote, - libshared]], + libshared], + [], + [journal_includes]], ] diff --git a/src/journal/journald-file.c b/src/journal/journald-file.c new file mode 100644 index 0000000000..7f607d4aa6 --- /dev/null +++ b/src/journal/journald-file.c @@ -0,0 +1,489 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <pthread.h> +#include <unistd.h> + +#include "chattr-util.h" +#include "copy.h" +#include "fd-util.h" +#include "format-util.h" +#include "journal-authenticate.h" +#include "journald-file.h" +#include "path-util.h" +#include "random-util.h" +#include "set.h" +#include "stat-util.h" +#include "sync-util.h" + +static int journald_file_truncate(JournalFile *f) { + uint64_t p; + int r; + + /* truncate excess from the end of archives */ + r = journal_file_tail_end(f, &p); + if (r < 0) + return log_debug_errno(r, "Failed to determine end of tail object: %m"); + + /* arena_size can't exceed the file size, ensure it's updated before truncating */ + f->header->arena_size = htole64(p - le64toh(f->header->header_size)); + + if (ftruncate(f->fd, p) < 0) + log_debug_errno(errno, "Failed to truncate %s: %m", f->path); + + return 0; +} + +static int journald_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint64_t n_entries) { + Object o; + uint64_t offset, sz, n_items = 0, n_unused; + int r; + + if (n_entries == 0) + return 0; + + for (uint64_t q = p; q != 0; q = le64toh(o.entry_array.next_entry_array_offset)) { + r = journal_file_read_object(f, OBJECT_ENTRY_ARRAY, q, &o); + if (r < 0) + return r; + + n_items += journal_file_entry_array_n_items(&o); + p = q; + } + + if (p == 0) + return 0; + + if (n_entries > n_items) + return -EBADMSG; + + /* Amount of unused items in the final entry array. */ + n_unused = n_items - n_entries; + + if (n_unused == 0) + return 0; + + offset = p + offsetof(Object, entry_array.items) + + (journal_file_entry_array_n_items(&o) - n_unused) * sizeof(le64_t); + sz = p + le64toh(o.object.size) - offset; + + if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, sz) < 0) + return log_debug_errno(errno, "Failed to punch hole in entry array of %s: %m", f->path); + + return 0; +} + +static int journald_file_punch_holes(JournalFile *f) { + HashItem items[4096 / sizeof(HashItem)]; + uint64_t p, sz; + size_t to_read; + int r; + + r = journald_file_entry_array_punch_hole( + f, le64toh(f->header->entry_array_offset), le64toh(f->header->n_entries)); + if (r < 0) + return r; + + p = le64toh(f->header->data_hash_table_offset); + sz = le64toh(f->header->data_hash_table_size); + to_read = MIN((size_t) f->last_stat.st_blksize, sizeof(items)); + + for (uint64_t i = p; i < p + sz; i += sizeof(items)) { + ssize_t n_read; + + n_read = pread(f->fd, items, MIN(to_read, p + sz - i), i); + if (n_read < 0) + return n_read; + + for (size_t j = 0; j < (size_t) n_read / sizeof(HashItem); j++) { + Object o; + + for (uint64_t q = le64toh(items[j].head_hash_offset); q != 0; + q = le64toh(o.data.next_hash_offset)) { + + r = journal_file_read_object(f, OBJECT_DATA, q, &o); + if (r < 0) { + log_debug_errno(r, "Invalid data object: %m, ignoring"); + break; + } + + if (le64toh(o.data.n_entries) == 0) + continue; + + (void) journald_file_entry_array_punch_hole( + f, le64toh(o.data.entry_array_offset), le64toh(o.data.n_entries) - 1); + } + } + } + + return 0; +} + +/* This may be called from a separate thread to prevent blocking the caller for the duration of fsync(). + * As a result we use atomic operations on f->offline_state for inter-thread communications with + * journal_file_set_offline() and journal_file_set_online(). */ +static void journald_file_set_offline_internal(JournaldFile *f) { + int r; + + assert(f); + assert(f->file->fd >= 0); + assert(f->file->header); + + for (;;) { + switch (f->file->offline_state) { + case OFFLINE_CANCEL: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_CANCEL, OFFLINE_DONE)) + continue; + return; + + case OFFLINE_AGAIN_FROM_SYNCING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_AGAIN_FROM_SYNCING, OFFLINE_SYNCING)) + continue; + break; + + case OFFLINE_AGAIN_FROM_OFFLINING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_AGAIN_FROM_OFFLINING, OFFLINE_SYNCING)) + continue; + break; + + case OFFLINE_SYNCING: + if (f->file->archive) { + (void) journald_file_truncate(f->file); + (void) journald_file_punch_holes(f->file); + } + + (void) fsync(f->file->fd); + + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_SYNCING, OFFLINE_OFFLINING)) + continue; + + f->file->header->state = f->file->archive ? STATE_ARCHIVED : STATE_OFFLINE; + (void) fsync(f->file->fd); + + /* If we've archived the journal file, first try to re-enable COW on the file. If the + * FS_NOCOW_FL flag was never set or we succesfully removed it, continue. If we fail + * to remove the flag on the archived file, rewrite the file without the NOCOW flag. + * We need this fallback because on some filesystems (BTRFS), the NOCOW flag cannot + * be removed after data has been written to a file. The only way to remove it is to + * copy all data to a new file without the NOCOW flag set. */ + + if (f->file->archive) { + r = chattr_fd(f->file->fd, 0, FS_NOCOW_FL, NULL); + if (r >= 0) + continue; + + log_debug_errno(r, "Failed to re-enable copy-on-write for %s: %m, rewriting file", f->file->path); + + r = copy_file_atomic(f->file->path, f->file->path, f->file->mode, 0, FS_NOCOW_FL, COPY_REPLACE | COPY_FSYNC); + if (r < 0) { + log_debug_errno(r, "Failed to rewrite %s: %m", f->file->path); + continue; + } + } + + break; + + case OFFLINE_OFFLINING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_OFFLINING, OFFLINE_DONE)) + continue; + _fallthrough_; + case OFFLINE_DONE: + return; + + case OFFLINE_JOINED: + log_debug("OFFLINE_JOINED unexpected offline state for journal_file_set_offline_internal()"); + return; + } + } +} + +static void * journald_file_set_offline_thread(void *arg) { + JournaldFile *f = arg; + + (void) pthread_setname_np(pthread_self(), "journal-offline"); + + journald_file_set_offline_internal(f); + + return NULL; +} + +/* Trigger a restart if the offline thread is mid-flight in a restartable state. */ +static bool journald_file_set_offline_try_restart(JournaldFile *f) { + for (;;) { + switch (f->file->offline_state) { + case OFFLINE_AGAIN_FROM_SYNCING: + case OFFLINE_AGAIN_FROM_OFFLINING: + return true; + + case OFFLINE_CANCEL: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_CANCEL, OFFLINE_AGAIN_FROM_SYNCING)) + continue; + return true; + + case OFFLINE_SYNCING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_SYNCING, OFFLINE_AGAIN_FROM_SYNCING)) + continue; + return true; + + case OFFLINE_OFFLINING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_OFFLINING, OFFLINE_AGAIN_FROM_OFFLINING)) + continue; + return true; + + default: + return false; + } + } +} + +/* Sets a journal offline. + * + * If wait is false then an offline is dispatched in a separate thread for a + * subsequent journal_file_set_offline() or journal_file_set_online() of the + * same journal to synchronize with. + * + * If wait is true, then either an existing offline thread will be restarted + * and joined, or if none exists the offline is simply performed in this + * context without involving another thread. + */ +int journald_file_set_offline(JournaldFile *f, bool wait) { + int target_state; + bool restarted; + int r; + + assert(f); + + if (!f->file->writable) + return -EPERM; + + if (f->file->fd < 0 || !f->file->header) + return -EINVAL; + + target_state = f->file->archive ? STATE_ARCHIVED : STATE_OFFLINE; + + /* An offlining journal is implicitly online and may modify f->header->state, + * we must also join any potentially lingering offline thread when already in + * the desired offline state. + */ + if (!journald_file_is_offlining(f) && f->file->header->state == target_state) + return journal_file_set_offline_thread_join(f->file); + + /* Restart an in-flight offline thread and wait if needed, or join a lingering done one. */ + restarted = journald_file_set_offline_try_restart(f); + if ((restarted && wait) || !restarted) { + r = journal_file_set_offline_thread_join(f->file); + if (r < 0) + return r; + } + + if (restarted) + return 0; + + /* Initiate a new offline. */ + f->file->offline_state = OFFLINE_SYNCING; + + if (wait) /* Without using a thread if waiting. */ + journald_file_set_offline_internal(f); + else { + sigset_t ss, saved_ss; + int k; + + assert_se(sigfillset(&ss) >= 0); + /* Don't block SIGBUS since the offlining thread accesses a memory mapped file. + * Asynchronous SIGBUS signals can safely be handled by either thread. */ + assert_se(sigdelset(&ss, SIGBUS) >= 0); + + r = pthread_sigmask(SIG_BLOCK, &ss, &saved_ss); + if (r > 0) + return -r; + + r = pthread_create(&f->file->offline_thread, NULL, journald_file_set_offline_thread, f); + + k = pthread_sigmask(SIG_SETMASK, &saved_ss, NULL); + if (r > 0) { + f->file->offline_state = OFFLINE_JOINED; + return -r; + } + if (k > 0) + return -k; + } + + return 0; +} + +bool journald_file_is_offlining(JournaldFile *f) { + assert(f); + + __sync_synchronize(); + + if (IN_SET(f->file->offline_state, OFFLINE_DONE, OFFLINE_JOINED)) + return false; + + return true; +} + +JournaldFile* journald_file_close(JournaldFile *f) { + if (!f) + return NULL; + +#if HAVE_GCRYPT + /* Write the final tag */ + if (f->file->seal && f->file->writable) { + int r; + + r = journal_file_append_tag(f->file); + if (r < 0) + log_error_errno(r, "Failed to append tag when closing journal: %m"); + } +#endif + + if (f->file->post_change_timer) { + if (sd_event_source_get_enabled(f->file->post_change_timer, NULL) > 0) + journal_file_post_change(f->file); + + sd_event_source_disable_unref(f->file->post_change_timer); + } + + journald_file_set_offline(f, true); + + journal_file_close(f->file); + + return mfree(f); +} + +int journald_file_open( + int fd, + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret) { + _cleanup_free_ JournaldFile *f = NULL; + int r; + + set_clear_with_destructor(deferred_closes, journald_file_close); + + f = new0(JournaldFile, 1); + if (!f) + return -ENOMEM; + + r = journal_file_open(fd, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, + mmap_cache, template ? template->file : NULL, &f->file); + if (r < 0) + return r; + + *ret = TAKE_PTR(f); + + return 0; +} + + +JournaldFile* journald_file_initiate_close(JournaldFile *f, Set *deferred_closes) { + int r; + + assert(f); + + if (deferred_closes) { + r = set_put(deferred_closes, f); + if (r < 0) + log_debug_errno(r, "Failed to add file to deferred close set, closing immediately."); + else { + (void) journald_file_set_offline(f, false); + return NULL; + } + } + + return journald_file_close(f); +} + +int journald_file_rotate( + JournaldFile **f, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + Set *deferred_closes) { + + _cleanup_free_ char *path = NULL; + JournaldFile *new_file = NULL; + int r; + + assert(f); + assert(*f); + + r = journal_file_archive((*f)->file, &path); + if (r < 0) + return r; + + r = journald_file_open( + -1, + path, + (*f)->file->flags, + (*f)->file->mode, + compress, + compress_threshold_bytes, + seal, + NULL, /* metrics */ + (*f)->file->mmap, + deferred_closes, + *f, /* template */ + &new_file); + + journald_file_initiate_close(*f, deferred_closes); + *f = new_file; + + return r; +} + +int journald_file_open_reliably( + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret) { + + int r; + + r = journald_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, + mmap_cache, deferred_closes, template, ret); + if (!IN_SET(r, + -EBADMSG, /* Corrupted */ + -ENODATA, /* Truncated */ + -EHOSTDOWN, /* Other machine */ + -EPROTONOSUPPORT, /* Incompatible feature */ + -EBUSY, /* Unclean shutdown */ + -ESHUTDOWN, /* Already archived */ + -EIO, /* IO error, including SIGBUS on mmap */ + -EIDRM, /* File has been deleted */ + -ETXTBSY)) /* File is from the future */ + return r; + + if ((flags & O_ACCMODE) == O_RDONLY) + return r; + + if (!(flags & O_CREAT)) + return r; + + if (!endswith(fname, ".journal")) + return r; + + /* The file is corrupted. Rotate it away and try it again (but only once) */ + log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); + + r = journal_file_dispose(AT_FDCWD, fname); + if (r < 0) + return r; + + return journald_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, + mmap_cache, deferred_closes, template, ret); +} diff --git a/src/journal/journald-file.h b/src/journal/journald-file.h new file mode 100644 index 0000000000..341043c836 --- /dev/null +++ b/src/journal/journald-file.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "journal-file.h" + +typedef struct { + JournalFile *file; +} JournaldFile; + +int journald_file_open( + int fd, + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret); + +int journald_file_set_offline(JournaldFile *f, bool wait); +bool journald_file_is_offlining(JournaldFile *f); +JournaldFile* journald_file_close(JournaldFile *f); +DEFINE_TRIVIAL_CLEANUP_FUNC(JournaldFile*, journald_file_close); + +int journald_file_open_reliably( + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret); + +JournaldFile* journald_file_initiate_close(JournaldFile *f, Set *deferred_closes); +int journald_file_rotate(JournaldFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes); diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 5ba9b3765b..a13bbeeee9 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -29,7 +29,7 @@ #include "id128-util.h" #include "io-util.h" #include "journal-authenticate.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-internal.h" #include "journal-vacuum.h" #include "journald-audit.h" @@ -243,7 +243,7 @@ static bool uid_for_system_journal(uid_t uid) { return uid_is_system(uid) || uid_is_dynamic(uid) || uid == UID_NOBODY; } -static void server_add_acls(JournalFile *f, uid_t uid) { +static void server_add_acls(JournaldFile *f, uid_t uid) { assert(f); #if HAVE_ACL @@ -252,9 +252,9 @@ static void server_add_acls(JournalFile *f, uid_t uid) { if (uid_for_system_journal(uid)) return; - r = fd_add_uid_acl_permission(f->fd, uid, ACL_READ); + r = fd_add_uid_acl_permission(f->file->fd, uid, ACL_READ); if (r < 0) - log_warning_errno(r, "Failed to set ACL on %s, ignoring: %m", f->path); + log_warning_errno(r, "Failed to set ACL on %s, ignoring: %m", f->file->path); #endif } @@ -265,9 +265,9 @@ static int open_journal( int flags, bool seal, JournalMetrics *metrics, - JournalFile **ret) { + JournaldFile **ret) { - _cleanup_(journal_file_closep) JournalFile *f = NULL; + _cleanup_(journald_file_closep) JournaldFile *f = NULL; int r; assert(s); @@ -275,16 +275,18 @@ static int open_journal( assert(ret); if (reliably) - r = journal_file_open_reliably(fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, - seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journald_file_open_reliably(fname, flags, 0640, s->compress.enabled, + s->compress.threshold_bytes, seal, metrics, s->mmap, + s->deferred_closes, NULL, &f); else - r = journal_file_open(-1, fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, seal, - metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journald_file_open(-1, fname, flags, 0640, s->compress.enabled, + s->compress.threshold_bytes, seal, metrics, s->mmap, + s->deferred_closes, NULL, &f); if (r < 0) return r; - r = journal_file_enable_post_change_timer(f, s->event, POST_CHANGE_TIMER_INTERVAL_USEC); + r = journal_file_enable_post_change_timer(f->file, s->event, POST_CHANGE_TIMER_INTERVAL_USEC); if (r < 0) return r; @@ -388,9 +390,9 @@ static int system_journal_open(Server *s, bool flush_requested, bool relinquish_ return r; } -static JournalFile* find_journal(Server *s, uid_t uid) { +static JournaldFile* find_journal(Server *s, uid_t uid) { _cleanup_free_ char *p = NULL; - JournalFile *f; + JournaldFile *f; int r; assert(s); @@ -433,7 +435,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { /* Too many open? Then let's close one (or more) */ while (ordered_hashmap_size(s->user_journals) >= USER_JOURNALS_MAX) { assert_se(f = ordered_hashmap_steal_first(s->user_journals)); - (void) journal_file_close(f); + (void) journald_file_close(f); } r = open_journal(s, true, p, O_RDWR|O_CREAT, s->seal, &s->system_storage.metrics, &f); @@ -442,7 +444,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { r = ordered_hashmap_put(s->user_journals, UID_TO_PTR(uid), f); if (r < 0) { - (void) journal_file_close(f); + (void) journald_file_close(f); return s->system_journal; } @@ -452,7 +454,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { static int do_rotate( Server *s, - JournalFile **f, + JournaldFile **f, const char* name, bool seal, uint32_t uid) { @@ -463,10 +465,10 @@ static int do_rotate( if (!*f) return -EINVAL; - r = journal_file_rotate(f, s->compress.enabled, s->compress.threshold_bytes, seal, s->deferred_closes); + r = journald_file_rotate(f, s->compress.enabled, s->compress.threshold_bytes, seal, s->deferred_closes); if (r < 0) { if (*f) - return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); + return log_error_errno(r, "Failed to rotate %s: %m", (*f)->file->path); else return log_error_errno(r, "Failed to create new %s journal: %m", name); } @@ -476,15 +478,15 @@ static int do_rotate( } static void server_process_deferred_closes(Server *s) { - JournalFile *f; + JournaldFile *f; /* Perform any deferred closes which aren't still offlining. */ SET_FOREACH(f, s->deferred_closes) { - if (journal_file_is_offlining(f)) + if (journald_file_is_offlining(f)) continue; (void) set_remove(s->deferred_closes, f); - (void) journal_file_close(f); + (void) journald_file_close(f); } } @@ -500,10 +502,10 @@ static void server_vacuum_deferred_closes(Server *s) { /* And now, let's close some more until we reach the limit again. */ while (set_size(s->deferred_closes) >= DEFERRED_CLOSES_MAX) { - JournalFile *f; + JournaldFile *f; assert_se(f = set_steal_first(s->deferred_closes)); - journal_file_close(f); + journald_file_close(f); } } @@ -526,7 +528,7 @@ static int vacuum_offline_user_journals(Server *s) { _cleanup_close_ int fd = -1; const char *a, *b; struct dirent *de; - JournalFile *f; + JournaldFile *f; uid_t uid; errno = 0; @@ -574,18 +576,18 @@ static int vacuum_offline_user_journals(Server *s) { server_vacuum_deferred_closes(s); /* Open the file briefly, so that we can archive it */ - r = journal_file_open(fd, - full, - O_RDWR, - 0640, - s->compress.enabled, - s->compress.threshold_bytes, - s->seal, - &s->system_storage.metrics, - s->mmap, - s->deferred_closes, - NULL, - &f); + r = journald_file_open(fd, + full, + O_RDWR, + 0640, + s->compress.enabled, + s->compress.threshold_bytes, + s->seal, + &s->system_storage.metrics, + s->mmap, + s->deferred_closes, + NULL, + &f); if (r < 0) { log_warning_errno(r, "Failed to read journal file %s for rotation, trying to move it out of the way: %m", full); @@ -598,20 +600,21 @@ static int vacuum_offline_user_journals(Server *s) { continue; } - TAKE_FD(fd); /* Donated to journal_file_open() */ + TAKE_FD(fd); /* Donated to journald_file_open() */ - r = journal_file_archive(f); + r = journal_file_archive(f->file, NULL); if (r < 0) log_debug_errno(r, "Failed to archive journal file '%s', ignoring: %m", full); - f = journal_initiate_close(f, s->deferred_closes); + journald_file_initiate_close(f, s->deferred_closes); + f = NULL; } return 0; } void server_rotate(Server *s) { - JournalFile *f; + JournaldFile *f; void *k; int r; @@ -640,17 +643,17 @@ void server_rotate(Server *s) { } void server_sync(Server *s) { - JournalFile *f; + JournaldFile *f; int r; if (s->system_journal) { - r = journal_file_set_offline(s->system_journal, false); + r = journald_file_set_offline(s->system_journal, false); if (r < 0) log_warning_errno(r, "Failed to sync system journal, ignoring: %m"); } ORDERED_HASHMAP_FOREACH(f, s->user_journals) { - r = journal_file_set_offline(f, false); + r = journald_file_set_offline(f, false); if (r < 0) log_warning_errno(r, "Failed to sync user journal, ignoring: %m"); } @@ -795,7 +798,7 @@ static bool shall_try_append_again(JournalFile *f, int r) { static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n, int priority) { bool vacuumed = false, rotate = false; struct dual_timestamp ts; - JournalFile *f; + JournaldFile *f; int r; assert(s); @@ -822,8 +825,8 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n if (!f) return; - if (journal_file_rotate_suggested(f, s->max_file_usec, LOG_INFO)) { - log_info("%s: Journal header limits reached or header out-of-date, rotating.", f->path); + if (journal_file_rotate_suggested(f->file, s->max_file_usec, LOG_INFO)) { + log_info("%s: Journal header limits reached or header out-of-date, rotating.", f->file->path); rotate = true; } } @@ -840,13 +843,13 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n s->last_realtime_clock = ts.realtime; - r = journal_file_append_entry(f, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); + r = journal_file_append_entry(f->file, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); if (r >= 0) { server_schedule_sync(s, priority); return; } - if (vacuumed || !shall_try_append_again(f, r)) { + if (vacuumed || !shall_try_append_again(f->file, r)) { log_error_errno(r, "Failed to write entry (%zu items, %zu bytes), ignoring: %m", n, IOVEC_TOTAL_SIZE(iovec, n)); return; } @@ -861,7 +864,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n return; log_debug("Retrying write."); - r = journal_file_append_entry(f, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); + r = journal_file_append_entry(f->file, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); if (r < 0) log_error_errno(r, "Failed to write entry (%zu items, %zu bytes) despite vacuuming, ignoring: %m", n, IOVEC_TOTAL_SIZE(iovec, n)); else @@ -1168,11 +1171,11 @@ int server_flush_to_var(Server *s, bool require_flag_file) { goto finish; } - r = journal_file_copy_entry(f, s->system_journal, o, f->current_offset); + r = journal_file_copy_entry(f, s->system_journal->file, o, f->current_offset); if (r >= 0) continue; - if (!shall_try_append_again(s->system_journal, r)) { + if (!shall_try_append_again(s->system_journal->file, r)) { log_error_errno(r, "Can't write entry: %m"); goto finish; } @@ -1189,7 +1192,7 @@ int server_flush_to_var(Server *s, bool require_flag_file) { } log_debug("Retrying write."); - r = journal_file_copy_entry(f, s->system_journal, o, f->current_offset); + r = journal_file_copy_entry(f, s->system_journal->file, o, f->current_offset); if (r < 0) { log_error_errno(r, "Can't write entry: %m"); goto finish; @@ -1200,9 +1203,9 @@ int server_flush_to_var(Server *s, bool require_flag_file) { finish: if (s->system_journal) - journal_file_post_change(s->system_journal); + journal_file_post_change(s->system_journal->file); - s->runtime_journal = journal_file_close(s->runtime_journal); + s->runtime_journal = journald_file_close(s->runtime_journal); if (r >= 0) (void) rm_rf(s->runtime_storage.path, REMOVE_ROOT); @@ -1242,9 +1245,9 @@ static int server_relinquish_var(Server *s) { (void) system_journal_open(s, false, true); - s->system_journal = journal_file_close(s->system_journal); - ordered_hashmap_clear_with_destructor(s->user_journals, journal_file_close); - set_clear_with_destructor(s->deferred_closes, journal_file_close); + s->system_journal = journald_file_close(s->system_journal); + ordered_hashmap_clear_with_destructor(s->user_journals, journald_file_close); + set_clear_with_destructor(s->deferred_closes, journald_file_close); fn = strjoina(s->runtime_directory, "/flushed"); if (unlink(fn) < 0 && errno != ENOENT) @@ -2439,16 +2442,16 @@ int server_init(Server *s, const char *namespace) { void server_maybe_append_tags(Server *s) { #if HAVE_GCRYPT - JournalFile *f; + JournaldFile *f; usec_t n; n = now(CLOCK_REALTIME); if (s->system_journal) - journal_file_maybe_append_tag(s->system_journal, n); + journal_file_maybe_append_tag(s->system_journal->file, n); ORDERED_HASHMAP_FOREACH(f, s->user_journals) - journal_file_maybe_append_tag(f, n); + journal_file_maybe_append_tag(f->file, n); #endif } @@ -2458,17 +2461,17 @@ void server_done(Server *s) { free(s->namespace); free(s->namespace_field); - set_free_with_destructor(s->deferred_closes, journal_file_close); + set_free_with_destructor(s->deferred_closes, journald_file_close); while (s->stdout_streams) stdout_stream_free(s->stdout_streams); client_context_flush_all(s); - (void) journal_file_close(s->system_journal); - (void) journal_file_close(s->runtime_journal); + (void) journald_file_close(s->system_journal); + (void) journald_file_close(s->runtime_journal); - ordered_hashmap_free_with_destructor(s->user_journals, journal_file_close); + ordered_hashmap_free_with_destructor(s->user_journals, journald_file_close); varlink_server_unref(s->varlink_server); diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index 5b7e59cada..92c78a0d74 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -10,7 +10,7 @@ typedef struct Server Server; #include "conf-parser.h" #include "hashmap.h" -#include "journal-file.h" +#include "journald-file.h" #include "journald-context.h" #include "journald-rate-limit.h" #include "journald-stream.h" @@ -89,8 +89,8 @@ struct Server { sd_event_source *watchdog_event_source; sd_event_source *idle_event_source; - JournalFile *runtime_journal; - JournalFile *system_journal; + JournaldFile *runtime_journal; + JournaldFile *system_journal; OrderedHashmap *user_journals; uint64_t seqnum; diff --git a/src/journal/journald.c b/src/journal/journald.c index 94aad05de9..3d4044295e 100644 --- a/src/journal/journald.c +++ b/src/journal/journald.c @@ -102,7 +102,7 @@ int main(int argc, char *argv[]) { if (server.system_journal) { usec_t u; - if (journal_file_next_evolve_usec(server.system_journal, &u)) { + if (journal_file_next_evolve_usec(server.system_journal->file, &u)) { if (n >= u) t = 0; else diff --git a/src/journal/meson.build b/src/journal/meson.build index 171e276736..383ed355fd 100644 --- a/src/journal/meson.build +++ b/src/journal/meson.build @@ -7,6 +7,8 @@ sources = files(''' journald-console.h journald-context.c journald-context.h + journald-file.c + journald-file.h journald-kmsg.c journald-kmsg.h journald-native.c @@ -33,6 +35,7 @@ libjournal_core = static_library( 'journal-core', sources, include_directories : includes, + dependencies: threads, install : false) journal_includes = [includes, include_directories('.')] @@ -92,6 +95,26 @@ tests += [ [libxz, liblz4, libselinux]], + + [['src/journal/test-journal.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-stream.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-flush.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-verify.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-interleaving.c'], + [libjournal_core, + libshared]], ] fuzzers += [ diff --git a/src/libsystemd/sd-journal/test-journal-flush.c b/src/journal/test-journal-flush.c index c6fb65791e..f0a7024002 100644 --- a/src/libsystemd/sd-journal/test-journal-flush.c +++ b/src/journal/test-journal-flush.c @@ -7,7 +7,7 @@ #include "alloc-util.h" #include "chattr-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-internal.h" #include "macro.h" #include "path-util.h" @@ -16,7 +16,7 @@ int main(int argc, char *argv[]) { _cleanup_free_ char *fn = NULL; char dn[] = "/var/tmp/test-journal-flush.XXXXXX"; - JournalFile *new_journal = NULL; + JournaldFile *new_journal = NULL; sd_journal *j = NULL; unsigned n = 0; int r; @@ -26,7 +26,7 @@ int main(int argc, char *argv[]) { fn = path_join(dn, "test.journal"); - r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal); + r = journald_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal); assert_se(r >= 0); if (argc > 1) @@ -49,7 +49,7 @@ int main(int argc, char *argv[]) { log_error_errno(r, "journal_file_move_to_object failed: %m"); assert_se(r >= 0); - r = journal_file_copy_entry(f, new_journal, o, f->current_offset); + r = journal_file_copy_entry(f, new_journal->file, o, f->current_offset); if (r < 0) log_warning_errno(r, "journal_file_copy_entry failed: %m"); assert_se(r >= 0 || @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) { sd_journal_close(j); - (void) journal_file_close(new_journal); + (void) journald_file_close(new_journal); unlink(fn); assert_se(rmdir(dn) == 0); diff --git a/src/libsystemd/sd-journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index 62db2266c7..48be6a14bc 100644 --- a/src/libsystemd/sd-journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -8,7 +8,7 @@ #include "alloc-util.h" #include "chattr-util.h" #include "io-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-vacuum.h" #include "log.h" #include "parse-util.h" @@ -33,17 +33,17 @@ _noreturn_ static void log_assert_errno(const char *text, int error, const char log_assert_errno(#expr, -_r_, PROJECT_FILE, __LINE__, __PRETTY_FUNCTION__); \ } while (false) -static JournalFile *test_open(const char *name) { - JournalFile *f; - assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f)); +static JournaldFile *test_open(const char *name) { + JournaldFile *f; + assert_ret(journald_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f)); return f; } -static void test_close(JournalFile *f) { - (void) journal_file_close (f); +static void test_close(JournaldFile *f) { + (void) journald_file_close(f); } -static void append_number(JournalFile *f, int n, uint64_t *seqnum) { +static void append_number(JournaldFile *f, int n, uint64_t *seqnum) { char *p; dual_timestamp ts; static dual_timestamp previous_ts = {}; @@ -61,7 +61,7 @@ static void append_number(JournalFile *f, int n, uint64_t *seqnum) { assert_se(asprintf(&p, "NUMBER=%d", n) >= 0); iovec[0] = IOVEC_MAKE_STRING(p); - assert_ret(journal_file_append_entry(f, &ts, NULL, iovec, 1, seqnum, NULL, NULL)); + assert_ret(journal_file_append_entry(f->file, &ts, NULL, iovec, 1, seqnum, NULL, NULL)); free(p); } @@ -108,7 +108,7 @@ static void test_check_numbers_up (sd_journal *j, int count) { } static void setup_sequential(void) { - JournalFile *one, *two; + JournaldFile *one, *two; one = test_open("one.journal"); two = test_open("two.journal"); append_number(one, 1, NULL); @@ -120,7 +120,7 @@ static void setup_sequential(void) { } static void setup_interleaved(void) { - JournalFile *one, *two; + JournaldFile *one, *two; one = test_open("one.journal"); two = test_open("two.journal"); append_number(one, 1, NULL); @@ -199,14 +199,14 @@ static void test_skip(void (*setup)(void)) { static void test_sequence_numbers(void) { char t[] = "/var/tmp/journal-seq-XXXXXX"; - JournalFile *one, *two; + JournaldFile *one, *two; uint64_t seqnum = 0; sd_id128_t seqnum_id; mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644, - true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); + assert_se(journald_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644, + true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); append_number(one, 1, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -215,21 +215,21 @@ static void test_sequence_numbers(void) { printf("seqnum=%"PRIu64"\n", seqnum); assert_se(seqnum == 2); - assert_se(one->header->state == STATE_ONLINE); - assert_se(!sd_id128_equal(one->header->file_id, one->header->machine_id)); - assert_se(!sd_id128_equal(one->header->file_id, one->header->boot_id)); - assert_se(sd_id128_equal(one->header->file_id, one->header->seqnum_id)); + assert_se(one->file->header->state == STATE_ONLINE); + assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->machine_id)); + assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->boot_id)); + assert_se(sd_id128_equal(one->file->header->file_id, one->file->header->seqnum_id)); - memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t)); + memcpy(&seqnum_id, &one->file->header->seqnum_id, sizeof(sd_id128_t)); - assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644, - true, UINT64_MAX, false, NULL, NULL, NULL, one, &two) == 0); + assert_se(journald_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644, + true, UINT64_MAX, false, NULL, NULL, NULL, one, &two) == 0); - assert_se(two->header->state == STATE_ONLINE); - assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id)); - assert_se(sd_id128_equal(one->header->machine_id, one->header->machine_id)); - assert_se(sd_id128_equal(one->header->boot_id, one->header->boot_id)); - assert_se(sd_id128_equal(one->header->seqnum_id, one->header->seqnum_id)); + assert_se(two->file->header->state == STATE_ONLINE); + assert_se(!sd_id128_equal(two->file->header->file_id, one->file->header->file_id)); + assert_se(sd_id128_equal(one->file->header->machine_id, one->file->header->machine_id)); + assert_se(sd_id128_equal(one->file->header->boot_id, one->file->header->boot_id)); + assert_se(sd_id128_equal(one->file->header->seqnum_id, one->file->header->seqnum_id)); append_number(two, 3, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -253,10 +253,10 @@ static void test_sequence_numbers(void) { /* restart server */ seqnum = 0; - assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0, - true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); + assert_se(journald_file_open(-1, "two.journal", O_RDWR, 0, + true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); - assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id)); + assert_se(sd_id128_equal(two->file->header->seqnum_id, seqnum_id)); append_number(two, 7, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -281,7 +281,7 @@ static void test_sequence_numbers(void) { int main(int argc, char *argv[]) { test_setup_logging(LOG_DEBUG); - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); diff --git a/src/libsystemd/sd-journal/test-journal-stream.c b/src/journal/test-journal-stream.c index ca1eaaebfc..f7fc4332a9 100644 --- a/src/libsystemd/sd-journal/test-journal-stream.c +++ b/src/journal/test-journal-stream.c @@ -8,7 +8,7 @@ #include "alloc-util.h" #include "chattr-util.h" #include "io-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-internal.h" #include "log.h" #include "macro.h" @@ -60,7 +60,7 @@ static void verify_contents(sd_journal *j, unsigned skip) { } static void run_test(void) { - JournalFile *one, *two, *three; + JournaldFile *one, *two, *three; char t[] = "/var/tmp/journal-stream-XXXXXX"; unsigned i; _cleanup_(sd_journal_closep) sd_journal *j = NULL; @@ -73,9 +73,9 @@ static void run_test(void) { assert_se(chdir(t) >= 0); (void) chattr_path(t, FS_NOCOW_FL, FS_NOCOW_FL, NULL); - assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); - assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); - assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &three) == 0); + assert_se(journald_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); + assert_se(journald_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); + assert_se(journald_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &three) == 0); for (i = 0; i < N_ENTRIES; i++) { char *p, *q; @@ -100,21 +100,21 @@ static void run_test(void) { iovec[1] = IOVEC_MAKE(q, strlen(q)); if (i % 10 == 0) - assert_se(journal_file_append_entry(three, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(three->file, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); else { if (i % 3 == 0) - assert_se(journal_file_append_entry(two, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(two->file, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); - assert_se(journal_file_append_entry(one, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(one->file, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); } free(p); free(q); } - (void) journal_file_close(one); - (void) journal_file_close(two); - (void) journal_file_close(three); + (void) journald_file_close(one); + (void) journald_file_close(two); + (void) journald_file_close(three); assert_se(sd_journal_open_directory(&j, t, 0) >= 0); @@ -174,7 +174,7 @@ static void run_test(void) { int main(int argc, char *argv[]) { - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); diff --git a/src/libsystemd/sd-journal/test-journal-verify.c b/src/journal/test-journal-verify.c index 8eb9ca88f7..cf9692b3d2 100644 --- a/src/libsystemd/sd-journal/test-journal-verify.c +++ b/src/journal/test-journal-verify.c @@ -7,7 +7,7 @@ #include "chattr-util.h" #include "fd-util.h" #include "io-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-verify.h" #include "log.h" #include "rm-rf.h" @@ -41,7 +41,7 @@ static int raw_verify(const char *fn, const char *verification_key) { JournalFile *f; int r; - r = journal_file_open(-1, fn, O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &f); + r = journal_file_open(-1, fn, O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, &f); if (r < 0) return r; @@ -55,12 +55,13 @@ int main(int argc, char *argv[]) { char t[] = "/var/tmp/journal-XXXXXX"; unsigned n; JournalFile *f; + JournaldFile *df; const char *verification_key = argv[1]; usec_t from = 0, to = 0, total = 0; struct stat st; uint64_t p; - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); @@ -72,7 +73,7 @@ int main(int argc, char *argv[]) { log_info("Generating..."); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &df) == 0); for (n = 0; n < N_ENTRIES; n++) { struct iovec iovec; @@ -85,16 +86,16 @@ int main(int argc, char *argv[]) { iovec = IOVEC_MAKE_STRING(test); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(df->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); free(test); } - (void) journal_file_close(f); + (void) journald_file_close(df); log_info("Verifying..."); - assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, &f) == 0); /* journal_file_print_header(f); */ journal_file_dump(f); diff --git a/src/libsystemd/sd-journal/test-journal.c b/src/journal/test-journal.c index bf69111d44..c2c4a75fef 100644 --- a/src/libsystemd/sd-journal/test-journal.c +++ b/src/journal/test-journal.c @@ -6,7 +6,7 @@ #include "chattr-util.h" #include "io-util.h" #include "journal-authenticate.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-vacuum.h" #include "log.h" #include "rm-rf.h" @@ -25,7 +25,7 @@ static void mkdtemp_chdir_chattr(char *path) { static void test_non_empty(void) { dual_timestamp ts; - JournalFile *f; + JournaldFile *f; struct iovec iovec; static const char test[] = "TEST1=1", test2[] = "TEST2=2"; Object *o; @@ -37,71 +37,71 @@ static void test_non_empty(void) { mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f) == 0); assert_se(dual_timestamp_get(&ts)); assert_se(sd_id128_randomize(&fake_boot_id) == 0); iovec = IOVEC_MAKE_STRING(test); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); iovec = IOVEC_MAKE_STRING(test2); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); iovec = IOVEC_MAKE_STRING(test); - assert_se(journal_file_append_entry(f, &ts, &fake_boot_id, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, &fake_boot_id, &iovec, 1, NULL, NULL, NULL) == 0); #if HAVE_GCRYPT - journal_file_append_tag(f); + journal_file_append_tag(f->file); #endif - journal_file_dump(f); + journal_file_dump(f->file); - assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, 0, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, p, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, p, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 3); assert_se(sd_id128_equal(o->entry.boot_id, fake_boot_id)); - assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 0); + assert_se(journal_file_next_entry(f->file, p, DIRECTION_DOWN, &o, &p) == 0); - assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, 0, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_find_data_object(f, test, strlen(test), NULL, &p) == 1); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_find_data_object(f->file, test, strlen(test), NULL, &p) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 3); - assert_se(journal_file_find_data_object(f, test2, strlen(test2), NULL, &p) == 1); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); + assert_se(journal_file_find_data_object(f->file, test2, strlen(test2), NULL, &p) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_find_data_object(f, "quux", 4, NULL, &p) == 0); + assert_se(journal_file_find_data_object(f->file, "quux", 4, NULL, &p) == 0); - assert_se(journal_file_move_to_entry_by_seqnum(f, 1, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 1, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_move_to_entry_by_seqnum(f, 3, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 3, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 3); - assert_se(journal_file_move_to_entry_by_seqnum(f, 2, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 2, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 10, DIRECTION_DOWN, &o, NULL) == 0); - journal_file_rotate(&f, true, UINT64_MAX, true, NULL); - journal_file_rotate(&f, true, UINT64_MAX, true, NULL); + journald_file_rotate(&f, true, UINT64_MAX, true, NULL); + journald_file_rotate(&f, true, UINT64_MAX, true, NULL); - (void) journal_file_close(f); + (void) journald_file_close(f); log_info("Done..."); @@ -117,28 +117,25 @@ static void test_non_empty(void) { } static void test_empty(void) { - JournalFile *f1, *f2, *f3, *f4; + JournaldFile *f1, *f2, *f3, *f4; char t[] = "/var/tmp/journal-XXXXXX"; test_setup_logging(LOG_DEBUG); mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f1) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f1) == 0); + assert_se(journald_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f2) == 0); + assert_se(journald_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f3) == 0); + assert_se(journald_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f4) == 0); - assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f2) == 0); - - assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f3) == 0); - - assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f4) == 0); - - journal_file_print_header(f1); + journal_file_print_header(f1->file); puts(""); - journal_file_print_header(f2); + journal_file_print_header(f2->file); puts(""); - journal_file_print_header(f3); + journal_file_print_header(f3->file); puts(""); - journal_file_print_header(f4); + journal_file_print_header(f4->file); puts(""); log_info("Done..."); @@ -151,16 +148,16 @@ static void test_empty(void) { assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } - (void) journal_file_close(f1); - (void) journal_file_close(f2); - (void) journal_file_close(f3); - (void) journal_file_close(f4); + (void) journald_file_close(f1); + (void) journald_file_close(f2); + (void) journald_file_close(f3); + (void) journald_file_close(f4); } #if HAVE_COMPRESSION static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { dual_timestamp ts; - JournalFile *f; + JournaldFile *f; struct iovec iovec; Object *o; uint64_t p; @@ -175,34 +172,34 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0); dual_timestamp_get(&ts); iovec = IOVEC_MAKE(data, data_size); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); #if HAVE_GCRYPT - journal_file_append_tag(f); + journal_file_append_tag(f->file); #endif - journal_file_dump(f); + journal_file_dump(f->file); /* We have to partially reimplement some of the dump logic, because the normal next_entry does the * decompression for us. */ - p = le64toh(f->header->header_size); + p = le64toh(f->file->header->header_size); for (;;) { - r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o); + r = journal_file_move_to_object(f->file, OBJECT_UNUSED, p, &o); assert_se(r == 0); if (o->object.type == OBJECT_DATA) break; - assert_se(p < le64toh(f->header->tail_object_offset)); + assert_se(p < le64toh(f->file->header->tail_object_offset)); p = p + ALIGN64(le64toh(o->object.size)); } is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0; - (void) journal_file_close(f); + (void) journald_file_close(f); log_info("Done..."); @@ -245,7 +242,7 @@ int main(int argc, char *argv[]) { test_setup_logging(LOG_INFO); - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); diff --git a/src/libsystemd/meson.build b/src/libsystemd/meson.build index 6ee78fd195..73c0228248 100644 --- a/src/libsystemd/meson.build +++ b/src/libsystemd/meson.build @@ -190,8 +190,6 @@ custom_target( ############################################################ tests += [ - [['src/libsystemd/sd-journal/test-journal.c']], - [['src/libsystemd/sd-journal/test-journal-send.c']], [['src/libsystemd/sd-journal/test-journal-match.c']], @@ -199,16 +197,8 @@ tests += [ [['src/libsystemd/sd-journal/test-journal-enum.c'], [], [], [], '', 'timeout=360'], - [['src/libsystemd/sd-journal/test-journal-stream.c']], - - [['src/libsystemd/sd-journal/test-journal-flush.c']], - [['src/libsystemd/sd-journal/test-journal-init.c']], - [['src/libsystemd/sd-journal/test-journal-verify.c']], - - [['src/libsystemd/sd-journal/test-journal-interleaving.c']], - [['src/libsystemd/sd-journal/test-mmap-cache.c']], [['src/libsystemd/sd-journal/test-catalog.c']], diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 611c8c0d28..11b9da1cb5 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -91,7 +91,7 @@ # pragma GCC diagnostic ignored "-Waddress-of-packed-member" #endif -static int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) { +int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) { Object tail; uint64_t p; int r; @@ -126,174 +126,7 @@ static int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) { return 0; } -static int journal_file_truncate(JournalFile *f) { - uint64_t p; - int r; - - /* truncate excess from the end of archives */ - r = journal_file_tail_end(f, &p); - if (r < 0) - return log_debug_errno(r, "Failed to determine end of tail object: %m"); - - /* arena_size can't exceed the file size, ensure it's updated before truncating */ - f->header->arena_size = htole64(p - le64toh(f->header->header_size)); - - if (ftruncate(f->fd, p) < 0) - log_debug_errno(errno, "Failed to truncate %s: %m", f->path); - - return 0; -} - -static int journal_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint64_t n_entries) { - Object o; - uint64_t offset, sz, n_items = 0, n_unused; - int r; - - if (n_entries == 0) - return 0; - - for (uint64_t q = p; q != 0; q = le64toh(o.entry_array.next_entry_array_offset)) { - r = journal_file_read_object(f, OBJECT_ENTRY_ARRAY, q, &o); - if (r < 0) - return r; - - n_items += journal_file_entry_array_n_items(&o); - p = q; - } - - if (p == 0) - return 0; - - if (n_entries > n_items) - return -EBADMSG; - - /* Amount of unused items in the final entry array. */ - n_unused = n_items - n_entries; - - if (n_unused == 0) - return 0; - - offset = p + offsetof(Object, entry_array.items) + - (journal_file_entry_array_n_items(&o) - n_unused) * sizeof(le64_t); - sz = p + le64toh(o.object.size) - offset; - - if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, sz) < 0) - return log_debug_errno(errno, "Failed to punch hole in entry array of %s: %m", f->path); - - return 0; -} - -static int journal_file_punch_holes(JournalFile *f) { - HashItem items[4096 / sizeof(HashItem)]; - uint64_t p, sz; - size_t to_read; - int r; - - r = journal_file_entry_array_punch_hole( - f, le64toh(f->header->entry_array_offset), le64toh(f->header->n_entries)); - if (r < 0) - return r; - - p = le64toh(f->header->data_hash_table_offset); - sz = le64toh(f->header->data_hash_table_size); - to_read = MIN((size_t) f->last_stat.st_blksize, sizeof(items)); - - for (uint64_t i = p; i < p + sz; i += sizeof(items)) { - ssize_t n_read; - - n_read = pread(f->fd, items, MIN(to_read, p + sz - i), i); - if (n_read < 0) - return n_read; - - for (size_t j = 0; j < (size_t) n_read / sizeof(HashItem); j++) { - Object o; - - for (uint64_t q = le64toh(items[j].head_hash_offset); q != 0; - q = le64toh(o.data.next_hash_offset)) { - - r = journal_file_read_object(f, OBJECT_DATA, q, &o); - if (r < 0) { - log_debug_errno(r, "Invalid data object: %m, ignoring"); - break; - } - - if (le64toh(o.data.n_entries) == 0) - continue; - - (void) journal_file_entry_array_punch_hole( - f, le64toh(o.data.entry_array_offset), le64toh(o.data.n_entries) - 1); - } - } - } - - return 0; -} - -/* This may be called from a separate thread to prevent blocking the caller for the duration of fsync(). - * As a result we use atomic operations on f->offline_state for inter-thread communications with - * journal_file_set_offline() and journal_file_set_online(). */ -static void journal_file_set_offline_internal(JournalFile *f) { - assert(f); - assert(f->fd >= 0); - assert(f->header); - - for (;;) { - switch (f->offline_state) { - case OFFLINE_CANCEL: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_CANCEL, OFFLINE_DONE)) - continue; - return; - - case OFFLINE_AGAIN_FROM_SYNCING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_AGAIN_FROM_SYNCING, OFFLINE_SYNCING)) - continue; - break; - - case OFFLINE_AGAIN_FROM_OFFLINING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_AGAIN_FROM_OFFLINING, OFFLINE_SYNCING)) - continue; - break; - - case OFFLINE_SYNCING: - if (f->archive) { - (void) journal_file_truncate(f); - (void) journal_file_punch_holes(f); - } - - (void) fsync(f->fd); - - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_SYNCING, OFFLINE_OFFLINING)) - continue; - - f->header->state = f->archive ? STATE_ARCHIVED : STATE_OFFLINE; - (void) fsync(f->fd); - break; - - case OFFLINE_OFFLINING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_OFFLINING, OFFLINE_DONE)) - continue; - _fallthrough_; - case OFFLINE_DONE: - return; - - case OFFLINE_JOINED: - log_debug("OFFLINE_JOINED unexpected offline state for journal_file_set_offline_internal()"); - return; - } - } -} - -static void * journal_file_set_offline_thread(void *arg) { - JournalFile *f = arg; - - (void) pthread_setname_np(pthread_self(), "journal-offline"); - - journal_file_set_offline_internal(f); - - return NULL; -} - -static int journal_file_set_offline_thread_join(JournalFile *f) { +int journal_file_set_offline_thread_join(JournalFile *f) { int r; assert(f); @@ -313,110 +146,6 @@ static int journal_file_set_offline_thread_join(JournalFile *f) { return 0; } -/* Trigger a restart if the offline thread is mid-flight in a restartable state. */ -static bool journal_file_set_offline_try_restart(JournalFile *f) { - for (;;) { - switch (f->offline_state) { - case OFFLINE_AGAIN_FROM_SYNCING: - case OFFLINE_AGAIN_FROM_OFFLINING: - return true; - - case OFFLINE_CANCEL: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_CANCEL, OFFLINE_AGAIN_FROM_SYNCING)) - continue; - return true; - - case OFFLINE_SYNCING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_SYNCING, OFFLINE_AGAIN_FROM_SYNCING)) - continue; - return true; - - case OFFLINE_OFFLINING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_OFFLINING, OFFLINE_AGAIN_FROM_OFFLINING)) - continue; - return true; - - default: - return false; - } - } -} - -/* Sets a journal offline. - * - * If wait is false then an offline is dispatched in a separate thread for a - * subsequent journal_file_set_offline() or journal_file_set_online() of the - * same journal to synchronize with. - * - * If wait is true, then either an existing offline thread will be restarted - * and joined, or if none exists the offline is simply performed in this - * context without involving another thread. - */ -int journal_file_set_offline(JournalFile *f, bool wait) { - int target_state; - bool restarted; - int r; - - assert(f); - - if (!f->writable) - return -EPERM; - - if (f->fd < 0 || !f->header) - return -EINVAL; - - target_state = f->archive ? STATE_ARCHIVED : STATE_OFFLINE; - - /* An offlining journal is implicitly online and may modify f->header->state, - * we must also join any potentially lingering offline thread when already in - * the desired offline state. - */ - if (!journal_file_is_offlining(f) && f->header->state == target_state) - return journal_file_set_offline_thread_join(f); - - /* Restart an in-flight offline thread and wait if needed, or join a lingering done one. */ - restarted = journal_file_set_offline_try_restart(f); - if ((restarted && wait) || !restarted) { - r = journal_file_set_offline_thread_join(f); - if (r < 0) - return r; - } - - if (restarted) - return 0; - - /* Initiate a new offline. */ - f->offline_state = OFFLINE_SYNCING; - - if (wait) /* Without using a thread if waiting. */ - journal_file_set_offline_internal(f); - else { - sigset_t ss, saved_ss; - int k; - - assert_se(sigfillset(&ss) >= 0); - /* Don't block SIGBUS since the offlining thread accesses a memory mapped file. - * Asynchronous SIGBUS signals can safely be handled by either thread. */ - assert_se(sigdelset(&ss, SIGBUS) >= 0); - - r = pthread_sigmask(SIG_BLOCK, &ss, &saved_ss); - if (r > 0) - return -r; - - r = pthread_create(&f->offline_thread, NULL, journal_file_set_offline_thread, f); - - k = pthread_sigmask(SIG_SETMASK, &saved_ss, NULL); - if (r > 0) { - f->offline_state = OFFLINE_JOINED; - return -r; - } - if (k > 0) - return -k; - } - - return 0; -} - static int journal_file_set_online(JournalFile *f) { bool wait = true; @@ -484,56 +213,13 @@ static int journal_file_set_online(JournalFile *f) { } } -bool journal_file_is_offlining(JournalFile *f) { - assert(f); - - __sync_synchronize(); - - if (IN_SET(f->offline_state, OFFLINE_DONE, OFFLINE_JOINED)) - return false; - - return true; -} - JournalFile* journal_file_close(JournalFile *f) { if (!f) return NULL; -#if HAVE_GCRYPT - /* Write the final tag */ - if (f->seal && f->writable) { - int r; - - r = journal_file_append_tag(f); - if (r < 0) - log_error_errno(r, "Failed to append tag when closing journal: %m"); - } -#endif - - if (f->post_change_timer) { - if (sd_event_source_get_enabled(f->post_change_timer, NULL) > 0) - journal_file_post_change(f); - - sd_event_source_disable_unref(f->post_change_timer); - } - - journal_file_set_offline(f, true); - if (f->mmap && f->cache_fd) mmap_cache_fd_free(f->cache_fd); - if (f->fd >= 0 && f->defrag_on_close) { - - /* Be friendly to btrfs: turn COW back on again now, - * and defragment the file. We won't write to the file - * ever again, hence remove all fragmentation, and - * reenable all the good bits COW usually provides - * (such as data checksumming). */ - - (void) chattr_fd(f->fd, 0, FS_NOCOW_FL, NULL); - (void) btrfs_defrag_fd(f->fd); - } - if (f->close_fd) safe_close(f->fd); free(f->path); @@ -3562,7 +3248,6 @@ int journal_file_open( bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, - Set *deferred_closes, JournalFile *template, JournalFile **ret) { @@ -3747,8 +3432,6 @@ int journal_file_open( f->header = h; if (!newly_created) { - set_clear_with_destructor(deferred_closes, journal_file_close); - r = journal_file_verify_header(f); if (r < 0) goto fail; @@ -3826,7 +3509,7 @@ fail: return r; } -int journal_file_archive(JournalFile *f) { +int journal_file_archive(JournalFile *f, char **ret_previous_path) { _cleanup_free_ char *p = NULL; assert(f); @@ -3857,6 +3540,13 @@ int journal_file_archive(JournalFile *f) { /* Sync the rename to disk */ (void) fsync_directory_of_file(f->fd); + if (ret_previous_path) + *ret_previous_path = f->path; + else + free(f->path); + + f->path = TAKE_PTR(p); + /* Set as archive so offlining commits w/state=STATE_ARCHIVED. Previously we would set old_file->header->state * to STATE_ARCHIVED directly here, but journal_file_set_offline() short-circuits when state != STATE_ONLINE, * which would result in the rotated journal never getting fsync() called before closing. Now we simply queue @@ -3864,75 +3554,11 @@ int journal_file_archive(JournalFile *f) { * occurs. */ f->archive = true; - /* Currently, btrfs is not very good with out write patterns and fragments heavily. Let's defrag our journal - * files when we archive them */ - f->defrag_on_close = true; - return 0; } -JournalFile* journal_initiate_close( - JournalFile *f, - Set *deferred_closes) { - - int r; - - assert(f); - - if (deferred_closes) { - - r = set_put(deferred_closes, f); - if (r < 0) - log_debug_errno(r, "Failed to add file to deferred close set, closing immediately."); - else { - (void) journal_file_set_offline(f, false); - return NULL; - } - } - - return journal_file_close(f); -} - -int journal_file_rotate( - JournalFile **f, - bool compress, - uint64_t compress_threshold_bytes, - bool seal, - Set *deferred_closes) { - - JournalFile *new_file = NULL; - int r; - - assert(f); - assert(*f); - - r = journal_file_archive(*f); - if (r < 0) - return r; - - r = journal_file_open( - -1, - (*f)->path, - (*f)->flags, - (*f)->mode, - compress, - compress_threshold_bytes, - seal, - NULL, /* metrics */ - (*f)->mmap, - deferred_closes, - *f, /* template */ - &new_file); - - journal_initiate_close(*f, deferred_closes); - *f = new_file; - - return r; -} - int journal_file_dispose(int dir_fd, const char *fname) { _cleanup_free_ char *p = NULL; - _cleanup_close_ int fd = -1; assert(fname); @@ -3953,67 +3579,9 @@ int journal_file_dispose(int dir_fd, const char *fname) { if (renameat(dir_fd, fname, dir_fd, p) < 0) return -errno; - /* btrfs doesn't cope well with our write pattern and fragments heavily. Let's defrag all files we rotate */ - fd = openat(dir_fd, p, O_RDONLY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW); - if (fd < 0) - log_debug_errno(errno, "Failed to open file for defragmentation/FS_NOCOW_FL, ignoring: %m"); - else { - (void) chattr_fd(fd, 0, FS_NOCOW_FL, NULL); - (void) btrfs_defrag_fd(fd); - } - return 0; } -int journal_file_open_reliably( - const char *fname, - int flags, - mode_t mode, - bool compress, - uint64_t compress_threshold_bytes, - bool seal, - JournalMetrics *metrics, - MMapCache *mmap_cache, - Set *deferred_closes, - JournalFile *template, - JournalFile **ret) { - - int r; - - r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache, - deferred_closes, template, ret); - if (!IN_SET(r, - -EBADMSG, /* Corrupted */ - -ENODATA, /* Truncated */ - -EHOSTDOWN, /* Other machine */ - -EPROTONOSUPPORT, /* Incompatible feature */ - -EBUSY, /* Unclean shutdown */ - -ESHUTDOWN, /* Already archived */ - -EIO, /* IO error, including SIGBUS on mmap */ - -EIDRM, /* File has been deleted */ - -ETXTBSY)) /* File is from the future */ - return r; - - if ((flags & O_ACCMODE) == O_RDONLY) - return r; - - if (!(flags & O_CREAT)) - return r; - - if (!endswith(fname, ".journal")) - return r; - - /* The file is corrupted. Rotate it away and try it again (but only once) */ - log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); - - r = journal_file_dispose(AT_FDCWD, fname); - if (r < 0) - return r; - - return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache, - deferred_closes, template, ret); -} - int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p) { uint64_t q, n, xor_hash = 0; const sd_id128_t *boot_id; diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 81b2ff23e1..ecda2b3cc0 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -68,7 +68,6 @@ typedef struct JournalFile { bool compress_lz4:1; bool compress_zstd:1; bool seal:1; - bool defrag_on_close:1; bool close_fd:1; bool archive:1; bool keyed_hash:1; @@ -138,29 +137,14 @@ int journal_file_open( bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, - Set *deferred_closes, JournalFile *template, JournalFile **ret); -int journal_file_set_offline(JournalFile *f, bool wait); -bool journal_file_is_offlining(JournalFile *f); +int journal_file_set_offline_thread_join(JournalFile *f); JournalFile* journal_file_close(JournalFile *j); int journal_file_fstat(JournalFile *f); DEFINE_TRIVIAL_CLEANUP_FUNC(JournalFile*, journal_file_close); -int journal_file_open_reliably( - const char *fname, - int flags, - mode_t mode, - bool compress, - uint64_t compress_threshold_bytes, - bool seal, - JournalMetrics *metrics, - MMapCache *mmap_cache, - Set *deferred_closes, - JournalFile *template, - JournalFile **ret); - #define ALIGN64(x) (((x) + 7ULL) & ~7ULL) #define VALID64(x) (((x) & 7ULL) == 0ULL) @@ -204,6 +188,8 @@ static inline bool VALID_EPOCH(uint64_t u) { int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret); int journal_file_read_object(JournalFile *f, ObjectType type, uint64_t offset, Object *ret); +int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset); + uint64_t journal_file_entry_n_items(Object *o) _pure_; uint64_t journal_file_entry_array_n_items(Object *o) _pure_; uint64_t journal_file_hash_table_n_items(Object *o) _pure_; @@ -245,9 +231,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 void journal_file_dump(JournalFile *f); void journal_file_print_header(JournalFile *f); -int journal_file_archive(JournalFile *f); -JournalFile* journal_initiate_close(JournalFile *f, Set *deferred_closes); -int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes); +int journal_file_archive(JournalFile *f, char **ret_previous_path); int journal_file_dispose(int dir_fd, const char *fname); diff --git a/src/libsystemd/sd-journal/sd-journal.c b/src/libsystemd/sd-journal/sd-journal.c index 4861726673..b3f14cc548 100644 --- a/src/libsystemd/sd-journal/sd-journal.c +++ b/src/libsystemd/sd-journal/sd-journal.c @@ -1334,7 +1334,7 @@ static int add_any_file( goto finish; } - r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f); + r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, &f); if (r < 0) { log_debug_errno(r, "Failed to open journal file %s: %m", path); goto finish; |