diff options
author | OpenShift Merge Robot <openshift-merge-robot@users.noreply.github.com> | 2020-01-27 08:25:59 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-01-27 08:25:59 -0800 |
commit | 8a9a4965012f034dba81e68d5cc1bc11957d2a84 (patch) | |
tree | 77da87683c5e0b652b2f83d25cb2e587bc854ae5 | |
parent | 08e292bc1469f5405ac91df8771b472553cf8ba9 (diff) | |
parent | 97c831dd5fe509483939cdd40703a0ce8e0e9bfd (diff) | |
download | ostree-8a9a4965012f034dba81e68d5cc1bc11957d2a84.tar.gz |
Merge pull request #1957 from dbnicholson/commit-sizes
Upstream Endless sizes metadata changes
-rw-r--r-- | Makefile-tests.am | 6 | ||||
-rw-r--r-- | apidoc/ostree-sections.txt | 7 | ||||
-rw-r--r-- | bash/ostree | 1 | ||||
-rw-r--r-- | man/ostree-show.xml | 11 | ||||
-rw-r--r-- | src/libostree/libostree-devel.sym | 6 | ||||
-rw-r--r-- | src/libostree/ostree-autocleanups.h | 1 | ||||
-rw-r--r-- | src/libostree/ostree-core-private.h | 3 | ||||
-rw-r--r-- | src/libostree/ostree-core.c | 182 | ||||
-rw-r--r-- | src/libostree/ostree-core.h | 37 | ||||
-rw-r--r-- | src/libostree/ostree-repo-commit.c | 125 | ||||
-rw-r--r-- | src/libostree/ostree-repo-libarchive.c | 2 | ||||
-rw-r--r-- | src/libostree/ostree-repo-private.h | 14 | ||||
-rw-r--r-- | src/ostree/ot-builtin-show.c | 69 | ||||
-rwxr-xr-x | tests/test-libarchive.sh | 16 | ||||
-rwxr-xr-x | tests/test-pull-sizes.sh | 58 | ||||
-rwxr-xr-x | tests/test-sizes.js | 189 |
16 files changed, 668 insertions, 59 deletions
diff --git a/Makefile-tests.am b/Makefile-tests.am index 553f535c..83b0f1a2 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -86,6 +86,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-resume.sh \ tests/test-pull-basicauth.sh \ tests/test-pull-repeated.sh \ + tests/test-pull-sizes.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ tests/test-pull-localcache.sh \ @@ -354,7 +355,10 @@ tests_test_varint_LDADD = $(TESTS_LDADD) tests_test_bsdiff_CFLAGS = $(TESTS_CFLAGS) tests_test_bsdiff_LDADD = libbsdiff.la $(TESTS_LDADD) -tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c +tests_test_checksum_SOURCES = \ + src/libostree/ostree-core.c \ + src/libostree/ostree-varint.c \ + tests/test-checksum.c tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) tests_test_checksum_LDADD = $(TESTS_LDADD) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 1ef4bbf6..32cf5228 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -151,7 +151,14 @@ ostree_validate_structureof_dirmeta ostree_commit_get_parent ostree_commit_get_timestamp ostree_commit_get_content_checksum +ostree_commit_get_object_sizes +OstreeCommitSizesEntry +ostree_commit_sizes_entry_new +ostree_commit_sizes_entry_copy +ostree_commit_sizes_entry_free ostree_check_version +<SUBSECTION Standard> +ostree_commit_sizes_entry_get_type </SECTION> <SECTION> diff --git a/bash/ostree b/bash/ostree index fc429983..4aec588b 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1445,6 +1445,7 @@ _ostree_show() { local boolean_options=" $main_boolean_options --print-related + --print-sizes --raw " diff --git a/man/ostree-show.xml b/man/ostree-show.xml index a3d9aa4a..a28f704c 100644 --- a/man/ostree-show.xml +++ b/man/ostree-show.xml @@ -100,6 +100,17 @@ Boston, MA 02111-1307, USA. </varlistentry> <varlistentry> + <term><option>--print-sizes</option></term> + + <listitem><para> + Show the commit size metadata. This in only supported for + commits that contain <varname>ostree.sizes</varname> + metadata. This can be included when creating commits with + <command>ostree commit --generate-sizes</command>. + </para></listitem> + </varlistentry> + + <varlistentry> <term><option>--raw</option></term> <listitem><para> diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index d1666176..ff5f52c4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,12 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.7 { +global: + ostree_commit_get_object_sizes; + ostree_commit_sizes_entry_copy; + ostree_commit_sizes_entry_free; + ostree_commit_sizes_entry_get_type; + ostree_commit_sizes_entry_new; ostree_sysroot_initialize; ostree_sysroot_is_booted; ostree_sysroot_set_mount_namespace_in_use; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index c07f88a8..c9692ebe 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -49,6 +49,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoDevInoCache, ostree_repo_devino_cache_u G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeAsyncProgress, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeBootconfigParser, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCommitSizesEntry, ostree_commit_sizes_entry_free) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeDeployment, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeGpgVerifyResult, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeKernelArgs, ostree_kernel_args_free) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 43cf22c4..c1a82386 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -102,6 +102,9 @@ _ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf) */ #define _OSTREE_LOOSE_PATH_MAX (256) +/* GVariant format for ostree.sizes metadata entries. */ +#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" + char * _ostree_get_relative_object_path (const char *checksum, OstreeObjectType type, diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 3d16757e..4667dd8f 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -32,6 +32,7 @@ #include "ostree.h" #include "ostree-core-private.h" #include "ostree-chain-input-stream.h" +#include "ostree-varint.h" #include "otutil.h" /* Generic ABI checks */ @@ -2430,6 +2431,187 @@ ostree_commit_get_content_checksum (GVariant *commit_variant) return g_strdup (hexdigest); } +G_DEFINE_BOXED_TYPE (OstreeCommitSizesEntry, ostree_commit_sizes_entry, + ostree_commit_sizes_entry_copy, ostree_commit_sizes_entry_free) + +/** + * ostree_commit_sizes_entry_new: + * @checksum: (not nullable): object checksum + * @objtype: object type + * @unpacked: unpacked object size + * @archived: compressed object size + * + * Create a new #OstreeCommitSizesEntry for representing an object in a + * commit's "ostree.sizes" metadata. + * + * Returns: (transfer full) (nullable): a new #OstreeCommitSizesEntry + * Since: 2019.7 + */ +OstreeCommitSizesEntry * +ostree_commit_sizes_entry_new (const gchar *checksum, + OstreeObjectType objtype, + guint64 unpacked, + guint64 archived) +{ + g_return_val_if_fail (checksum == NULL || ostree_validate_checksum_string (checksum, NULL), NULL); + + g_autoptr(OstreeCommitSizesEntry) entry = g_new0 (OstreeCommitSizesEntry, 1); + entry->checksum = g_strdup (checksum); + entry->objtype = objtype; + entry->unpacked = unpacked; + entry->archived = archived; + + return g_steal_pointer (&entry); +} + +/** + * ostree_commit_sizes_entry_copy: + * @entry: (not nullable): an #OstreeCommitSizesEntry + * + * Create a copy of the given @entry. + * + * Returns: (transfer full) (nullable): a new copy of @entry + * Since: 2019.7 + */ +OstreeCommitSizesEntry * +ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry) +{ + g_return_val_if_fail (entry != NULL, NULL); + + return ostree_commit_sizes_entry_new (entry->checksum, + entry->objtype, + entry->unpacked, + entry->archived); +} + +/** + * ostree_commit_sizes_entry_free: + * @entry: (transfer full): an #OstreeCommitSizesEntry + * + * Free given @entry. + * + * Since: 2019.7 + */ +void +ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry) +{ + g_return_if_fail (entry != NULL); + + g_free (entry->checksum); + g_free (entry); +} + +static gboolean +read_sizes_entry (GVariant *entry, + OstreeCommitSizesEntry **out_sizes, + GError **error) +{ + gsize entry_size = g_variant_get_size (entry); + g_return_val_if_fail (entry_size >= OSTREE_SHA256_DIGEST_LEN + 2, FALSE); + + const guchar *buffer = g_variant_get_data (entry); + if (buffer == NULL) + return glnx_throw (error, "Could not read ostree.sizes metadata entry"); + + char checksum[OSTREE_SHA256_STRING_LEN + 1]; + ostree_checksum_inplace_from_bytes (buffer, checksum); + buffer += OSTREE_SHA256_DIGEST_LEN; + entry_size -= OSTREE_SHA256_DIGEST_LEN; + + gsize bytes_read = 0; + guint64 archived = 0; + if (!_ostree_read_varuint64 (buffer, entry_size, &archived, &bytes_read)) + return glnx_throw (error, "Unexpected EOF reading ostree.sizes varint"); + buffer += bytes_read; + entry_size -= bytes_read; + + guint64 unpacked = 0; + if (!_ostree_read_varuint64 (buffer, entry_size, &unpacked, &bytes_read)) + return glnx_throw (error, "Unexpected EOF reading ostree.sizes varint"); + buffer += bytes_read; + entry_size -= bytes_read; + + /* On newer commits, an additional byte is used for the object type. */ + OstreeObjectType objtype; + if (entry_size > 0) + { + objtype = *buffer; + if (objtype < OSTREE_OBJECT_TYPE_FILE || objtype > OSTREE_OBJECT_TYPE_LAST) + return glnx_throw (error, "Unexpected ostree.sizes object type %u", + objtype); + buffer++; + entry_size--; + } + else + { + /* Assume the object is a file. */ + objtype = OSTREE_OBJECT_TYPE_FILE; + } + + g_autoptr(OstreeCommitSizesEntry) sizes = ostree_commit_sizes_entry_new (checksum, + objtype, + unpacked, + archived); + + if (out_sizes != NULL) + *out_sizes = g_steal_pointer (&sizes); + + return TRUE; +} + +/** + * ostree_commit_get_object_sizes: + * @commit_variant: (not nullable): variant of type %OSTREE_OBJECT_TYPE_COMMIT + * @out_sizes_entries: (out) (element-type OstreeCommitSizesEntry) (transfer container) (optional): + * return location for an array of object size entries + * @error: Error + * + * Reads a commit's "ostree.sizes" metadata and returns an array of + * #OstreeCommitSizesEntry in @out_sizes_entries. Each element + * represents an object in the commit. If the commit does not contain + * the "ostree.sizes" metadata, a %G_IO_ERROR_NOT_FOUND error will be + * returned. + * + * Since: 2019.7 + */ +gboolean +ostree_commit_get_object_sizes (GVariant *commit_variant, + GPtrArray **out_sizes_entries, + GError **error) +{ + g_return_val_if_fail (commit_variant != NULL, FALSE); + + g_autoptr(GVariant) metadata = g_variant_get_child_value (commit_variant, 0); + g_autoptr(GVariant) sizes_variant = + g_variant_lookup_value (metadata, "ostree.sizes", + G_VARIANT_TYPE ("a" _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE)); + if (sizes_variant == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No metadata key ostree.sizes in commit"); + return FALSE; + } + + g_autoptr(GPtrArray) sizes_entries = + g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_commit_sizes_entry_free); + g_autoptr(GVariant) entry = NULL; + GVariantIter entry_iter; + g_variant_iter_init (&entry_iter, sizes_variant); + while ((entry = g_variant_iter_next_value (&entry_iter))) + { + OstreeCommitSizesEntry *sizes_entry = NULL; + if (!read_sizes_entry (entry, &sizes_entry, error)) + return FALSE; + g_clear_pointer (&entry, g_variant_unref); + g_ptr_array_add (sizes_entries, sizes_entry); + } + + if (out_sizes_entries != NULL) + *out_sizes_entries = g_steal_pointer (&sizes_entries); + + return TRUE; +} + /* Used in pull/deploy to validate we're not being downgraded */ gboolean _ostree_compare_timestamps (const char *current_rev, diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 69477a75..10601123 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -521,6 +521,43 @@ guint64 ostree_commit_get_timestamp (GVariant *commit_variant); _OSTREE_PUBLIC gchar * ostree_commit_get_content_checksum (GVariant *commit_variant); +/** + * OstreeCommitSizesEntry: + * @checksum: (not nullable): object checksum + * @objtype: object type + * @unpacked: unpacked object size + * @archived: compressed object size + * + * Structure representing an entry in the "ostree.sizes" commit metadata. Each + * entry corresponds to an object in the associated commit. + * + * Since: 2019.5 + */ +typedef struct { + gchar *checksum; + OstreeObjectType objtype; + guint64 unpacked; + guint64 archived; +} OstreeCommitSizesEntry; + +_OSTREE_PUBLIC +GType ostree_commit_sizes_entry_get_type (void); + +_OSTREE_PUBLIC +OstreeCommitSizesEntry *ostree_commit_sizes_entry_new (const gchar *checksum, + OstreeObjectType objtype, + guint64 unpacked, + guint64 archived); +_OSTREE_PUBLIC +OstreeCommitSizesEntry *ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry); +_OSTREE_PUBLIC +void ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry); + +_OSTREE_PUBLIC +gboolean ostree_commit_get_object_sizes (GVariant *commit_variant, + GPtrArray **out_sizes_entries, + GError **error); + _OSTREE_PUBLIC gboolean ostree_check_version (guint required_year, guint required_release); diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8c5d9411..87b585fd 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -322,16 +322,19 @@ commit_loose_regfile_object (OstreeRepo *self, /* This is used by OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES */ typedef struct { + OstreeObjectType objtype; goffset unpacked; goffset archived; } OstreeContentSizeCacheEntry; static OstreeContentSizeCacheEntry * -content_size_cache_entry_new (goffset unpacked, - goffset archived) +content_size_cache_entry_new (OstreeObjectType objtype, + goffset unpacked, + goffset archived) { OstreeContentSizeCacheEntry *entry = g_slice_new0 (OstreeContentSizeCacheEntry); + entry->objtype = objtype; entry->unpacked = unpacked; entry->archived = archived; @@ -345,19 +348,61 @@ content_size_cache_entry_free (gpointer entry) g_slice_free (OstreeContentSizeCacheEntry, entry); } +void +_ostree_repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier) +{ + if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) + { + if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE) + { + self->generate_sizes = TRUE; + + /* Clear any stale data in the object sizes hash table */ + if (self->object_sizes != NULL) + g_hash_table_remove_all (self->object_sizes); + } + else + g_debug ("Not generating sizes for non-archive repo"); + } +} + +static void +repo_ensure_size_entries (OstreeRepo *self) +{ + if (G_UNLIKELY (self->object_sizes == NULL)) + self->object_sizes = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, content_size_cache_entry_free); +} + +static gboolean +repo_has_size_entry (OstreeRepo *self, + OstreeObjectType objtype, + const gchar *checksum) +{ + /* Only file, dirtree and dirmeta objects appropriate for size metadata */ + if (objtype > OSTREE_OBJECT_TYPE_DIR_META) + return TRUE; + + repo_ensure_size_entries (self); + return (g_hash_table_lookup (self->object_sizes, checksum) != NULL); +} + static void repo_store_size_entry (OstreeRepo *self, + OstreeObjectType objtype, const gchar *checksum, goffset unpacked, goffset archived) { - if (G_UNLIKELY (self->object_sizes == NULL)) - self->object_sizes = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, content_size_cache_entry_free); + /* Only file, dirtree and dirmeta objects appropriate for size metadata */ + if (objtype > OSTREE_OBJECT_TYPE_DIR_META) + return; + repo_ensure_size_entries (self); g_hash_table_replace (self->object_sizes, g_strdup (checksum), - content_size_cache_entry_new (unpacked, archived)); + content_size_cache_entry_new (objtype, unpacked, archived)); } static int @@ -408,6 +453,7 @@ add_size_index_to_metadata (OstreeRepo *self, g_hash_table_lookup (self->object_sizes, e_checksum); _ostree_write_varuint64 (buffer, e_size->archived); _ostree_write_varuint64 (buffer, e_size->unpacked); + g_string_append_c (buffer, (gchar) e_size->objtype); g_variant_builder_add (&index_builder, "@ay", ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len)); @@ -415,6 +461,9 @@ add_size_index_to_metadata (OstreeRepo *self, g_variant_builder_add (builder, "{sv}", "ostree.sizes", g_variant_builder_end (&index_builder)); + + /* Clear the object sizes hash table for a subsequent commit. */ + g_hash_table_remove_all (self->object_sizes); } return g_variant_ref_sink (g_variant_builder_end (builder)); @@ -956,7 +1005,6 @@ write_content_object (OstreeRepo *self, g_auto(OtCleanupUnlinkat) tmp_unlinker = { commit_tmp_dfd (self), NULL }; g_auto(GLnxTmpfile) tmpf = { 0, }; goffset unpacked_size = 0; - gboolean indexable = FALSE; /* Is it a symlink physically? */ if (phys_object_is_symlink) { @@ -982,9 +1030,6 @@ write_content_object (OstreeRepo *self, g_assert (repo_mode == OSTREE_REPO_MODE_ARCHIVE); - if (self->generate_sizes) - indexable = TRUE; - if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, &tmpf, error)) return FALSE; @@ -1013,6 +1058,11 @@ write_content_object (OstreeRepo *self, unpacked_size = g_file_info_get_size (file_info); } + else + { + /* For a symlink, the size is the length of the target */ + unpacked_size = strlen (g_file_info_get_symlink_target (file_info)); + } if (!g_output_stream_flush (temp_out, cancellable, error)) return FALSE; @@ -1042,6 +1092,19 @@ write_content_object (OstreeRepo *self, g_assert (actual_checksum != NULL); /* Pacify static analysis */ + /* Update size metadata if configured and entry missing */ + if (self->generate_sizes && + !repo_has_size_entry (self, OSTREE_OBJECT_TYPE_FILE, actual_checksum)) + { + struct stat stbuf; + + if (!glnx_fstat (tmpf.fd, &stbuf, error)) + return FALSE; + + repo_store_size_entry (self, OSTREE_OBJECT_TYPE_FILE, actual_checksum, + unpacked_size, stbuf.st_size); + } + /* See whether or not we have the object, now that we know the * checksum. */ @@ -1107,17 +1170,6 @@ write_content_object (OstreeRepo *self, } else { - /* Update size metadata if configured */ - if (indexable && object_file_type == G_FILE_TYPE_REGULAR) - { - struct stat stbuf; - - if (!glnx_fstat (tmpf.fd, &stbuf, error)) - return FALSE; - - repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); - } - /* Check if a file with the same payload is present in the repository, and in case try to reflink it */ if (actual_payload_checksum && !_try_clone_from_payload_link (self, self, actual_payload_checksum, file_info, &tmpf, cancellable, error)) @@ -1309,6 +1361,11 @@ write_metadata_object (OstreeRepo *self, */ if (have_obj) { + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, actual_checksum)) + repo_store_size_entry (self, objtype, actual_checksum, len, len); + g_mutex_lock (&self->txn_lock); self->txn.stats.metadata_objects_total++; g_mutex_unlock (&self->txn_lock); @@ -1330,6 +1387,11 @@ write_metadata_object (OstreeRepo *self, gsize len; const guint8 *bufp = g_bytes_get_data (buf, &len); + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, actual_checksum)) + repo_store_size_entry (self, objtype, actual_checksum, len, len); + /* Write the metadata to a temporary file */ g_auto(GLnxTmpfile) tmpf = { 0, }; if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, @@ -2345,6 +2407,16 @@ ostree_repo_write_metadata (OstreeRepo *self, return FALSE; if (have_obj) { + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, expected_checksum)) + { + /* Make sure we have a fully serialized object */ + g_autoptr(GVariant) trusted = g_variant_get_normal_form (object); + gsize size = g_variant_get_size (trusted); + repo_store_size_entry (self, objtype, expected_checksum, size, size); + } + if (out_csum) *out_csum = ostree_checksum_to_bytes (expected_checksum); return TRUE; @@ -2620,8 +2692,11 @@ ostree_repo_write_content (OstreeRepo *self, { /* First, if we have an expected checksum, see if we already have this * object. This mirrors the same logic in ostree_repo_write_metadata(). + * + * If size metadata is needed, fall through to write_content_object() + * where the entries are made. */ - if (expected_checksum) + if (expected_checksum && !self->generate_sizes) { gboolean have_obj; if (!_ostree_repo_has_loose_object (self, expected_checksum, @@ -3848,8 +3923,7 @@ ostree_repo_write_directory_to_mtree (OstreeRepo *self, } else { - if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) - self->generate_sizes = TRUE; + _ostree_repo_setup_generate_sizes (self, modifier); g_autoptr(GPtrArray) path = g_ptr_array_new (); if (!write_directory_to_mtree_internal (self, dir, mtree, modifier, path, @@ -3883,8 +3957,7 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error) { - if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) - self->generate_sizes = TRUE; + _ostree_repo_setup_generate_sizes (self, modifier); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error)) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 1850f99f..d55459f4 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -844,6 +844,8 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, .modifier = modifier }; + _ostree_repo_setup_generate_sizes (self, modifier); + while (TRUE) { int r = archive_read_next_header (a, &aictx.entry); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b57ad799..0465327c 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -31,8 +31,6 @@ G_BEGIN_DECLS #define OSTREE_DELTAPART_VERSION (0) -#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" - #define _OSTREE_SUMMARY_CACHE_DIR "summaries" #define _OSTREE_CACHE_DIR "cache" @@ -143,6 +141,14 @@ struct OstreeRepo { guint zlib_compression_level; GHashTable *loose_object_devino_hash; GHashTable *updated_uncompressed_dirs; + + /* FIXME: The object sizes hash table is really per-commit state, not repo + * state. Using a single table for the repo means that commits cannot be + * built simultaneously if they're adding size information. This data should + * probably be in OstreeMutableTree, but that's gone by the time the actual + * commit is constructed. At that point the only commit state is in the root + * OstreeRepoFile. + */ GHashTable *object_sizes; /* Cache the repo's device/inode to use for comparisons elsewhere */ @@ -329,6 +335,10 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, GFileInfo *file_info, GFileInfo **out_modified_info); +void +_ostree_repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier); + gboolean _ostree_repo_remote_name_is_file (const char *remote_name); diff --git a/src/ostree/ot-builtin-show.c b/src/ostree/ot-builtin-show.c index 5091a93c..96e2d4c6 100644 --- a/src/ostree/ot-builtin-show.c +++ b/src/ostree/ot-builtin-show.c @@ -33,6 +33,7 @@ static gboolean opt_print_related; static char* opt_print_variant_type; static char* opt_print_metadata_key; static char* opt_print_detached_metadata_key; +static gboolean opt_print_sizes; static gboolean opt_raw; static gboolean opt_no_byteswap; static char *opt_gpg_homedir; @@ -48,6 +49,7 @@ static GOptionEntry options[] = { { "print-variant-type", 0, 0, G_OPTION_ARG_STRING, &opt_print_variant_type, "Memory map OBJECT (in this case a filename) to the GVariant type string", "TYPE" }, { "print-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_metadata_key, "Print string value of metadata key", "KEY" }, { "print-detached-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_detached_metadata_key, "Print string value of detached metadata key", "KEY" }, + { "print-sizes", 0, 0, G_OPTION_ARG_NONE, &opt_print_sizes, "Show the commit size metadata", NULL }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data" }, { "no-byteswap", 'B', 0, G_OPTION_ARG_NONE, &opt_no_byteswap, "Do not automatically convert variant data from big endian" }, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, @@ -147,6 +149,65 @@ do_print_metadata_key (OstreeRepo *repo, } static gboolean +do_print_sizes (OstreeRepo *repo, + const char *rev, + GError **error) +{ + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, + &commit, error)) + { + g_prefix_error (error, "Failed to read commit: "); + return FALSE; + } + + g_autoptr(GPtrArray) sizes = NULL; + if (!ostree_commit_get_object_sizes (commit, &sizes, error)) + return FALSE; + + gint64 new_archived = 0; + gint64 new_unpacked = 0; + gsize new_objects = 0; + gint64 archived = 0; + gint64 unpacked = 0; + gsize objects = 0; + for (guint i = 0; i < sizes->len; i++) + { + OstreeCommitSizesEntry *entry = sizes->pdata[i]; + + archived += entry->archived; + unpacked += entry->unpacked; + objects++; + + gboolean exists; + if (!ostree_repo_has_object (repo, entry->objtype, entry->checksum, + &exists, NULL, error)) + return FALSE; + + if (!exists) + { + /* Object not in local repo */ + new_archived += entry->archived; + new_unpacked += entry->unpacked; + new_objects++; + } + } + + g_autofree char *new_archived_str = g_format_size (new_archived); + g_autofree char *archived_str = g_format_size (archived); + g_autofree char *new_unpacked_str = g_format_size (new_unpacked); + g_autofree char *unpacked_str = g_format_size (unpacked); + g_print ("Compressed size (needed/total): %s/%s\n" + "Unpacked size (needed/total): %s/%s\n" + "Number of objects (needed/total): %" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT "\n", + new_archived_str, archived_str, + new_unpacked_str, unpacked_str, + new_objects, objects); + + return TRUE; +} + +static gboolean print_object (OstreeRepo *repo, OstreeObjectType objtype, const char *checksum, @@ -279,6 +340,14 @@ ostree_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, if (!do_print_variant_generic (G_VARIANT_TYPE (opt_print_variant_type), rev, error)) return FALSE; } + else if (opt_print_sizes) + { + if (!ostree_repo_resolve_rev (repo, rev, FALSE, &resolved_rev, error)) + return FALSE; + + if (!do_print_sizes (repo, resolved_rev, error)) + return FALSE; + } else { gboolean found = FALSE; diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index 24de55b2..174be800 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -28,7 +28,7 @@ fi . $(dirname $0)/libtest.sh -echo "1..17" +echo "1..18" setup_test_repository "bare" @@ -234,3 +234,17 @@ for filter in '^usr/bin/,usr/sbin/' '/bin/,/sbin/'; do assert_file_has_content usr/lib/libfoo.so 'a library' echo "ok tar pathname filter modification: ${filter}" done + +# Test sizes metadata. This needs an archive repo, so a separate repo is used. +cd ${test_tmpdir} +rm -rf repo2 +ostree_repo_init repo2 --mode=archive +${CMD_PREFIX} ostree --repo=repo2 commit \ + -s "from tar" -b test-tar \ + --generate-sizes \ + --tree=tar=foo.tar.gz +${CMD_PREFIX} ostree --repo=repo2 show --print-sizes test-tar > sizes.txt +assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[ ]bytes/1.1[ ]kB' +assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[ ]bytes/900[ ]bytes' +assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/12' +echo "ok tar sizes metadata" diff --git a/tests/test-pull-sizes.sh b/tests/test-pull-sizes.sh new file mode 100755 index 00000000..8ee07cc8 --- /dev/null +++ b/tests/test-pull-sizes.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (C) 2019 Endless Mobile, Inc. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" "--generate-sizes" + +echo '1..3' + +cd ${test_tmpdir} +mkdir repo +ostree_repo_init repo +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo + +# Pull commit metadata only. All size and objects will be needed. +${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 637[ ]bytes/637[ ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 457[ ]bytes/457[ ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 10/10' +echo "ok sizes commit metadata only" + +# Pull the parent commit so we get most of the objects +parent=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main^) +${CMD_PREFIX} ostree --repo=repo pull origin ${parent} +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 501[ ]bytes/637[ ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 429[ ]bytes/457[ ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 6/10' +echo "ok sizes commit partial" + +# Finish pulling the commit and check that no objects needed +${CMD_PREFIX} ostree --repo=repo pull origin main +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 0[ ]bytes/637[ ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 0[ ]bytes/457[ ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 0/10' +echo "ok sizes commit full" diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 73b179c5..a2246536 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -28,11 +28,110 @@ function assertEquals(a, b) { throw new Error("assertion failed " + JSON.stringify(a) + " == " + JSON.stringify(b)); } -print('1..1') +function assertGreater(a, b) { + if (a <= b) + throw new Error("assertion failed " + JSON.stringify(a) + " > " + JSON.stringify(b)); +} + +function assertGreaterEquals(a, b) { + if (a < b) + throw new Error("assertion failed " + JSON.stringify(a) + " >= " + JSON.stringify(b)); +} + +// Adapted from _ostree_read_varuint64() +function readVarint(buffer) { + let result = 0; + let count = 0; + let len = buffer.length; + let cur; + + do { + assertGreater(len, 0); + cur = buffer[count]; + result = result | ((cur & 0x7F) << (7 * count)); + count++; + len--; + } while (cur & 0x80); + + return [result, count]; +} + +// There have been various bugs with byte array unpacking in GJS, so +// just do it manually. +function unpackByteArray(variant) { + let array = []; + let nBytes = variant.n_children(); + for (let i = 0; i < nBytes; i++) { + array.push(variant.get_child_value(i).get_byte()); + } + return array; +} + +function validateSizes(repo, commit, expectedObjects) { + let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); + let metadata = commitVariant.get_child_value(0); + let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); + let nObjects = sizes.n_children(); + let expectedNObjects = Object.keys(expectedObjects).length + assertEquals(nObjects, expectedNObjects); + + for (let i = 0; i < nObjects; i++) { + let sizeEntry = sizes.get_child_value(i); + assertGreaterEquals(sizeEntry.n_children(), 34); + let entryBytes = unpackByteArray(sizeEntry); + let checksumBytes = entryBytes.slice(0, 32); + let checksumString = OSTree.checksum_from_bytes(checksumBytes); + print("checksum = " + checksumString); + + // Read the sizes from the next 2 varints + let remainingBytes = entryBytes.slice(32); + assertGreaterEquals(remainingBytes.length, 2); + let varintRead; + let compressedSize; + let uncompressedSize; + [compressedSize, varintRead] = readVarint(remainingBytes); + remainingBytes = remainingBytes.slice(varintRead); + assertGreaterEquals(remainingBytes.length, 1); + [uncompressedSize, varintRead] = readVarint(remainingBytes); + remainingBytes = remainingBytes.slice(varintRead); + assertEquals(remainingBytes.length, 1); + let objectType = remainingBytes[0]; + let objectTypeString = OSTree.object_type_to_string(objectType); + print("compressed = " + compressedSize); + print("uncompressed = " + uncompressedSize); + print("objtype = " + objectTypeString + " (" + objectType + ")"); + let objectName = OSTree.object_to_string(checksumString, objectType); + print("object = " + objectName); + + if (!(objectName in expectedObjects)) { + throw new Error("Object " + objectName + " not in " + + JSON.stringify(expectedObjects)); + } + let expectedSizes = expectedObjects[objectName]; + let expectedCompressedSize = expectedSizes[0]; + let expectedUncompressedSize = expectedSizes[1]; + if (compressedSize != expectedCompressedSize) { + throw new Error("Compressed size " + compressedSize + + " for checksum " + checksumString + + " does not match expected " + expectedCompressedSize); + } + if (uncompressedSize != expectedUncompressedSize) { + throw new Error("Uncompressed size " + uncompressedSize + + " for checksum " + checksumString + + " does not match expected " + expectedUncompressedSize); + } + } +} + +print('1..3') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); testDataDir.get_child('some-file').replace_contents("hello world!", null, false, 0, null); +testDataDir.get_child('some-file').copy(testDataDir.get_child('duplicate-file'), + Gio.FileCopyFlags.OVERWRITE, + null, null); +testDataDir.get_child('link-file').make_symbolic_link('some-file', null); testDataDir.get_child('another-file').replace_contents("hello world again!", null, false, 0, null); let repoPath = Gio.File.new_for_path('repo'); @@ -41,7 +140,10 @@ repo.create(OSTree.RepoMode.ARCHIVE_Z2, null); repo.open(null); -let commitModifier = OSTree.RepoCommitModifier.new(OSTree.RepoCommitModifierFlags.GENERATE_SIZES, null); +let commitModifierFlags = (OSTree.RepoCommitModifierFlags.GENERATE_SIZES | + OSTree.RepoCommitModifierFlags.SKIP_XATTRS | + OSTree.RepoCommitModifierFlags.CANONICAL_PERMISSIONS); +let commitModifier = OSTree.RepoCommitModifier.new(commitModifierFlags, null); assertEquals(repo.get_mode(), OSTree.RepoMode.ARCHIVE_Z2); @@ -55,32 +157,61 @@ print("commit => " + commit); repo.commit_transaction(null); -// Test the sizes metadata -let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); -let metadata = commitVariant.get_child_value(0); -let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); -let nSizes = sizes.n_children(); -assertEquals(nSizes, 2); -let expectedUncompressedSizes = [12, 18]; -let foundExpectedUncompressedSizes = 0; -for (let i = 0; i < nSizes; i++) { - let sizeEntry = sizes.get_child_value(i); - assertEquals(sizeEntry.n_children(), 34); - let compressedSize = sizeEntry.get_child_value(32).get_byte(); - let uncompressedSize = sizeEntry.get_child_value(33).get_byte(); - print("compressed = " + compressedSize); - print("uncompressed = " + uncompressedSize); - for (let j = 0; j < expectedUncompressedSizes.length; j++) { - let expected = expectedUncompressedSizes[j]; - if (expected == uncompressedSize) { - print("Matched expected uncompressed size " + expected); - expectedUncompressedSizes.splice(j, 1); - break; - } - } -} -if (expectedUncompressedSizes.length > 0) { - throw new Error("Failed to match expectedUncompressedSizes: " + JSON.stringify(expectedUncompressedSizes)); -} +// Test the sizes metadata. The key is the object and the value is an +// array of compressed size and uncompressed size. +let expectedObjects = { + 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [ + 54, 18 + ], + 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [ + 48, 12 + ], + '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [ + 43, 9 + ], + '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [ + 185, 185 + ], + '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [ + 12, 12 + ], +}; +validateSizes(repo, commit, expectedObjects); print("ok test-sizes"); + +// Remove a file to make sure that metadata is not reused from the +// previous commit. Remove that file from the expected metadata and +// replace the dirtree object. +testDataDir.get_child('another-file').delete(null); +delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file']; +delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree']; +expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [ + 138, 138 +]; + +repo.prepare_transaction(null); +mtree = OSTree.MutableTree.new(); +repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); +[,dirTree] = repo.write_mtree(mtree, null); +[,commit] = repo.write_commit(null, 'Some subject', 'Some body', null, dirTree, null); +print("commit => " + commit); +repo.commit_transaction(null); + +validateSizes(repo, commit, expectedObjects); + +print("ok test-sizes file deleted"); + +// Repeat the commit now that all the objects are cached and ensure the +// metadata is still correct +repo.prepare_transaction(null); +mtree = OSTree.MutableTree.new(); +repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); +[,dirTree] = repo.write_mtree(mtree, null); +[,commit] = repo.write_commit(null, 'Another subject', 'Another body', null, dirTree, null); +print("commit => " + commit); +repo.commit_transaction(null); + +validateSizes(repo, commit, expectedObjects); + +print("ok test-sizes repeated"); |