diff options
author | Colin Walters <walters@verbum.org> | 2011-12-21 21:42:13 -0500 |
---|---|---|
committer | Colin Walters <walters@verbum.org> | 2011-12-21 21:42:13 -0500 |
commit | e9fd921afe09266047d8ebbff633ada95c3f55f1 (patch) | |
tree | 643c929e6d6a9a552e67304d5f6ba40f992a38ff | |
parent | add55849abf4f9f18442beebe0067aa50b55aa7f (diff) | |
download | ostree-e9fd921afe09266047d8ebbff633ada95c3f55f1.tar.gz |
core: Change compose to operate purely in-memory
This is *significantly* faster than checking out each branch into the
real filesystem, then importing it again.
-rw-r--r-- | src/libostree/ostree-mutable-tree.c | 27 | ||||
-rw-r--r-- | src/libostree/ostree-repo-file.c | 6 | ||||
-rw-r--r-- | src/libostree/ostree-repo-file.h | 2 | ||||
-rw-r--r-- | src/libostree/ostree-repo.c | 14 | ||||
-rw-r--r-- | src/ostree/ot-builtin-compose.c | 176 | ||||
-rwxr-xr-x | tests/t0004-compose.sh | 38 |
6 files changed, 128 insertions, 135 deletions
diff --git a/src/libostree/ostree-mutable-tree.c b/src/libostree/ostree-mutable-tree.c index 1c698b91..ca8be1ee 100644 --- a/src/libostree/ostree-mutable-tree.c +++ b/src/libostree/ostree-mutable-tree.c @@ -95,6 +95,31 @@ ostree_mutable_tree_set_contents_checksum (OstreeMutableTree *self, const char * ostree_mutable_tree_get_contents_checksum (OstreeMutableTree *self) { + GHashTableIter iter; + gpointer key, value; + + if (!self->contents_checksum) + return NULL; + + /* Ensure the cache is valid; this implementation is a bit + * lame in that we walk the whole tree every time this + * getter is called; a better approach would be to invalidate + * all of the parents whenever a child is modified. + * + * However, we only call this function once right now. + */ + g_hash_table_iter_init (&iter, self->subdirs); + while (g_hash_table_iter_next (&iter, &key, &value)) + { + OstreeMutableTree *subdir = value; + if (!ostree_mutable_tree_get_contents_checksum (subdir)) + { + g_free (self->contents_checksum); + self->contents_checksum = NULL; + return NULL; + } + } + return self->contents_checksum; } @@ -122,6 +147,7 @@ ostree_mutable_tree_replace_file (OstreeMutableTree *self, goto out; } + ostree_mutable_tree_set_contents_checksum (self, NULL); g_hash_table_replace (self->files, g_strdup (name), g_strdup (checksum)); @@ -153,6 +179,7 @@ ostree_mutable_tree_ensure_dir (OstreeMutableTree *self, if (!ret_dir) { ret_dir = ostree_mutable_tree_new (); + ostree_mutable_tree_set_contents_checksum (self, NULL); g_hash_table_insert (self->subdirs, g_strdup (name), g_object_ref (ret_dir)); } diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c index 083e770a..818bd6dc 100644 --- a/src/libostree/ostree-repo-file.c +++ b/src/libostree/ostree-repo-file.c @@ -280,6 +280,12 @@ ostree_repo_file_ensure_resolved (OstreeRepoFile *self, return TRUE; } +const char * +ostree_repo_file_get_commit (OstreeRepoFile *self) +{ + return ostree_repo_file_get_root (self)->commit; +} + gboolean ostree_repo_file_get_xattrs (OstreeRepoFile *self, GVariant **out_xattrs, diff --git a/src/libostree/ostree-repo-file.h b/src/libostree/ostree-repo-file.h index 1d3f7f47..44dda4c3 100644 --- a/src/libostree/ostree-repo-file.h +++ b/src/libostree/ostree-repo-file.h @@ -75,6 +75,8 @@ gboolean ostree_repo_file_is_tree (OstreeRepoFile *self); const char * ostree_repo_file_get_checksum (OstreeRepoFile *self); +const char * ostree_repo_file_get_commit (OstreeRepoFile *self); + GFile *ostree_repo_file_nontree_get_local (OstreeRepoFile *self); int ostree_repo_file_tree_find_child (OstreeRepoFile *self, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 493c7bcc..569dff5f 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1380,6 +1380,7 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo *self, GChecksum *child_file_checksum = NULL; GVariant *xattrs = NULL; GInputStream *file_input = NULL; + gboolean repo_dir_was_empty = FALSE; /* We can only reuse checksums directly if there's no modifier */ if (OSTREE_IS_REPO_FILE (dir) && modifier == NULL) @@ -1388,7 +1389,9 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo *self, if (repo_dir) { ostree_mutable_tree_set_metadata_checksum (mtree, ostree_repo_file_get_checksum (repo_dir)); - ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir)); + repo_dir_was_empty = + g_hash_table_size (ostree_mutable_tree_get_files (mtree)) == 0 + && g_hash_table_size (ostree_mutable_tree_get_subdirs (mtree)) == 0; } else { @@ -1484,6 +1487,9 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo *self, goto out; } + if (repo_dir && repo_dir_was_empty) + ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir)); + ret = TRUE; out: g_clear_object (&dir_enum); @@ -1593,10 +1599,12 @@ ostree_repo_stage_mtree (OstreeRepo *self, GVariant *serialized_tree = NULL; GHashTableIter hash_iter; gpointer key, value; + const char *existing_checksum; - if (ostree_mutable_tree_get_contents_checksum (mtree)) + existing_checksum = ostree_mutable_tree_get_contents_checksum (mtree); + if (existing_checksum) { - ret_contents_checksum = g_strdup (ostree_mutable_tree_get_contents_checksum (mtree)); + ret_contents_checksum = g_strdup (existing_checksum); } else { diff --git a/src/ostree/ot-builtin-compose.c b/src/ostree/ot-builtin-compose.c index a1769f66..03d3bc34 100644 --- a/src/ostree/ot-builtin-compose.c +++ b/src/ostree/ot-builtin-compose.c @@ -27,97 +27,43 @@ #include <glib/gi18n.h> -static char *compose_metadata_path; +static char *subject; +static char *body; +static char *branch; static GOptionEntry options[] = { - { "out-metadata", 0, 0, G_OPTION_ARG_FILENAME, &compose_metadata_path, "Output a file containing serialized metadata about the compose, in host endianness", "path" }, + { "subject", 's', 0, G_OPTION_ARG_STRING, &subject, "One line subject", "subject" }, + { "body", 'm', 0, G_OPTION_ARG_STRING, &body, "Full description", "body" }, + { "branch", 'b', 0, G_OPTION_ARG_STRING, &branch, "Branch", "branch" }, { NULL } }; -static void -rm_rf (GFile *path) -{ - GFileInfo *finfo = NULL; - GFileEnumerator *path_enum = NULL; - guint32 type; - - finfo = g_file_query_info (path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, NULL); - if (!finfo) - goto out; - - type = g_file_info_get_attribute_uint32 (finfo, "standard::type"); - if (type == G_FILE_TYPE_DIRECTORY) - { - path_enum = g_file_enumerate_children (path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, NULL); - if (!path_enum) - goto out; - - - g_clear_object (&finfo); - while ((finfo = g_file_enumerator_next_file (path_enum, NULL, NULL)) != NULL) - { - GFile *child = g_file_get_child (path, g_file_info_get_attribute_byte_string (finfo, "standard::name")); - rm_rf (child); - g_clear_object (&child); - g_clear_object (&finfo); - } - } - - (void) unlink (ot_gfile_get_path_cached (path)); - - out: - g_clear_object (&finfo); - g_clear_object (&path_enum); -} - static gboolean -compose_branch_on_dir (OstreeRepo *repo, - GFile *destination, - const char *branch, - GVariantBuilder *metadata_builder, - GError **error) +add_branch (OstreeRepo *repo, + OstreeMutableTree *mtree, + const char *branch, + GVariantBuilder *metadata_builder, + GError **error) { - char *destpath = NULL; - char *branchpath = NULL; - GFile *branchf = NULL; - GFileEnumerator *enumerator = NULL; gboolean ret = FALSE; - char *branchrev = NULL; + GFile *branchf = NULL; + const char *branch_rev; - if (!ostree_repo_resolve_rev (repo, branch, FALSE, &branchrev, error)) - goto out; - - destpath = g_strdup (ot_gfile_get_path_cached (destination)); - if (g_str_has_suffix (destpath, "/")) - destpath[strlen (destpath) - 1] = '\0'; - branchpath = g_strconcat (destpath, "-tmp-checkout-", branchrev, NULL); - branchf = ot_gfile_new_for_path (branchpath); - - g_print ("Checking out %s (commit %s)...\n", branch, branchrev); - if (!ostree_repo_checkout (repo, OSTREE_REPO_CHECKOUT_MODE_NONE, - branchrev, branchf, NULL, error)) - goto out; - g_print ("...done\n"); - g_print ("Merging over destination...\n"); - if (!ot_gfile_merge_dirs (destination, branchf, NULL, error)) + if (!ostree_repo_read_commit (repo, branch, &branchf, NULL, error)) goto out; + branch_rev = ostree_repo_file_get_commit ((OstreeRepoFile*)branchf); + + if (!ostree_repo_stage_directory_to_mtree (repo, branchf, mtree, NULL, + NULL, error)) + goto out; + if (metadata_builder) - g_variant_builder_add (metadata_builder, "(ss)", branch, branchrev); + g_variant_builder_add (metadata_builder, "(ss)", branch, branch_rev); ret = TRUE; out: - if (branchf) - rm_rf (branchf); - g_free (destpath); - g_clear_object (&enumerator); g_clear_object (&branchf); - g_free (branchrev); - g_free (branchpath); return ret; } @@ -128,17 +74,21 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e gboolean ret = FALSE; OstreeRepo *repo = NULL; OstreeCheckout *checkout = NULL; - const char *destination; + char *parent = NULL; GFile *destf = NULL; gboolean compose_metadata_builder_initialized = FALSE; GVariantBuilder compose_metadata_builder; gboolean commit_metadata_builder_initialized = FALSE; GVariantBuilder commit_metadata_builder; GVariant *commit_metadata = NULL; + char *contents_checksum = NULL; + char *commit_checksum = NULL; + GCancellable *cancellable = NULL; GFile *metadata_f = NULL; + OstreeMutableTree *mtree = NULL; int i; - context = g_option_context_new ("DESTINATION BRANCH1 BRANCH2 ... - Merge multiple commits into a single filesystem tree"); + context = g_option_context_new ("BRANCH1 BRANCH2 ... - Merge multiple commits into a single commit tree"); g_option_context_add_main_entries (context, options, NULL); if (!g_option_context_parse (context, &argc, &argv, error)) @@ -148,50 +98,66 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e if (!ostree_repo_check (repo, error)) goto out; - if (argc < 3) + if (!branch) { - gchar *help = g_option_context_get_help (context, TRUE, NULL); - g_printerr ("%s\n", help); - g_free (help); g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "DESTINATION and at least one COMMIT must be specified"); + "A branch must be specified with --branch"); goto out; } - destination = argv[1]; - destf = ot_gfile_new_for_path (destination); - - if (compose_metadata_path) + if (!subject) { - compose_metadata_builder_initialized = TRUE; - g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)")); + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "A subject must be specified with --subject"); + goto out; } + + compose_metadata_builder_initialized = TRUE; + g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)")); + + if (!ostree_repo_resolve_rev (repo, branch, TRUE, &parent, error)) + goto out; + + if (!ostree_repo_prepare_transaction (repo, cancellable, error)) + goto out; + + mtree = ostree_mutable_tree_new (); - for (i = 2; i < argc; i++) + for (i = 1; i < argc; i++) { - const char *branch = argv[i]; + const char *src_branch = argv[i]; - if (!compose_branch_on_dir (repo, destf, branch, compose_metadata_builder_initialized ? &compose_metadata_builder : NULL, error)) + if (!add_branch (repo, mtree, src_branch, + compose_metadata_builder_initialized ? &compose_metadata_builder : NULL, + error)) goto out; } - if (compose_metadata_path) - { - commit_metadata_builder_initialized = TRUE; - g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}")); + commit_metadata_builder_initialized = TRUE; + g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (&commit_metadata_builder, "{sv}", - "ostree-compose", g_variant_builder_end (&compose_metadata_builder)); - compose_metadata_builder_initialized = FALSE; + g_variant_builder_add (&commit_metadata_builder, "{sv}", + "ostree-compose", g_variant_builder_end (&compose_metadata_builder)); + commit_metadata = g_variant_builder_end (&commit_metadata_builder); + g_variant_ref_sink (commit_metadata); - metadata_f = ot_gfile_new_for_path (compose_metadata_path); + if (!ostree_repo_stage_mtree (repo, mtree, &contents_checksum, cancellable, error)) + goto out; - commit_metadata = g_variant_builder_end (&commit_metadata_builder); - if (!ot_util_variant_save (metadata_f, commit_metadata, NULL, error)) - goto out; - } + if (!ostree_repo_stage_commit (repo, branch, parent, subject, body, commit_metadata, + contents_checksum, + ostree_mutable_tree_get_metadata_checksum (mtree), + &commit_checksum, cancellable, error)) + goto out; + + if (!ostree_repo_commit_transaction (repo, cancellable, error)) + goto out; + + if (!ostree_repo_write_ref (repo, NULL, branch, commit_checksum, error)) + goto out; ret = TRUE; + g_print ("%s\n", commit_checksum); out: if (compose_metadata_builder_initialized) g_variant_builder_clear (&compose_metadata_builder); @@ -199,10 +165,14 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e g_variant_builder_clear (&commit_metadata_builder); if (context) g_option_context_free (context); + g_free (parent); + g_free (contents_checksum); + g_free (commit_checksum); ot_clear_gvariant (&commit_metadata); g_clear_object (&repo); g_clear_object (&checkout); g_clear_object (&destf); g_clear_object (&metadata_f); + g_clear_object (&mtree); return ret; } diff --git a/tests/t0004-compose.sh b/tests/t0004-compose.sh index 51d2f8d8..58e880a2 100755 --- a/tests/t0004-compose.sh +++ b/tests/t0004-compose.sh @@ -19,7 +19,7 @@ set -e -echo "1..8" +echo "1..5" . libtest.sh @@ -58,10 +58,11 @@ $OSTREE commit -b artifact-barapp -s 'Build 42 of barapp' echo 'ok artifacts committed' cd "${test_tmpdir}" -$OSTREE compose some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp -echo 'ok compose command' +$OSTREE compose -s "compose 1" -b some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp +echo 'ok compose' -cd some-compose +$OSTREE checkout some-compose some-compose-checkout +cd some-compose-checkout assert_file_has_content ./usr/bin/bar 'another ELF file' assert_file_has_content ./usr/share/doc/foo.txt 'some documentation' find | md5sum > ../some-compose-md5 @@ -70,36 +71,15 @@ assert_file_has_content ../some-compose-md5 9038703e43d2ff2745fb7dd844de65c8 echo 'ok compose content' cd "${test_tmpdir}" -rm -rf some-compose -$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp -echo 'ok compose output metadata' - -cd some-compose -$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Initial commit of some-compose' -echo 'ok compose commit with metadata' - -$OSTREE show --print-compose some-compose > ${test_tmpdir}/some-compose-contents -assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-runtime -assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-devel -echo 'ok compose verify metadata' - -cd "${test_tmpdir}" -rm -rf some-compose some-compose-metadata +rm -rf some-compose-checkout some-compose-metadata cd "${test_tmpdir}"/artifact-barapp echo 'updated bar ELF file' > usr/bin/bar $OSTREE commit -b artifact-barapp -s 'Build 43 of barapp' +$OSTREE compose -s "compose 2" -b some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp +echo 'ok compose update commit' cd "${test_tmpdir}" -$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp -cd some-compose -assert_file_has_content ./usr/bin/bar 'updated bar ELF file' - -echo 'ok updated artifact barapp' -$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Updated some-compose' -cd ${test_tmpdir} -rm -rf some-compose - $OSTREE checkout some-compose some-compose-checkout cd some-compose-checkout assert_file_has_content ./usr/bin/bar 'updated bar ELF file' -echo 'ok updated compose commit' +echo 'ok compose update contents' |