Add an `export` builtin, and API to write to libarchive

At the moment I'm looking at using rpm-ostree to manage RPM inputs
which can then be converted into Docker images.  It's most convenient
if we can stream directly out of libostree rather than doing a
checkout + tar combination.

There are also backup/debugging etc. reasons to implement `export` as
well.

10 files changed, 484 insertions, 2 deletions

diff --git a/Makefile-man.am b/Makefile-man.am
index a6090bf4..615bf0f0 100644
--- a/Makefile-man.am
+++ b/Makefile-man.am
@@ -19,7 +19,7 @@
 
 if ENABLE_MAN
 
-man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1
+man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1
 
 if BUILDOPT_FUSE
 man1_files += rofiles-fuse.1
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index 76fb26f3..ab4485c6 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -28,6 +28,7 @@ ostree_SOURCES = src/ostree/main.c \
 	src/ostree/ot-builtin-checksum.c \
 	src/ostree/ot-builtin-commit.c \
 	src/ostree/ot-builtin-diff.c \
+	src/ostree/ot-builtin-export.c \
 	src/ostree/ot-builtin-fsck.c \
 	src/ostree/ot-builtin-gpg-sign.c \
 	src/ostree/ot-builtin-init.c \
diff --git a/Makefile-tests.am b/Makefile-tests.am
index b0466840..292699ef 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -27,6 +27,7 @@ testfiles = test-basic \
 	test-remote-add \
 	test-remote-gpg-import \
         test-commit-sign \
+        test-export \
 	test-help \
 	test-libarchive \
 	test-pull-archive-z \
diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c
index e86e4026..3b1b0b83 100644
--- a/src/libostree/ostree-repo-libarchive.c
+++ b/src/libostree/ostree-repo-libarchive.c
@@ -24,6 +24,7 @@
 
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
+#include "ostree-repo-file.h"
 #include "ostree-mutable-tree.h"
 
 #ifdef HAVE_LIBARCHIVE
@@ -356,3 +357,251 @@ ostree_repo_write_archive_to_mtree (OstreeRepo                *self,
   return FALSE;
 #endif
 }
+
+#ifdef HAVE_LIBARCHIVE
+
+static gboolean
+file_to_archive_entry_common (GFile         *root,
+                              OstreeRepoArchiveOptions *opts,
+                              GFile         *path,
+                              GFileInfo  *file_info,
+                              struct archive_entry *entry,
+                              GError            **error)
+{
+  gboolean ret = FALSE;
+  g_autofree char *pathstr = g_file_get_relative_path (root, path);
+  g_autoptr(GVariant) xattrs = NULL;
+  time_t ts = (time_t) opts->timestamp_secs;
+
+  if (pathstr && !pathstr[0])
+    {
+      g_free (pathstr);
+      pathstr = g_strdup (".");
+    }
+
+  archive_entry_update_pathname_utf8 (entry, pathstr);
+  archive_entry_set_ctime (entry, ts, 0);
+  archive_entry_set_mtime (entry, ts, 0);
+  archive_entry_set_atime (entry, ts, 0);
+  archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid"));
+  archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid"));
+  archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode"));
+
+  if (!ostree_repo_file_get_xattrs ((OstreeRepoFile*)path, &xattrs, NULL, error))
+    goto out;
+
+  if (!opts->disable_xattrs)
+    {
+      int i, n;
+      
+      n = g_variant_n_children (xattrs);
+      for (i = 0; i < n; i++)
+        {
+          const guint8* name;
+          g_autoptr(GVariant) value = NULL;
+          const guint8* value_data;
+          gsize value_len;
+
+          g_variant_get_child (xattrs, i, "(^&ay@ay)", &name, &value);
+          value_data = g_variant_get_fixed_array (value, &value_len, 1);
+
+          archive_entry_xattr_add_entry (entry, (char*)name,
+                                         (char*) value_data, value_len);
+        }
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
+static gboolean
+write_header_free_entry (struct archive *a,
+                         struct archive_entry **entryp,
+                         GError **error)
+{
+  struct archive_entry *entry = *entryp;
+  gboolean ret = FALSE;
+
+  if (archive_write_header (a, entry) != ARCHIVE_OK)
+    {
+      propagate_libarchive_error (error, a);
+      goto out;
+    }
+
+  ret = TRUE;
+ out:
+  archive_entry_free (entry);
+  *entryp = NULL;
+  return ret;
+}
+
+static gboolean
+write_directory_to_libarchive_recurse (OstreeRepo               *self,
+                                       OstreeRepoArchiveOptions *opts,
+                                       GFile                    *root,
+                                       GFile                    *dir,
+                                       struct archive           *a,
+                                       GCancellable             *cancellable,
+                                       GError                  **error)
+{
+  gboolean ret = FALSE;
+  g_autoptr(GFileInfo) dir_info = NULL;
+  g_autoptr(GFileEnumerator) dir_enum = NULL;
+  struct archive_entry *entry = NULL;
+
+  dir_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO,
+                                G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                cancellable, error);
+  if (!dir_info)
+    goto out;
+
+  entry = archive_entry_new2 (a);
+  if (!file_to_archive_entry_common (root, opts, dir, dir_info, entry, error))
+    goto out;
+  if (!write_header_free_entry (a, &entry, error))
+    goto out;
+
+  dir_enum = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, 
+                                        G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                        cancellable, error);
+  if (!dir_enum)
+    goto out;
+
+  while (TRUE)
+    {
+      GFileInfo *file_info;
+      GFile *path;
+
+      if (!gs_file_enumerator_iterate (dir_enum, &file_info, &path,
+                                       cancellable, error))
+        goto out;
+      if (file_info == NULL)
+        break;
+
+      /* First, handle directories recursively */
+      if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
+        {
+          if (!write_directory_to_libarchive_recurse (self, opts, root, path, a,
+                                                      cancellable, error))
+            goto out;
+
+          /* Go to the next entry */
+          continue;
+        }
+
+      /* Past here, should be a regular file or a symlink */
+
+      entry = archive_entry_new2 (a);
+      if (!file_to_archive_entry_common (root, opts, path, file_info, entry, error))
+        goto out;
+
+      switch (g_file_info_get_file_type (file_info))
+        {
+        case G_FILE_TYPE_SYMBOLIC_LINK:
+          {
+            archive_entry_set_symlink (entry, g_file_info_get_symlink_target (file_info));
+            if (!write_header_free_entry (a, &entry, error))
+              goto out;
+          }
+          break;
+        case G_FILE_TYPE_REGULAR:
+          {
+            guint8 buf[8192];
+            g_autoptr(GInputStream) file_in = NULL;
+            g_autoptr(GFileInfo) file_info = NULL;
+            const char *checksum;
+
+            checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path);
+
+            if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL,
+                                        cancellable, error))
+              goto out;
+
+            archive_entry_set_size (entry, g_file_info_get_size (file_info));
+
+            if (archive_write_header (a, entry) != ARCHIVE_OK)
+              {
+                propagate_libarchive_error (error, a);
+                goto out;
+              }
+
+            while (TRUE)
+              {
+                gssize bytes_read = g_input_stream_read (file_in, buf, sizeof (buf),
+                                                         cancellable, error);
+                if (bytes_read < 0)
+                  goto out;
+                if (bytes_read == 0)
+                  break;
+
+                { ssize_t r = archive_write_data (a, buf, bytes_read);
+                  if (r != bytes_read)
+                    {
+                      propagate_libarchive_error (error, a);
+                      g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", bytes_read, r);
+                      goto out;
+                    }
+                }
+              }
+
+            if (archive_write_finish_entry (a) != ARCHIVE_OK)
+              {
+                propagate_libarchive_error (error, a);
+                goto out;
+              }
+
+            archive_entry_free (entry);
+            entry = NULL;
+          }
+          break;
+        default:
+          g_assert_not_reached ();
+        }
+    }
+
+  ret = TRUE;
+ out:
+  if (entry)
+    archive_entry_free (entry);
+  return ret;
+}
+#endif
+
+/**
+ * ostree_repo_write_tree_to_archive:
+ * @self: An #OstreeRepo
+ * @opts: Options controlling conversion
+ * @root: An #OstreeRepoFile for the base directory
+ * @archive: A `struct archive`, but specified as void to avoid a dependency on the libarchive headers
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Import an archive file @archive into the repository, and write its
+ * file structure to @mtree.
+ */
+gboolean
+ostree_repo_write_tree_to_archive (OstreeRepo                *self,
+                                   OstreeRepoArchiveOptions *opts,
+                                   OstreeRepoFile            *root,
+                                   void                      *archive,
+                                   GCancellable             *cancellable,
+                                   GError                  **error)
+{
+#ifdef HAVE_LIBARCHIVE
+  gboolean ret = FALSE;
+  struct archive *a = archive;
+
+  if (!write_directory_to_libarchive_recurse (self, opts, (GFile*)root, (GFile*)root,
+                                              a, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  return ret;
+#else
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+               "This version of ostree is not compiled with libarchive support");
+  return FALSE;
+#endif
+}
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index d4d0f418..64e8a028 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -441,6 +441,23 @@ gboolean      ostree_repo_write_dfd_to_mtree (OstreeRepo                 *self,
                                               GCancellable               *cancellable,
                                               GError                    **error);
 
+/**
+ * OstreeRepoWriteArchiveOptions:
+ *
+ * An extensible options structure controlling archive creation.  Ensure that
+ * you have entirely zeroed the structure, then set just the desired
+ * options.  This is used by ostree_repo_write_tree_to_archive().
+ */
+typedef struct {
+  guint disable_xattrs : 1;
+  guint reserved : 31;
+
+  guint64 timestamp_secs;
+
+  guint unused_uint[8];
+  gpointer unused_ptrs[8];
+} OstreeRepoArchiveOptions;
+
 gboolean      ostree_repo_write_archive_to_mtree (OstreeRepo                   *self,
                                                   GFile                        *archive,
                                                   OstreeMutableTree            *mtree,
@@ -449,6 +466,13 @@ gboolean      ostree_repo_write_archive_to_mtree (OstreeRepo                   *
                                                   GCancellable                 *cancellable,
                                                   GError                      **error);
 
+gboolean ostree_repo_write_tree_to_archive (OstreeRepo                *self,
+                                            OstreeRepoArchiveOptions  *opts,
+                                            OstreeRepoFile            *root,
+                                            void                      *archive,  /* Really struct archive * */
+                                            GCancellable             *cancellable,
+                                            GError                  **error);
+
 gboolean      ostree_repo_write_mtree (OstreeRepo         *self,
                                        OstreeMutableTree  *mtree,
                                        GFile             **out_file,
diff --git a/src/ostree/main.c b/src/ostree/main.c
index 99d7d916..eff3082d 100644
--- a/src/ostree/main.c
+++ b/src/ostree/main.c
@@ -40,6 +40,7 @@ static OstreeCommand commands[] = {
   { "commit", ostree_builtin_commit },
   { "config", ostree_builtin_config },
   { "diff", ostree_builtin_diff },
+  { "export", ostree_builtin_export },
   { "fsck", ostree_builtin_fsck },
   { "gpg-sign", ostree_builtin_gpg_sign },
   { "init", ostree_builtin_init },
diff --git a/src/ostree/ot-builtin-diff.c b/src/ostree/ot-builtin-diff.c
index b5e0c5a8..a23ed83f 100644
--- a/src/ostree/ot-builtin-diff.c
+++ b/src/ostree/ot-builtin-diff.c
@@ -29,10 +29,12 @@
 
 static gboolean opt_stats;
 static gboolean opt_fs_diff;
+static gboolean opt_no_xattrs;
 
 static GOptionEntry options[] = {
   { "stats", 0, 0, G_OPTION_ARG_NONE, &opt_stats, "Print various statistics", NULL },
   { "fs-diff", 0, 0, G_OPTION_ARG_NONE, &opt_fs_diff, "Print filesystem diff", NULL },
+  { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL },
   { NULL }
 };
 
@@ -162,6 +164,11 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **
 
   if (opt_fs_diff)
     {
+      OstreeDiffFlags diff_flags = OSTREE_DIFF_FLAGS_NONE; 
+
+      if (opt_no_xattrs)
+        diff_flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS;
+      
       if (!parse_file_or_commit (repo, src, &srcf, cancellable, error))
         goto out;
       if (!parse_file_or_commit (repo, target, &targetf, cancellable, error))
@@ -171,7 +178,7 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **
       removed = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
       added = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
       
-      if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, srcf, targetf, modified, removed, added, cancellable, error))
+      if (!ostree_diff_dirs (diff_flags, srcf, targetf, modified, removed, added, cancellable, error))
         goto out;
 
       ostree_diff_print (srcf, targetf, modified, removed, added);
diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c
new file mode 100644
index 00000000..2d350b8c
--- /dev/null
+++ b/src/ostree/ot-builtin-export.c
@@ -0,0 +1,148 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2016 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ot-main.h"
+#include "ot-builtins.h"
+#include "ostree.h"
+#include "ostree-repo-file.h"
+#include "otutil.h"
+
+#ifdef HAVE_LIBARCHIVE
+#include <archive.h>
+#include <archive_entry.h>
+#endif
+
+static char *opt_output_path;
+static gboolean opt_no_xattrs;
+
+static GOptionEntry options[] = {
+  { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL },
+  { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" },
+  { NULL }
+};
+
+#ifdef HAVE_LIBARCHIVE
+
+static void
+propagate_libarchive_error (GError      **error,
+                            struct archive *a)
+{
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+               "%s", archive_error_string (a));
+}
+
+#endif
+
+gboolean
+ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError **error)
+{
+  GOptionContext *context;
+  glnx_unref_object OstreeRepo *repo = NULL;
+  gboolean ret = FALSE;
+  const char *rev;
+  g_autoptr(GFile) root = NULL;
+  g_autofree char *commit = NULL;
+  g_autoptr(GVariant) commit_data = NULL;
+  struct archive *a;
+  OstreeRepoArchiveOptions opts = { 0, };
+
+  context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format");
+
+  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+    goto out;
+
+#ifdef HAVE_LIBARCHIVE  
+
+  if (argc <= 1)
+    {
+      ot_util_usage_error (context, "A COMMIT argument is required", error);
+      goto out;
+    }
+  rev = argv[1];
+
+  a = archive_write_new ();
+  /* Yes, this is hardcoded for now.  There is
+   * archive_write_set_format_filter_by_ext() but it's fairly magic.
+   * Many programs have support now for GNU tar, so should be a good
+   * default.  I also don't want to lock us into everything libarchive
+   * supports.
+   */
+  if (archive_write_set_format_gnutar (a) != ARCHIVE_OK)
+    {
+      propagate_libarchive_error (error, a);
+      goto out;
+    }
+  if (archive_write_add_filter_none (a) != ARCHIVE_OK)
+    {
+      propagate_libarchive_error (error, a);
+      goto out;
+    }
+  if (opt_output_path)
+    {
+      if (archive_write_open_filename (a, opt_output_path) != ARCHIVE_OK)
+        {
+          propagate_libarchive_error (error, a);
+          goto out;
+        }
+    }
+  else
+    {
+      if (archive_write_open_FILE (a, stdout) != ARCHIVE_OK)
+        {
+          propagate_libarchive_error (error, a);
+          goto out;
+        }
+    }
+
+  if (opt_no_xattrs)
+    opts.disable_xattrs = TRUE;
+
+  if (!ostree_repo_read_commit (repo, rev, &root, &commit, cancellable, error))
+    goto out;
+
+  if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_data, error))
+    goto out;
+
+  opts.timestamp_secs = ostree_commit_get_timestamp (commit_data);
+
+  if (!ostree_repo_write_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a,
+                                          cancellable, error))
+    goto out;
+
+  if (archive_write_close (a) != ARCHIVE_OK)
+    {
+      propagate_libarchive_error (error, a);
+      goto out;
+    }
+
+#else
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+               "This version of ostree is not compiled with libarchive support");
+  goto out;
+#endif  
+  
+  ret = TRUE;
+ out:
+  if (context)
+    g_option_context_free (context);
+  return ret;
+}
diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h
index 95262ec4..1c862925 100644
--- a/src/ostree/ot-builtins.h
+++ b/src/ostree/ot-builtins.h
@@ -35,6 +35,7 @@ BUILTINPROTO(checkout);
 BUILTINPROTO(checksum);
 BUILTINPROTO(commit);
 BUILTINPROTO(diff);
+BUILTINPROTO(export);
 BUILTINPROTO(gpg_sign);
 BUILTINPROTO(init);
 BUILTINPROTO(log);
diff --git a/tests/test-export.sh b/tests/test-export.sh
new file mode 100755
index 00000000..18f2f7ae
--- /dev/null
+++ b/tests/test-export.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (C) 2016 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_test_repository "archive-z2"
+
+echo '1..2'
+
+$OSTREE checkout test2 test2-co
+$OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co
+rm test2-co -rf
+
+cd ${test_tmpdir}
+${OSTREE} 'export' test2-noxattrs -o test2.tar
+mkdir t
+(cd t && tar xf ../test2.tar)
+ostree --repo=repo diff --no-xattrs test2-noxattrs ./t > diff.txt
+assert_file_empty diff.txt
+rm test2.tar diff.txt t -rf
+
+echo 'ok export gnutar diff (no xattrs)'
+
+cd ${test_tmpdir}
+${OSTREE} 'export' test2 -o test2.tar
+${OSTREE} commit -b test2-from-tar -s 'Import from tar' --tree=tar=test2.tar
+ostree --repo=repo diff test2 test2-from-tar
+assert_file_empty diff.txt
+rm test2.tar diff.txt t -rf
+
+echo 'ok export import'
+