Support for "bare-user" repo format

This format is pretty much the same as the "bare" format, except the
file ownership and xattrs is not stored in the actual filesystem object, but
rather on the side in a user xattr. This means two things:

1) An unprivileged user can store such a repo independent of the types
   of files in it or their xattrs. And you can later (as root)
   reconstruct the real filesystem tree with ownership. Although you
   can't do that using hardlink-sharing. This also means ostree
   fsck does a full verification.

2) Such a repository can be checked out with user-mode (checkout -U)
   as an unprivileged user using hardlinks for space sharing.

Additionally, symlinks are stored as regular files (with the content
being the symlink target) because user xattrs are not supported on
symlinks. We know at checkout time if the file is a symlink because
the original st_mode is stored in the xattr metadata.

https://bugzilla.gnome.org/show_bug.cgi?id=741125

4 files changed, 210 insertions, 21 deletions

diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index e54e45d2..c5c940e6 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -94,6 +94,22 @@ typedef enum {
 #define OSTREE_DIRMETA_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_DIRMETA_GVARIANT_STRING)
 
 /**
+ * OSTREE_FILEMETA_GVARIANT_FORMAT:
+ *
+ * This is not a regular object type, but used as an xattr on a .file object
+ * in bare-user repositories. This allows us to store metadata information that we
+ * can't store in the real filesystem but we can still use a regular .file object
+ * that we can hardlink to in the case of a user-mode checkout.
+ *
+ * u - uid
+ * u - gid
+ * u - mode
+ * a(ayay) - xattrs
+ */
+#define OSTREE_FILEMETA_GVARIANT_STRING "(uuua(ayay))"
+#define OSTREE_FILEMETA_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_FILEMETA_GVARIANT_STRING)
+
+/**
  * OSTREE_TREE_GVARIANT_FORMAT:
  *
  * a(say) - array of (filename, checksum) for files
@@ -130,13 +146,15 @@ typedef enum {
  * OstreeRepoMode:
  * @OSTREE_REPO_MODE_BARE: Files are stored as themselves; can only be written as root
  * @OSTREE_REPO_MODE_ARCHIVE_Z2: Files are compressed, should be owned by non-root.  Can be served via HTTP
+ * @OSTREE_REPO_MODE_BARE_USER: Files are stored as themselves, except ownership; can be written by user
  *
  * See the documentation of #OstreeRepo for more information about the
  * possible modes.
  */
 typedef enum {
   OSTREE_REPO_MODE_BARE,
-  OSTREE_REPO_MODE_ARCHIVE_Z2
+  OSTREE_REPO_MODE_ARCHIVE_Z2,
+  OSTREE_REPO_MODE_BARE_USER
 } OstreeRepoMode;
 
 const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype);
diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c
index 531fdf9e..e9c420c3 100644
--- a/src/libostree/ostree-repo-checkout.c
+++ b/src/libostree/ostree-repo-checkout.c
@@ -324,7 +324,7 @@ checkout_file_hardlink (OstreeRepo                          *self,
 {
   gboolean ret = FALSE;
   gboolean ret_was_supported = FALSE;
-  int srcfd = self->mode == OSTREE_REPO_MODE_BARE ?
+  int srcfd = (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER) ?
     self->objects_dir_fd : self->uncompressed_objects_dir_fd;
 
  again:
@@ -401,8 +401,10 @@ checkout_one_file_at (OstreeRepo                        *repo,
 
       while (current_repo)
         {
-          gboolean is_bare = (current_repo->mode == OSTREE_REPO_MODE_BARE
-                              && mode == OSTREE_REPO_CHECKOUT_MODE_NONE);
+          gboolean is_bare = ((current_repo->mode == OSTREE_REPO_MODE_BARE
+                               && mode == OSTREE_REPO_CHECKOUT_MODE_NONE) ||
+                              (current_repo->mode == OSTREE_REPO_MODE_BARE_USER
+                               && mode == OSTREE_REPO_CHECKOUT_MODE_USER));
           gboolean is_archive_z2_with_cache = (current_repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2
                                                && mode == OSTREE_REPO_CHECKOUT_MODE_USER
                                                && current_repo->enable_uncompressed_cache);
diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index 342272a6..9d970e4a 100644
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -33,6 +33,7 @@
 #include "ostree-checksum-input-stream.h"
 #include "ostree-mutable-tree.h"
 #include "ostree-varint.h"
+#include <attr/xattr.h>
 
 gboolean
 _ostree_repo_ensure_loose_objdir_at (int             dfd,
@@ -57,13 +58,60 @@ _ostree_repo_ensure_loose_objdir_at (int             dfd,
   return TRUE;
 }
 
+static GVariant *
+create_file_metadata (GFileInfo *file_info,
+                      GVariant     *xattrs)
+{
+  GVariant *ret_metadata = NULL;
+  gs_unref_variant GVariant *tmp_xattrs = NULL;
+
+  if (xattrs == NULL)
+    tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0));
+
+  ret_metadata = g_variant_new ("(uuu@a(ayay))",
+                                GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::uid")),
+                                GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::gid")),
+                                GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::mode")),
+                                xattrs ? xattrs : tmp_xattrs);
+  g_variant_ref_sink (ret_metadata);
+
+  return ret_metadata;
+}
+
+static gboolean
+write_file_metadata_to_xattr (int fd,
+                              GFileInfo    *file_info,
+                              GVariant     *xattrs,
+                              GError       **error)
+{
+  gs_unref_variant GVariant *filemeta = NULL;
+  int res;
+
+  filemeta = create_file_metadata (file_info, xattrs);
+
+  do
+    res = fsetxattr (fd, "user.ostreemeta",
+                     (char*)g_variant_get_data (filemeta),
+                     g_variant_get_size (filemeta),
+                     0);
+  while (G_UNLIKELY (res == -1 && errno == EINTR));
+  if (G_UNLIKELY (res == -1))
+    {
+      ot_util_set_error_from_errno (error, errno);
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+
 static gboolean
 commit_loose_object_trusted (OstreeRepo        *self,
                              OstreeObjectType   objtype,
                              const char        *loose_path,
                              GFile             *temp_file,
                              const char        *temp_filename,
-                             gboolean           is_symlink,
+                             gboolean           object_is_symlink,
                              GFileInfo         *file_info,
                              GVariant          *xattrs,
                              GOutputStream     *temp_out,
@@ -89,10 +137,13 @@ commit_loose_object_trusted (OstreeRepo        *self,
     }
 
   /* Special handling for symlinks in bare repositories */
-  if (is_symlink && self->mode == OSTREE_REPO_MODE_BARE)
+  if (object_is_symlink && self->mode == OSTREE_REPO_MODE_BARE)
     {
       /* Now that we know the checksum is valid, apply uid/gid, mode bits,
        * and extended attributes.
+       *
+       * Note, this does not apply for bare-user repos, as they store symlinks
+       * as regular files.
        */
       if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename,
                                 g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
@@ -102,7 +153,7 @@ commit_loose_object_trusted (OstreeRepo        *self,
           ot_util_set_error_from_errno (error, errno);
           goto out;
         }
-              
+
       if (xattrs != NULL)
         {
           if (!gs_dfd_and_name_set_all_xattrs (self->tmp_dir_fd, temp_filename,
@@ -143,13 +194,41 @@ commit_loose_object_trusted (OstreeRepo        *self,
               ot_util_set_error_from_errno (error, errno);
               goto out;
             }
-          
+
           if (xattrs)
             {
               if (!gs_fd_set_all_xattrs (fd, xattrs, cancellable, error))
                 goto out;
             }
+        }
+
+      if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER)
+        {
+          g_assert (file_info != NULL);
+
+          if (!write_file_metadata_to_xattr (fd, file_info, xattrs, error))
+            goto out;
+
+          if (!object_is_symlink)
+            {
+              /* We need to apply at least some mode bits, because the repo file was created
+                 with mode 644, and we need e.g. exec bits to be right when we do a user-mode
+                 checkout. To make this work we apply all user bits and the read bits for
+                 group/other */
+              do
+                res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode") | 0744);
+              while (G_UNLIKELY (res == -1 && errno == EINTR));
+              if (G_UNLIKELY (res == -1))
+                {
+                  ot_util_set_error_from_errno (error, errno);
+                  goto out;
+                }
+            }
+        }
 
+      if (objtype == OSTREE_OBJECT_TYPE_FILE && (self->mode == OSTREE_REPO_MODE_BARE ||
+                                                 self->mode == OSTREE_REPO_MODE_BARE_USER))
+        {
           /* To satisfy tools such as guile which compare mtimes
            * to determine whether or not source files need to be compiled,
            * set the modification time to 0.
@@ -379,7 +458,8 @@ write_object (OstreeRepo         *self,
   gboolean have_obj;
   GChecksum *checksum = NULL;
   gboolean temp_file_is_regular;
-  gboolean is_symlink = FALSE;
+  gboolean temp_file_is_symlink;
+  gboolean object_is_symlink = FALSE;
   char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
   gssize unpacked_size = 0;
   gboolean indexable = FALSE;
@@ -420,9 +500,27 @@ write_object (OstreeRepo         *self,
         goto out;
 
       temp_file_is_regular = g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR;
-      is_symlink = g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK;
+      temp_file_is_symlink = object_is_symlink =
+        g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK;
+
+      if (repo_mode == OSTREE_REPO_MODE_BARE_USER && object_is_symlink)
+        {
+          const char *target_str = g_file_info_get_symlink_target (file_info);
+          gs_unref_bytes GBytes *target = g_bytes_new (target_str, strlen (target_str) + 1);
+
+          /* For bare-user we can't store symlinks as symlinks, as symlinks don't
+             support user xattrs to store the ownership. So, instead store them
+             as regular files */
+          temp_file_is_regular = TRUE;
+          temp_file_is_symlink = FALSE;
+          if (file_input != NULL)
+            g_object_unref (file_input);
+
+          /* Include the terminating zero so we can e.g. mmap this file */
+          file_input = g_memory_input_stream_new_from_bytes (target);
+        }
 
-      if (!(temp_file_is_regular || is_symlink))
+      if (!(temp_file_is_regular || temp_file_is_symlink))
         {
           g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                        "Unsupported file type %u", g_file_info_get_file_type (file_info));
@@ -436,7 +534,7 @@ write_object (OstreeRepo         *self,
        * binary with trailing garbage, creating a window on the local
        * system where a malicious setuid binary exists.
        */
-      if (repo_mode == OSTREE_REPO_MODE_BARE && temp_file_is_regular)
+      if ((repo_mode == OSTREE_REPO_MODE_BARE || repo_mode == OSTREE_REPO_MODE_BARE_USER) && temp_file_is_regular)
         {
           guint64 size = g_file_info_get_size (file_info);
 
@@ -453,7 +551,7 @@ write_object (OstreeRepo         *self,
                                       cancellable, error) < 0)
             goto out;
         }
-      else if (repo_mode == OSTREE_REPO_MODE_BARE && is_symlink)
+      else if (repo_mode == OSTREE_REPO_MODE_BARE && temp_file_is_symlink)
         {
           if (!_ostree_make_temporary_symlink_at (self->tmp_dir_fd,
                                                   g_file_info_get_symlink_target (file_info),
@@ -564,7 +662,7 @@ write_object (OstreeRepo         *self,
     {
       if (!commit_loose_object_trusted (self, objtype, loose_objpath,
                                         temp_file, temp_filename,
-                                        is_symlink, file_info,
+                                        object_is_symlink, file_info,
                                         xattrs, temp_out,
                                         cancellable, error))
         goto out;
@@ -703,6 +801,7 @@ scan_loose_devino (OstreeRepo                     *self,
             {
             case OSTREE_REPO_MODE_ARCHIVE_Z2:
             case OSTREE_REPO_MODE_BARE:
+            case OSTREE_REPO_MODE_BARE_USER:
               skip = !g_str_has_suffix (name, ".file");
               break;
             default:
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 2ae893f1..a3e0ef66 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -48,9 +48,11 @@
  * The #OstreeRepo is like git, a content-addressed object store.
  * Unlike git, it records uid, gid, and extended attributes.
  *
- * There are two possible "modes" for an #OstreeRepo;
+ * There are three possible "modes" for an #OstreeRepo;
  * %OSTREE_REPO_MODE_BARE is very simple - content files are
  * represented exactly as they are, and checkouts are just hardlinks.
+ * %OSTREE_REPO_MODE_BARE_USER is similar, except the uid/gids are not
+ * set on the files, and checkouts as hardlinks hardlinks work only for user checkouts.
  * A %OSTREE_REPO_MODE_ARCHIVE_Z2 repository in contrast stores
  * content files zlib-compressed.  It is suitable for non-root-owned
  * repositories that can be served via a static HTTP server.
@@ -560,6 +562,9 @@ ostree_repo_mode_to_string (OstreeRepoMode   mode,
     case OSTREE_REPO_MODE_BARE:
       ret_mode = "bare";
       break;
+    case OSTREE_REPO_MODE_BARE_USER:
+      ret_mode = "bare-user";
+      break;
     case OSTREE_REPO_MODE_ARCHIVE_Z2:
       ret_mode ="archive-z2";
       break;
@@ -585,6 +590,8 @@ ostree_repo_mode_from_string (const char      *mode,
 
   if (strcmp (mode, "bare") == 0)
     ret_mode = OSTREE_REPO_MODE_BARE;
+  else if (strcmp (mode, "bare-user") == 0)
+    ret_mode = OSTREE_REPO_MODE_BARE_USER;
   else if (strcmp (mode, "archive-z2") == 0)
     ret_mode = OSTREE_REPO_MODE_ARCHIVE_Z2;
   else
@@ -1123,7 +1130,7 @@ list_loose_objects_at (OstreeRepo             *self,
 
       if ((self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2
            && strcmp (dot, ".filez") == 0) ||
-          (self->mode == OSTREE_REPO_MODE_BARE
+          ((self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER)
            && strcmp (dot, ".file") == 0))
         objtype = OSTREE_OBJECT_TYPE_FILE;
       else if (strcmp (dot, ".dirtree") == 0)
@@ -1393,6 +1400,26 @@ query_info_for_bare_content_object (OstreeRepo      *self,
   return ret;
 }
 
+static GVariant  *
+set_info_from_filemeta (GFileInfo  *info,
+                        GVariant   *metadata)
+{
+  guint32 uid, gid, mode;
+  GVariant *xattrs;
+
+  g_variant_get (metadata, "(uuu@a(ayay))",
+                 &uid, &gid, &mode, &xattrs);
+  uid = GUINT32_FROM_BE (uid);
+  gid = GUINT32_FROM_BE (gid);
+  mode = GUINT32_FROM_BE (mode);
+
+  g_file_info_set_attribute_uint32 (info, "unix::uid", uid);
+  g_file_info_set_attribute_uint32 (info, "unix::gid", gid);
+  g_file_info_set_attribute_uint32 (info, "unix::mode", mode);
+
+  return xattrs;
+}
+
 /**
  * ostree_repo_load_file:
  * @self: Repo
@@ -1468,14 +1495,57 @@ ostree_repo_load_file (OstreeRepo         *self,
 
       if (ret_file_info)
         {
-          if (out_xattrs)
+          if (repo_mode == OSTREE_REPO_MODE_BARE_USER)
             {
-              gs_unref_object GFile *full_path =
-                    _ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_FILE);
+              gs_unref_variant GVariant *metadata = NULL;
+              gs_unref_bytes GBytes *bytes = NULL;
 
-              if (!gs_file_get_all_xattrs (full_path, &ret_xattrs,
-                                           cancellable, error))
+              bytes = ot_lgetxattrat (self->objects_dir_fd, loose_path_buf,
+                                      "user.ostreemeta", error);
+              if (bytes == NULL)
                 goto out;
+
+              metadata = g_variant_new_from_bytes (OSTREE_FILEMETA_GVARIANT_FORMAT,
+                                                   bytes, FALSE);
+              g_variant_ref_sink (metadata);
+
+              ret_xattrs = set_info_from_filemeta (ret_file_info, metadata);
+
+              if (S_ISLNK (g_file_info_get_attribute_uint32 (ret_file_info, "unix::mode")))
+                {
+                  int fd = -1;
+                  gs_unref_object GInputStream *target_input = NULL;
+                  char targetbuf[PATH_MAX+1];
+                  gsize target_size;
+
+                  g_file_info_set_file_type (ret_file_info, G_FILE_TYPE_SYMBOLIC_LINK);
+                  g_file_info_set_size (ret_file_info, 0);
+
+                  if (!gs_file_openat_noatime (self->objects_dir_fd, loose_path_buf, &fd,
+                                               cancellable, error))
+                    goto out;
+
+                  target_input = g_unix_input_stream_new (fd, TRUE);
+
+                  if (!g_input_stream_read_all (target_input, targetbuf, sizeof (targetbuf),
+                                                &target_size, cancellable, error))
+                    goto out;
+
+                  g_file_info_set_symlink_target (ret_file_info, targetbuf);
+                }
+            }
+
+          if (repo_mode == OSTREE_REPO_MODE_BARE)
+            {
+              if (out_xattrs)
+                {
+                  gs_unref_object GFile *full_path =
+                    _ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_FILE);
+
+                  if (!gs_file_get_all_xattrs (full_path, &ret_xattrs,
+                                               cancellable, error))
+                    goto out;
+                }
             }
 
           if (out_input && g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)