core: Optimize commits to raw repositories

Avoid creating a temporary intermediate file in the case where we have
the total object length available.

7 files changed, 208 insertions, 75 deletions

diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index 0bbf4030..72906ffe 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -340,6 +340,7 @@ ostree_raw_file_to_content_stream (GInputStream       *input,
                                    GFileInfo          *file_info,
                                    GVariant           *xattrs,
                                    GInputStream      **out_input,
+                                   guint64            *out_length,
                                    GCancellable       *cancellable,
                                    GError            **error)
 {
@@ -377,21 +378,23 @@ ostree_raw_file_to_content_stream (GInputStream       *input,
 
   ret = TRUE;
   ot_transfer_out_value (out_input, &ret_input);
+  if (out_length)
+    *out_length = header_size + g_file_info_get_size (file_info);
  out:
   return ret;
 }
 
 gboolean
-ostree_content_file_parse (GFile                  *content_path,
-                           gboolean                trusted,
-                           GInputStream          **out_input,
-                           GFileInfo             **out_file_info,
-                           GVariant              **out_xattrs,
-                           GCancellable           *cancellable,
-                           GError                **error)
+ostree_content_stream_parse (GInputStream           *input,
+                             guint64                 input_length,
+                             gboolean                trusted,
+                             GInputStream          **out_input,
+                             GFileInfo             **out_file_info,
+                             GVariant              **out_xattrs,
+                             GCancellable           *cancellable,
+                             GError                **error)
 {
   gboolean ret = FALSE;
-  guint64 length;
   guint32 archive_header_size;
   guchar dummy[4];
   gsize bytes_read;
@@ -402,39 +405,27 @@ ostree_content_file_parse (GFile                  *content_path,
   ot_lvariant GVariant *file_header = NULL;
   ot_lfree guchar *buf = NULL;
 
-  ret_input = (GInputStream*)g_file_read (content_path, cancellable, error);
-  if (!ret_input)
-    goto out;
-
-  content_file_info = g_file_input_stream_query_info ((GFileInputStream*)ret_input,
-                                                      OSTREE_GIO_FAST_QUERYINFO,
-                                                      cancellable, error);
-  if (!content_file_info)
-    goto out;
-
-  length = g_file_info_get_size (content_file_info);
-
-  if (!g_input_stream_read_all (ret_input,
+  if (!g_input_stream_read_all (input,
                                 &archive_header_size, 4, &bytes_read,
                                 cancellable, error))
     goto out;
   archive_header_size = GUINT32_FROM_BE (archive_header_size);
-  if (archive_header_size > length)
+  if (archive_header_size > input_length)
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                    "File header size %u exceeds size %" G_GUINT64_FORMAT,
-                   (guint)archive_header_size, length);
+                   (guint)archive_header_size, input_length);
       goto out;
     }
 
   /* Skip over padding */
-  if (!g_input_stream_read_all (ret_input,
+  if (!g_input_stream_read_all (input,
                                 dummy, 4, &bytes_read,
                                 cancellable, error))
     goto out;
 
   buf = g_malloc (archive_header_size);
-  if (!g_input_stream_read_all (ret_input, buf, archive_header_size, &bytes_read,
+  if (!g_input_stream_read_all (input, buf, archive_header_size, &bytes_read,
                                 cancellable, error))
     goto out;
   file_header = g_variant_new_from_data (OSTREE_FILE_HEADER_GVARIANT_FORMAT,
@@ -448,14 +439,61 @@ ostree_content_file_parse (GFile                  *content_path,
                                  error))
     goto out;
   if (ret_file_info)
-    g_file_info_set_size (ret_file_info, length - archive_header_size - 8);
+    g_file_info_set_size (ret_file_info, input_length - archive_header_size - 8);
   
   if (g_file_info_get_file_type (ret_file_info) != G_FILE_TYPE_REGULAR)
     {
       g_clear_object (&ret_input);
     }
 
-  /* Now give the input stream at its current position as return value */
+  ret = TRUE;
+  /* Give the input stream at its current position as return value;
+   * assuming the caller doesn't seek, this should be fine.  We might
+   * want to wrap it though in a non-seekable stream.
+   **/
+  g_object_ref (input);
+  ot_transfer_out_value (out_input, &input);
+  ot_transfer_out_value (out_file_info, &ret_file_info);
+  ot_transfer_out_value (out_xattrs, &ret_xattrs);
+ out:
+  return ret;
+
+}
+
+gboolean
+ostree_content_file_parse (GFile                  *content_path,
+                           gboolean                trusted,
+                           GInputStream          **out_input,
+                           GFileInfo             **out_file_info,
+                           GVariant              **out_xattrs,
+                           GCancellable           *cancellable,
+                           GError                **error)
+{
+  gboolean ret = FALSE;
+  guint64 length;
+  ot_lobj GInputStream *file_input = NULL;
+  ot_lobj GInputStream *ret_input = NULL;
+  ot_lobj GFileInfo *content_file_info = NULL;
+  ot_lobj GFileInfo *ret_file_info = NULL;
+  ot_lvariant GVariant *ret_xattrs = NULL;
+
+  file_input = (GInputStream*)g_file_read (content_path, cancellable, error);
+  if (!file_input)
+    goto out;
+
+  content_file_info = g_file_input_stream_query_info ((GFileInputStream*)file_input,
+                                                      OSTREE_GIO_FAST_QUERYINFO,
+                                                      cancellable, error);
+  if (!content_file_info)
+    goto out;
+
+  length = g_file_info_get_size (content_file_info);
+
+  if (!ostree_content_stream_parse (file_input, length, trusted,
+                                    out_input ? &ret_input : NULL,
+                                    &ret_file_info, &ret_xattrs,
+                                    cancellable, error))
+    goto out;
 
   ret = TRUE;
   ot_transfer_out_value (out_input, &ret_input);
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index 32de9057..a7b6a365 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -207,6 +207,15 @@ gboolean ostree_file_header_parse (GVariant         *data,
                                    GFileInfo       **out_file_info,
                                    GVariant        **out_xattrs,
                                    GError          **error);
+gboolean
+ostree_content_stream_parse (GInputStream           *input,
+                             guint64                 input_length,
+                             gboolean                trusted,
+                             GInputStream          **out_input,
+                             GFileInfo             **out_file_info,
+                             GVariant              **out_xattrs,
+                             GCancellable           *cancellable,
+                             GError                **error);
 
 gboolean ostree_content_file_parse (GFile                  *content_path,
                                     gboolean                trusted,
@@ -226,6 +235,7 @@ gboolean ostree_raw_file_to_content_stream (GInputStream       *input,
                                             GFileInfo          *file_info,
                                             GVariant           *xattrs,
                                             GInputStream      **out_input,
+                                            guint64            *out_length,
                                             GCancellable       *cancellable,
                                             GError            **error);
 
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index b2a87550..4fd7f6d7 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -781,16 +781,6 @@ ostree_repo_get_file_object_path (OstreeRepo   *self,
 }
 
 static gboolean
-stage_object_impl (OstreeRepo         *self,
-                   OstreeObjectType    objtype,
-                   gboolean            store_if_packed,
-                   GInputStream       *input,
-                   const char         *expected_checksum,
-                   guchar            **out_csum,
-                   GCancellable       *cancellable,
-                   GError            **error);
-
-static gboolean
 commit_loose_object_trusted (OstreeRepo        *self,
                              const char        *checksum,
                              OstreeObjectType   objtype,
@@ -826,11 +816,17 @@ commit_loose_object_trusted (OstreeRepo        *self,
   return ret;
 }
 
+typedef enum {
+  OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED = (1<<0),
+  OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<1)
+} OstreeRepoStageFlags;
+
 static gboolean
 stage_object_impl (OstreeRepo         *self,
+                   OstreeRepoStageFlags flags,
                    OstreeObjectType    objtype,
-                   gboolean            store_if_packed,
                    GInputStream       *input,
+                   guint64             file_object_length,
                    const char         *expected_checksum,
                    guchar            **out_csum,
                    GCancellable       *cancellable,
@@ -860,7 +856,7 @@ stage_object_impl (OstreeRepo         *self,
 
   if (expected_checksum)
     {
-      if (!store_if_packed)
+      if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED))
         {
           if (!repo_find_object (self, objtype, expected_checksum,
                                  &stored_path, &pack_checksum, &pack_offset,
@@ -879,6 +875,7 @@ stage_object_impl (OstreeRepo         *self,
   if (stored_path == NULL && pack_checksum == NULL)
     {
       ot_lvariant GVariant *file_header = NULL;
+      gboolean do_stage_bare_file;
 
       if (out_csum)
         {
@@ -887,14 +884,39 @@ stage_object_impl (OstreeRepo         *self,
             checksum_input = ostree_checksum_input_stream_new (input, checksum);
         }
 
-      if (!ostree_create_temp_file_from_input (priv->tmp_dir,
-                                               ostree_object_type_to_string (objtype), NULL,
-                                               NULL, NULL,
-                                               checksum_input ? (GInputStream*)checksum_input : input,
-                                               &temp_file,
-                                               cancellable, error))
-        goto out;
+      do_stage_bare_file = (flags & OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID)
+        && objtype == OSTREE_OBJECT_TYPE_FILE
+        && priv->mode == OSTREE_REPO_MODE_BARE;
+      if (do_stage_bare_file)
+        {
+          ot_lobj GInputStream *file_input = NULL;
+          ot_lobj GFileInfo *file_info = NULL;
+          ot_lvariant GVariant *xattrs = NULL;
 
+          if (!ostree_content_stream_parse (checksum_input ? (GInputStream*)checksum_input : input,
+                                            file_object_length, FALSE,
+                                            &file_input, &file_info, &xattrs,
+                                            cancellable, error))
+            goto out;
+
+          if (!ostree_create_temp_file_from_input (priv->tmp_dir,
+                                                   ostree_object_type_to_string (objtype), NULL,
+                                                   file_info, xattrs, file_input,
+                                                   &temp_file,
+                                                   cancellable, error))
+            goto out;
+        }
+      else
+        {
+          if (!ostree_create_temp_file_from_input (priv->tmp_dir,
+                                                   ostree_object_type_to_string (objtype), NULL,
+                                                   NULL, NULL,
+                                                   checksum_input ? (GInputStream*)checksum_input : input,
+                                                   &temp_file,
+                                                   cancellable, error))
+            goto out;
+        }
+          
       if (!checksum)
         actual_checksum = expected_checksum;
       else
@@ -911,7 +933,7 @@ stage_object_impl (OstreeRepo         *self,
           
         }
           
-      if (!store_if_packed)
+      if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED))
         {
           gboolean have_obj;
           
@@ -926,7 +948,9 @@ stage_object_impl (OstreeRepo         *self,
 
       if (do_commit)
         {
-          if (objtype == OSTREE_OBJECT_TYPE_FILE && priv->mode == OSTREE_REPO_MODE_BARE)
+          /* Only do this if we *didn't* stage a bare file above */
+          if (!do_stage_bare_file
+              && objtype == OSTREE_OBJECT_TYPE_FILE && priv->mode == OSTREE_REPO_MODE_BARE)
             {
               ot_lobj GInputStream *file_input = NULL;
               ot_lobj GFileInfo *file_info = NULL;
@@ -1039,14 +1063,12 @@ stage_metadata_object (OstreeRepo         *self,
                        GError            **error)
 {
   gboolean ret = FALSE;
-  ot_lobj GInputStream *mem = NULL;
+  ot_lobj GInputStream *input = NULL;
 
-  mem = g_memory_input_stream_new_from_data (g_variant_get_data (variant),
-                                             g_variant_get_size (variant),
-                                             NULL);
+  input = ot_variant_read (variant);
   
-  if (!stage_object_impl (self, type, FALSE, mem,
-                          NULL, out_csum, cancellable, error))
+  if (!stage_object_impl (self, 0, type, input, 0, NULL,
+                          out_csum, cancellable, error))
     goto out;
 
   ret = TRUE;
@@ -1104,8 +1126,11 @@ ostree_repo_stage_object_trusted (OstreeRepo   *self,
                                   GCancellable *cancellable,
                                   GError      **error)
 {
-  return stage_object_impl (self, objtype, store_if_packed,
-                            object_input, checksum, NULL,
+  int flags = 0;
+  if (store_if_packed)
+    flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED;
+  return stage_object_impl (self, flags, objtype,
+                            object_input, 0, checksum, NULL,
                             cancellable, error);
 }
 
@@ -1120,8 +1145,47 @@ ostree_repo_stage_object (OstreeRepo       *self,
   gboolean ret = FALSE;
   ot_lfree guchar *actual_csum = NULL;
   
-  if (!stage_object_impl (self, objtype, FALSE,
-                          object_input, expected_checksum, &actual_csum,
+  if (!stage_object_impl (self, 0, objtype, 
+                          object_input, 0, expected_checksum, &actual_csum,
+                          cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
+gboolean      
+ostree_repo_stage_file_object_trusted (OstreeRepo       *self,
+                                       const char       *checksum,
+                                       gboolean          store_if_packed,
+                                       GInputStream     *object_input,
+                                       guint64           length,
+                                       GCancellable     *cancellable,
+                                       GError          **error)
+{
+  int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
+  if (store_if_packed)
+    flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED;
+  return stage_object_impl (self, flags, OSTREE_OBJECT_TYPE_FILE,
+                            object_input, length, checksum, NULL,
+                            cancellable, error);
+}
+
+gboolean
+ostree_repo_stage_file_object (OstreeRepo       *self,
+                               const char       *expected_checksum,
+                               GInputStream     *object_input,
+                               guint64           length,
+                               GCancellable     *cancellable,
+                               GError          **error)
+{
+  gboolean ret = FALSE;
+  int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
+  ot_lfree guchar *actual_csum = NULL;
+  
+  if (!stage_object_impl (self, flags, OSTREE_OBJECT_TYPE_FILE, 
+                          object_input, length, expected_checksum, &actual_csum,
                           cancellable, error))
     goto out;
 
@@ -2416,6 +2480,7 @@ stage_directory_to_mtree_internal (OstreeRepo           *self,
               else
                 {
                   ot_lobj GInputStream *file_object_input = NULL;
+                  guint64 file_obj_length;
 
                   g_clear_object (&file_input);
                   if (g_file_info_get_file_type (modified_info) == G_FILE_TYPE_REGULAR)
@@ -2435,12 +2500,14 @@ stage_directory_to_mtree_internal (OstreeRepo           *self,
                   g_free (child_file_csum);
                   child_file_csum = NULL;
 
-                  if (!ostree_raw_file_to_content_stream (file_input, modified_info, xattrs,
-                                                          &file_object_input, cancellable, error))
+                  if (!ostree_raw_file_to_content_stream (file_input,
+                                                          modified_info, xattrs,
+                                                          &file_object_input, &file_obj_length,
+                                                          cancellable, error))
                     goto out;
-                  if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_FILE, FALSE,
-                                          file_object_input, NULL,
-                                          &child_file_csum, cancellable, error))
+                  if (!stage_object_impl (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID,
+                                          OSTREE_OBJECT_TYPE_FILE, file_object_input, file_obj_length,
+                                          NULL, &child_file_csum, cancellable, error))
                     goto out;
 
                   g_free (tmp_checksum);
@@ -2619,6 +2686,7 @@ import_libarchive_entry_file (OstreeRepo           *self,
   gboolean ret = FALSE;
   ot_lobj GInputStream *file_object_input = NULL;
   ot_lobj GInputStream *archive_stream = NULL;
+  guint64 length;
   
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;
@@ -2627,11 +2695,11 @@ import_libarchive_entry_file (OstreeRepo           *self,
     archive_stream = ostree_libarchive_input_stream_new (a);
   
   if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL,
-                                          &file_object_input, cancellable, error))
+                                          &file_object_input, &length, cancellable, error))
     goto out;
   
-  if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_FILE, FALSE,
-                          file_object_input, NULL, out_csum,
+  if (!stage_object_impl (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID, OSTREE_OBJECT_TYPE_FILE,
+                          file_object_input, length, NULL, out_csum,
                           cancellable, error))
     goto out;
 
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 45b7a1e2..1bff3e44 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -106,6 +106,13 @@ gboolean      ostree_repo_stage_object (OstreeRepo       *self,
                                         GCancellable     *cancellable,
                                         GError          **error);
 
+gboolean      ostree_repo_stage_file_object (OstreeRepo       *self,
+                                             const char       *expected_checksum,
+                                             GInputStream     *content,
+                                             guint64           content_length,
+                                             GCancellable     *cancellable,
+                                             GError          **error);
+
 gboolean      ostree_repo_stage_object_trusted (OstreeRepo   *self,
                                                 OstreeObjectType objtype,
                                                 const char   *checksum,
@@ -114,6 +121,14 @@ gboolean      ostree_repo_stage_object_trusted (OstreeRepo   *self,
                                                 GCancellable *cancellable,
                                                 GError      **error);
 
+gboolean      ostree_repo_stage_file_object_trusted (OstreeRepo   *self,
+                                                     const char   *checksum,
+                                                     gboolean          store_if_packed,
+                                                     GInputStream     *content,
+                                                     guint64           content_length,
+                                                     GCancellable *cancellable,
+                                                     GError      **error);
+
 gboolean      ostree_repo_resolve_rev (OstreeRepo  *self,
                                        const char  *rev,
                                        gboolean     allow_noent,
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index b2003a28..71f8305e 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -843,7 +843,7 @@ store_file_from_pack (OtPullData          *pull_data,
     goto out;
 
   if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
-                                          &file_object_input, cancellable, error))
+                                          &file_object_input, NULL, cancellable, error))
     goto out;
   
   if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum,
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index 3af95768..eb3f0f34 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -55,19 +55,21 @@ import_one_object (OtLocalCloneData *data,
   if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
       ot_lobj GInputStream *file_object = NULL;
+      guint64 length;
 
       if (!ostree_repo_load_file (data->src_repo, checksum,
                                   &input, &file_info, &xattrs,
                                   cancellable, error))
         goto out;
 
-      if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, 
+      if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
+                                              &file_object, &length,
                                               cancellable, error))
         goto out;
 
-      if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_FILE,
-                                             checksum, FALSE, file_object,
-                                             cancellable, error))
+      if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, FALSE,
+                                                  file_object, length,
+                                                  cancellable, error))
         goto out;
     }
   else
diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c
index 42bd3b65..df5ef5ee 100644
--- a/src/ostree/ot-builtin-unpack.c
+++ b/src/ostree/ot-builtin-unpack.c
@@ -105,19 +105,19 @@ unpack_one_object (OstreeRepo        *repo,
   if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
       ot_lobj GInputStream *file_object = NULL;
+      guint64 length;
 
       if (!ostree_repo_load_file (repo, checksum,
                                   &input, &file_info, &xattrs,
                                   cancellable, error))
         goto out;
 
-      if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, 
+      if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, &length,
                                               cancellable, error))
         goto out;
 
-      if (!ostree_repo_stage_object_trusted (repo, OSTREE_OBJECT_TYPE_FILE,
-                                             checksum, TRUE, file_object,
-                                             cancellable, error))
+      if (!ostree_repo_stage_file_object_trusted (repo, checksum, TRUE, file_object, length,
+                                                  cancellable, error))
         goto out;
     }
   else