Merge pull request #1025 from ethomson/fetchodb

fetch: Write packs to the ODB instead of directly to disk

7 files changed, 150 insertions, 21 deletions

diff --git a/include/git2/odb.h b/include/git2/odb.h
index c6e73571b..4afa3b788 100644
--- a/include/git2/odb.h
+++ b/include/git2/odb.h
@@ -11,6 +11,7 @@
 #include "types.h"
 #include "oid.h"
 #include "odb_backend.h"
+#include "indexer.h"
 
 /**
  * @file git2/odb.h
@@ -263,6 +264,26 @@ GIT_EXTERN(int) git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_
 GIT_EXTERN(int) git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid);
 
 /**
+ * Open a stream for writing a pack file to the ODB.
+ *
+ * If the ODB layer understands pack files, then the given
+ * packfile will likely be streamed directly to disk (and a
+ * corresponding index created).  If the ODB layer does not
+ * understand pack files, the objects will be stored in whatever
+ * format the ODB layer uses.
+ *
+ * @see git_odb_writepack
+ *
+ * @param writepack pointer to the writepack functions
+ * @param db object database where the stream will read from
+ * @param progress_cb function to call with progress information.
+ * Be aware that this is called inline with network and indexing operations,
+ * so performance may be affected.
+ * @param progress_payload payload for the progress callback
+ */
+GIT_EXTERN(int) git_odb_write_pack(git_odb_writepack **writepack, git_odb *db, git_transfer_progress_callback progress_cb, void *progress_payload);
+
+/**
  * Determine the object-ID (sha1 hash) of a data buffer
  *
  * The resulting SHA-1 OID will be the identifier for the data
diff --git a/include/git2/odb_backend.h b/include/git2/odb_backend.h
index cb8069787..4df48d77e 100644
--- a/include/git2/odb_backend.h
+++ b/include/git2/odb_backend.h
@@ -10,6 +10,7 @@
 #include "common.h"
 #include "types.h"
 #include "oid.h"
+#include "indexer.h"
 
 /**
  * @file git2/backend.h
@@ -21,6 +22,7 @@
 GIT_BEGIN_DECL
 
 struct git_odb_stream;
+struct git_odb_writepack;
 
 /** An instance for a custom backend */
 struct git_odb_backend {
@@ -75,11 +77,16 @@ struct git_odb_backend {
 			struct git_odb_backend *,
 			const git_oid *);
 
-	int (*foreach)(
-		       struct git_odb_backend *,
-		       int (*cb)(git_oid *oid, void *data),
-		       void *data
-		       );
+	int (* foreach)(
+			struct git_odb_backend *,
+			int (*cb)(git_oid *oid, void *data),
+			void *data);
+
+	int (* writepack)(
+			struct git_odb_writepack **,
+			struct git_odb_backend *,
+			git_transfer_progress_callback progress_cb,
+			void *progress_payload);
 
 	void (* free)(struct git_odb_backend *);
 };
@@ -102,6 +109,15 @@ struct git_odb_stream {
 	void (*free)(struct git_odb_stream *stream);
 };
 
+/** A stream to write a pack file to the ODB */
+struct git_odb_writepack {
+	struct git_odb_backend *backend;
+
+	int (*add)(struct git_odb_writepack *writepack, const void *data, size_t size, git_transfer_progress *stats);
+	int (*commit)(struct git_odb_writepack *writepack, git_transfer_progress *stats);
+	void (*free)(struct git_odb_writepack *writepack);
+};
+
 GIT_EXTERN(int) git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir);
 GIT_EXTERN(int) git_odb_backend_loose(git_odb_backend **backend_out, const char *objects_dir, int compression_level, int do_fsync);
 GIT_EXTERN(int) git_odb_backend_one_pack(git_odb_backend **backend_out, const char *index_file);
diff --git a/include/git2/types.h b/include/git2/types.h
index 01ddbf3d6..58cbaecc5 100644
--- a/include/git2/types.h
+++ b/include/git2/types.h
@@ -89,6 +89,9 @@ typedef struct git_odb_object git_odb_object;
 /** A stream to read/write from the ODB */
 typedef struct git_odb_stream git_odb_stream;
 
+/** A stream to write a packfile to the ODB */
+typedef struct git_odb_writepack git_odb_writepack;
+
 /**
  * Representation of an existing git repository,
  * including all its object contents
diff --git a/src/fetch.c b/src/fetch.c
index 4f9f0c6f9..81136fc5f 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -8,7 +8,6 @@
 #include "git2/oid.h"
 #include "git2/refs.h"
 #include "git2/revwalk.h"
-#include "git2/indexer.h"
 #include "git2/transport.h"
 
 #include "common.h"
diff --git a/src/odb.c b/src/odb.c
index 7c21598f0..d6b1de946 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -766,6 +766,31 @@ int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oi
 	return error;
 }
 
+int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_callback progress_cb, void *progress_payload)
+{
+	unsigned int i;
+	int error = GIT_ERROR;
+
+	assert(out && db);
+
+	for (i = 0; i < db->backends.length && error < 0; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		/* we don't write in alternates! */
+		if (internal->is_alternate)
+			continue;
+
+		if (b->writepack != NULL)
+			error = b->writepack(out, b, progress_cb, progress_payload);
+	}
+
+	if (error == GIT_PASSTHROUGH)
+		error = 0;
+
+	return error;
+}
+
 void * git_odb_backend_malloc(git_odb_backend *backend, size_t len)
 {
 	GIT_UNUSED(backend);
diff --git a/src/odb_pack.c b/src/odb_pack.c
index 964e82afb..9f7a6ee1f 100644
--- a/src/odb_pack.c
+++ b/src/odb_pack.c
@@ -26,6 +26,11 @@ struct pack_backend {
 	char *pack_folder;
 };
 
+struct pack_writepack {
+	struct git_odb_writepack parent;
+	git_indexer_stream *indexer_stream;
+};
+
 /**
  * The wonderful tale of a Packed Object lookup query
  * ===================================================
@@ -475,6 +480,67 @@ static int pack_backend__foreach(git_odb_backend *_backend, int (*cb)(git_oid *o
 	return 0;
 }
 
+static int pack_backend__writepack_add(struct git_odb_writepack *_writepack, const void *data, size_t size, git_transfer_progress *stats)
+{
+	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
+
+	assert(writepack);
+
+	return git_indexer_stream_add(writepack->indexer_stream, data, size, stats);
+}
+
+static int pack_backend__writepack_commit(struct git_odb_writepack *_writepack, git_transfer_progress *stats)
+{
+	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
+
+	assert(writepack);
+
+	return git_indexer_stream_finalize(writepack->indexer_stream, stats);
+}
+
+static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
+{
+	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
+
+	assert(writepack);
+
+	git_indexer_stream_free(writepack->indexer_stream);
+	git__free(writepack);
+}
+
+static int pack_backend__writepack(struct git_odb_writepack **out,
+	git_odb_backend *_backend,
+	git_transfer_progress_callback progress_cb,
+	void *progress_payload)
+{
+	struct pack_backend *backend;
+	struct pack_writepack *writepack;
+
+	assert(out && _backend);
+
+	*out = NULL;
+
+	backend = (struct pack_backend *)_backend;
+
+	writepack = git__calloc(1, sizeof(struct pack_writepack));
+	GITERR_CHECK_ALLOC(writepack);
+
+	if (git_indexer_stream_new(&writepack->indexer_stream,
+		backend->pack_folder, progress_cb, progress_payload) < 0) {
+		git__free(writepack);
+		return -1;
+	}
+
+	writepack->parent.backend = _backend;
+	writepack->parent.add = pack_backend__writepack_add;
+	writepack->parent.commit = pack_backend__writepack_commit;
+	writepack->parent.free = pack_backend__writepack_free;
+
+	*out = (git_odb_writepack *)writepack;
+
+	return 0;
+}
+
 static void pack_backend__free(git_odb_backend *_backend)
 {
 	struct pack_backend *backend;
@@ -553,6 +619,7 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
 	backend->parent.read_header = NULL;
 	backend->parent.exists = &pack_backend__exists;
 	backend->parent.foreach = &pack_backend__foreach;
+	backend->parent.writepack = &pack_backend__writepack;
 	backend->parent.free = &pack_backend__free;
 
 	*backend_out = (git_odb_backend *)backend;
diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c
index 4fdd72d69..e24eb2783 100644
--- a/src/transports/smart_protocol.c
+++ b/src/transports/smart_protocol.c
@@ -6,6 +6,7 @@
  */
 #include "smart.h"
 #include "refs.h"
+#include "repository.h"
 
 #define NETWORK_XFER_THRESHOLD (100*1024)
 
@@ -341,7 +342,7 @@ on_error:
 	return error;
 }
 
-static int no_sideband(transport_smart *t, git_indexer_stream *idx, gitno_buffer *buf, git_transfer_progress *stats)
+static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack, gitno_buffer *buf, git_transfer_progress *stats)
 {
 	int recvd;
 
@@ -351,7 +352,7 @@ static int no_sideband(transport_smart *t, git_indexer_stream *idx, gitno_buffer
 			return GIT_EUSER;
 		}
 
-		if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
+		if (writepack->add(writepack, buf->data, buf->offset, stats) < 0)
 			return -1;
 
 		gitno_consume_n(buf, buf->offset);
@@ -360,7 +361,7 @@ static int no_sideband(transport_smart *t, git_indexer_stream *idx, gitno_buffer
 			return -1;
 	} while(recvd > 0);
 
-	if (git_indexer_stream_finalize(idx, stats))
+	if (writepack->commit(writepack, stats))
 		return -1;
 
 	return 0;
@@ -396,9 +397,9 @@ int git_smart__download_pack(
 	void *progress_payload)
 {
 	transport_smart *t = (transport_smart *)transport;
-	git_buf path = GIT_BUF_INIT;
 	gitno_buffer *buf = &t->buffer;
-	git_indexer_stream *idx = NULL;
+	git_odb *odb;
+	struct git_odb_writepack *writepack = NULL;
 	int error = -1;
 	struct network_packetsize_payload npp = {0};
 
@@ -416,19 +417,17 @@ int git_smart__download_pack(
 			t->packetsize_cb(t->buffer.offset, t->packetsize_payload);
 	}
 
-	if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
-		return -1;
-
-	if (git_indexer_stream_new(&idx, git_buf_cstr(&path), progress_cb, progress_payload) < 0)
+	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
+		((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) < 0))
 		goto on_error;
 
 	/*
 	 * If the remote doesn't support the side-band, we can feed
-	 * the data directly to the indexer. Otherwise, we need to
+	 * the data directly to the pack writer. Otherwise, we need to
 	 * check which one belongs there.
 	 */
 	if (!t->caps.side_band && !t->caps.side_band_64k) {
-		if (no_sideband(t, idx, buf, stats) < 0)
+		if (no_sideband(t, writepack, buf, stats) < 0)
 			goto on_error;
 
 		goto on_success;
@@ -454,7 +453,7 @@ int git_smart__download_pack(
 			git__free(pkt);
 		} else if (pkt->type == GIT_PKT_DATA) {
 			git_pkt_data *p = (git_pkt_data *) pkt;
-			if (git_indexer_stream_add(idx, p->data, p->len, stats) < 0)
+			if (writepack->add(writepack, p->data, p->len, stats) < 0)
 				goto on_error;
 
 			git__free(pkt);
@@ -465,15 +464,14 @@ int git_smart__download_pack(
 		}
 	} while (1);
 
-	if (git_indexer_stream_finalize(idx, stats) < 0)
+	if (writepack->commit(writepack, stats) < 0)
 		goto on_error;
 
 on_success:
 	error = 0;
 
 on_error:
-	git_buf_free(&path);
-	git_indexer_stream_free(idx);
+	writepack->free(writepack);
 
 	/* Trailing execution of progress_cb, if necessary */
 	if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes)