I broke your bindings

Hey. Apologies in advance -- I broke your bindings.

This is a major commit that includes a long-overdue redesign of the
whole object-database structure. This is expected to be the last major
external API redesign of the library until the first non-alpha release.

Please get your bindings up to date with these changes. They will be
included in the next minor release. Sorry again!

Major features include:

	- Real caching and refcounting on parsed objects
	- Real caching and refcounting on objects read from the ODB
	- Streaming writes & reads from the ODB
	- Single-method writes for all object types
	- The external API is now partially thread-safe

The speed increases are significant in all aspects, specially when
reading an object several times from the ODB (revwalking) and when
writing big objects to the ODB.

Here's a full changelog for the external API:

blob.h
------

	- Remove `git_blob_new`
	- Remove `git_blob_set_rawcontent`
	- Remove `git_blob_set_rawcontent_fromfile`
	- Rename `git_blob_writefile` -> `git_blob_create_fromfile`
	- Change `git_blob_create_fromfile`:
		The `path` argument is now relative to the repository's working dir
	- Add `git_blob_create_frombuffer`

commit.h
--------

	- Remove `git_commit_new`
	- Remove `git_commit_add_parent`
	- Remove `git_commit_set_message`
	- Remove `git_commit_set_committer`
	- Remove `git_commit_set_author`
	- Remove `git_commit_set_tree`

	- Add `git_commit_create`
	- Add `git_commit_create_v`
	- Add `git_commit_create_o`
	- Add `git_commit_create_ov`

tag.h
-----

	- Remove `git_tag_new`
	- Remove `git_tag_set_target`
	- Remove `git_tag_set_name`
	- Remove `git_tag_set_tagger`
	- Remove `git_tag_set_message`

	- Add `git_tag_create`
	- Add `git_tag_create_o`

tree.h
------

	- Change `git_tree_entry_2object`:
		New signature is `(git_object **object_out, git_repository *repo, git_tree_entry *entry)`

	- Remove `git_tree_new`
	- Remove `git_tree_add_entry`
	- Remove `git_tree_remove_entry_byindex`
	- Remove `git_tree_remove_entry_byname`
	- Remove `git_tree_clearentries`
	- Remove `git_tree_entry_set_id`
	- Remove `git_tree_entry_set_name`
	- Remove `git_tree_entry_set_attributes`

object.h
------------

	- Remove `git_object_new
	- Remove `git_object_write`

	- Change `git_object_close`:
		This method is now *mandatory*. Not closing an object causes a
		memory leak.

odb.h
-----

	- Remove type `git_rawobj`
	- Remove `git_rawobj_close`
	- Rename `git_rawobj_hash` -> `git_odb_hash`
	- Change `git_odb_hash`:
		New signature is `(git_oid *id, const void *data, size_t len, git_otype type)`

	- Add type `git_odb_object`
	- Add `git_odb_object_close`

	- Change `git_odb_read`:
		New signature is `(git_odb_object **out, git_odb *db, const git_oid *id)`
	- Change `git_odb_read_header`:
		New signature is `(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)`
	- Remove `git_odb_write`
	- Add `git_odb_open_wstream`
	- Add `git_odb_open_rstream`

odb_backend.h
-------------

	- Change type `git_odb_backend`:
		New internal signatures are as follows

			int (* read)(void **, size_t *, git_otype *, struct git_odb_backend *, const git_oid *)
			int (* read_header)(size_t *, git_otype *, struct git_odb_backend *, const git_oid *)
			int (* writestream)(struct git_odb_stream **, struct git_odb_backend *, size_t, git_otype)
			int (* readstream)( struct git_odb_stream **, struct git_odb_backend *, const git_oid *)

	- Add type `git_odb_stream`
	- Add enum `git_odb_streammode`

Signed-off-by: Vicent Marti <tanoku@gmail.com>

1 files changed, 144 insertions, 44 deletions

diff --git a/src/filebuf.c b/src/filebuf.c
index 4fc4f1486..607ad618d 100644
--- a/src/filebuf.c
+++ b/src/filebuf.c
@@ -77,43 +77,81 @@ void git_filebuf_cleanup(git_filebuf *file)
 	if (file->fd >= 0)
 		gitfo_close(file->fd);
 
-	if (gitfo_exists(file->path_lock) == GIT_SUCCESS)
+	if (file->path_lock && gitfo_exists(file->path_lock) == GIT_SUCCESS)
 		gitfo_unlink(file->path_lock);
 
 	if (file->digest)
 		git_hash_free_ctx(file->digest);
 
 	free(file->buffer);
+	free(file->z_buf);
 
-#ifdef GIT_FILEBUF_THREADS
-	free(file->buffer_back);
-#endif
+	deflateEnd(&file->zs);
 
 	free(file->path_original);
 	free(file->path_lock);
 }
 
-static int flush_buffer(git_filebuf *file)
+GIT_INLINE(int) flush_buffer(git_filebuf *file)
 {
-	int result = GIT_SUCCESS;
+	int result = file->write(file, file->buffer, file->buf_pos);
+	file->buf_pos = 0;
+	return result;
+}
 
-	if (file->buf_pos > 0) {
-		result = gitfo_write(file->fd, file->buffer, file->buf_pos);
-		if (file->digest)
-			git_hash_update(file->digest, file->buffer, file->buf_pos);
+static int write_normal(git_filebuf *file, const void *source, size_t len)
+{
+	int result = 0;
 
-		file->buf_pos = 0;
+	if (len > 0) {
+		result = gitfo_write(file->fd, (void *)source, len);
+		if (file->digest)
+			git_hash_update(file->digest, source, len);
 	}
 
 	return result;
 }
 
+static int write_deflate(git_filebuf *file, const void *source, size_t len)
+{
+	int result = Z_OK;
+	z_stream *zs = &file->zs;
+
+	if (len > 0 || file->flush_mode == Z_FINISH) {
+		zs->next_in = (void *)source;
+		zs->avail_in = len;
+
+		do {
+			int have;
+
+			zs->next_out = file->z_buf;
+			zs->avail_out = file->buf_size;
+
+            result = deflate(zs, file->flush_mode);
+            assert(result != Z_STREAM_ERROR);
+
+            have = file->buf_size - zs->avail_out;
+
+			if (gitfo_write(file->fd, file->z_buf, have) < GIT_SUCCESS)
+				return GIT_EOSERR;
+
+        } while (zs->avail_out == 0);
+
+		assert(zs->avail_in == 0);
+
+		if (file->digest)
+			git_hash_update(file->digest, source, len);
+	}
+
+	return GIT_SUCCESS;
+}
+
 int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 {
 	int error;
 	size_t path_len;
 
-	if (file == NULL || path == NULL)
+	if (file == NULL)
 		return GIT_ERROR;
 
 	memset(file, 0x0, sizeof(git_filebuf));
@@ -122,46 +160,93 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 	file->buf_pos = 0;
 	file->fd = -1;
 
-	path_len = strlen(path);
-
-	file->path_original = git__strdup(path);
-	if (file->path_original == NULL) {
+	/* Allocate the main cache buffer */
+	file->buffer = git__malloc(file->buf_size);
+	if (file->buffer == NULL){
 		error = GIT_ENOMEM;
 		goto cleanup;
 	}
 
-	file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH);
-	if (file->path_lock == NULL) {
-		error = GIT_ENOMEM;
-		goto cleanup;
+	/* If we are hashing on-write, allocate a new hash context */
+	if (flags & GIT_FILEBUF_HASH_CONTENTS) {
+		if ((file->digest = git_hash_new_ctx()) == NULL) {
+			error = GIT_ENOMEM;
+			goto cleanup;
+		}
 	}
 
-	memcpy(file->path_lock, file->path_original, path_len);
-	memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH);
+	/* If we are deflating on-write, */
+	if (flags & GIT_FILEBUF_DEFLATE_CONTENTS) {
 
-	file->buffer = git__malloc(file->buf_size);
-	if (file->buffer == NULL){
-		error = GIT_ENOMEM;
-		goto cleanup;
-	}
+		/* Initialize the ZLib stream */
+		if (deflateInit(&file->zs, Z_DEFAULT_COMPRESSION) != Z_OK) {
+			error = GIT_EZLIB;
+			goto cleanup;
+		}
 
-#ifdef GIT_FILEBUF_THREADS
-	file->buffer_back = git__malloc(file->buf_size);
-	if (file->buffer_back == NULL){
-		error = GIT_ENOMEM;
-		goto cleanup;
+		/* Allocate the Zlib cache buffer */
+		file->z_buf = git__malloc(file->buf_size);
+		if (file->z_buf == NULL){
+			error = GIT_ENOMEM;
+			goto cleanup;
+		}
+
+		/* Never flush */
+		file->flush_mode = Z_NO_FLUSH;
+		file->write = &write_deflate;
+	} else {
+		file->write = &write_normal;
 	}
-#endif
 
-	if (flags & GIT_FILEBUF_HASH_CONTENTS) {
-		if ((file->digest = git_hash_new_ctx()) == NULL) {
+	/* If we are writing to a temp file */
+	if (flags & GIT_FILEBUF_TEMPORARY) {
+		char tmp_path[GIT_PATH_MAX];
+
+		/* Open the file as temporary for locking */
+		file->fd = gitfo_creat_tmp(tmp_path, "_filebuf_"); 
+		if (file->fd < 0) {
+			error = GIT_EOSERR;
+			goto cleanup;
+		}
+
+		/* No original path */
+		file->path_original = NULL;
+		file->path_lock = git__strdup(tmp_path);
+
+		if (file->path_lock == NULL) {
 			error = GIT_ENOMEM;
 			goto cleanup;
 		}
-	}
+	} else {
+		/* If the file is not temporary, make sure we have a path */
+		if (path == NULL) {
+			error = GIT_ERROR;
+			goto cleanup;
+		}
 
-	if ((error = lock_file(file, flags)) < GIT_SUCCESS)
-		goto cleanup;
+		path_len = strlen(path);
+
+		/* Save the original path of the file */
+		file->path_original = git__strdup(path);
+		if (file->path_original == NULL) {
+			error = GIT_ENOMEM;
+			goto cleanup;
+		}
+
+		/* create the locking path by appending ".lock" to the original */
+		file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH);
+		if (file->path_lock == NULL) {
+			error = GIT_ENOMEM;
+			goto cleanup;
+		}
+
+		memcpy(file->path_lock, file->path_original, path_len);
+		memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH);
+
+		/* open the file for locking */
+		if ((error = lock_file(file, flags)) < GIT_SUCCESS)
+			goto cleanup;
+	}
 
 	return GIT_SUCCESS;
 
@@ -187,10 +272,25 @@ int git_filebuf_hash(git_oid *oid, git_filebuf *file)
 	return GIT_SUCCESS;
 }
 
+int git_filebuf_commit_at(git_filebuf *file, const char *path)
+{
+	free(file->path_original);
+	file->path_original = git__strdup(path);
+	if (file->path_original == NULL)
+		return GIT_ENOMEM;
+
+	return git_filebuf_commit(file);
+}
+
 int git_filebuf_commit(git_filebuf *file)
 {
 	int error;
 
+	/* tmp file cannot be commited */
+	if (file->path_original == NULL)
+		return GIT_EOSERR;
+
+	file->flush_mode = Z_FINISH;
 	if ((error = flush_buffer(file)) < GIT_SUCCESS)
 		goto cleanup;
 
@@ -204,16 +304,16 @@ cleanup:
 	return error;
 }
 
-GIT_INLINE(void) add_to_cache(git_filebuf *file, void *buf, size_t len)
+GIT_INLINE(void) add_to_cache(git_filebuf *file, const void *buf, size_t len)
 {
 	memcpy(file->buffer + file->buf_pos, buf, len);
 	file->buf_pos += len;
 }
 
-int git_filebuf_write(git_filebuf *file, void *buff, size_t len)
+int git_filebuf_write(git_filebuf *file, const void *buff, size_t len)
 {
 	int error;
-	unsigned char *buf = buff;
+	const unsigned char *buf = buff;
 
 	for (;;) {
 		size_t space_left = file->buf_size - file->buf_pos;
@@ -237,9 +337,9 @@ int git_filebuf_write(git_filebuf *file, void *buff, size_t len)
 
 		/* write too-large chunks immediately */
 		if (len > file->buf_size) {
-			error = gitfo_write(file->fd, buf, len);
-			if (file->digest)
-				git_hash_update(file->digest, buf, len);
+			error = file->write(file, buf, len);
+			if (error < GIT_SUCCESS)
+				return error;
 		}
 	}
 }