Merge pull request #1454 from libgit2/vmg/new-cache

New caching

9 files changed, 394 insertions, 47 deletions

diff --git a/tests-clar/commit/parse.c b/tests-clar/commit/parse.c
index b99d27991..8de4401dc 100644
--- a/tests-clar/commit/parse.c
+++ b/tests-clar/commit/parse.c
@@ -264,37 +264,40 @@ gpgsig -----BEGIN PGP SIGNATURE-----\n\
 a simple commit which works\n",
 };
 
-void test_commit_parse__entire_commit(void)
+static int parse_commit(git_commit **out, const char *buffer)
 {
-	const int broken_commit_count = sizeof(failing_commit_cases) / sizeof(*failing_commit_cases);
-	const int working_commit_count = sizeof(passing_commit_cases) / sizeof(*passing_commit_cases);
-	int i;
+	git_commit *commit;
+	git_odb_object fake_odb_object;
+	int error;
 
-	for (i = 0; i < broken_commit_count; ++i) {
-		git_commit *commit;
-		commit = (git_commit*)git__malloc(sizeof(git_commit));
-		memset(commit, 0x0, sizeof(git_commit));
-		commit->object.repo = g_repo;
+	commit = (git_commit*)git__malloc(sizeof(git_commit));
+	memset(commit, 0x0, sizeof(git_commit));
+	commit->object.repo = g_repo;
 
-		cl_git_fail(git_commit__parse_buffer(
-			commit, failing_commit_cases[i], strlen(failing_commit_cases[i]))
-		);
+	memset(&fake_odb_object, 0x0, sizeof(git_odb_object));
+	fake_odb_object.buffer = (char *)buffer;
+	fake_odb_object.cached.size = strlen(fake_odb_object.buffer);
 
-		git_commit__free(commit);
-	}
+	error = git_commit__parse(commit, &fake_odb_object);
 
-	for (i = 0; i < working_commit_count; ++i) {
-		git_commit *commit;
+	*out = commit;
+	return error;
+}
+
+void test_commit_parse__entire_commit(void)
+{
+	const int failing_commit_count = ARRAY_SIZE(failing_commit_cases);
+	const int passing_commit_count = ARRAY_SIZE(passing_commit_cases);
+	int i;
+	git_commit *commit;
 
-		commit = (git_commit*)git__malloc(sizeof(git_commit));
-		memset(commit, 0x0, sizeof(git_commit));
-		commit->object.repo = g_repo;
+	for (i = 0; i < failing_commit_count; ++i) {
+		cl_git_fail(parse_commit(&commit, failing_commit_cases[i]));
+		git_commit__free(commit);
+	}
 
-		cl_git_pass(git_commit__parse_buffer(
-                     commit,
-                     passing_commit_cases[i],
-                     strlen(passing_commit_cases[i]))
-         );
+	for (i = 0; i < passing_commit_count; ++i) {
+		cl_git_pass(parse_commit(&commit, passing_commit_cases[i]));
 
 		if (!i)
 			cl_assert_equal_s("", git_commit_message(commit));
@@ -387,9 +390,7 @@ This commit has a few LF at the start of the commit message";
 	memset(commit, 0x0, sizeof(git_commit));
 	commit->object.repo = g_repo;
 
-	cl_git_pass(git_commit__parse_buffer(commit, buffer, strlen(buffer)));
-
+	cl_git_pass(parse_commit(&commit, buffer));
 	cl_assert_equal_s(message, git_commit_message(commit));
-
 	git_commit__free(commit);
 }
diff --git a/tests-clar/core/oidmap.c b/tests-clar/core/oidmap.c
new file mode 100644
index 000000000..ec4b5e775
--- /dev/null
+++ b/tests-clar/core/oidmap.c
@@ -0,0 +1,110 @@
+#include "clar_libgit2.h"
+#include "oidmap.h"
+
+GIT__USE_OIDMAP;
+
+typedef struct {
+	git_oid oid;
+	size_t extra;
+} oidmap_item;
+
+#define NITEMS 0x0fff
+
+void test_core_oidmap__basic(void)
+{
+	git_oidmap *map;
+	oidmap_item items[NITEMS];
+	uint32_t i, j;
+
+	for (i = 0; i < NITEMS; ++i) {
+		items[i].extra = i;
+		for (j = 0; j < GIT_OID_RAWSZ / 4; ++j) {
+			items[i].oid.id[j * 4    ] = (unsigned char)i;
+			items[i].oid.id[j * 4 + 1] = (unsigned char)(i >> 8);
+			items[i].oid.id[j * 4 + 2] = (unsigned char)(i >> 16);
+			items[i].oid.id[j * 4 + 3] = (unsigned char)(i >> 24);
+		}
+	}
+
+	map = git_oidmap_alloc();
+	cl_assert(map != NULL);
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+		int ret;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos == kh_end(map));
+
+		pos = kh_put(oid, map, &items[i].oid, &ret);
+		cl_assert(ret != 0);
+
+		kh_val(map, pos) = &items[i];
+	}
+
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos != kh_end(map));
+
+		cl_assert_equal_p(kh_val(map, pos), &items[i]);
+	}
+
+	git_oidmap_free(map);
+}
+
+void test_core_oidmap__hash_collision(void)
+{
+	git_oidmap *map;
+	oidmap_item items[NITEMS];
+	uint32_t i, j;
+
+	for (i = 0; i < NITEMS; ++i) {
+		uint32_t segment = i / 8;
+		int modi = i - (segment * 8);
+
+		items[i].extra = i;
+
+		for (j = 0; j < GIT_OID_RAWSZ / 4; ++j) {
+			items[i].oid.id[j * 4    ] = (unsigned char)modi;
+			items[i].oid.id[j * 4 + 1] = (unsigned char)(modi >> 8);
+			items[i].oid.id[j * 4 + 2] = (unsigned char)(modi >> 16);
+			items[i].oid.id[j * 4 + 3] = (unsigned char)(modi >> 24);
+		}
+
+		items[i].oid.id[ 8] = (unsigned char)i;
+		items[i].oid.id[ 9] = (unsigned char)(i >> 8);
+		items[i].oid.id[10] = (unsigned char)(i >> 16);
+		items[i].oid.id[11] = (unsigned char)(i >> 24);
+	}
+
+	map = git_oidmap_alloc();
+	cl_assert(map != NULL);
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+		int ret;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos == kh_end(map));
+
+		pos = kh_put(oid, map, &items[i].oid, &ret);
+		cl_assert(ret != 0);
+
+		kh_val(map, pos) = &items[i];
+	}
+
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos != kh_end(map));
+
+		cl_assert_equal_p(kh_val(map, pos), &items[i]);
+	}
+
+	git_oidmap_free(map);
+}
diff --git a/tests-clar/core/opts.c b/tests-clar/core/opts.c
index 907339d51..3173c648b 100644
--- a/tests-clar/core/opts.c
+++ b/tests-clar/core/opts.c
@@ -16,15 +16,4 @@ void test_core_opts__readwrite(void)
 	git_libgit2_opts(GIT_OPT_GET_MWINDOW_SIZE, &new_val);
 
 	cl_assert(new_val == old_val);
-
-	git_libgit2_opts(GIT_OPT_GET_ODB_CACHE_SIZE, &old_val);
-
-	cl_assert(old_val == GIT_DEFAULT_CACHE_SIZE);
-
-	git_libgit2_opts(GIT_OPT_SET_ODB_CACHE_SIZE, (size_t)GIT_DEFAULT_CACHE_SIZE*2);
-	git_libgit2_opts(GIT_OPT_GET_ODB_CACHE_SIZE, &new_val);
-
-	cl_assert(new_val == (GIT_DEFAULT_CACHE_SIZE*2));
-
-	git_libgit2_opts(GIT_OPT_GET_ODB_CACHE_SIZE, &old_val);
 }
diff --git a/tests-clar/diff/workdir.c b/tests-clar/diff/workdir.c
index 9d92d8d60..435bd4f2c 100644
--- a/tests-clar/diff/workdir.c
+++ b/tests-clar/diff/workdir.c
@@ -908,7 +908,6 @@ void test_diff_workdir__can_diff_empty_file(void)
 	/* baseline - make sure there are no outstanding diffs */
 
 	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, tree, &opts));
-	git_tree_free(tree);
 	cl_assert_equal_i(2, (int)git_diff_num_deltas(diff));
 	git_diff_list_free(diff);
 
@@ -935,6 +934,8 @@ void test_diff_workdir__can_diff_empty_file(void)
 	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 1));
 	git_diff_patch_free(patch);
 	git_diff_list_free(diff);
+
+	git_tree_free(tree);
 }
 
 void test_diff_workdir__to_index_issue_1397(void)
diff --git a/tests-clar/object/cache.c b/tests-clar/object/cache.c
new file mode 100644
index 000000000..a3eba8737
--- /dev/null
+++ b/tests-clar/object/cache.c
@@ -0,0 +1,235 @@
+#include "clar_libgit2.h"
+#include "repository.h"
+
+static git_repository *g_repo;
+
+void test_object_cache__initialize(void)
+{
+   cl_git_pass(git_repository_open(&g_repo, cl_fixture("testrepo.git")));
+}
+
+void test_object_cache__cleanup(void)
+{
+	git_repository_free(g_repo);
+	g_repo = NULL;
+
+	git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, (int)GIT_OBJ_BLOB, (size_t)0);
+}
+
+static struct {
+	git_otype type;
+	const char *sha;
+} g_data[] = {
+	/* HEAD */
+	{ GIT_OBJ_BLOB, "a8233120f6ad708f843d861ce2b7228ec4e3dec6" }, /* README */
+	{ GIT_OBJ_BLOB, "3697d64be941a53d4ae8f6a271e4e3fa56b022cc" }, /* branch_file.txt */
+	{ GIT_OBJ_BLOB, "a71586c1dfe8a71c6cbf6c129f404c5642ff31bd" }, /* new.txt */
+
+	/* refs/heads/subtrees */
+	{ GIT_OBJ_BLOB, "1385f264afb75a56a5bec74243be9b367ba4ca08" }, /* README */
+	{ GIT_OBJ_TREE, "f1425cef211cc08caa31e7b545ffb232acb098c3" }, /* ab */
+	{ GIT_OBJ_BLOB, "d6c93164c249c8000205dd4ec5cbca1b516d487f" }, /* ab/4.txt */
+	{ GIT_OBJ_TREE, "9a03079b8a8ee85a0bee58bf9be3da8b62414ed4" }, /* ab/c */
+	{ GIT_OBJ_BLOB, "270b8ea76056d5cad83af921837702d3e3c2924d" }, /* ab/c/3.txt */
+	{ GIT_OBJ_TREE, "b6361fc6a97178d8fc8639fdeed71c775ab52593" }, /* ab/de */
+	{ GIT_OBJ_BLOB, "e7b4ad382349ff96dd8199000580b9b1e2042eb0" }, /* ab/de/2.txt */
+	{ GIT_OBJ_TREE, "3259a6bd5b57fb9c1281bb7ed3167b50f224cb54" }, /* ab/de/fgh */
+	{ GIT_OBJ_BLOB, "1f67fc4386b2d171e0d21be1c447e12660561f9b" }, /* ab/de/fgh/1.txt */
+	{ GIT_OBJ_BLOB, "45b983be36b73c0788dc9cbcb76cbb80fc7bb057" }, /* branch_file.txt */
+	{ GIT_OBJ_BLOB, "fa49b077972391ad58037050f2a75f74e3671e92" }, /* new.txt */
+
+	/* refs/heads/chomped */
+	{ GIT_OBJ_BLOB, "0266163a49e280c4f5ed1e08facd36a2bd716bcf" }, /* readme.txt */
+
+	{ 0, NULL },
+	{ 0, NULL }
+};
+
+void test_object_cache__cache_everything(void)
+{
+	int i, start;
+	git_oid oid;
+	git_odb_object *odb_obj;
+	git_object *obj;
+	git_odb *odb;
+
+	git_libgit2_opts(
+		GIT_OPT_SET_CACHE_OBJECT_LIMIT, (int)GIT_OBJ_BLOB, (size_t)32767);
+
+	cl_git_pass(git_repository_odb(&odb, g_repo));
+
+	start = (int)git_cache_size(&g_repo->objects);
+
+	for (i = 0; g_data[i].sha != NULL; ++i) {
+		int count = (int)git_cache_size(&g_repo->objects);
+
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+
+		/* alternate between loading raw and parsed objects */
+		if ((i & 1) == 0) {
+			cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+			cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+			git_odb_object_free(odb_obj);
+		} else {
+			cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+			cl_assert(g_data[i].type == git_object_type(obj));
+			git_object_free(obj);
+		}
+
+		cl_assert_equal_i(count + 1, (int)git_cache_size(&g_repo->objects));
+	}
+
+	cl_assert_equal_i(i, git_cache_size(&g_repo->objects) - start);
+
+	git_odb_free(odb);
+
+	for (i = 0; g_data[i].sha != NULL; ++i) {
+		int count = (int)git_cache_size(&g_repo->objects);
+
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+		cl_assert(g_data[i].type == git_object_type(obj));
+		git_object_free(obj);
+
+		cl_assert_equal_i(count, (int)git_cache_size(&g_repo->objects));
+	}
+}
+
+void test_object_cache__cache_no_blobs(void)
+{
+	int i, start, nonblobs = 0;
+	git_oid oid;
+	git_odb_object *odb_obj;
+	git_object *obj;
+	git_odb *odb;
+
+	git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, (int)GIT_OBJ_BLOB, (size_t)0);
+
+	cl_git_pass(git_repository_odb(&odb, g_repo));
+
+	start = (int)git_cache_size(&g_repo->objects);
+
+	for (i = 0; g_data[i].sha != NULL; ++i) {
+		int count = (int)git_cache_size(&g_repo->objects);
+
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+
+		/* alternate between loading raw and parsed objects */
+		if ((i & 1) == 0) {
+			cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+			cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+			git_odb_object_free(odb_obj);
+		} else {
+			cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+			cl_assert(g_data[i].type == git_object_type(obj));
+			git_object_free(obj);
+		}
+
+		if (g_data[i].type == GIT_OBJ_BLOB)
+			cl_assert_equal_i(count, (int)git_cache_size(&g_repo->objects));
+		else {
+			cl_assert_equal_i(count + 1, (int)git_cache_size(&g_repo->objects));
+			nonblobs++;
+		}
+	}
+
+	cl_assert_equal_i(nonblobs, git_cache_size(&g_repo->objects) - start);
+
+	git_odb_free(odb);
+}
+
+static void *cache_parsed(void *arg)
+{
+	int i;
+	git_oid oid;
+	git_object *obj;
+
+	for (i = ((int *)arg)[1]; g_data[i].sha != NULL; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+		cl_assert(g_data[i].type == git_object_type(obj));
+		git_object_free(obj);
+	}
+
+	for (i = 0; i < ((int *)arg)[1]; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+		cl_assert(g_data[i].type == git_object_type(obj));
+		git_object_free(obj);
+	}
+
+	return arg;
+}
+
+static void *cache_raw(void *arg)
+{
+	int i;
+	git_oid oid;
+	git_odb *odb;
+	git_odb_object *odb_obj;
+
+	cl_git_pass(git_repository_odb(&odb, g_repo));
+
+	for (i = ((int *)arg)[1]; g_data[i].sha != NULL; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+		cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+		git_odb_object_free(odb_obj);
+	}
+
+	for (i = 0; i < ((int *)arg)[1]; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+		cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+		git_odb_object_free(odb_obj);
+	}
+
+	git_odb_free(odb);
+
+	return arg;
+}
+
+#define REPEAT 50
+#define THREADCOUNT 20
+
+void test_object_cache__threadmania(void)
+{
+	int try, th, max_i;
+	void *data;
+	void *(*fn)(void *);
+
+#ifdef GIT_THREADS
+	git_thread t[THREADCOUNT];
+#endif
+
+	for (max_i = 0; g_data[max_i].sha != NULL; ++max_i)
+		/* count up */;
+
+	for (try = 0; try < REPEAT; ++try) {
+
+		for (th = 0; th < THREADCOUNT; ++th) {
+			data = git__malloc(2 * sizeof(int));
+
+			((int *)data)[0] = th;
+			((int *)data)[1] = th % max_i;
+
+			fn = (th & 1) ? cache_parsed : cache_raw;
+
+#ifdef GIT_THREADS
+			cl_git_pass(git_thread_create(&t[th], NULL, fn, data));
+#else
+			cl_assert(fn(data) == data);
+			git__free(data);
+#endif
+		}
+
+#ifdef GIT_THREADS
+		for (th = 0; th < THREADCOUNT; ++th) {
+			cl_git_pass(git_thread_join(t[th], &data));
+			cl_assert_equal_i(th, ((int *)data)[0]);
+			git__free(data);
+		}
+#endif
+
+	}
+}
diff --git a/tests-clar/object/raw/write.c b/tests-clar/object/raw/write.c
index 60aa31f6a..9709c0302 100644
--- a/tests-clar/object/raw/write.c
+++ b/tests-clar/object/raw/write.c
@@ -63,6 +63,7 @@ void test_body(object_data *d, git_rawobj *o)
    git_odb *db;
    git_oid id1, id2;
    git_odb_object *obj;
+   git_rawobj tmp;
 
    make_odb_dir();
    cl_git_pass(git_odb_open(&db, odb_dir));
@@ -73,7 +74,12 @@ void test_body(object_data *d, git_rawobj *o)
    check_object_files(d);
 
    cl_git_pass(git_odb_read(&obj, db, &id1));
-   cmp_objects(&obj->raw, o);
+
+   tmp.data = obj->buffer;
+   tmp.len = obj->cached.size;
+   tmp.type = obj->cached.type;
+
+   cmp_objects(&tmp, o);
 
    git_odb_object_free(obj);
    git_odb_free(db);
diff --git a/tests-clar/odb/loose.c b/tests-clar/odb/loose.c
index f95dc28d4..9539bb24c 100644
--- a/tests-clar/odb/loose.c
+++ b/tests-clar/odb/loose.c
@@ -30,6 +30,7 @@ static void test_read_object(object_data *data)
     git_oid id;
     git_odb_object *obj;
 	git_odb *odb;
+	git_rawobj tmp;
 
     write_object_files(data);
 
@@ -37,7 +38,11 @@ static void test_read_object(object_data *data)
     cl_git_pass(git_oid_fromstr(&id, data->id));
     cl_git_pass(git_odb_read(&obj, odb, &id));
 
-    cmp_objects((git_rawobj *)&obj->raw, data);
+	tmp.data = obj->buffer;
+	tmp.len = obj->cached.size;
+	tmp.type = obj->cached.type;
+
+    cmp_objects(&tmp, data);
 
     git_odb_object_free(obj);
 	git_odb_free(odb);
diff --git a/tests-clar/odb/packed.c b/tests-clar/odb/packed.c
index 90e9f3abd..b4f549b58 100644
--- a/tests-clar/odb/packed.c
+++ b/tests-clar/odb/packed.c
@@ -46,8 +46,8 @@ void test_odb_packed__read_header_0(void)
 		cl_git_pass(git_odb_read(&obj, _odb, &id));
 		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
 
-		cl_assert(obj->raw.len == len);
-		cl_assert(obj->raw.type == type);
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
 
 		git_odb_object_free(obj);
 	}
@@ -70,8 +70,8 @@ void test_odb_packed__read_header_1(void)
 		cl_git_pass(git_odb_read(&obj, _odb, &id));
 		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
 
-		cl_assert(obj->raw.len == len);
-		cl_assert(obj->raw.type == type);
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
 
 		git_odb_object_free(obj);
 	}
diff --git a/tests-clar/odb/packed_one.c b/tests-clar/odb/packed_one.c
index 4f9bde9ed..0c6ed387b 100644
--- a/tests-clar/odb/packed_one.c
+++ b/tests-clar/odb/packed_one.c
@@ -52,8 +52,8 @@ void test_odb_packed_one__read_header_0(void)
 		cl_git_pass(git_odb_read(&obj, _odb, &id));
 		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
 
-		cl_assert(obj->raw.len == len);
-		cl_assert(obj->raw.type == type);
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
 
 		git_odb_object_free(obj);
 	}