libgit2 v0.21.0v0.21.0

167 files changed, 13891 insertions, 7116 deletions

diff --git a/src/amiga/map.c b/src/amiga/map.c
deleted file mode 100644
index 0ba7995c6..000000000
--- a/src/amiga/map.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (C) the libgit2 contributors. All rights reserved.
- *
- * This file is part of libgit2, distributed under the GNU GPL v2 with
- * a Linking Exception. For full terms see the included COPYING file.
- */
-#include <git2/common.h>
-
-#ifndef GIT_WIN32
-
-#include "posix.h"
-#include "map.h"
-#include <errno.h>
-
-int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
-{
-	GIT_MMAP_VALIDATE(out, len, prot, flags);
-
-	out->data = NULL;
-	out->len = 0;
-
-	if ((prot & GIT_PROT_WRITE) && ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED)) {
-		giterr_set(GITERR_OS, "Trying to map shared-writeable");
-		return -1;
-	}
-
-	out->data = malloc(len);
-	GITERR_CHECK_ALLOC(out->data);
-
-	if ((p_lseek(fd, offset, SEEK_SET) < 0) || ((size_t)p_read(fd, out->data, len) != len)) {
-		giterr_set(GITERR_OS, "mmap emulation failed");
-		return -1;
-	}
-
-	out->len = len;
-	return 0;
-}
-
-int p_munmap(git_map *map)
-{
-	assert(map != NULL);
-	free(map->data);
-
-	return 0;
-}
-
-#endif
-
diff --git a/src/array.h b/src/array.h
index 1d4e1c224..f8a48722a 100644
--- a/src/array.h
+++ b/src/array.h
@@ -7,7 +7,7 @@
 #ifndef INCLUDE_array_h__
 #define INCLUDE_array_h__
 
-#include "util.h"
+#include "common.h"
 
 /*
  * Use this to declare a typesafe resizable array of items, a la:
diff --git a/src/attr.c b/src/attr.c
index 98a328a55..05b0c1b3c 100644
--- a/src/attr.c
+++ b/src/attr.c
@@ -1,8 +1,8 @@
 #include "common.h"
 #include "repository.h"
-#include "fileops.h"
+#include "sysdir.h"
 #include "config.h"
-#include "attr.h"
+#include "attr_file.h"
 #include "ignore.h"
 #include "git2/oid.h"
 #include <ctype.h>
@@ -33,6 +33,7 @@ static int collect_attr_files(
 	const char *path,
 	git_vector *files);
 
+static void release_attr_files(git_vector *files);
 
 int git_attr_get(
 	const char **value,
@@ -49,6 +50,8 @@ int git_attr_get(
 	git_attr_name attr;
 	git_attr_rule *rule;
 
+	assert(value && repo && name);
+
 	*value = NULL;
 
 	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
@@ -57,6 +60,7 @@ int git_attr_get(
 	if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0)
 		goto cleanup;
 
+	memset(&attr, 0, sizeof(attr));
 	attr.name = name;
 	attr.name_hash = git_attr_file__name_hash(name);
 
@@ -74,7 +78,7 @@ int git_attr_get(
 	}
 
 cleanup:
-	git_vector_free(&files);
+	release_attr_files(&files);
 	git_attr_path__free(&path);
 
 	return error;
@@ -103,6 +107,11 @@ int git_attr_get_many(
 	attr_get_many_info *info = NULL;
 	size_t num_found = 0;
 
+	if (!num_attr)
+		return 0;
+
+	assert(values && repo && names);
+
 	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
 		return -1;
 
@@ -145,7 +154,7 @@ int git_attr_get_many(
 	}
 
 cleanup:
-	git_vector_free(&files);
+	release_attr_files(&files);
 	git_attr_path__free(&path);
 	git__free(info);
 
@@ -169,15 +178,15 @@ int git_attr_foreach(
 	git_attr_assignment *assign;
 	git_strmap *seen = NULL;
 
+	assert(repo && callback);
+
 	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
 		return -1;
 
-	if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0)
+	if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0 ||
+		(error = git_strmap_alloc(&seen)) < 0)
 		goto cleanup;
 
-	seen = git_strmap_alloc();
-	GITERR_CHECK_ALLOC(seen);
-
 	git_vector_foreach(&files, i, file) {
 
 		git_attr_file__foreach_matching_rule(file, &path, j, rule) {
@@ -193,8 +202,7 @@ int git_attr_foreach(
 
 				error = callback(assign->name, assign->value, payload);
 				if (error) {
-					giterr_clear();
-					error = GIT_EUSER;
+					giterr_set_after_callback(error);
 					goto cleanup;
 				}
 			}
@@ -203,278 +211,114 @@ int git_attr_foreach(
 
 cleanup:
 	git_strmap_free(seen);
-	git_vector_free(&files);
+	release_attr_files(&files);
 	git_attr_path__free(&path);
 
 	return error;
 }
 
-
-int git_attr_add_macro(
+static int preload_attr_file(
 	git_repository *repo,
-	const char *name,
-	const char *values)
+	git_attr_file_source source,
+	const char *base,
+	const char *file)
 {
 	int error;
-	git_attr_rule *macro = NULL;
-	git_pool *pool;
-
-	if (git_attr_cache__init(repo) < 0)
-		return -1;
-
-	macro = git__calloc(1, sizeof(git_attr_rule));
-	GITERR_CHECK_ALLOC(macro);
-
-	pool = &git_repository_attr_cache(repo)->pool;
-
-	macro->match.pattern = git_pool_strdup(pool, name);
-	GITERR_CHECK_ALLOC(macro->match.pattern);
-
-	macro->match.length = strlen(macro->match.pattern);
-	macro->match.flags = GIT_ATTR_FNMATCH_MACRO;
+	git_attr_file *preload = NULL;
 
-	error = git_attr_assignment__parse(repo, pool, &macro->assigns, &values);
-
-	if (!error)
-		error = git_attr_cache__insert_macro(repo, macro);
-
-	if (error < 0)
-		git_attr_rule__free(macro);
+	if (!file)
+		return 0;
+	if (!(error = git_attr_cache__get(
+			&preload, repo, source, base, file, git_attr_file__parse_buffer)))
+		git_attr_file__free(preload);
 
 	return error;
 }
 
-bool git_attr_cache__is_cached(
-	git_repository *repo, git_attr_file_source source, const char *path)
+static int attr_setup(git_repository *repo)
 {
-	git_buf cache_key = GIT_BUF_INIT;
-	git_strmap *files = git_repository_attr_cache(repo)->files;
+	int error = 0;
 	const char *workdir = git_repository_workdir(repo);
-	bool rval;
-
-	if (workdir && git__prefixcmp(path, workdir) == 0)
-		path += strlen(workdir);
-	if (git_buf_printf(&cache_key, "%d#%s", (int)source, path) < 0)
-		return false;
-
-	rval = git_strmap_exists(files, git_buf_cstr(&cache_key));
-
-	git_buf_free(&cache_key);
-
-	return rval;
-}
+	git_index *idx = NULL;
+	git_buf sys = GIT_BUF_INIT;
 
-static int load_attr_file(
-	const char **data,
-	git_futils_filestamp *stamp,
-	const char *filename)
-{
-	int error;
-	git_buf content = GIT_BUF_INIT;
-
-	error = git_futils_filestamp_check(stamp, filename);
-	if (error < 0)
+	if ((error = git_attr_cache__init(repo)) < 0)
 		return error;
 
-	/* if error == 0, then file is up to date. By returning GIT_ENOTFOUND,
-	 * we tell the caller not to reparse this file...
+	/* preload attribute files that could contain macros so the
+	 * definitions will be available for later file parsing
 	 */
-	if (!error)
-		return GIT_ENOTFOUND;
 
-	error = git_futils_readbuffer(&content, filename);
+	if (!(error = git_sysdir_find_system_file(&sys, GIT_ATTR_FILE_SYSTEM))) {
+		error = preload_attr_file(
+			repo, GIT_ATTR_FILE__FROM_FILE, NULL, sys.ptr);
+		git_buf_free(&sys);
+	}
 	if (error < 0) {
-		/* convert error into ENOTFOUND so failed permissions / invalid
-		 * file type don't actually stop the operation in progress.
-		 */
-		return GIT_ENOTFOUND;
-
-		/* TODO: once warnings are available, issue a warning callback */
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			error = 0;
+		} else
+			return error;
 	}
 
-	*data = git_buf_detach(&content);
-
-	return 0;
-}
-
-static int load_attr_blob_from_index(
-	const char **content,
-	git_blob **blob,
-	git_repository *repo,
-	const git_oid *old_oid,
-	const char *relfile)
-{
-	int error;
-	size_t pos;
-	git_index *index;
-	const git_index_entry *entry;
-
-	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
-		(error = git_index_find(&pos, index, relfile)) < 0)
+	if ((error = preload_attr_file(
+			repo, GIT_ATTR_FILE__FROM_FILE,
+			NULL, git_repository_attr_cache(repo)->cfg_attr_file)) < 0)
 		return error;
 
-	entry = git_index_get_byindex(index, pos);
-
-	if (old_oid && git_oid__cmp(old_oid, &entry->oid) == 0)
-		return GIT_ENOTFOUND;
-
-	if ((error = git_blob_lookup(blob, repo, &entry->oid)) < 0)
+	if ((error = preload_attr_file(
+			repo, GIT_ATTR_FILE__FROM_FILE,
+			git_repository_path(repo), GIT_ATTR_FILE_INREPO)) < 0)
 		return error;
 
-	*content = git_blob_rawcontent(*blob);
-	return 0;
-}
-
-static int load_attr_from_cache(
-	git_attr_file **file,
-	git_attr_cache *cache,
-	git_attr_file_source source,
-	const char *relative_path)
-{
-	git_buf  cache_key = GIT_BUF_INIT;
-	khiter_t cache_pos;
-
-	*file = NULL;
-
-	if (!cache || !cache->files)
-		return 0;
-
-	if (git_buf_printf(&cache_key, "%d#%s", (int)source, relative_path) < 0)
-		return -1;
-
-	cache_pos = git_strmap_lookup_index(cache->files, cache_key.ptr);
-
-	git_buf_free(&cache_key);
-
-	if (git_strmap_valid_index(cache->files, cache_pos))
-		*file = git_strmap_value_at(cache->files, cache_pos);
-
-	return 0;
-}
-
-int git_attr_cache__internal_file(
-	git_repository *repo,
-	const char *filename,
-	git_attr_file **file)
-{
-	int error = 0;
-	git_attr_cache *cache = git_repository_attr_cache(repo);
-	khiter_t cache_pos = git_strmap_lookup_index(cache->files, filename);
-
-	if (git_strmap_valid_index(cache->files, cache_pos)) {
-		*file = git_strmap_value_at(cache->files, cache_pos);
-		return 0;
-	}
-
-	if (git_attr_file__new(file, 0, filename, &cache->pool) < 0)
-		return -1;
+	if (workdir != NULL &&
+		(error = preload_attr_file(
+			repo, GIT_ATTR_FILE__FROM_FILE, workdir, GIT_ATTR_FILE)) < 0)
+		return error;
 
-	git_strmap_insert(cache->files, (*file)->key + 2, *file, error);
-	if (error > 0)
-		error = 0;
+	if ((error = git_repository_index__weakptr(&idx, repo)) < 0 ||
+		(error = preload_attr_file(
+			repo, GIT_ATTR_FILE__FROM_INDEX, NULL, GIT_ATTR_FILE)) < 0)
+		return error;
 
 	return error;
 }
 
-int git_attr_cache__push_file(
+int git_attr_add_macro(
 	git_repository *repo,
-	const char *base,
-	const char *filename,
-	git_attr_file_source source,
-	git_attr_file_parser parse,
-	void* parsedata,
-	git_vector *stack)
+	const char *name,
+	const char *values)
 {
-	int error = 0;
-	git_buf path = GIT_BUF_INIT;
-	const char *workdir = git_repository_workdir(repo);
-	const char *relfile, *content = NULL;
-	git_attr_cache *cache = git_repository_attr_cache(repo);
-	git_attr_file *file = NULL;
-	git_blob *blob = NULL;
-	git_futils_filestamp stamp;
-
-	assert(filename && stack);
-
-	/* join base and path as needed */
-	if (base != NULL && git_path_root(filename) < 0) {
-		if (git_buf_joinpath(&path, base, filename) < 0)
-			return -1;
-		filename = path.ptr;
-	}
-
-	relfile = filename;
-	if (workdir && git__prefixcmp(relfile, workdir) == 0)
-		relfile += strlen(workdir);
-
-	/* check cache */
-	if (load_attr_from_cache(&file, cache, source, relfile) < 0)
-		return -1;
-
-	/* if not in cache, load data, parse, and cache */
-
-	if (source == GIT_ATTR_FILE_FROM_FILE) {
-		git_futils_filestamp_set(
-			&stamp, file ? &file->cache_data.stamp : NULL);
-
-		error = load_attr_file(&content, &stamp, filename);
-	} else {
-		error = load_attr_blob_from_index(&content, &blob,
-			repo, file ? &file->cache_data.oid : NULL, relfile);
-	}
-
-	if (error) {
-		/* not finding a file is not an error for this function */
-		if (error == GIT_ENOTFOUND) {
-			giterr_clear();
-			error = 0;
-		}
-		goto finish;
-	}
+	int error;
+	git_attr_rule *macro = NULL;
+	git_pool *pool;
 
-	/* if we got here, we have to parse and/or reparse the file */
-	if (file)
-		git_attr_file__clear_rules(file);
-	else {
-		error = git_attr_file__new(&file, source, relfile, &cache->pool);
-		if (error < 0)
-			goto finish;
-	}
+	if ((error = git_attr_cache__init(repo)) < 0)
+		return error;
 
-	if (parse && (error = parse(repo, parsedata, content, file)) < 0)
-		goto finish;
+	macro = git__calloc(1, sizeof(git_attr_rule));
+	GITERR_CHECK_ALLOC(macro);
 
-	git_strmap_insert(cache->files, file->key, file, error); //-V595
-	if (error > 0)
-		error = 0;
+	pool = &git_repository_attr_cache(repo)->pool;
 
-	/* remember "cache buster" file signature */
-	if (blob)
-		git_oid_cpy(&file->cache_data.oid, git_object_id((git_object *)blob));
-	else
-		git_futils_filestamp_set(&file->cache_data.stamp, &stamp);
+	macro->match.pattern = git_pool_strdup(pool, name);
+	GITERR_CHECK_ALLOC(macro->match.pattern);
 
-finish:
-	/* push file onto vector if we found one*/
-	if (!error && file != NULL)
-		error = git_vector_insert(stack, file);
+	macro->match.length = strlen(macro->match.pattern);
+	macro->match.flags = GIT_ATTR_FNMATCH_MACRO;
 
-	if (error != 0)
-		git_attr_file__free(file);
+	error = git_attr_assignment__parse(repo, pool, &macro->assigns, &values);
 
-	if (blob)
-		git_blob_free(blob);
-	else
-		git__free((void *)content);
+	if (!error)
+		error = git_attr_cache__insert_macro(repo, macro);
 
-	git_buf_free(&path);
+	if (error < 0)
+		git_attr_rule__free(macro);
 
 	return error;
 }
 
-#define push_attr_file(R,S,B,F) \
-	git_attr_cache__push_file((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,git_attr_file__parse_buffer,NULL,(S))
-
 typedef struct {
 	git_repository *repo;
 	uint32_t flags;
@@ -483,7 +327,7 @@ typedef struct {
 	git_vector *files;
 } attr_walk_up_info;
 
-int git_attr_cache__decide_sources(
+static int attr_decide_sources(
 	uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs)
 {
 	int count = 0;
@@ -491,42 +335,76 @@ int git_attr_cache__decide_sources(
 	switch (flags & 0x03) {
 	case GIT_ATTR_CHECK_FILE_THEN_INDEX:
 		if (has_wd)
-			srcs[count++] = GIT_ATTR_FILE_FROM_FILE;
+			srcs[count++] = GIT_ATTR_FILE__FROM_FILE;
 		if (has_index)
-			srcs[count++] = GIT_ATTR_FILE_FROM_INDEX;
+			srcs[count++] = GIT_ATTR_FILE__FROM_INDEX;
 		break;
 	case GIT_ATTR_CHECK_INDEX_THEN_FILE:
 		if (has_index)
-			srcs[count++] = GIT_ATTR_FILE_FROM_INDEX;
+			srcs[count++] = GIT_ATTR_FILE__FROM_INDEX;
 		if (has_wd)
-			srcs[count++] = GIT_ATTR_FILE_FROM_FILE;
+			srcs[count++] = GIT_ATTR_FILE__FROM_FILE;
 		break;
 	case GIT_ATTR_CHECK_INDEX_ONLY:
 		if (has_index)
-			srcs[count++] = GIT_ATTR_FILE_FROM_INDEX;
+			srcs[count++] = GIT_ATTR_FILE__FROM_INDEX;
 		break;
 	}
 
 	return count;
 }
 
+static int push_attr_file(
+	git_repository *repo,
+	git_vector *list,
+	git_attr_file_source source,
+	const char *base,
+	const char *filename)
+{
+	int error = 0;
+	git_attr_file *file = NULL;
+
+	error = git_attr_cache__get(
+		&file, repo, source, base, filename, git_attr_file__parse_buffer);
+	if (error < 0)
+		return error;
+
+	if (file != NULL) {
+		if ((error = git_vector_insert(list, file)) < 0)
+			git_attr_file__free(file);
+	}
+
+	return error;
+}
+
 static int push_one_attr(void *ref, git_buf *path)
 {
 	int error = 0, n_src, i;
 	attr_walk_up_info *info = (attr_walk_up_info *)ref;
 	git_attr_file_source src[2];
 
-	n_src = git_attr_cache__decide_sources(
+	n_src = attr_decide_sources(
 		info->flags, info->workdir != NULL, info->index != NULL, src);
 
 	for (i = 0; !error && i < n_src; ++i)
-		error = git_attr_cache__push_file(
-			info->repo, path->ptr, GIT_ATTR_FILE, src[i],
-			git_attr_file__parse_buffer, NULL, info->files);
+		error = push_attr_file(
+			info->repo, info->files, src[i], path->ptr, GIT_ATTR_FILE);
 
 	return error;
 }
 
+static void release_attr_files(git_vector *files)
+{
+	size_t i;
+	git_attr_file *file;
+
+	git_vector_foreach(files, i, file) {
+		git_attr_file__free(file);
+		files->contents[i] = NULL;
+	}
+	git_vector_free(files);
+}
+
 static int collect_attr_files(
 	git_repository *repo,
 	uint32_t flags,
@@ -536,11 +414,10 @@ static int collect_attr_files(
 	int error;
 	git_buf dir = GIT_BUF_INIT;
 	const char *workdir = git_repository_workdir(repo);
-	attr_walk_up_info info;
+	attr_walk_up_info info = { NULL };
 
-	if (git_attr_cache__init(repo) < 0 ||
-		git_vector_init(files, 4, NULL) < 0)
-		return -1;
+	if ((error = attr_setup(repo)) < 0)
+		return error;
 
 	/* Resolve path in a non-bare repo */
 	if (workdir != NULL)
@@ -558,7 +435,8 @@ static int collect_attr_files(
 	 */
 
 	error = push_attr_file(
-		repo, files, git_repository_path(repo), GIT_ATTR_FILE_INREPO);
+		repo, files, GIT_ATTR_FILE__FROM_FILE,
+		git_repository_path(repo), GIT_ATTR_FILE_INREPO);
 	if (error < 0)
 		goto cleanup;
 
@@ -575,15 +453,17 @@ static int collect_attr_files(
 
 	if (git_repository_attr_cache(repo)->cfg_attr_file != NULL) {
 		error = push_attr_file(
-			repo, files, NULL, git_repository_attr_cache(repo)->cfg_attr_file);
+			repo, files, GIT_ATTR_FILE__FROM_FILE,
+			NULL, git_repository_attr_cache(repo)->cfg_attr_file);
 		if (error < 0)
 			goto cleanup;
 	}
 
 	if ((flags & GIT_ATTR_CHECK_NO_SYSTEM) == 0) {
-		error = git_futils_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM);
+		error = git_sysdir_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM);
 		if (!error)
-			error = push_attr_file(repo, files, NULL, dir.ptr);
+			error = push_attr_file(
+				repo, files, GIT_ATTR_FILE__FROM_FILE, NULL, dir.ptr);
 		else if (error == GIT_ENOTFOUND) {
 			giterr_clear();
 			error = 0;
@@ -592,153 +472,8 @@ static int collect_attr_files(
 
  cleanup:
 	if (error < 0)
-		git_vector_free(files);
+		release_attr_files(files);
 	git_buf_free(&dir);
 
 	return error;
 }
-
-static int attr_cache__lookup_path(
-	char **out, git_config *cfg, const char *key, const char *fallback)
-{
-	git_buf buf = GIT_BUF_INIT;
-	int error;
-	const char *cfgval = NULL;
-
-	*out = NULL;
-
-	if (!(error = git_config_get_string(&cfgval, cfg, key))) {
-
-		/* expand leading ~/ as needed */
-		if (cfgval && cfgval[0] == '~' && cfgval[1] == '/' &&
-			!git_futils_find_global_file(&buf, &cfgval[2]))
-			*out = git_buf_detach(&buf);
-		else if (cfgval)
-			*out = git__strdup(cfgval);
-
-	} else if (error == GIT_ENOTFOUND) {
-		giterr_clear();
-		error = 0;
-
-		if (!git_futils_find_xdg_file(&buf, fallback))
-			*out = git_buf_detach(&buf);
-	}
-
-	git_buf_free(&buf);
-
-	return error;
-}
-
-int git_attr_cache__init(git_repository *repo)
-{
-	int ret;
-	git_attr_cache *cache = git_repository_attr_cache(repo);
-	git_config *cfg;
-
-	if (cache->initialized)
-		return 0;
-
-	/* cache config settings for attributes and ignores */
-	if (git_repository_config__weakptr(&cfg, repo) < 0)
-		return -1;
-
-	ret = attr_cache__lookup_path(
-		&cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG, GIT_ATTR_FILE_XDG);
-	if (ret < 0)
-		return ret;
-
-	ret = attr_cache__lookup_path(
-		&cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG, GIT_IGNORE_FILE_XDG);
-	if (ret < 0)
-		return ret;
-
-	/* allocate hashtable for attribute and ignore file contents */
-	if (cache->files == NULL) {
-		cache->files = git_strmap_alloc();
-		GITERR_CHECK_ALLOC(cache->files);
-	}
-
-	/* allocate hashtable for attribute macros */
-	if (cache->macros == NULL) {
-		cache->macros = git_strmap_alloc();
-		GITERR_CHECK_ALLOC(cache->macros);
-	}
-
-	/* allocate string pool */
-	if (git_pool_init(&cache->pool, 1, 0) < 0)
-		return -1;
-
-	cache->initialized = 1;
-
-	/* insert default macros */
-	return git_attr_add_macro(repo, "binary", "-diff -crlf -text");
-}
-
-void git_attr_cache_flush(
-	git_repository *repo)
-{
-	git_attr_cache *cache;
-
-	if (!repo)
-		return;
-
-	cache = git_repository_attr_cache(repo);
-
-	if (cache->files != NULL) {
-		git_attr_file *file;
-
-		git_strmap_foreach_value(cache->files, file, {
-			git_attr_file__free(file);
-		});
-
-		git_strmap_free(cache->files);
-	}
-
-	if (cache->macros != NULL) {
-		git_attr_rule *rule;
-
-		git_strmap_foreach_value(cache->macros, rule, {
-			git_attr_rule__free(rule);
-		});
-
-		git_strmap_free(cache->macros);
-	}
-
-	git_pool_clear(&cache->pool);
-
-	git__free(cache->cfg_attr_file);
-	cache->cfg_attr_file = NULL;
-
-	git__free(cache->cfg_excl_file);
-	cache->cfg_excl_file = NULL;
-
-	cache->initialized = 0;
-}
-
-int git_attr_cache__insert_macro(git_repository *repo, git_attr_rule *macro)
-{
-	git_strmap *macros = git_repository_attr_cache(repo)->macros;
-	int error;
-
-	/* TODO: generate warning log if (macro->assigns.length == 0) */
-	if (macro->assigns.length == 0)
-		return 0;
-
-	git_strmap_insert(macros, macro->match.pattern, macro, error);
-	return (error < 0) ? -1 : 0;
-}
-
-git_attr_rule *git_attr_cache__lookup_macro(
-	git_repository *repo, const char *name)
-{
-	git_strmap *macros = git_repository_attr_cache(repo)->macros;
-	khiter_t pos;
-
-	pos = git_strmap_lookup_index(macros, name);
-
-	if (!git_strmap_valid_index(macros, pos))
-		return NULL;
-
-	return (git_attr_rule *)git_strmap_value_at(macros, pos);
-}
-
diff --git a/src/attr.h b/src/attr.h
index 19c979bcd..f9f216d07 100644
--- a/src/attr.h
+++ b/src/attr.h
@@ -8,38 +8,6 @@
 #define INCLUDE_attr_h__
 
 #include "attr_file.h"
-
-#define GIT_ATTR_CONFIG       "core.attributesfile"
-#define GIT_IGNORE_CONFIG     "core.excludesfile"
-
-typedef int (*git_attr_file_parser)(
-	git_repository *, void *, const char *, git_attr_file *);
-
-extern int git_attr_cache__insert_macro(
-	git_repository *repo, git_attr_rule *macro);
-
-extern git_attr_rule *git_attr_cache__lookup_macro(
-	git_repository *repo, const char *name);
-
-extern int git_attr_cache__push_file(
-	git_repository *repo,
-	const char *base,
-	const char *filename,
-	git_attr_file_source source,
-	git_attr_file_parser parse,
-	void *parsedata, /* passed through to parse function */
-	git_vector *stack);
-
-extern int git_attr_cache__internal_file(
-	git_repository *repo,
-	const char *key,
-	git_attr_file **file_ptr);
-
-/* returns true if path is in cache */
-extern bool git_attr_cache__is_cached(
-	git_repository *repo, git_attr_file_source source, const char *path);
-
-extern int git_attr_cache__decide_sources(
-	uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs);
+#include "attrcache.h"
 
 #endif
diff --git a/src/attr_file.c b/src/attr_file.c
index 4eb732436..3e95a2134 100644
--- a/src/attr_file.c
+++ b/src/attr_file.c
@@ -1,103 +1,253 @@
 #include "common.h"
 #include "repository.h"
 #include "filebuf.h"
-#include "attr.h"
+#include "attr_file.h"
+#include "attrcache.h"
 #include "git2/blob.h"
 #include "git2/tree.h"
+#include "index.h"
 #include <ctype.h>
 
-static int sort_by_hash_and_name(const void *a_raw, const void *b_raw);
-static void git_attr_rule__clear(git_attr_rule *rule);
-static bool parse_optimized_patterns(
-	git_attr_fnmatch *spec,
-	git_pool *pool,
-	const char *pattern);
+static void attr_file_free(git_attr_file *file)
+{
+	bool unlock = !git_mutex_lock(&file->lock);
+	git_attr_file__clear_rules(file, false);
+	git_pool_clear(&file->pool);
+	if (unlock)
+		git_mutex_unlock(&file->lock);
+	git_mutex_free(&file->lock);
+
+	git__memzero(file, sizeof(*file));
+	git__free(file);
+}
 
 int git_attr_file__new(
-	git_attr_file **attrs_ptr,
-	git_attr_file_source from,
-	const char *path,
-	git_pool *pool)
+	git_attr_file **out,
+	git_attr_file_entry *entry,
+	git_attr_file_source source)
 {
-	git_attr_file *attrs = NULL;
-
-	attrs = git__calloc(1, sizeof(git_attr_file));
+	git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file));
 	GITERR_CHECK_ALLOC(attrs);
 
-	if (pool)
-		attrs->pool = pool;
-	else {
-		attrs->pool = git__calloc(1, sizeof(git_pool));
-		if (!attrs->pool || git_pool_init(attrs->pool, 1, 0) < 0)
-			goto fail;
-		attrs->pool_is_allocated = true;
+	if (git_mutex_init(&attrs->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to initialize lock");
+		git__free(attrs);
+		return -1;
 	}
 
-	if (path) {
-		size_t len = strlen(path);
+	if (git_pool_init(&attrs->pool, 1, 0) < 0) {
+		attr_file_free(attrs);
+		return -1;
+	}
+
+	GIT_REFCOUNT_INC(attrs);
+	attrs->entry  = entry;
+	attrs->source = source;
+	*out = attrs;
+	return 0;
+}
 
-		attrs->key = git_pool_malloc(attrs->pool, (uint32_t)len + 3);
-		GITERR_CHECK_ALLOC(attrs->key);
+int git_attr_file__clear_rules(git_attr_file *file, bool need_lock)
+{
+	unsigned int i;
+	git_attr_rule *rule;
 
-		attrs->key[0] = '0' + (char)from;
-		attrs->key[1] = '#';
-		memcpy(&attrs->key[2], path, len);
-		attrs->key[len + 2] = '\0';
+	if (need_lock && git_mutex_lock(&file->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock attribute file");
+		return -1;
 	}
 
-	if (git_vector_init(&attrs->rules, 4, NULL) < 0)
-		goto fail;
+	git_vector_foreach(&file->rules, i, rule)
+		git_attr_rule__free(rule);
+	git_vector_free(&file->rules);
+
+	if (need_lock)
+		git_mutex_unlock(&file->lock);
 
-	*attrs_ptr = attrs;
 	return 0;
+}
 
-fail:
-	git_attr_file__free(attrs);
-	attrs_ptr = NULL;
-	return -1;
+void git_attr_file__free(git_attr_file *file)
+{
+	if (!file)
+		return;
+	GIT_REFCOUNT_DEC(file, attr_file_free);
 }
 
-int git_attr_file__parse_buffer(
-	git_repository *repo, void *parsedata, const char *buffer, git_attr_file *attrs)
+static int attr_file_oid_from_index(
+	git_oid *oid, git_repository *repo, const char *path)
+{
+	int error;
+	git_index *idx;
+	size_t pos;
+	const git_index_entry *entry;
+
+	if ((error = git_repository_index__weakptr(&idx, repo)) < 0 ||
+		(error = git_index__find_pos(&pos, idx, path, 0, 0)) < 0)
+		return error;
+
+	if (!(entry = git_index_get_byindex(idx, pos)))
+		return GIT_ENOTFOUND;
+
+	*oid = entry->id;
+	return 0;
+}
+
+int git_attr_file__load(
+	git_attr_file **out,
+	git_repository *repo,
+	git_attr_file_entry *entry,
+	git_attr_file_source source,
+	git_attr_file_parser parser)
 {
 	int error = 0;
-	const char *scan = NULL;
-	char *context = NULL;
-	git_attr_rule *rule = NULL;
+	git_blob *blob = NULL;
+	git_buf content = GIT_BUF_INIT;
+	const char *data = NULL;
+	git_attr_file *file;
+	struct stat st;
+
+	*out = NULL;
+
+	switch (source) {
+	case GIT_ATTR_FILE__IN_MEMORY:
+		/* in-memory attribute file doesn't need data */
+		break;
+	case GIT_ATTR_FILE__FROM_INDEX: {
+		git_oid id;
+
+		if ((error = attr_file_oid_from_index(&id, repo, entry->path)) < 0 ||
+			(error = git_blob_lookup(&blob, repo, &id)) < 0)
+			return error;
+
+		data = git_blob_rawcontent(blob);
+		break;
+	}
+	case GIT_ATTR_FILE__FROM_FILE: {
+		int fd;
+
+		if (p_stat(entry->fullpath, &st) < 0)
+			return git_path_set_error(errno, entry->fullpath, "stat");
+		if (S_ISDIR(st.st_mode))
+			return GIT_ENOTFOUND;
+
+		/* For open or read errors, return ENOTFOUND to skip item */
+		/* TODO: issue warning when warning API is available */
+
+		if ((fd = git_futils_open_ro(entry->fullpath)) < 0)
+			return GIT_ENOTFOUND;
+
+		error = git_futils_readbuffer_fd(&content, fd, (size_t)st.st_size);
+		p_close(fd);
+
+		if (error < 0)
+			return GIT_ENOTFOUND;
+
+		data = content.ptr;
+		break;
+	}
+	default:
+		giterr_set(GITERR_INVALID, "Unknown file source %d", source);
+		return -1;
+	}
+
+	if ((error = git_attr_file__new(&file, entry, source)) < 0)
+		goto cleanup;
+
+	if (parser && (error = parser(repo, file, data)) < 0) {
+		git_attr_file__free(file);
+		goto cleanup;
+	}
 
-	GIT_UNUSED(parsedata);
+	/* write cache breaker */
+	if (source == GIT_ATTR_FILE__FROM_INDEX)
+		git_oid_cpy(&file->cache_data.oid, git_blob_id(blob));
+	else if (source == GIT_ATTR_FILE__FROM_FILE)
+		git_futils_filestamp_set_from_stat(&file->cache_data.stamp, &st);
+	/* else always cacheable */
+
+	*out = file;
+
+cleanup:
+	git_blob_free(blob);
+	git_buf_free(&content);
+
+	return error;
+}
+
+int git_attr_file__out_of_date(git_repository *repo, git_attr_file *file)
+{
+	if (!file)
+		return 1;
+
+	switch (file->source) {
+	case GIT_ATTR_FILE__IN_MEMORY:
+		return 0;
+
+	case GIT_ATTR_FILE__FROM_FILE:
+		return git_futils_filestamp_check(
+			&file->cache_data.stamp, file->entry->fullpath);
+
+	case GIT_ATTR_FILE__FROM_INDEX: {
+		int error;
+		git_oid id;
+
+		if ((error = attr_file_oid_from_index(
+				&id, repo, file->entry->path)) < 0)
+			return error;
+
+		return (git_oid__cmp(&file->cache_data.oid, &id) != 0);
+	}
+
+	default:
+		giterr_set(GITERR_INVALID, "Invalid file type %d", file->source);
+		return -1;
+	}
+}
 
-	assert(buffer && attrs);
+static int sort_by_hash_and_name(const void *a_raw, const void *b_raw);
+static void git_attr_rule__clear(git_attr_rule *rule);
+static bool parse_optimized_patterns(
+	git_attr_fnmatch *spec,
+	git_pool *pool,
+	const char *pattern);
 
-	scan = buffer;
+int git_attr_file__parse_buffer(
+	git_repository *repo, git_attr_file *attrs, const char *data)
+{
+	int error = 0;
+	const char *scan = data, *context = NULL;
+	git_attr_rule *rule = NULL;
 
 	/* if subdir file path, convert context for file paths */
-	if (attrs->key && git__suffixcmp(attrs->key, "/" GIT_ATTR_FILE) == 0) {
-		context = attrs->key + 2;
-		context[strlen(context) - strlen(GIT_ATTR_FILE)] = '\0';
+	if (attrs->entry &&
+		git_path_root(attrs->entry->path) < 0 &&
+		!git__suffixcmp(attrs->entry->path, "/" GIT_ATTR_FILE))
+		context = attrs->entry->path;
+
+	if (git_mutex_lock(&attrs->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock attribute file");
+		return -1;
 	}
 
 	while (!error && *scan) {
 		/* allocate rule if needed */
-		if (!rule) {
-			if (!(rule = git__calloc(1, sizeof(git_attr_rule)))) {
-				error = -1;
-				break;
-			}
-			rule->match.flags = GIT_ATTR_FNMATCH_ALLOWNEG |
-				GIT_ATTR_FNMATCH_ALLOWMACRO;
+		if (!rule && !(rule = git__calloc(1, sizeof(*rule)))) {
+			error = -1;
+			break;
 		}
 
+		rule->match.flags =
+			GIT_ATTR_FNMATCH_ALLOWNEG | GIT_ATTR_FNMATCH_ALLOWMACRO;
+
 		/* parse the next "pattern attr attr attr" line */
 		if (!(error = git_attr_fnmatch__parse(
-				&rule->match, attrs->pool, context, &scan)) &&
+				&rule->match, &attrs->pool, context, &scan)) &&
 			!(error = git_attr_assignment__parse(
-				repo, attrs->pool, &rule->assigns, &scan)))
+				repo, &attrs->pool, &rule->assigns, &scan)))
 		{
 			if (rule->match.flags & GIT_ATTR_FNMATCH_MACRO)
-				/* should generate error/warning if this is coming from any
-				 * file other than .gitattributes at repo root.
-				 */
+				/* TODO: warning if macro found in file below repo root */
 				error = git_attr_cache__insert_macro(repo, rule);
 			else
 				error = git_vector_insert(&attrs->rules, rule);
@@ -113,66 +263,12 @@ int git_attr_file__parse_buffer(
 		}
 	}
 
+	git_mutex_unlock(&attrs->lock);
 	git_attr_rule__free(rule);
 
-	/* restore file path used for context */
-	if (context)
-		context[strlen(context)] = '.'; /* first char of GIT_ATTR_FILE */
-
 	return error;
 }
 
-int git_attr_file__new_and_load(
-	git_attr_file **attrs_ptr,
-	const char *path)
-{
-	int error;
-	git_buf content = GIT_BUF_INIT;
-
-	if ((error = git_attr_file__new(attrs_ptr, 0, path, NULL)) < 0)
-		return error;
-
-	if (!(error = git_futils_readbuffer(&content, path)))
-		error = git_attr_file__parse_buffer(
-			NULL, NULL, git_buf_cstr(&content), *attrs_ptr);
-
-	git_buf_free(&content);
-
-	if (error) {
-		git_attr_file__free(*attrs_ptr);
-		*attrs_ptr = NULL;
-	}
-
-	return error;
-}
-
-void git_attr_file__clear_rules(git_attr_file *file)
-{
-	unsigned int i;
-	git_attr_rule *rule;
-
-	git_vector_foreach(&file->rules, i, rule)
-		git_attr_rule__free(rule);
-
-	git_vector_free(&file->rules);
-}
-
-void git_attr_file__free(git_attr_file *file)
-{
-	if (!file)
-		return;
-
-	git_attr_file__clear_rules(file);
-
-	if (file->pool_is_allocated) {
-		git_pool_clear(file->pool);
-		git__free(file->pool);
-	}
-	file->pool = NULL;
-
-	git__free(file);
-}
-
 uint32_t git_attr_file__name_hash(const char *name)
 {
 	uint32_t h = 5381;
@@ -183,10 +279,9 @@ uint32_t git_attr_file__name_hash(const char *name)
 	return h;
 }
 
-
 int git_attr_file__lookup_one(
 	git_attr_file *file,
-	const git_attr_path *path,
+	git_attr_path *path,
 	const char *attr,
 	const char **value)
 {
@@ -212,30 +307,83 @@ int git_attr_file__lookup_one(
 	return 0;
 }
 
+int git_attr_file__load_standalone(git_attr_file **out, const char *path)
+{
+	int error;
+	git_attr_file *file;
+	git_buf content = GIT_BUF_INIT;
+
+	error = git_attr_file__new(&file, NULL, GIT_ATTR_FILE__FROM_FILE);
+	if (error < 0)
+		return error;
+
+	error = git_attr_cache__alloc_file_entry(
+		&file->entry, NULL, path, &file->pool);
+	if (error < 0) {
+		git_attr_file__free(file);
+		return error;
+	}
+	/* because the cache entry is allocated from the file's own pool, we
+	 * don't have to free it - freeing file+pool will free cache entry, too.
+	 */
+
+	if (!(error = git_futils_readbuffer(&content, path))) {
+		error = git_attr_file__parse_buffer(NULL, file, content.ptr);
+		git_buf_free(&content);
+	}
+
+	if (error < 0)
+		git_attr_file__free(file);
+	else
+		*out = file;
+
+	return error;
+}
 
 bool git_attr_fnmatch__match(
 	git_attr_fnmatch *match,
-	const git_attr_path *path)
+	git_attr_path *path)
 {
-	int fnm;
-	int icase_flags = (match->flags & GIT_ATTR_FNMATCH_ICASE) ? FNM_CASEFOLD : 0;
+	const char *filename;
+	int flags = 0;
 
-	if (match->flags & GIT_ATTR_FNMATCH_DIRECTORY && !path->is_dir)
-		return false;
+	if (match->flags & GIT_ATTR_FNMATCH_ICASE)
+		flags |= FNM_CASEFOLD;
+	if (match->flags & GIT_ATTR_FNMATCH_LEADINGDIR)
+		flags |= FNM_LEADING_DIR;
 
-	if (match->flags & GIT_ATTR_FNMATCH_FULLPATH)
-		fnm = p_fnmatch(match->pattern, path->path, FNM_PATHNAME | icase_flags);
-	else if (path->is_dir)
-		fnm = p_fnmatch(match->pattern, path->basename, FNM_LEADING_DIR | icase_flags);
-	else
-		fnm = p_fnmatch(match->pattern, path->basename, icase_flags);
+	if (match->flags & GIT_ATTR_FNMATCH_FULLPATH) {
+		filename = path->path;
+		flags |= FNM_PATHNAME;
+	} else {
+		filename = path->basename;
+
+		if (path->is_dir)
+			flags |= FNM_LEADING_DIR;
+	}
+
+	if ((match->flags & GIT_ATTR_FNMATCH_DIRECTORY) && !path->is_dir) {
+		int matchval;
+
+		/* for attribute checks or root ignore checks, fail match */
+		if (!(match->flags & GIT_ATTR_FNMATCH_IGNORE) ||
+			path->basename == path->path)
+			return false;
 
-	return (fnm == FNM_NOMATCH) ? false : true;
+		/* for ignore checks, use container of current item for check */
+		path->basename[-1] = '\0';
+		flags |= FNM_LEADING_DIR;
+		matchval = p_fnmatch(match->pattern, path->path, flags);
+		path->basename[-1] = '/';
+		return (matchval != FNM_NOMATCH);
+	}
+
+	return (p_fnmatch(match->pattern, filename, flags) != FNM_NOMATCH);
 }
 
 bool git_attr_rule__match(
 	git_attr_rule *rule,
-	const git_attr_path *path)
+	git_attr_path *path)
 {
 	bool matched = git_attr_fnmatch__match(&rule->match, path);
 
@@ -245,7 +393,6 @@ bool git_attr_rule__match(
 	return matched;
 }
 
-
 git_attr_assignment *git_attr_rule__lookup_assignment(
 	git_attr_rule *rule, const char *name)
 {
@@ -344,7 +491,7 @@ void git_attr_path__free(git_attr_path *info)
 int git_attr_fnmatch__parse(
 	git_attr_fnmatch *spec,
 	git_pool *pool,
-	const char *source,
+	const char *context,
 	const char **base)
 {
 	const char *pattern, *scan;
@@ -410,20 +557,31 @@ int git_attr_fnmatch__parse(
 		if (--slash_count <= 0)
 			spec->flags = spec->flags & ~GIT_ATTR_FNMATCH_FULLPATH;
 	}
+	if ((spec->flags & GIT_ATTR_FNMATCH_NOLEADINGDIR) == 0 &&
+		spec->length >= 2 &&
+		pattern[spec->length - 1] == '*' &&
+		pattern[spec->length - 2] == '/') {
+		spec->length -= 2;
+		spec->flags = spec->flags | GIT_ATTR_FNMATCH_LEADINGDIR;
+		/* leave FULLPATH match on, however */
+	}
 
 	if ((spec->flags & GIT_ATTR_FNMATCH_FULLPATH) != 0 &&
-		source != NULL && git_path_root(pattern) < 0)
+		context != NULL && git_path_root(pattern) < 0)
 	{
-		size_t sourcelen = strlen(source);
+		/* use context path minus the trailing filename */
+		char *slash = strrchr(context, '/');
+		size_t contextlen = slash ? slash - context + 1 : 0;
+
 		/* given an unrooted fullpath match from a file inside a repo,
 		 * prefix the pattern with the relative directory of the source file
 		 */
 		spec->pattern = git_pool_malloc(
-			pool, (uint32_t)(sourcelen + spec->length + 1));
+			pool, (uint32_t)(contextlen + spec->length + 1));
 		if (spec->pattern) {
-			memcpy(spec->pattern, source, sourcelen);
-			memcpy(spec->pattern + sourcelen, pattern, spec->length);
-			spec->length += sourcelen;
+			memcpy(spec->pattern, context, contextlen);
+			memcpy(spec->pattern + contextlen, pattern, spec->length);
+			spec->length += contextlen;
 			spec->pattern[spec->length] = '\0';
 		}
 	} else {
@@ -436,6 +594,7 @@ int git_attr_fnmatch__parse(
 	} else {
 		/* strip '\' that might have be used for internal whitespace */
 		spec->length = git__unescape(spec->pattern);
+		/* TODO: convert remaining '\' into '/' for POSIX ??? */
 	}
 
 	return 0;
diff --git a/src/attr_file.h b/src/attr_file.h
index 3bc7c6cb8..87cde7e35 100644
--- a/src/attr_file.h
+++ b/src/attr_file.h
@@ -30,10 +30,20 @@
 #define GIT_ATTR_FNMATCH_MATCH_ALL	(1U << 8)
 #define GIT_ATTR_FNMATCH_ALLOWNEG   (1U << 9)
 #define GIT_ATTR_FNMATCH_ALLOWMACRO (1U << 10)
+#define GIT_ATTR_FNMATCH_LEADINGDIR (1U << 11)
+#define GIT_ATTR_FNMATCH_NOLEADINGDIR (1U << 12)
 
 #define GIT_ATTR_FNMATCH__INCOMING \
-	(GIT_ATTR_FNMATCH_ALLOWSPACE | \
-	 GIT_ATTR_FNMATCH_ALLOWNEG | GIT_ATTR_FNMATCH_ALLOWMACRO)
+	(GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG | \
+	 GIT_ATTR_FNMATCH_ALLOWMACRO | GIT_ATTR_FNMATCH_NOLEADINGDIR)
+
+typedef enum {
+	GIT_ATTR_FILE__IN_MEMORY  = 0,
+	GIT_ATTR_FILE__FROM_FILE  = 1,
+	GIT_ATTR_FILE__FROM_INDEX = 2,
+
+	GIT_ATTR_FILE_NUM_SOURCES = 3
+} git_attr_file_source;
 
 extern const char *git_attr__true;
 extern const char *git_attr__false;
@@ -63,17 +73,32 @@ typedef struct {
 	const char *value;
 } git_attr_assignment;
 
+typedef struct git_attr_file_entry git_attr_file_entry;
+
 typedef struct {
-	char *key;				/* cache "source#path" this was loaded from */
-	git_vector rules;		/* vector of <rule*> or <fnmatch*> */
-	git_pool *pool;
-	bool pool_is_allocated;
+	git_refcount rc;
+	git_mutex lock;
+	git_attr_file_entry *entry;
+	git_attr_file_source source;
+	git_vector rules;			/* vector of <rule*> or <fnmatch*> */
+	git_pool pool;
 	union {
 		git_oid oid;
 		git_futils_filestamp stamp;
 	} cache_data;
 } git_attr_file;
 
+struct git_attr_file_entry {
+	git_attr_file *file[GIT_ATTR_FILE_NUM_SOURCES];
+	const char *path; /* points into fullpath */
+	char fullpath[GIT_FLEX_ARRAY];
+};
+
+typedef int (*git_attr_file_parser)(
+	git_repository *repo,
+	git_attr_file *file,
+	const char *data);
+
 typedef struct {
 	git_buf  full;
 	char    *path;
@@ -81,31 +106,39 @@ typedef struct {
 	int      is_dir;
 } git_attr_path;
 
-typedef enum {
-	GIT_ATTR_FILE_FROM_FILE = 0,
-	GIT_ATTR_FILE_FROM_INDEX = 1
-} git_attr_file_source;
-
 /*
  * git_attr_file API
  */
 
-extern int git_attr_file__new(
-	git_attr_file **attrs_ptr, git_attr_file_source src, const char *path, git_pool *pool);
+int git_attr_file__new(
+	git_attr_file **out,
+	git_attr_file_entry *entry,
+	git_attr_file_source source);
+
+void git_attr_file__free(git_attr_file *file);
+
+int git_attr_file__load(
+	git_attr_file **out,
+	git_repository *repo,
+	git_attr_file_entry *ce,
+	git_attr_file_source source,
+	git_attr_file_parser parser);
 
-extern int git_attr_file__new_and_load(
-	git_attr_file **attrs_ptr, const char *path);
+int git_attr_file__load_standalone(
+	git_attr_file **out, const char *path);
 
-extern void git_attr_file__free(git_attr_file *file);
+int git_attr_file__out_of_date(
+	git_repository *repo, git_attr_file *file);
 
-extern void git_attr_file__clear_rules(git_attr_file *file);
+int git_attr_file__parse_buffer(
+	git_repository *repo, git_attr_file *attrs, const char *data);
 
-extern int git_attr_file__parse_buffer(
-	git_repository *repo, void *parsedata, const char *buf, git_attr_file *file);
+int git_attr_file__clear_rules(
+	git_attr_file *file, bool need_lock);
 
-extern int git_attr_file__lookup_one(
+int git_attr_file__lookup_one(
 	git_attr_file *file,
-	const git_attr_path *path,
+	git_attr_path *path,
 	const char *attr,
 	const char **value);
 
@@ -114,7 +147,7 @@ extern int git_attr_file__lookup_one(
 	git_vector_rforeach(&(file)->rules, (iter), (rule)) \
 		if (git_attr_rule__match((rule), (path)))
 
-extern uint32_t git_attr_file__name_hash(const char *name);
+uint32_t git_attr_file__name_hash(const char *name);
 
 
 /*
@@ -129,13 +162,13 @@ extern int git_attr_fnmatch__parse(
 
 extern bool git_attr_fnmatch__match(
 	git_attr_fnmatch *rule,
-	const git_attr_path *path);
+	git_attr_path *path);
 
 extern void git_attr_rule__free(git_attr_rule *rule);
 
 extern bool git_attr_rule__match(
 	git_attr_rule *rule,
-	const git_attr_path *path);
+	git_attr_path *path);
 
 extern git_attr_assignment *git_attr_rule__lookup_assignment(
 	git_attr_rule *rule, const char *name);
diff --git a/src/attrcache.c b/src/attrcache.c
new file mode 100644
index 000000000..b4579bfc0
--- /dev/null
+++ b/src/attrcache.c
@@ -0,0 +1,449 @@
+#include "common.h"
+#include "repository.h"
+#include "attr_file.h"
+#include "config.h"
+#include "sysdir.h"
+#include "ignore.h"
+
+GIT__USE_STRMAP;
+
+GIT_INLINE(int) attr_cache_lock(git_attr_cache *cache)
+{
+	GIT_UNUSED(cache); /* avoid warning if threading is off */
+
+	if (git_mutex_lock(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to get attr cache lock");
+		return -1;
+	}
+	return 0;
+}
+
+GIT_INLINE(void) attr_cache_unlock(git_attr_cache *cache)
+{
+	GIT_UNUSED(cache); /* avoid warning if threading is off */
+	git_mutex_unlock(&cache->lock);
+}
+
+GIT_INLINE(git_attr_file_entry *) attr_cache_lookup_entry(
+	git_attr_cache *cache, const char *path)
+{
+	khiter_t pos = git_strmap_lookup_index(cache->files, path);
+
+	if (git_strmap_valid_index(cache->files, pos))
+		return git_strmap_value_at(cache->files, pos);
+	else
+		return NULL;
+}
+
+int git_attr_cache__alloc_file_entry(
+	git_attr_file_entry **out,
+	const char *base,
+	const char *path,
+	git_pool *pool)
+{
+	size_t baselen = 0, pathlen = strlen(path);
+	size_t cachesize = sizeof(git_attr_file_entry) + pathlen + 1;
+	git_attr_file_entry *ce;
+
+	if (base != NULL && git_path_root(path) < 0) {
+		baselen = strlen(base);
+		cachesize += baselen;
+
+		if (baselen && base[baselen - 1] != '/')
+			cachesize++;
+	}
+
+	ce = git_pool_mallocz(pool, (uint32_t)cachesize);
+	GITERR_CHECK_ALLOC(ce);
+
+	if (baselen) {
+		memcpy(ce->fullpath, base, baselen);
+
+		if (base[baselen - 1] != '/')
+			ce->fullpath[baselen++] = '/';
+	}
+	memcpy(&ce->fullpath[baselen], path, pathlen);
+
+	ce->path = &ce->fullpath[baselen];
+	*out = ce;
+
+	return 0;
+}
+
+/* call with attrcache locked */
+static int attr_cache_make_entry(
+	git_attr_file_entry **out, git_repository *repo, const char *path)
+{
+	int error = 0;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_attr_file_entry *entry = NULL;
+
+	error = git_attr_cache__alloc_file_entry(
+		&entry, git_repository_workdir(repo), path, &cache->pool);
+
+	if (!error) {
+		git_strmap_insert(cache->files, entry->path, entry, error);
+		if (error > 0)
+			error = 0;
+	}
+
+	*out = entry;
+	return error;
+}
+
+/* insert entry or replace existing if we raced with another thread */
+static int attr_cache_upsert(git_attr_cache *cache, git_attr_file *file)
+{
+	git_attr_file_entry *entry;
+	git_attr_file *old;
+
+	if (attr_cache_lock(cache) < 0)
+		return -1;
+
+	entry = attr_cache_lookup_entry(cache, file->entry->path);
+
+	GIT_REFCOUNT_OWN(file, entry);
+	GIT_REFCOUNT_INC(file);
+
+	old = git__compare_and_swap(
+		&entry->file[file->source], entry->file[file->source], file);
+
+	if (old) {
+		GIT_REFCOUNT_OWN(old, NULL);
+		git_attr_file__free(old);
+	}
+
+	attr_cache_unlock(cache);
+	return 0;
+}
+
+static int attr_cache_remove(git_attr_cache *cache, git_attr_file *file)
+{
+	int error = 0;
+	git_attr_file_entry *entry;
+
+	if (!file)
+		return 0;
+	if ((error = attr_cache_lock(cache)) < 0)
+		return error;
+
+	if ((entry = attr_cache_lookup_entry(cache, file->entry->path)) != NULL)
+		file = git__compare_and_swap(&entry->file[file->source], file, NULL);
+
+	attr_cache_unlock(cache);
+
+	if (file) {
+		GIT_REFCOUNT_OWN(file, NULL);
+		git_attr_file__free(file);
+	}
+
+	return error;
+}
+
+/* Look up cache entry and file.
+ * - If entry is not present, create it while the cache is locked.
+ * - If file is present, increment refcount before returning it, so the
+ *   cache can be unlocked and it won't go away.
+ */
+static int attr_cache_lookup(
+	git_attr_file **out_file,
+	git_attr_file_entry **out_entry,
+	git_repository *repo,
+	git_attr_file_source source,
+	const char *base,
+	const char *filename)
+{
+	int error = 0;
+	git_buf path = GIT_BUF_INIT;
+	const char *wd = git_repository_workdir(repo), *relfile;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_attr_file_entry *entry = NULL;
+	git_attr_file *file = NULL;
+
+	/* join base and path as needed */
+	if (base != NULL && git_path_root(filename) < 0) {
+		if (git_buf_joinpath(&path, base, filename) < 0)
+			return -1;
+		filename = path.ptr;
+	}
+
+	relfile = filename;
+	if (wd && !git__prefixcmp(relfile, wd))
+		relfile += strlen(wd);
+
+	/* check cache for existing entry */
+	if ((error = attr_cache_lock(cache)) < 0)
+		goto cleanup;
+
+	entry = attr_cache_lookup_entry(cache, relfile);
+	if (!entry)
+		error = attr_cache_make_entry(&entry, repo, relfile);
+	else if (entry->file[source] != NULL) {
+		file = entry->file[source];
+		GIT_REFCOUNT_INC(file);
+	}
+
+	attr_cache_unlock(cache);
+
+cleanup:
+	*out_file  = file;
+	*out_entry = entry;
+
+	git_buf_free(&path);
+	return error;
+}
+
+int git_attr_cache__get(
+	git_attr_file **out,
+	git_repository *repo,
+	git_attr_file_source source,
+	const char *base,
+	const char *filename,
+	git_attr_file_parser parser)
+{
+	int error = 0;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_attr_file_entry *entry = NULL;
+	git_attr_file *file = NULL, *updated = NULL;
+
+	if ((error = attr_cache_lookup(
+			&file, &entry, repo, source, base, filename)) < 0)
+		return error;
+
+	/* load file if we don't have one or if existing one is out of date */
+	if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0)
+		error = git_attr_file__load(&updated, repo, entry, source, parser);
+
+	/* if we loaded the file, insert into and/or update cache */
+	if (updated) {
+		if ((error = attr_cache_upsert(cache, updated)) < 0)
+			git_attr_file__free(updated);
+		else {
+			git_attr_file__free(file); /* offset incref from lookup */
+			file = updated;
+		}
+	}
+
+	/* if file could not be loaded */
+	if (error < 0) {
+		/* remove existing entry */
+		if (file) {
+			attr_cache_remove(cache, file);
+			git_attr_file__free(file); /* offset incref from lookup */
+			file = NULL;
+		}
+		/* no error if file simply doesn't exist */
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			error = 0;
+		}
+	}
+
+	*out = file;
+	return error;
+}
+
+bool git_attr_cache__is_cached(
+	git_repository *repo,
+	git_attr_file_source source,
+	const char *filename)
+{
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_strmap *files;
+	khiter_t pos;
+	git_attr_file_entry *entry;
+
+	if (!cache || !(files = cache->files))
+		return false;
+
+	pos = git_strmap_lookup_index(files, filename);
+	if (!git_strmap_valid_index(files, pos))
+		return false;
+
+	entry = git_strmap_value_at(files, pos);
+
+	return entry && (entry->file[source] != NULL);
+}
+
+
+static int attr_cache__lookup_path(
+	char **out, git_config *cfg, const char *key, const char *fallback)
+{
+	git_buf buf = GIT_BUF_INIT;
+	int error;
+	const git_config_entry *entry = NULL;
+
+	*out = NULL;
+
+	if ((error = git_config__lookup_entry(&entry, cfg, key, false)) < 0)
+		return error;
+
+	if (entry) {
+		const char *cfgval = entry->value;
+
+		/* expand leading ~/ as needed */
+		if (cfgval && cfgval[0] == '~' && cfgval[1] == '/' &&
+			!git_sysdir_find_global_file(&buf, &cfgval[2]))
+			*out = git_buf_detach(&buf);
+		else if (cfgval)
+			*out = git__strdup(cfgval);
+	}
+	else if (!git_sysdir_find_xdg_file(&buf, fallback))
+		*out = git_buf_detach(&buf);
+
+	git_buf_free(&buf);
+
+	return error;
+}
+
+static void attr_cache__free(git_attr_cache *cache)
+{
+	bool unlock;
+
+	if (!cache)
+		return;
+
+	unlock = (git_mutex_lock(&cache->lock) == 0);
+
+	if (cache->files != NULL) {
+		git_attr_file_entry *entry;
+		git_attr_file *file;
+		int i;
+
+		git_strmap_foreach_value(cache->files, entry, {
+			for (i = 0; i < GIT_ATTR_FILE_NUM_SOURCES; ++i) {
+				if ((file = git__swap(entry->file[i], NULL)) != NULL) {
+					GIT_REFCOUNT_OWN(file, NULL);
+					git_attr_file__free(file);
+				}
+			}
+		});
+		git_strmap_free(cache->files);
+	}
+
+	if (cache->macros != NULL) {
+		git_attr_rule *rule;
+
+		git_strmap_foreach_value(cache->macros, rule, {
+			git_attr_rule__free(rule);
+		});
+		git_strmap_free(cache->macros);
+	}
+
+	git_pool_clear(&cache->pool);
+
+	git__free(cache->cfg_attr_file);
+	cache->cfg_attr_file = NULL;
+
+	git__free(cache->cfg_excl_file);
+	cache->cfg_excl_file = NULL;
+
+	if (unlock)
+		git_mutex_unlock(&cache->lock);
+	git_mutex_free(&cache->lock);
+
+	git__free(cache);
+}
+
+int git_attr_cache__do_init(git_repository *repo)
+{
+	int ret = 0;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_config *cfg = NULL;
+
+	if (cache)
+		return 0;
+
+	cache = git__calloc(1, sizeof(git_attr_cache));
+	GITERR_CHECK_ALLOC(cache);
+
+	/* set up lock */
+	if (git_mutex_init(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to initialize lock for attr cache");
+		git__free(cache);
+		return -1;
+	}
+
+	if ((ret = git_repository_config_snapshot(&cfg, repo)) < 0)
+		goto cancel;
+
+	/* cache config settings for attributes and ignores */
+	ret = attr_cache__lookup_path(
+		&cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG, GIT_ATTR_FILE_XDG);
+	if (ret < 0)
+		goto cancel;
+
+	ret = attr_cache__lookup_path(
+		&cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG, GIT_IGNORE_FILE_XDG);
+	if (ret < 0)
+		goto cancel;
+
+	/* allocate hashtable for attribute and ignore file contents,
+	 * hashtable for attribute macros, and string pool
+	 */
+	if ((ret = git_strmap_alloc(&cache->files)) < 0 ||
+		(ret = git_strmap_alloc(&cache->macros)) < 0 ||
+		(ret = git_pool_init(&cache->pool, 1, 0)) < 0)
+		goto cancel;
+
+	cache = git__compare_and_swap(&repo->attrcache, NULL, cache);
+	if (cache)
+		goto cancel; /* raced with another thread, free this but no error */
+
+	git_config_free(cfg);
+
+	/* insert default macros */
+	return git_attr_add_macro(repo, "binary", "-diff -crlf -text");
+
+cancel:
+	attr_cache__free(cache);
+	git_config_free(cfg);
+	return ret;
+}
+
+void git_attr_cache_flush(git_repository *repo)
+{
+	git_attr_cache *cache;
+
+	/* this could be done less expensively, but for now, we'll just free
+	 * the entire attrcache and let the next use reinitialize it...
+	 */
+	if (repo && (cache = git__swap(repo->attrcache, NULL)) != NULL)
+		attr_cache__free(cache);
+}
+
+int git_attr_cache__insert_macro(git_repository *repo, git_attr_rule *macro)
+{
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_strmap *macros = cache->macros;
+	int error;
+
+	/* TODO: generate warning log if (macro->assigns.length == 0) */
+	if (macro->assigns.length == 0)
+		return 0;
+
+	if (git_mutex_lock(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to get attr cache lock");
+		error = -1;
+	} else {
+		git_strmap_insert(macros, macro->match.pattern, macro, error);
+		git_mutex_unlock(&cache->lock);
+	}
+
+	return (error < 0) ? -1 : 0;
+}
+
+git_attr_rule *git_attr_cache__lookup_macro(
+	git_repository *repo, const char *name)
+{
+	git_strmap *macros = git_repository_attr_cache(repo)->macros;
+	khiter_t pos;
+
+	pos = git_strmap_lookup_index(macros, name);
+
+	if (!git_strmap_valid_index(macros, pos))
+		return NULL;
+
+	return (git_attr_rule *)git_strmap_value_at(macros, pos);
+}
+
diff --git a/src/attrcache.h b/src/attrcache.h
index 077633b87..be0a22f5c 100644
--- a/src/attrcache.h
+++ b/src/attrcache.h
@@ -7,18 +7,50 @@
 #ifndef INCLUDE_attrcache_h__
 #define INCLUDE_attrcache_h__
 
-#include "pool.h"
+#include "attr_file.h"
 #include "strmap.h"
 
+#define GIT_ATTR_CONFIG       "core.attributesfile"
+#define GIT_IGNORE_CONFIG     "core.excludesfile"
+
 typedef struct {
-	int initialized;
-	git_pool pool;
-	git_strmap *files;	 /* hash path to git_attr_file of rules */
-	git_strmap *macros;	 /* hash name to vector<git_attr_assignment> */
 	char *cfg_attr_file; /* cached value of core.attributesfile */
 	char *cfg_excl_file; /* cached value of core.excludesfile */
+	git_strmap *files;	 /* hash path to git_attr_cache_entry records */
+	git_strmap *macros;	 /* hash name to vector<git_attr_assignment> */
+	git_mutex lock;
+	git_pool  pool;
 } git_attr_cache;
 
-extern int git_attr_cache__init(git_repository *repo);
+extern int git_attr_cache__do_init(git_repository *repo);
+
+#define git_attr_cache__init(REPO) \
+	(git_repository_attr_cache(REPO) ? 0 : git_attr_cache__do_init(REPO))
+
+/* get file - loading and reload as needed */
+extern int git_attr_cache__get(
+	git_attr_file **file,
+	git_repository *repo,
+	git_attr_file_source source,
+	const char *base,
+	const char *filename,
+	git_attr_file_parser parser);
+
+extern bool git_attr_cache__is_cached(
+	git_repository *repo,
+	git_attr_file_source source,
+	const char *path);
+
+extern int git_attr_cache__alloc_file_entry(
+	git_attr_file_entry **out,
+	const char *base,
+	const char *path,
+	git_pool *pool);
+
+extern int git_attr_cache__insert_macro(
+	git_repository *repo, git_attr_rule *macro);
+
+extern git_attr_rule *git_attr_cache__lookup_macro(
+	git_repository *repo, const char *name);
 
 #endif
diff --git a/src/bitvec.h b/src/bitvec.h
index fd6f0ccf8..544832d95 100644
--- a/src/bitvec.h
+++ b/src/bitvec.h
@@ -7,7 +7,7 @@
 #ifndef INCLUDE_bitvec_h__
 #define INCLUDE_bitvec_h__
 
-#include "util.h"
+#include "common.h"
 
 /*
  * This is a silly little fixed length bit vector type that will store
diff --git a/src/blame.c b/src/blame.c
index 219a6bfe4..eb977c287 100644
--- a/src/blame.c
+++ b/src/blame.c
@@ -20,12 +20,15 @@
 
 static int hunk_byfinalline_search_cmp(const void *key, const void *entry)
 {
-	uint16_t lineno = (uint16_t)*(size_t*)key;
 	git_blame_hunk *hunk = (git_blame_hunk*)entry;
 
-	if (lineno < hunk->final_start_line_number)
+	size_t lineno = *(size_t*)key;
+	size_t lines_in_hunk = (size_t)hunk->lines_in_hunk;
+	size_t final_start_line_number = (size_t)hunk->final_start_line_number;
+
+	if (lineno < final_start_line_number)
 		return -1;
-	if (lineno >= hunk->final_start_line_number + hunk->lines_in_hunk)
+	if (lineno >= final_start_line_number + lines_in_hunk)
 		return 1;
 	return 0;
 }
@@ -76,8 +79,8 @@ static git_blame_hunk* dup_hunk(git_blame_hunk *hunk)
 	git_oid_cpy(&newhunk->orig_commit_id, &hunk->orig_commit_id);
 	git_oid_cpy(&newhunk->final_commit_id, &hunk->final_commit_id);
 	newhunk->boundary = hunk->boundary;
-	newhunk->final_signature = git_signature_dup(hunk->final_signature);
-	newhunk->orig_signature = git_signature_dup(hunk->orig_signature);
+	git_signature_dup(&newhunk->final_signature, hunk->final_signature);
+	git_signature_dup(&newhunk->orig_signature, hunk->orig_signature);
 	return newhunk;
 }
 
@@ -95,7 +98,7 @@ static void shift_hunks_by(git_vector *v, size_t start_line, int shift_by)
 {
 	size_t i;
 
-	if (!git_vector_bsearch2( &i, v, hunk_byfinalline_search_cmp, &start_line)) {
+	if (!git_vector_bsearch2(&i, v, hunk_byfinalline_search_cmp, &start_line)) {
 		for (; i < v->length; i++) {
 			git_blame_hunk *hunk = (git_blame_hunk*)v->contents[i];
 			hunk->final_start_line_number += shift_by;
@@ -108,17 +111,22 @@ git_blame* git_blame__alloc(
 	git_blame_options opts,
 	const char *path)
 {
-	git_blame *gbr = (git_blame*)git__calloc(1, sizeof(git_blame));
-	if (!gbr) {
-		giterr_set_oom();
+	git_blame *gbr = git__calloc(1, sizeof(git_blame));
+	if (!gbr)
 		return NULL;
-	}
-	git_vector_init(&gbr->hunks, 8, hunk_cmp);
-	git_vector_init(&gbr->paths, 8, paths_cmp);
+
 	gbr->repository = repo;
 	gbr->options = opts;
-	gbr->path = git__strdup(path);
-	git_vector_insert(&gbr->paths, git__strdup(path));
+
+	if (git_vector_init(&gbr->hunks, 8, hunk_cmp) < 0 ||
+		git_vector_init(&gbr->paths, 8, paths_cmp) < 0 ||
+		(gbr->path = git__strdup(path)) == NULL ||
+		git_vector_insert(&gbr->paths, git__strdup(path)) < 0)
+	{
+		git_blame_free(gbr);
+		return NULL;
+	}
+
 	return gbr;
 }
 
@@ -126,7 +134,6 @@ void git_blame_free(git_blame *blame)
 {
 	size_t i;
 	git_blame_hunk *hunk;
-	char *path;
 
 	if (!blame) return;
 
@@ -134,13 +141,11 @@ void git_blame_free(git_blame *blame)
 		free_hunk(hunk);
 	git_vector_free(&blame->hunks);
 
-	git_vector_foreach(&blame->paths, i, path)
-		git__free(path);
-	git_vector_free(&blame->paths);
+	git_vector_free_deep(&blame->paths);
 
 	git_array_clear(blame->line_index);
 
-	git__free((void*)blame->path);
+	git__free(blame->path);
 	git_blob_free(blame->final_blob);
 	git__free(blame);
 }
@@ -159,10 +164,10 @@ const git_blame_hunk *git_blame_get_hunk_byindex(git_blame *blame, uint32_t inde
 
 const git_blame_hunk *git_blame_get_hunk_byline(git_blame *blame, uint32_t lineno)
 {
-	size_t i;
+	size_t i, new_lineno = (size_t)lineno;
 	assert(blame);
 
-	if (!git_vector_bsearch2( &i, &blame->hunks, hunk_byfinalline_search_cmp, &lineno)) {
+	if (!git_vector_bsearch2(&i, &blame->hunks, hunk_byfinalline_search_cmp, &new_lineno)) {
 		return git_blame_get_hunk_byindex(blame, (uint32_t)i);
 	}
 
@@ -239,7 +244,7 @@ static int index_blob_lines(git_blame *blame)
     git_off_t len = blame->final_buf_size;
     int num = 0, incomplete = 0, bol = 1;
     size_t *i;
- 
+
     if (len && buf[len-1] != '\n')
         incomplete++; /* incomplete line at the end */
     while (len--) {
@@ -260,13 +265,15 @@ static int index_blob_lines(git_blame *blame)
     blame->num_lines = num + incomplete;
     return blame->num_lines;
 }
- 
+
 static git_blame_hunk* hunk_from_entry(git_blame__entry *e)
 {
 	git_blame_hunk *h = new_hunk(
 			e->lno+1, e->num_lines, e->s_lno+1, e->suspect->path);
 	git_oid_cpy(&h->final_commit_id, git_commit_id(e->suspect->commit));
-	h->final_signature = git_signature_dup(git_commit_author(e->suspect->commit));
+	git_oid_cpy(&h->orig_commit_id, git_commit_id(e->suspect->commit));
+	git_signature_dup(&h->final_signature, git_commit_author(e->suspect->commit));
+	git_signature_dup(&h->orig_signature, git_commit_author(e->suspect->commit));
 	h->boundary = e->is_boundary ? 1 : 0;
 	return h;
 }
@@ -282,8 +289,6 @@ static int load_blob(git_blame *blame)
 		goto cleanup;
 	error = git_object_lookup_bypath((git_object**)&blame->final_blob,
 			(git_object*)blame->final, blame->path, GIT_OBJ_BLOB);
-	if (error < 0)
-		goto cleanup;
 
 cleanup:
 	return error;
@@ -448,7 +453,7 @@ int git_blame_buffer(
 		git_blame **out,
 		git_blame *reference,
 		const char *buffer,
-		uint32_t buffer_len)
+		size_t buffer_len)
 {
 	git_blame *blame;
 	git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT;
@@ -474,3 +479,10 @@ int git_blame_buffer(
 	*out = blame;
 	return 0;
 }
+
+int git_blame_init_options(git_blame_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_blame_options, GIT_BLAME_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/blame.h b/src/blame.h
index 637e43985..7e23de808 100644
--- a/src/blame.h
+++ b/src/blame.h
@@ -64,7 +64,7 @@ typedef struct git_blame__entry {
 } git_blame__entry;
 
 struct git_blame {
-	const char *path;
+	char *path;
 	git_repository *repository;
 	git_blame_options options;
 
diff --git a/src/blame_git.c b/src/blame_git.c
index 800f1f039..72afb852b 100644
--- a/src/blame_git.c
+++ b/src/blame_git.c
@@ -485,12 +485,14 @@ static void pass_blame(git_blame *blame, git_blame__origin *origin, uint32_t opt
 	git_blame__origin *sg_buf[16];
 	git_blame__origin *porigin, **sg_origin = sg_buf;
 
-	GIT_UNUSED(opt);
-
 	num_parents = git_commit_parentcount(commit);
 	if (!git_oid_cmp(git_commit_id(commit), &blame->options.oldest_commit))
 		/* Stop at oldest specified commit */
 		num_parents = 0;
+	else if (opt & GIT_BLAME_FIRST_PARENT && num_parents > 1)
+		/* Limit search to the first parent */
+		num_parents = 1;
+
 	if (!num_parents) {
 		git_oid_cpy(&blame->options.oldest_commit, git_commit_id(commit));
 		goto finish;
diff --git a/src/blob.c b/src/blob.c
index 2c6d52800..30d5b705b 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -50,25 +50,28 @@ int git_blob__parse(void *blob, git_odb_object *odb_obj)
 	return 0;
 }
 
-int git_blob_create_frombuffer(git_oid *oid, git_repository *repo, const void *buffer, size_t len)
+int git_blob_create_frombuffer(
+	git_oid *id, git_repository *repo, const void *buffer, size_t len)
 {
 	int error;
 	git_odb *odb;
 	git_odb_stream *stream;
 
+	assert(id && repo);
+
 	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
 		(error = git_odb_open_wstream(&stream, odb, len, GIT_OBJ_BLOB)) < 0)
 		return error;
 
 	if ((error = git_odb_stream_write(stream, buffer, len)) == 0)
-		error = git_odb_stream_finalize_write(oid, stream);
+		error = git_odb_stream_finalize_write(id, stream);
 
 	git_odb_stream_free(stream);
 	return error;
 }
 
 static int write_file_stream(
-	git_oid *oid, git_odb *odb, const char *path, git_off_t file_size)
+	git_oid *id, git_odb *odb, const char *path, git_off_t file_size)
 {
 	int fd, error;
 	char buffer[4096];
@@ -97,14 +100,14 @@ static int write_file_stream(
 	}
 
 	if (!error)
-		error = git_odb_stream_finalize_write(oid, stream);
+		error = git_odb_stream_finalize_write(id, stream);
 
 	git_odb_stream_free(stream);
 	return error;
 }
 
 static int write_file_filtered(
-	git_oid *oid,
+	git_oid *id,
 	git_off_t *size,
 	git_odb *odb,
 	const char *full_path,
@@ -119,7 +122,7 @@ static int write_file_filtered(
 	if (!error) {
 		*size = tgt.size;
 
-		error = git_odb_write(oid, odb, tgt.ptr, tgt.size, GIT_OBJ_BLOB);
+		error = git_odb_write(id, odb, tgt.ptr, tgt.size, GIT_OBJ_BLOB);
 	}
 
 	git_buf_free(&tgt);
@@ -127,7 +130,7 @@ static int write_file_filtered(
 }
 
 static int write_symlink(
-	git_oid *oid, git_odb *odb, const char *path, size_t link_size)
+	git_oid *id, git_odb *odb, const char *path, size_t link_size)
 {
 	char *link_data;
 	ssize_t read_len;
@@ -143,13 +146,13 @@ static int write_symlink(
 		return -1;
 	}
 
-	error = git_odb_write(oid, odb, (void *)link_data, link_size, GIT_OBJ_BLOB);
+	error = git_odb_write(id, odb, (void *)link_data, link_size, GIT_OBJ_BLOB);
 	git__free(link_data);
 	return error;
 }
 
 int git_blob__create_from_paths(
-	git_oid *oid,
+	git_oid *id,
 	struct stat *out_st,
 	git_repository *repo,
 	const char *content_path,
@@ -188,24 +191,25 @@ int git_blob__create_from_paths(
 	mode = hint_mode ? hint_mode : st.st_mode;
 
 	if (S_ISLNK(mode)) {
-		error = write_symlink(oid, odb, content_path, (size_t)size);
+		error = write_symlink(id, odb, content_path, (size_t)size);
 	} else {
 		git_filter_list *fl = NULL;
 
 		if (try_load_filters)
 			/* Load the filters for writing this file to the ODB */
 			error = git_filter_list_load(
-				&fl, repo, NULL, hint_path, GIT_FILTER_TO_ODB);
+				&fl, repo, NULL, hint_path,
+				GIT_FILTER_TO_ODB, GIT_FILTER_OPT_DEFAULT);
 
 		if (error < 0)
 			/* well, that didn't work */;
 		else if (fl == NULL)
 			/* No filters need to be applied to the document: we can stream
 			 * directly from disk */
-			error = write_file_stream(oid, odb, content_path, size);
+			error = write_file_stream(id, odb, content_path, size);
 		else {
 			/* We need to apply one or more filters */
-			error = write_file_filtered(oid, &size, odb, content_path, fl);
+			error = write_file_filtered(id, &size, odb, content_path, fl);
 
 			git_filter_list_free(fl);
 		}
@@ -233,13 +237,13 @@ done:
 }
 
 int git_blob_create_fromworkdir(
-	git_oid *oid, git_repository *repo, const char *path)
+	git_oid *id, git_repository *repo, const char *path)
 {
-	return git_blob__create_from_paths(oid, NULL, repo, NULL, path, 0, true);
+	return git_blob__create_from_paths(id, NULL, repo, NULL, path, 0, true);
 }
 
 int git_blob_create_fromdisk(
-	git_oid *oid, git_repository *repo, const char *path)
+	git_oid *id, git_repository *repo, const char *path)
 {
 	int error;
 	git_buf full_path = GIT_BUF_INIT;
@@ -257,7 +261,7 @@ int git_blob_create_fromdisk(
 		hintpath += strlen(workdir);
 
 	error = git_blob__create_from_paths(
-		oid, NULL, repo, git_buf_cstr(&full_path), hintpath, 0, true);
+		id, NULL, repo, git_buf_cstr(&full_path), hintpath, 0, true);
 
 	git_buf_free(&full_path);
 	return error;
@@ -266,63 +270,72 @@ int git_blob_create_fromdisk(
 #define BUFFER_SIZE 4096
 
 int git_blob_create_fromchunks(
-	git_oid *oid,
+	git_oid *id,
 	git_repository *repo,
 	const char *hintpath,
 	int (*source_cb)(char *content, size_t max_length, void *payload),
 	void *payload)
 {
-	int error = -1, read_bytes;
+	int error;
 	char *content = NULL;
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf path = GIT_BUF_INIT;
 
-	if (git_buf_joinpath(
-			&path, git_repository_path(repo), GIT_OBJECTS_DIR "streamed") < 0)
+	assert(id && repo && source_cb);
+
+	if ((error = git_buf_joinpath(
+			&path, git_repository_path(repo), GIT_OBJECTS_DIR "streamed")) < 0)
 		goto cleanup;
 
 	content = git__malloc(BUFFER_SIZE);
 	GITERR_CHECK_ALLOC(content);
 
-	if (git_filebuf_open(&file, git_buf_cstr(&path), GIT_FILEBUF_TEMPORARY, 0666) < 0)
+	if ((error = git_filebuf_open(
+			&file, git_buf_cstr(&path), GIT_FILEBUF_TEMPORARY, 0666)) < 0)
 		goto cleanup;
 
 	while (1) {
-		read_bytes = source_cb(content, BUFFER_SIZE, payload);
+		int read_bytes = source_cb(content, BUFFER_SIZE, payload);
 
-		assert(read_bytes <= BUFFER_SIZE);
-
-		if (read_bytes <= 0)
+		if (!read_bytes)
 			break;
 
-		if (git_filebuf_write(&file, content, read_bytes) < 0)
+		if (read_bytes > BUFFER_SIZE) {
+			giterr_set(GITERR_OBJECT, "Invalid chunk size while creating blob");
+			error = GIT_EBUFS;
+		} else if (read_bytes < 0) {
+			error = giterr_set_after_callback(read_bytes);
+		} else {
+			error = git_filebuf_write(&file, content, read_bytes);
+		}
+
+		if (error < 0)
 			goto cleanup;
 	}
 
-	if (read_bytes < 0)
-		goto cleanup;
-
-	if (git_filebuf_flush(&file) < 0)
+	if ((error = git_filebuf_flush(&file)) < 0)
 		goto cleanup;
 
 	error = git_blob__create_from_paths(
-		oid, NULL, repo, file.path_lock, hintpath, 0, hintpath != NULL);
+		id, NULL, repo, file.path_lock, hintpath, 0, hintpath != NULL);
 
 cleanup:
 	git_buf_free(&path);
 	git_filebuf_cleanup(&file);
 	git__free(content);
+
 	return error;
 }
 
-int git_blob_is_binary(git_blob *blob)
+int git_blob_is_binary(const git_blob *blob)
 {
 	git_buf content;
 
 	assert(blob);
 
 	content.ptr   = blob->odb_object->buffer;
-	content.size  = min(blob->odb_object->cached.size, 4000);
+	content.size  =
+		min(blob->odb_object->cached.size, GIT_FILTER_BYTES_TO_CHECK_NUL);
 	content.asize = 0;
 
 	return git_buf_text_is_binary(&content);
@@ -339,11 +352,14 @@ int git_blob_filtered_content(
 
 	assert(blob && path && out);
 
+	git_buf_sanitize(out);
+
 	if (check_for_binary_data && git_blob_is_binary(blob))
 		return 0;
 
 	if (!(error = git_filter_list_load(
-			&fl, git_blob_owner(blob), blob, path, GIT_FILTER_TO_WORKTREE))) {
+			&fl, git_blob_owner(blob), blob, path,
+			GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) {
 
 		error = git_filter_list_apply_to_blob(out, fl, blob);
 
diff --git a/src/branch.c b/src/branch.c
index 95b3fd980..52760853b 100644
--- a/src/branch.c
+++ b/src/branch.c
@@ -21,27 +21,22 @@ static int retrieve_branch_reference(
 	const char *branch_name,
 	int is_remote)
 {
-	git_reference *branch;
-	int error = -1;
+	git_reference *branch = NULL;
+	int error = 0;
 	char *prefix;
 	git_buf ref_name = GIT_BUF_INIT;
 
-	*branch_reference_out = NULL;
-
 	prefix = is_remote ? GIT_REFS_REMOTES_DIR : GIT_REFS_HEADS_DIR;
 
-	if (git_buf_joinpath(&ref_name, prefix, branch_name) < 0)
-		goto cleanup;
-
-	if ((error = git_reference_lookup(&branch, repo, ref_name.ptr)) < 0) {
-		giterr_set(GITERR_REFERENCE,
-			"Cannot locate %s branch '%s'.", is_remote ? "remote-tracking" : "local", branch_name);
-		goto cleanup;
-	}
+	if ((error = git_buf_joinpath(&ref_name, prefix, branch_name)) < 0)
+		/* OOM */;
+	else if ((error = git_reference_lookup(&branch, repo, ref_name.ptr)) < 0)
+		giterr_set(
+			GITERR_REFERENCE, "Cannot locate %s branch '%s'",
+			is_remote ? "remote-tracking" : "local", branch_name);
 
-	*branch_reference_out = branch;
+	*branch_reference_out = branch; /* will be NULL on error */
 
-cleanup:
 	git_buf_free(&ref_name);
 	return error;
 }
@@ -59,26 +54,53 @@ int git_branch_create(
 	git_repository *repository,
 	const char *branch_name,
 	const git_commit *commit,
-	int force)
+	int force,
+	const git_signature *signature,
+	const char *log_message)
 {
+	int is_head = 0;
 	git_reference *branch = NULL;
-	git_buf canonical_branch_name = GIT_BUF_INIT;
+	git_buf canonical_branch_name = GIT_BUF_INIT,
+			  log_message_buf = GIT_BUF_INIT;
 	int error = -1;
 
 	assert(branch_name && commit && ref_out);
 	assert(git_object_owner((const git_object *)commit) == repository);
 
+	if (force && git_branch_lookup(&branch, repository, branch_name, GIT_BRANCH_LOCAL) == 0) {
+		error = git_branch_is_head(branch);
+		git_reference_free(branch);
+		branch = NULL;
+
+		if (error < 0)
+			goto cleanup;
+
+		is_head = error;
+	}
+
+	if (is_head && force) {
+		giterr_set(GITERR_REFERENCE, "Cannot force update branch '%s' as it is "
+			"the current HEAD of the repository.", branch_name);
+		error = -1;
+		goto cleanup;
+	}
+	
 	if (git_buf_joinpath(&canonical_branch_name, GIT_REFS_HEADS_DIR, branch_name) < 0)
 		goto cleanup;
 
+	if (git_buf_sets(&log_message_buf, log_message ? log_message : "Branch: created") < 0)
+		goto cleanup;
+
 	error = git_reference_create(&branch, repository,
-		git_buf_cstr(&canonical_branch_name), git_commit_id(commit), force);
+		git_buf_cstr(&canonical_branch_name), git_commit_id(commit), force, signature,
+		git_buf_cstr(&log_message_buf));
 
 	if (!error)
 		*ref_out = branch;
 
 cleanup:
 	git_buf_free(&canonical_branch_name);
+	git_buf_free(&log_message_buf);
 	return error;
 }
 
@@ -90,33 +112,35 @@ int git_branch_delete(git_reference *branch)
 
 	assert(branch);
 
-	if (!git_reference_is_branch(branch) &&
-		!git_reference_is_remote(branch)) {
-		giterr_set(GITERR_INVALID, "Reference '%s' is not a valid branch.", git_reference_name(branch));
-		return -1;
+	if (!git_reference_is_branch(branch) && !git_reference_is_remote(branch)) {
+		giterr_set(GITERR_INVALID, "Reference '%s' is not a valid branch.",
+			git_reference_name(branch));
+		return GIT_ENOTFOUND;
 	}
 
 	if ((is_head = git_branch_is_head(branch)) < 0)
 		return is_head;
 
 	if (is_head) {
-		giterr_set(GITERR_REFERENCE,
-				"Cannot delete branch '%s' as it is the current HEAD of the repository.", git_reference_name(branch));
+		giterr_set(GITERR_REFERENCE, "Cannot delete branch '%s' as it is "
+			"the current HEAD of the repository.", git_reference_name(branch));
 		return -1;
 	}
 
-	if (git_buf_printf(&config_section, "branch.%s", git_reference_name(branch) + strlen(GIT_REFS_HEADS_DIR)) < 0)
+	if (git_buf_join(&config_section, '.', "branch",
+			git_reference_name(branch) + strlen(GIT_REFS_HEADS_DIR)) < 0)
 		goto on_error;
 
 	if (git_config_rename_section(
-		git_reference_owner(branch), 
-		git_buf_cstr(&config_section),
-		NULL) < 0)
-			goto on_error;
+		git_reference_owner(branch), git_buf_cstr(&config_section), NULL) < 0)
+		goto on_error;
 
 	if (git_reference_delete(branch) < 0)
 		goto on_error;
 
+	if (git_reflog_delete(git_reference_owner(branch), git_reference_name(branch)) < 0)
+		goto on_error;
+
 	error = 0;
 
 on_error:
@@ -182,6 +206,9 @@ void git_branch_iterator_free(git_branch_iterator *_iter)
 {
 	branch_iter *iter = (branch_iter *) _iter;
 
+	if (iter == NULL)
+		return;
+
 	git_reference_iterator_free(iter->iter);
 	git__free(iter);
 }
@@ -190,11 +217,14 @@ int git_branch_move(
 	git_reference **out,
 	git_reference *branch,
 	const char *new_branch_name,
-	int force)
+	int force,
+	const git_signature *signature,
+	const char *log_message)
 {
 	git_buf new_reference_name = GIT_BUF_INIT,
-		old_config_section = GIT_BUF_INIT,
-		new_config_section = GIT_BUF_INIT;
+	        old_config_section = GIT_BUF_INIT,
+	        new_config_section = GIT_BUF_INIT,
+	        log_message_buf = GIT_BUF_INIT;
 	int error;
 
 	assert(branch && new_branch_name);
@@ -202,26 +232,40 @@ int git_branch_move(
 	if (!git_reference_is_branch(branch))
 		return not_a_local_branch(git_reference_name(branch));
 
-	error = git_buf_joinpath(&new_reference_name, GIT_REFS_HEADS_DIR, new_branch_name);
-	if (error < 0)
+	if ((error = git_buf_joinpath(&new_reference_name, GIT_REFS_HEADS_DIR, new_branch_name)) < 0)
 		goto done;
 
-	git_buf_printf(&old_config_section,
-		"branch.%s", git_reference_name(branch) + strlen(GIT_REFS_HEADS_DIR));
+	if (log_message) {
+		if ((error = git_buf_sets(&log_message_buf, log_message)) < 0)
+			goto done;
+	} else {
+		if ((error = git_buf_printf(&log_message_buf, "Branch: renamed %s to %s",
+						git_reference_name(branch), git_buf_cstr(&new_reference_name))) < 0)
+			goto done;
+	}
 
-	git_buf_printf(&new_config_section, "branch.%s", new_branch_name);
+	/* first update ref then config so failure won't trash config */
 
-	if ((error = git_config_rename_section(git_reference_owner(branch),
-		git_buf_cstr(&old_config_section),
-		git_buf_cstr(&new_config_section))) < 0)
+	error = git_reference_rename(
+		out, branch, git_buf_cstr(&new_reference_name), force,
+		signature, git_buf_cstr(&log_message_buf));
+	if (error < 0)
 		goto done;
 
-	error = git_reference_rename(out, branch, git_buf_cstr(&new_reference_name), force);
+	git_buf_join(&old_config_section, '.', "branch",
+		git_reference_name(branch) + strlen(GIT_REFS_HEADS_DIR));
+	git_buf_join(&new_config_section, '.', "branch", new_branch_name);
+
+	error = git_config_rename_section(
+		git_reference_owner(branch),
+		git_buf_cstr(&old_config_section),
+		git_buf_cstr(&new_config_section));
 
 done:
 	git_buf_free(&new_reference_name);
 	git_buf_free(&old_config_section);
 	git_buf_free(&new_config_section);
+	git_buf_free(&log_message_buf);
 
 	return error;
 }
@@ -237,7 +281,9 @@ int git_branch_lookup(
 	return retrieve_branch_reference(ref_out, repo, branch_name, branch_type == GIT_BRANCH_REMOTE);
 }
 
-int git_branch_name(const char **out, git_reference *ref)
+int git_branch_name(
+	const char **out,
+	const git_reference *ref)
 {
 	const char *branch_name;
 
@@ -260,17 +306,13 @@ int git_branch_name(const char **out, git_reference *ref)
 
 static int retrieve_upstream_configuration(
 	const char **out,
-	git_repository *repo,
+	const git_config *config,
 	const char *canonical_branch_name,
 	const char *format)
 {
-	git_config *config;
 	git_buf buf = GIT_BUF_INIT;
 	int error;
 
-	if (git_repository_config__weakptr(&config, repo) < 0)
-		return -1;
-
 	if (git_buf_printf(&buf, format,
 		canonical_branch_name + strlen(GIT_REFS_HEADS_DIR)) < 0)
 			return -1;
@@ -280,33 +322,39 @@ static int retrieve_upstream_configuration(
 	return error;
 }
 
-int git_branch_upstream__name(
-	git_buf *tracking_name,
+int git_branch_upstream_name(
+	git_buf *out,
 	git_repository *repo,
-	const char *canonical_branch_name)
+	const char *refname)
 {
 	const char *remote_name, *merge_name;
 	git_buf buf = GIT_BUF_INIT;
 	int error = -1;
 	git_remote *remote = NULL;
 	const git_refspec *refspec;
+	git_config *config;
+
+	assert(out && refname);
 
-	assert(tracking_name && canonical_branch_name);
+	git_buf_sanitize(out);
 
-	if (!git_reference__is_branch(canonical_branch_name))
-		return not_a_local_branch(canonical_branch_name);
+	if (!git_reference__is_branch(refname))
+		return not_a_local_branch(refname);
+
+	if ((error = git_repository_config_snapshot(&config, repo)) < 0)
+		return error;
 
 	if ((error = retrieve_upstream_configuration(
-		&remote_name, repo, canonical_branch_name, "branch.%s.remote")) < 0)
+		&remote_name, config, refname, "branch.%s.remote")) < 0)
 			goto cleanup;
 
 	if ((error = retrieve_upstream_configuration(
-		&merge_name, repo, canonical_branch_name, "branch.%s.merge")) < 0)
+		&merge_name, config, refname, "branch.%s.merge")) < 0)
 			goto cleanup;
 
 	if (!*remote_name || !*merge_name) {
 		giterr_set(GITERR_REFERENCE,
-			"branch '%s' does not have an upstream", canonical_branch_name);
+			"branch '%s' does not have an upstream", refname);
 		error = GIT_ENOTFOUND;
 		goto cleanup;
 	}
@@ -321,21 +369,22 @@ int git_branch_upstream__name(
 			goto cleanup;
 		}
 
-		if (git_refspec_transform_r(&buf, refspec, merge_name) < 0)
+		if (git_refspec_transform(&buf, refspec, merge_name) < 0)
 			goto cleanup;
 	} else
 		if (git_buf_sets(&buf, merge_name) < 0)
 			goto cleanup;
 
-	error = git_buf_set(tracking_name, git_buf_cstr(&buf), git_buf_len(&buf));
+	error = git_buf_set(out, git_buf_cstr(&buf), git_buf_len(&buf));
 
 cleanup:
+	git_config_free(config);
 	git_remote_free(remote);
 	git_buf_free(&buf);
 	return error;
 }
 
-static int remote_name(git_buf *buf, git_repository *repo, const char *canonical_branch_name)
+int git_branch_remote_name(git_buf *buf, git_repository *repo, const char *refname)
 {
 	git_strarray remote_list = {0};
 	size_t i;
@@ -344,12 +393,14 @@ static int remote_name(git_buf *buf, git_repository *repo, const char *canonical
 	int error = 0;
 	char *remote_name = NULL;
 
-	assert(buf && repo && canonical_branch_name);
+	assert(buf && repo && refname);
+
+	git_buf_sanitize(buf);
 
 	/* Verify that this is a remote branch */
-	if (!git_reference__is_remote(canonical_branch_name)) {
+	if (!git_reference__is_remote(refname)) {
 		giterr_set(GITERR_INVALID, "Reference '%s' is not a remote branch.",
-			canonical_branch_name);
+			refname);
 		error = GIT_ERROR;
 		goto cleanup;
 	}
@@ -363,7 +414,7 @@ static int remote_name(git_buf *buf, git_repository *repo, const char *canonical
 		if ((error = git_remote_load(&remote, repo, remote_list.strings[i])) < 0)
 			continue;
 
-		fetchspec = git_remote__matching_dst_refspec(remote, canonical_branch_name);
+		fetchspec = git_remote__matching_dst_refspec(remote, refname);
 		if (fetchspec) {
 			/* If we have not already set out yet, then set
 			 * it to the matching remote name. Otherwise
@@ -375,7 +426,7 @@ static int remote_name(git_buf *buf, git_repository *repo, const char *canonical
 				git_remote_free(remote);
 
 				giterr_set(GITERR_REFERENCE,
-					"Reference '%s' is ambiguous", canonical_branch_name);
+					"Reference '%s' is ambiguous", refname);
 				error = GIT_EAMBIGUOUS;
 				goto cleanup;
 			}
@@ -389,76 +440,26 @@ static int remote_name(git_buf *buf, git_repository *repo, const char *canonical
 		error = git_buf_puts(buf, remote_name);
 	} else {
 		giterr_set(GITERR_REFERENCE,
-			"Could not determine remote for '%s'", canonical_branch_name);
+			"Could not determine remote for '%s'", refname);
 		error = GIT_ENOTFOUND;
 	}
 
 cleanup:
+	if (error < 0)
+		git_buf_free(buf);
+
 	git_strarray_free(&remote_list);
 	return error;
 }
 
-int git_branch_remote_name(char *buffer, size_t buffer_len, git_repository *repo, const char *refname)
-{
-	int ret;
-	git_buf buf = GIT_BUF_INIT;
-
-	if ((ret = remote_name(&buf, repo, refname)) < 0)
-		return ret;
-
-	if (buffer)
-		git_buf_copy_cstr(buffer, buffer_len, &buf);
-
-	ret = (int)git_buf_len(&buf) + 1;
-	git_buf_free(&buf);
-
-	return ret;
-}
-
-int git_branch_upstream_name(
-	char *tracking_branch_name_out,
-	size_t buffer_size,
-	git_repository *repo,
-	const char *canonical_branch_name)
-{
-	git_buf buf = GIT_BUF_INIT;
-	int error;
-
-	assert(canonical_branch_name);
-
-	if (tracking_branch_name_out && buffer_size)
-		*tracking_branch_name_out = '\0';
-
-	if ((error = git_branch_upstream__name(
-		&buf, repo, canonical_branch_name)) < 0)
-			goto cleanup;
-
-	if (tracking_branch_name_out && buf.size + 1 > buffer_size) { /* +1 for NUL byte */
-		giterr_set(
-			GITERR_INVALID,
-			"Buffer too short to hold the tracked reference name.");
-		error = -1;
-		goto cleanup;
-	}
-
-	if (tracking_branch_name_out)
-		git_buf_copy_cstr(tracking_branch_name_out, buffer_size, &buf);
-
-	error = (int)buf.size + 1;
-
-cleanup:
-	git_buf_free(&buf);
-	return (int)error;
-}
-
 int git_branch_upstream(
-		git_reference **tracking_out,
-		git_reference *branch)
+	git_reference **tracking_out,
+	const git_reference *branch)
 {
 	int error;
 	git_buf tracking_name = GIT_BUF_INIT;
 
-	if ((error = git_branch_upstream__name(&tracking_name,
+	if ((error = git_branch_upstream_name(&tracking_name,
 		git_reference_owner(branch), git_reference_name(branch))) < 0)
 			return error;
 
@@ -541,7 +542,7 @@ int git_branch_set_upstream(git_reference *branch, const char *upstream_name)
 	if (local)
 		git_buf_puts(&value, ".");
 	else
-		remote_name(&value, repo, git_reference_name(upstream));
+		git_branch_remote_name(&value, repo, git_reference_name(upstream));
 
 	if (git_buf_printf(&key, "branch.%s.remote", shortname) < 0)
 		goto on_error;
@@ -560,7 +561,7 @@ int git_branch_set_upstream(git_reference *branch, const char *upstream_name)
 
 		fetchspec = git_remote__matching_dst_refspec(remote, git_reference_name(upstream));
 		git_buf_clear(&value);
-		if (!fetchspec || git_refspec_transform_l(&value, fetchspec, git_reference_name(upstream)) < 0)
+		if (!fetchspec || git_refspec_rtransform(&value, fetchspec, git_reference_name(upstream)) < 0)
 			goto on_error;
 
 		git_remote_free(remote);
@@ -590,7 +591,7 @@ on_error:
 }
 
 int git_branch_is_head(
-		git_reference *branch)
+		const git_reference *branch)
 {
 	git_reference *head;
 	bool is_same = false;
diff --git a/src/buffer.c b/src/buffer.c
index 20682322e..b8f8660ed 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -7,7 +7,6 @@
 #include "buffer.h"
 #include "posix.h"
 #include "git2/buffer.h"
-#include <stdarg.h>
 #include <ctype.h>
 
 /* Used as default value for git_buf->ptr so that people can always
@@ -66,8 +65,10 @@ int git_buf_try_grow(
 	new_ptr = git__realloc(new_ptr, new_size);
 
 	if (!new_ptr) {
-		if (mark_oom)
+		if (mark_oom) {
+			if (buf->ptr) git__free(buf->ptr);
 			buf->ptr = git_buf__oom;
+		}
 		return -1;
 	}
 
@@ -100,12 +101,23 @@ void git_buf_free(git_buf *buf)
 	git_buf_init(buf, 0);
 }
 
+void git_buf_sanitize(git_buf *buf)
+{
+	if (buf->ptr == NULL) {
+		assert(buf->size == 0 && buf->asize == 0);
+		buf->ptr = git_buf__initbuf;
+	} else if (buf->asize > buf->size)
+		buf->ptr[buf->size] = '\0';
+}
+
 void git_buf_clear(git_buf *buf)
 {
 	buf->size = 0;
 
-	if (!buf->ptr)
+	if (!buf->ptr) {
 		buf->ptr = git_buf__initbuf;
+		buf->asize = 0;
+	}
 
 	if (buf->asize > 0)
 		buf->ptr[0] = '\0';
@@ -120,8 +132,11 @@ int git_buf_set(git_buf *buf, const void *data, size_t len)
 			ENSURE_SIZE(buf, len + 1);
 			memmove(buf->ptr, data, len);
 		}
+
 		buf->size = len;
-		buf->ptr[buf->size] = '\0';
+		if (buf->asize > buf->size)
+			buf->ptr[buf->size] = '\0';
+
 	}
 	return 0;
 }
@@ -139,6 +154,15 @@ int git_buf_putc(git_buf *buf, char c)
 	return 0;
 }
 
+int git_buf_putcn(git_buf *buf, char c, size_t len)
+{
+	ENSURE_SIZE(buf, buf->size + len + 1);
+	memset(buf->ptr + buf->size, c, len);
+	buf->size += len;
+	buf->ptr[buf->size] = '\0';
+	return 0;
+}
+
 int git_buf_put(git_buf *buf, const char *data, size_t len)
 {
 	ENSURE_SIZE(buf, buf->size + len + 1);
@@ -194,6 +218,42 @@ int git_buf_put_base64(git_buf *buf, const char *data, size_t len)
 	return 0;
 }
 
+static const char b85str[] =
+	"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
+
+int git_buf_put_base85(git_buf *buf, const char *data, size_t len)
+{
+	ENSURE_SIZE(buf, buf->size + (5 * ((len / 4) + !!(len % 4))) + 1);
+
+	while (len) {
+		uint32_t acc = 0;
+		char b85[5];
+		int i;
+
+		for (i = 24; i >= 0; i -= 8) {
+			uint8_t ch = *data++;
+			acc |= ch << i;
+
+			if (--len == 0)
+				break;
+		}
+
+		for (i = 4; i >= 0; i--) {
+			int val = acc % 85;
+			acc /= 85;
+
+			b85[i] = b85str[val];
+		}
+
+		for (i = 0; i < 5; i++)
+			buf->ptr[buf->size++] = b85[i];
+	}
+
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
 int git_buf_vprintf(git_buf *buf, const char *format, va_list ap)
 {
 	int len;
@@ -272,19 +332,20 @@ void git_buf_consume(git_buf *buf, const char *end)
 
 void git_buf_truncate(git_buf *buf, size_t len)
 {
-	if (len < buf->size) {
-		buf->size = len;
+	if (len >= buf->size)
+		return;
+
+	buf->size = len;
+	if (buf->size < buf->asize)
 		buf->ptr[buf->size] = '\0';
-	}
 }
 
 void git_buf_shorten(git_buf *buf, size_t amount)
 {
-	if (amount > buf->size)
-		amount = buf->size;
-
-	buf->size = buf->size - amount;
-	buf->ptr[buf->size] = '\0';
+	if (buf->size > amount)
+		git_buf_truncate(buf, buf->size - amount);
+	else
+		git_buf_clear(buf);
 }
 
 void git_buf_rtruncate_at_char(git_buf *buf, char separator)
@@ -424,7 +485,7 @@ int git_buf_join(
 	ssize_t offset_a = -1;
 
 	/* not safe to have str_b point internally to the buffer */
-	assert(str_b < buf->ptr || str_b > buf->ptr + buf->size);
+	assert(str_b < buf->ptr || str_b >= buf->ptr + buf->size);
 
 	/* figure out if we need to insert a separator */
 	if (separator && strlen_a) {
@@ -439,13 +500,14 @@ int git_buf_join(
 
 	if (git_buf_grow(buf, strlen_a + strlen_b + need_sep + 1) < 0)
 		return -1;
+	assert(buf->ptr);
 
 	/* fix up internal pointers */
 	if (offset_a >= 0)
 		str_a = buf->ptr + offset_a;
 
 	/* do the actual copying */
-	if (offset_a != 0)
+	if (offset_a != 0 && str_a)
 		memmove(buf->ptr, str_a, strlen_a);
 	if (need_sep)
 		buf->ptr[strlen_a] = separator;
@@ -457,6 +519,59 @@ int git_buf_join(
 	return 0;
 }
 
+int git_buf_join3(
+	git_buf *buf,
+	char separator,
+	const char *str_a,
+	const char *str_b,
+	const char *str_c)
+{
+	size_t len_a = strlen(str_a), len_b = strlen(str_b), len_c = strlen(str_c);
+	int sep_a = 0, sep_b = 0;
+	char *tgt;
+
+	/* for this function, disallow pointers into the existing buffer */
+	assert(str_a < buf->ptr || str_a >= buf->ptr + buf->size);
+	assert(str_b < buf->ptr || str_b >= buf->ptr + buf->size);
+	assert(str_c < buf->ptr || str_c >= buf->ptr + buf->size);
+
+	if (separator) {
+		if (len_a > 0) {
+			while (*str_b == separator) { str_b++; len_b--; }
+			sep_a = (str_a[len_a - 1] != separator);
+		}
+		if (len_a > 0 || len_b > 0)
+			while (*str_c == separator) { str_c++; len_c--; }
+		if (len_b > 0)
+			sep_b = (str_b[len_b - 1] != separator);
+	}
+
+	if (git_buf_grow(buf, len_a + sep_a + len_b + sep_b + len_c + 1) < 0)
+		return -1;
+
+	tgt = buf->ptr;
+
+	if (len_a) {
+		memcpy(tgt, str_a, len_a);
+		tgt += len_a;
+	}
+	if (sep_a)
+		*tgt++ = separator;
+	if (len_b) {
+		memcpy(tgt, str_b, len_b);
+		tgt += len_b;
+	}
+	if (sep_b)
+		*tgt++ = separator;
+	if (len_c)
+		memcpy(tgt, str_c, len_c);
+
+	buf->size = len_a + sep_a + len_b + sep_b + len_c;
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
 void git_buf_rtrim(git_buf *buf)
 {
 	while (buf->size > 0) {
@@ -466,7 +581,8 @@ void git_buf_rtrim(git_buf *buf)
 		buf->size--;
 	}
 
-	buf->ptr[buf->size] = '\0';
+	if (buf->asize > buf->size)
+		buf->ptr[buf->size] = '\0';
 }
 
 int git_buf_cmp(const git_buf *a, const git_buf *b)
@@ -490,8 +606,7 @@ int git_buf_splice(
 	/* Ported from git.git
 	 * https://github.com/git/git/blob/16eed7c/strbuf.c#L159-176
 	 */
-	if (git_buf_grow(buf, git_buf_len(buf) + nb_to_insert - nb_to_remove) < 0)
-		return -1;
+	ENSURE_SIZE(buf, buf->size + nb_to_insert - nb_to_insert + 1);
 
 	memmove(buf->ptr + where + nb_to_insert,
 			buf->ptr + where + nb_to_remove,
diff --git a/src/buffer.h b/src/buffer.h
index 4ca9d4d94..70d6d73b3 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -10,7 +10,6 @@
 #include "common.h"
 #include "git2/strarray.h"
 #include "git2/buffer.h"
-#include <stdarg.h>
 
 /* typedef struct {
  *  	char   *ptr;
@@ -50,6 +49,15 @@ extern void git_buf_init(git_buf *buf, size_t initial_size);
 extern int git_buf_try_grow(
 	git_buf *buf, size_t target_size, bool mark_oom, bool preserve_external);
 
+/**
+ * Sanitizes git_buf structures provided from user input.  Users of the
+ * library, when providing git_buf's, may wish to provide a NULL ptr for
+ * ease of handling.  The buffer routines, however, expect a non-NULL ptr
+ * always.  This helper method simply handles NULL input, converting to a
+ * git_buf__initbuf.
+ */
+extern void git_buf_sanitize(git_buf *buf);
+
 extern void git_buf_swap(git_buf *buf_a, git_buf *buf_b);
 extern char *git_buf_detach(git_buf *buf);
 extern void git_buf_attach(git_buf *buf, char *ptr, size_t asize);
@@ -80,6 +88,7 @@ GIT_INLINE(bool) git_buf_oom(const git_buf *buf)
  */
 int git_buf_sets(git_buf *buf, const char *string);
 int git_buf_putc(git_buf *buf, char c);
+int git_buf_putcn(git_buf *buf, char c, size_t len);
 int git_buf_put(git_buf *buf, const char *data, size_t len);
 int git_buf_puts(git_buf *buf, const char *string);
 int git_buf_printf(git_buf *buf, const char *format, ...) GIT_FORMAT_PRINTF(2, 3);
@@ -90,8 +99,12 @@ void git_buf_truncate(git_buf *buf, size_t len);
 void git_buf_shorten(git_buf *buf, size_t amount);
 void git_buf_rtruncate_at_char(git_buf *path, char separator);
 
+/** General join with separator */
 int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...);
+/** Fast join of two strings - first may legally point into `buf` data */
 int git_buf_join(git_buf *buf, char separator, const char *str_a, const char *str_b);
+/** Fast join of three strings - cannot reference `buf` data */
+int git_buf_join3(git_buf *buf, char separator, const char *str_a, const char *str_b, const char *str_c);
 
 /**
  * Join two strings as paths, inserting a slash between as needed.
@@ -145,6 +158,9 @@ int git_buf_cmp(const git_buf *a, const git_buf *b);
 /* Write data as base64 encoded in buffer */
 int git_buf_put_base64(git_buf *buf, const char *data, size_t len);
 
+/* Write data as "base85" encoded in buffer */
+int git_buf_put_base85(git_buf *buf, const char *data, size_t len);
+
 /*
  * Insert, remove or replace a portion of the buffer.
  *
diff --git a/src/cc-compat.h b/src/cc-compat.h
index 37f1ea81e..e73cb6de8 100644
--- a/src/cc-compat.h
+++ b/src/cc-compat.h
@@ -7,6 +7,8 @@
 #ifndef INCLUDE_compat_h__
 #define INCLUDE_compat_h__
 
+#include <stdarg.h>
+
 /*
  * See if our compiler is known to support flexible array members.
  */
diff --git a/src/checkout.c b/src/checkout.c
index 6d7e3cfd4..20763fd35 100644
--- a/src/checkout.c
+++ b/src/checkout.c
@@ -47,7 +47,7 @@ enum {
 typedef struct {
 	git_repository *repo;
 	git_diff *diff;
-	git_checkout_opts opts;
+	git_checkout_options opts;
 	bool opts_free_baseline;
 	char *pfx;
 	git_index *index;
@@ -56,6 +56,7 @@ typedef struct {
 	git_vector conflicts;
 	git_buf path;
 	size_t workdir_len;
+	git_buf tmp;
 	unsigned int strategy;
 	int can_symlink;
 	bool reload_submodules;
@@ -70,7 +71,9 @@ typedef struct {
 
 	int name_collision:1,
 		directoryfile:1,
-		one_to_two:1;
+		one_to_two:1,
+		binary:1,
+		submodule:1;
 } checkout_conflictdata;
 
 static int checkout_notify(
@@ -83,19 +86,17 @@ static int checkout_notify(
 	const git_diff_file *baseline = NULL, *target = NULL, *workdir = NULL;
 	const char *path = NULL;
 
-	if (!data->opts.notify_cb)
-		return 0;
-
-	if ((why & data->opts.notify_flags) == 0)
+	if (!data->opts.notify_cb ||
+		(why & data->opts.notify_flags) == 0)
 		return 0;
 
 	if (wditem) {
 		memset(&wdfile, 0, sizeof(wdfile));
 
-		git_oid_cpy(&wdfile.oid, &wditem->oid);
+		git_oid_cpy(&wdfile.id, &wditem->id);
 		wdfile.path = wditem->path;
 		wdfile.size = wditem->file_size;
-		wdfile.flags = GIT_DIFF_FLAG_VALID_OID;
+		wdfile.flags = GIT_DIFF_FLAG_VALID_ID;
 		wdfile.mode = wditem->mode;
 
 		workdir = &wdfile;
@@ -125,8 +126,13 @@ static int checkout_notify(
 		path = delta->old_file.path;
 	}
 
-	return data->opts.notify_cb(
-		why, path, baseline, target, workdir, data->opts.notify_payload);
+	{
+		int error = data->opts.notify_cb(
+			why, path, baseline, target, workdir, data->opts.notify_payload);
+
+		return giterr_set_after_callback_function(
+			error, "git_checkout notification");
+	}
 }
 
 static bool checkout_is_workdir_modified(
@@ -142,19 +148,23 @@ static bool checkout_is_workdir_modified(
 		git_submodule *sm;
 		unsigned int sm_status = 0;
 		const git_oid *sm_oid = NULL;
+		bool rval = false;
 
-		if (git_submodule_lookup(&sm, data->repo, wditem->path) < 0 ||
-			git_submodule_status(&sm_status, sm) < 0)
-			return true;
-
-		if (GIT_SUBMODULE_STATUS_IS_WD_DIRTY(sm_status))
+		if (git_submodule_lookup(&sm, data->repo, wditem->path) < 0) {
+			giterr_clear();
 			return true;
+		}
 
-		sm_oid = git_submodule_wd_id(sm);
-		if (!sm_oid)
-			return false;
+		if (git_submodule_status(&sm_status, sm) < 0 ||
+			GIT_SUBMODULE_STATUS_IS_WD_DIRTY(sm_status))
+			rval = true;
+		else if ((sm_oid = git_submodule_wd_id(sm)) == NULL)
+			rval = false;
+		else
+			rval = (git_oid__cmp(&baseitem->id, sm_oid) != 0);
 
-		return (git_oid__cmp(&baseitem->oid, sm_oid) != 0);
+		git_submodule_free(sm);
+		return rval;
 	}
 
 	/* Look at the cache to decide if the workdir is modified.  If not,
@@ -165,7 +175,7 @@ static bool checkout_is_workdir_modified(
 		if (wditem->mtime.seconds == ie->mtime.seconds &&
 			wditem->mtime.nanoseconds == ie->mtime.nanoseconds &&
 			wditem->file_size == ie->file_size)
-			return (git_oid__cmp(&baseitem->oid, &ie->oid) != 0);
+			return (git_oid__cmp(&baseitem->id, &ie->id) != 0);
 	}
 
 	/* depending on where base is coming from, we may or may not know
@@ -174,113 +184,135 @@ static bool checkout_is_workdir_modified(
 	if (baseitem->size && wditem->file_size != baseitem->size)
 		return true;
 
-	if (git_diff__oid_for_file(
-			data->repo, wditem->path, wditem->mode,
-			wditem->file_size, &oid) < 0)
+	if (git_diff__oid_for_entry(&oid, data->diff, wditem, NULL) < 0)
 		return false;
 
-	return (git_oid__cmp(&baseitem->oid, &oid) != 0);
+	return (git_oid__cmp(&baseitem->id, &oid) != 0);
 }
 
 #define CHECKOUT_ACTION_IF(FLAG,YES,NO) \
 	((data->strategy & GIT_CHECKOUT_##FLAG) ? CHECKOUT_ACTION__##YES : CHECKOUT_ACTION__##NO)
 
 static int checkout_action_common(
+	int *action,
 	checkout_data *data,
-	int action,
 	const git_diff_delta *delta,
 	const git_index_entry *wd)
 {
 	git_checkout_notify_t notify = GIT_CHECKOUT_NOTIFY_NONE;
 
-	if (action <= 0)
-		return action;
-
 	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0)
-		action = (action & ~CHECKOUT_ACTION__REMOVE);
+		*action = (*action & ~CHECKOUT_ACTION__REMOVE);
 
-	if ((action & CHECKOUT_ACTION__UPDATE_BLOB) != 0) {
+	if ((*action & CHECKOUT_ACTION__UPDATE_BLOB) != 0) {
 		if (S_ISGITLINK(delta->new_file.mode))
-			action = (action & ~CHECKOUT_ACTION__UPDATE_BLOB) |
+			*action = (*action & ~CHECKOUT_ACTION__UPDATE_BLOB) |
 				CHECKOUT_ACTION__UPDATE_SUBMODULE;
 
 		/* to "update" a symlink, we must remove the old one first */
 		if (delta->new_file.mode == GIT_FILEMODE_LINK && wd != NULL)
-			action |= CHECKOUT_ACTION__REMOVE;
+			*action |= CHECKOUT_ACTION__REMOVE;
 
 		notify = GIT_CHECKOUT_NOTIFY_UPDATED;
 	}
 
-	if ((action & CHECKOUT_ACTION__CONFLICT) != 0)
+	if ((*action & CHECKOUT_ACTION__CONFLICT) != 0)
 		notify = GIT_CHECKOUT_NOTIFY_CONFLICT;
 
-	if (notify != GIT_CHECKOUT_NOTIFY_NONE &&
-		checkout_notify(data, notify, delta, wd) != 0)
-		return GIT_EUSER;
-
-	return action;
+	return checkout_notify(data, notify, delta, wd);
 }
 
 static int checkout_action_no_wd(
+	int *action,
 	checkout_data *data,
 	const git_diff_delta *delta)
 {
-	int action = CHECKOUT_ACTION__NONE;
+	int error = 0;
+
+	*action = CHECKOUT_ACTION__NONE;
 
 	switch (delta->status) {
 	case GIT_DELTA_UNMODIFIED: /* case 12 */
-		if (checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, NULL))
-			return GIT_EUSER;
-		action = CHECKOUT_ACTION_IF(SAFE_CREATE, UPDATE_BLOB, NONE);
+		error = checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, NULL);
+		if (error)
+			return error;
+		*action = CHECKOUT_ACTION_IF(SAFE_CREATE, UPDATE_BLOB, NONE);
 		break;
 	case GIT_DELTA_ADDED:    /* case 2 or 28 (and 5 but not really) */
-		action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
+		*action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
 		break;
 	case GIT_DELTA_MODIFIED: /* case 13 (and 35 but not really) */
-		action = CHECKOUT_ACTION_IF(SAFE_CREATE, UPDATE_BLOB, CONFLICT);
+		*action = CHECKOUT_ACTION_IF(SAFE_CREATE, UPDATE_BLOB, CONFLICT);
 		break;
 	case GIT_DELTA_TYPECHANGE: /* case 21 (B->T) and 28 (T->B)*/
 		if (delta->new_file.mode == GIT_FILEMODE_TREE)
-			action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
+			*action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
 		break;
 	case GIT_DELTA_DELETED: /* case 8 or 25 */
+		*action = CHECKOUT_ACTION_IF(SAFE, REMOVE, NONE);
+		break;
 	default: /* impossible */
 		break;
 	}
 
-	return checkout_action_common(data, action, delta, NULL);
+	return checkout_action_common(action, data, delta, NULL);
 }
 
+static bool wd_item_is_removable(git_iterator *iter, const git_index_entry *wd)
+{
+	git_buf *full = NULL;
+
+	if (wd->mode != GIT_FILEMODE_TREE)
+		return true;
+	if (git_iterator_current_workdir_path(&full, iter) < 0)
+		return true;
+	return !full || !git_path_contains(full, DOT_GIT);
+}
+
+static int checkout_queue_remove(checkout_data *data, const char *path)
+{
+	char *copy = git_pool_strdup(&data->pool, path);
+	GITERR_CHECK_ALLOC(copy);
+	return git_vector_insert(&data->removes, copy);
+}
+
+/* note that this advances the iterator over the wd item */
 static int checkout_action_wd_only(
 	checkout_data *data,
 	git_iterator *workdir,
-	const git_index_entry *wd,
+	const git_index_entry **wditem,
 	git_vector *pathspec)
 {
+	int error = 0;
 	bool remove = false;
 	git_checkout_notify_t notify = GIT_CHECKOUT_NOTIFY_NONE;
+	const git_index_entry *wd = *wditem;
 
 	if (!git_pathspec__match(
 			pathspec, wd->path,
 			(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
 			git_iterator_ignore_case(workdir), NULL, NULL))
-		return 0;
+		return git_iterator_advance(wditem, workdir);
 
 	/* check if item is tracked in the index but not in the checkout diff */
 	if (data->index != NULL) {
-		if (wd->mode != GIT_FILEMODE_TREE) {
-			int error;
+		size_t pos;
+
+		error = git_index__find_pos(
+			&pos, data->index, wd->path, 0, GIT_INDEX_STAGE_ANY);
 
-			if ((error = git_index_find(NULL, data->index, wd->path)) == 0) {
+		if (wd->mode != GIT_FILEMODE_TREE) {
+			if (!error) { /* found by git_index__find_pos call */
 				notify = GIT_CHECKOUT_NOTIFY_DIRTY;
 				remove = ((data->strategy & GIT_CHECKOUT_FORCE) != 0);
 			} else if (error != GIT_ENOTFOUND)
 				return error;
+			else
+				error = 0; /* git_index__find_pos does not set error msg */
 		} else {
 			/* for tree entries, we have to see if there are any index
 			 * entries that are contained inside that tree
 			 */
-			size_t pos = git_index__prefix_position(data->index, wd->path);
 			const git_index_entry *e = git_index_get_byindex(data->index, pos);
 
 			if (e != NULL && data->diff->pfxcomp(e->path, wd->path) == 0) {
@@ -290,29 +322,52 @@ static int checkout_action_wd_only(
 		}
 	}
 
-	if (notify != GIT_CHECKOUT_NOTIFY_NONE)
-		/* found in index */;
-	else if (git_iterator_current_is_ignored(workdir)) {
-		notify = GIT_CHECKOUT_NOTIFY_IGNORED;
-		remove = ((data->strategy & GIT_CHECKOUT_REMOVE_IGNORED) != 0);
-	}
-	else {
-		notify = GIT_CHECKOUT_NOTIFY_UNTRACKED;
-		remove = ((data->strategy & GIT_CHECKOUT_REMOVE_UNTRACKED) != 0);
-	}
+	if (notify != GIT_CHECKOUT_NOTIFY_NONE) {
+		/* if we found something in the index, notify and advance */
+		if ((error = checkout_notify(data, notify, NULL, wd)) != 0)
+			return error;
 
-	if (checkout_notify(data, notify, NULL, wd))
-		return GIT_EUSER;
+		if (remove && wd_item_is_removable(workdir, wd))
+			error = checkout_queue_remove(data, wd->path);
 
-	if (remove) {
-		char *path = git_pool_strdup(&data->pool, wd->path);
-		GITERR_CHECK_ALLOC(path);
+		if (!error)
+			error = git_iterator_advance(wditem, workdir);
+	} else {
+		/* untracked or ignored - can't know which until we advance through */
+		bool over = false, removable = wd_item_is_removable(workdir, wd);
+		git_iterator_status_t untracked_state;
+
+		/* copy the entry for issuing notification callback later */
+		git_index_entry saved_wd = *wd;
+		git_buf_sets(&data->tmp, wd->path);
+		saved_wd.path = data->tmp.ptr;
+
+		error = git_iterator_advance_over_with_status(
+			wditem, &untracked_state, workdir);
+		if (error == GIT_ITEROVER)
+			over = true;
+		else if (error < 0)
+			return error;
 
-		if (git_vector_insert(&data->removes, path) < 0)
-			return -1;
+		if (untracked_state == GIT_ITERATOR_STATUS_IGNORED) {
+			notify = GIT_CHECKOUT_NOTIFY_IGNORED;
+			remove = ((data->strategy & GIT_CHECKOUT_REMOVE_IGNORED) != 0);
+		} else {
+			notify = GIT_CHECKOUT_NOTIFY_UNTRACKED;
+			remove = ((data->strategy & GIT_CHECKOUT_REMOVE_UNTRACKED) != 0);
+		}
+
+		if ((error = checkout_notify(data, notify, NULL, &saved_wd)) != 0)
+			return error;
+
+		if (remove && removable)
+			error = checkout_queue_remove(data, saved_wd.path);
+
+		if (!error && over) /* restore ITEROVER if needed */
+			error = GIT_ITEROVER;
 	}
 
-	return 0;
+	return error;
 }
 
 static bool submodule_is_config_only(
@@ -321,45 +376,54 @@ static bool submodule_is_config_only(
 {
 	git_submodule *sm = NULL;
 	unsigned int sm_loc = 0;
+	bool rval = false;
 
-	if (git_submodule_lookup(&sm, data->repo, path) < 0 ||
-		git_submodule_location(&sm_loc, sm) < 0 ||
-		sm_loc == GIT_SUBMODULE_STATUS_IN_CONFIG)
+	if (git_submodule_lookup(&sm, data->repo, path) < 0)
 		return true;
 
-	return false;
+	if (git_submodule_location(&sm_loc, sm) < 0 ||
+		sm_loc == GIT_SUBMODULE_STATUS_IN_CONFIG)
+		rval = true;
+
+	git_submodule_free(sm);
+
+	return rval;
 }
 
 static int checkout_action_with_wd(
+	int *action,
 	checkout_data *data,
 	const git_diff_delta *delta,
+	git_iterator *workdir,
 	const git_index_entry *wd)
 {
-	int action = CHECKOUT_ACTION__NONE;
+	*action = CHECKOUT_ACTION__NONE;
 
 	switch (delta->status) {
 	case GIT_DELTA_UNMODIFIED: /* case 14/15 or 33 */
 		if (checkout_is_workdir_modified(data, &delta->old_file, wd)) {
-			if (checkout_notify(
-					data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, wd))
-				return GIT_EUSER;
-			action = CHECKOUT_ACTION_IF(FORCE, UPDATE_BLOB, NONE);
+			GITERR_CHECK_ERROR(
+				checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, wd) );
+			*action = CHECKOUT_ACTION_IF(FORCE, UPDATE_BLOB, NONE);
 		}
 		break;
 	case GIT_DELTA_ADDED: /* case 3, 4 or 6 */
-		action = CHECKOUT_ACTION_IF(FORCE, UPDATE_BLOB, CONFLICT);
+		if (git_iterator_current_is_ignored(workdir))
+			*action = CHECKOUT_ACTION_IF(DONT_OVERWRITE_IGNORED, CONFLICT, UPDATE_BLOB);
+		else
+			*action = CHECKOUT_ACTION_IF(FORCE, UPDATE_BLOB, CONFLICT);
 		break;
 	case GIT_DELTA_DELETED: /* case 9 or 10 (or 26 but not really) */
 		if (checkout_is_workdir_modified(data, &delta->old_file, wd))
-			action = CHECKOUT_ACTION_IF(FORCE, REMOVE, CONFLICT);
+			*action = CHECKOUT_ACTION_IF(FORCE, REMOVE, CONFLICT);
 		else
-			action = CHECKOUT_ACTION_IF(SAFE, REMOVE, NONE);
+			*action = CHECKOUT_ACTION_IF(SAFE, REMOVE, NONE);
 		break;
 	case GIT_DELTA_MODIFIED: /* case 16, 17, 18 (or 36 but not really) */
 		if (checkout_is_workdir_modified(data, &delta->old_file, wd))
-			action = CHECKOUT_ACTION_IF(FORCE, UPDATE_BLOB, CONFLICT);
+			*action = CHECKOUT_ACTION_IF(FORCE, UPDATE_BLOB, CONFLICT);
 		else
-			action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
+			*action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
 		break;
 	case GIT_DELTA_TYPECHANGE: /* case 22, 23, 29, 30 */
 		if (delta->old_file.mode == GIT_FILEMODE_TREE) {
@@ -367,92 +431,96 @@ static int checkout_action_with_wd(
 				/* either deleting items in old tree will delete the wd dir,
 				 * or we'll get a conflict when we attempt blob update...
 				 */
-				action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
+				*action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
 			else if (wd->mode == GIT_FILEMODE_COMMIT) {
 				/* workdir is possibly a "phantom" submodule - treat as a
 				 * tree if the only submodule info came from the config
 				 */
 				if (submodule_is_config_only(data, wd->path))
-					action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
+					*action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
 				else
-					action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
+					*action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
 			} else
-				action = CHECKOUT_ACTION_IF(FORCE, REMOVE, CONFLICT);
+				*action = CHECKOUT_ACTION_IF(FORCE, REMOVE, CONFLICT);
 		}
 		else if (checkout_is_workdir_modified(data, &delta->old_file, wd))
-			action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
+			*action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
 		else
-			action = CHECKOUT_ACTION_IF(SAFE, REMOVE_AND_UPDATE, NONE);
+			*action = CHECKOUT_ACTION_IF(SAFE, REMOVE_AND_UPDATE, NONE);
 
 		/* don't update if the typechange is to a tree */
 		if (delta->new_file.mode == GIT_FILEMODE_TREE)
-			action = (action & ~CHECKOUT_ACTION__UPDATE_BLOB);
+			*action = (*action & ~CHECKOUT_ACTION__UPDATE_BLOB);
 		break;
 	default: /* impossible */
 		break;
 	}
 
-	return checkout_action_common(data, action, delta, wd);
+	return checkout_action_common(action, data, delta, wd);
 }
 
 static int checkout_action_with_wd_blocker(
+	int *action,
 	checkout_data *data,
 	const git_diff_delta *delta,
 	const git_index_entry *wd)
 {
-	int action = CHECKOUT_ACTION__NONE;
+	*action = CHECKOUT_ACTION__NONE;
 
 	switch (delta->status) {
 	case GIT_DELTA_UNMODIFIED:
 		/* should show delta as dirty / deleted */
-		if (checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, wd))
-			return GIT_EUSER;
-		action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, NONE);
+		GITERR_CHECK_ERROR(
+			checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, wd) );
+		*action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, NONE);
 		break;
 	case GIT_DELTA_ADDED:
 	case GIT_DELTA_MODIFIED:
-		action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
+		*action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
 		break;
 	case GIT_DELTA_DELETED:
-		action = CHECKOUT_ACTION_IF(FORCE, REMOVE, CONFLICT);
+		*action = CHECKOUT_ACTION_IF(FORCE, REMOVE, CONFLICT);
 		break;
 	case GIT_DELTA_TYPECHANGE:
 		/* not 100% certain about this... */
-		action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
+		*action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
 		break;
 	default: /* impossible */
 		break;
 	}
 
-	return checkout_action_common(data, action, delta, wd);
+	return checkout_action_common(action, data, delta, wd);
 }
 
 static int checkout_action_with_wd_dir(
+	int *action,
 	checkout_data *data,
 	const git_diff_delta *delta,
+	git_iterator *workdir,
 	const git_index_entry *wd)
 {
-	int action = CHECKOUT_ACTION__NONE;
+	*action = CHECKOUT_ACTION__NONE;
 
 	switch (delta->status) {
 	case GIT_DELTA_UNMODIFIED: /* case 19 or 24 (or 34 but not really) */
-		if (checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, NULL) ||
-			checkout_notify(
-				data, GIT_CHECKOUT_NOTIFY_UNTRACKED, NULL, wd))
-			return GIT_EUSER;
+		GITERR_CHECK_ERROR(
+			checkout_notify(data, GIT_CHECKOUT_NOTIFY_DIRTY, delta, NULL));
+		GITERR_CHECK_ERROR(
+			checkout_notify(data, GIT_CHECKOUT_NOTIFY_UNTRACKED, NULL, wd));
 		break;
 	case GIT_DELTA_ADDED:/* case 4 (and 7 for dir) */
 	case GIT_DELTA_MODIFIED: /* case 20 (or 37 but not really) */
 		if (delta->old_file.mode == GIT_FILEMODE_COMMIT)
 			/* expected submodule (and maybe found one) */;
 		else if (delta->new_file.mode != GIT_FILEMODE_TREE)
-			action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
+			*action = git_iterator_current_is_ignored(workdir) ?
+				CHECKOUT_ACTION_IF(DONT_OVERWRITE_IGNORED, CONFLICT, REMOVE_AND_UPDATE) :
+				CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
 		break;
 	case GIT_DELTA_DELETED: /* case 11 (and 27 for dir) */
-		if (delta->old_file.mode != GIT_FILEMODE_TREE &&
-			checkout_notify(
-				data, GIT_CHECKOUT_NOTIFY_UNTRACKED, NULL, wd))
-			return GIT_EUSER;
+		if (delta->old_file.mode != GIT_FILEMODE_TREE)
+			GITERR_CHECK_ERROR(
+				checkout_notify(data, GIT_CHECKOUT_NOTIFY_UNTRACKED, NULL, wd));
 		break;
 	case GIT_DELTA_TYPECHANGE: /* case 24 or 31 */
 		if (delta->old_file.mode == GIT_FILEMODE_TREE) {
@@ -462,39 +530,41 @@ static int checkout_action_with_wd_dir(
 			 * directory if is it left empty, so we can defer removing the
 			 * dir and it will succeed if no children are left.
 			 */
-			action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
-			if (action != CHECKOUT_ACTION__NONE)
-				action |= CHECKOUT_ACTION__DEFER_REMOVE;
+			*action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
+			if (*action != CHECKOUT_ACTION__NONE)
+				*action |= CHECKOUT_ACTION__DEFER_REMOVE;
 		}
 		else if (delta->new_file.mode != GIT_FILEMODE_TREE)
 			/* For typechange to dir, dir is already created so no action */
-			action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
+			*action = CHECKOUT_ACTION_IF(FORCE, REMOVE_AND_UPDATE, CONFLICT);
 		break;
 	default: /* impossible */
 		break;
 	}
 
-	return checkout_action_common(data, action, delta, wd);
+	return checkout_action_common(action, data, delta, wd);
 }
 
 static int checkout_action(
+	int *action,
 	checkout_data *data,
 	git_diff_delta *delta,
 	git_iterator *workdir,
-	const git_index_entry **wditem_ptr,
+	const git_index_entry **wditem,
 	git_vector *pathspec)
 {
-	const git_index_entry *wd = *wditem_ptr;
-	int cmp = -1, act;
+	int cmp = -1, error;
 	int (*strcomp)(const char *, const char *) = data->diff->strcomp;
 	int (*pfxcomp)(const char *str, const char *pfx) = data->diff->pfxcomp;
-	int error;
+	int (*advance)(const git_index_entry **, git_iterator *) = NULL;
 
 	/* move workdir iterator to follow along with deltas */
 
 	while (1) {
+		const git_index_entry *wd = *wditem;
+
 		if (!wd)
-			return checkout_action_no_wd(data, delta);
+			return checkout_action_no_wd(action, data, delta);
 
 		cmp = strcomp(wd->path, delta->old_file.path);
 
@@ -512,79 +582,74 @@ static int checkout_action(
 			if (cmp == 0) {
 				if (wd->mode == GIT_FILEMODE_TREE) {
 					/* case 2 - entry prefixed by workdir tree */
-					error = git_iterator_advance_into_or_over(&wd, workdir);
-					if (error && error != GIT_ITEROVER)
-						goto fail;
-					*wditem_ptr = wd;
+					error = git_iterator_advance_into_or_over(wditem, workdir);
+					if (error < 0 && error != GIT_ITEROVER)
+						goto done;
 					continue;
 				}
 
 				/* case 3 maybe - wd contains non-dir where dir expected */
 				if (delta->old_file.path[strlen(wd->path)] == '/') {
-					act = checkout_action_with_wd_blocker(data, delta, wd);
-					*wditem_ptr =
-						git_iterator_advance(&wd, workdir) ? NULL : wd;
-					return act;
+					error = checkout_action_with_wd_blocker(
+						action, data, delta, wd);
+					advance = git_iterator_advance;
+					goto done;
 				}
 			}
 
 			/* case 1 - handle wd item (if it matches pathspec) */
-			if (checkout_action_wd_only(data, workdir, wd, pathspec) < 0)
-				goto fail;
-			if ((error = git_iterator_advance(&wd, workdir)) < 0 &&
-				error != GIT_ITEROVER)
-				goto fail;
-
-			*wditem_ptr = wd;
+			error = checkout_action_wd_only(data, workdir, wditem, pathspec);
+			if (error && error != GIT_ITEROVER)
+				goto done;
 			continue;
 		}
 
 		if (cmp == 0) {
 			/* case 4 */
-			act = checkout_action_with_wd(data, delta, wd);
-			*wditem_ptr = git_iterator_advance(&wd, workdir) ? NULL : wd;
-			return act;
+			error = checkout_action_with_wd(action, data, delta, workdir, wd);
+			advance = git_iterator_advance;
+			goto done;
 		}
 
 		cmp = pfxcomp(wd->path, delta->old_file.path);
 
 		if (cmp == 0) { /* case 5 */
 			if (wd->path[strlen(delta->old_file.path)] != '/')
-				return checkout_action_no_wd(data, delta);
+				return checkout_action_no_wd(action, data, delta);
 
 			if (delta->status == GIT_DELTA_TYPECHANGE) {
 				if (delta->old_file.mode == GIT_FILEMODE_TREE) {
-					act = checkout_action_with_wd(data, delta, wd);
-					if ((error = git_iterator_advance_into(&wd, workdir)) < 0 &&
-						error != GIT_ENOTFOUND)
-						goto fail;
-					*wditem_ptr = wd;
-					return act;
+					error = checkout_action_with_wd(action, data, delta, workdir, wd);
+					advance = git_iterator_advance_into;
+					goto done;
 				}
 
 				if (delta->new_file.mode == GIT_FILEMODE_TREE ||
 					delta->new_file.mode == GIT_FILEMODE_COMMIT ||
 					delta->old_file.mode == GIT_FILEMODE_COMMIT)
 				{
-					act = checkout_action_with_wd(data, delta, wd);
-					if ((error = git_iterator_advance(&wd, workdir)) < 0 &&
-						error != GIT_ITEROVER)
-						goto fail;
-					*wditem_ptr = wd;
-					return act;
+					error = checkout_action_with_wd(action, data, delta, workdir, wd);
+					advance = git_iterator_advance;
+					goto done;
 				}
 			}
 
-			return checkout_action_with_wd_dir(data, delta, wd);
+			return checkout_action_with_wd_dir(action, data, delta, workdir, wd);
 		}
 
 		/* case 6 - wd is after delta */
-		return checkout_action_no_wd(data, delta);
+		return checkout_action_no_wd(action, data, delta);
 	}
 
-fail:
-	*wditem_ptr = NULL;
-	return -1;
+done:
+	if (!error && advance != NULL &&
+		(error = advance(wditem, workdir)) < 0) {
+		*wditem = NULL;
+		if (error == GIT_ITEROVER)
+			error = 0;
+	}
+
+	return error;
 }
 
 static int checkout_remaining_wd_items(
@@ -595,10 +660,8 @@ static int checkout_remaining_wd_items(
 {
 	int error = 0;
 
-	while (wd && !error) {
-		if (!(error = checkout_action_wd_only(data, workdir, wd, spec)))
-			error = git_iterator_advance(&wd, workdir);
-	}
+	while (wd && !error)
+		error = checkout_action_wd_only(data, workdir, &wd, spec);
 
 	if (error == GIT_ITEROVER)
 		error = 0;
@@ -633,10 +696,13 @@ static int checkout_conflictdata_cmp(const void *a, const void *b)
 	return diff;
 }
 
-int checkout_conflictdata_empty(const git_vector *conflicts, size_t idx)
+int checkout_conflictdata_empty(
+	const git_vector *conflicts, size_t idx, void *payload)
 {
 	checkout_conflictdata *conflict;
 
+	GIT_UNUSED(payload);
+
 	if ((conflict = git_vector_get(conflicts, idx)) == NULL)
 		return -1;
 
@@ -674,6 +740,51 @@ GIT_INLINE(bool) conflict_pathspec_match(
 	return false;
 }
 
+GIT_INLINE(int) checkout_conflict_detect_submodule(checkout_conflictdata *conflict)
+{
+	conflict->submodule = ((conflict->ancestor && S_ISGITLINK(conflict->ancestor->mode)) ||
+		(conflict->ours && S_ISGITLINK(conflict->ours->mode)) ||
+		(conflict->theirs && S_ISGITLINK(conflict->theirs->mode)));
+	return 0;
+}
+
+GIT_INLINE(int) checkout_conflict_detect_binary(git_repository *repo, checkout_conflictdata *conflict)
+{
+	git_blob *ancestor_blob = NULL, *our_blob = NULL, *their_blob = NULL;
+	int error = 0;
+
+	if (conflict->submodule)
+		return 0;
+
+	if (conflict->ancestor) {
+		if ((error = git_blob_lookup(&ancestor_blob, repo, &conflict->ancestor->id)) < 0)
+			goto done;
+
+		conflict->binary = git_blob_is_binary(ancestor_blob);
+	}
+
+	if (!conflict->binary && conflict->ours) {
+		if ((error = git_blob_lookup(&our_blob, repo, &conflict->ours->id)) < 0)
+			goto done;
+
+		conflict->binary = git_blob_is_binary(our_blob);
+	}
+
+	if (!conflict->binary && conflict->theirs) {
+		if ((error = git_blob_lookup(&their_blob, repo, &conflict->theirs->id)) < 0)
+			goto done;
+
+		conflict->binary = git_blob_is_binary(their_blob);
+	}
+
+done:
+	git_blob_free(ancestor_blob);
+	git_blob_free(our_blob);
+	git_blob_free(their_blob);
+
+	return error;
+}
+
 static int checkout_conflicts_load(checkout_data *data, git_iterator *workdir, git_vector *pathspec)
 {
 	git_index_conflict_iterator *iterator = NULL;
@@ -698,6 +809,13 @@ static int checkout_conflicts_load(checkout_data *data, git_iterator *workdir, g
 		conflict->ours = ours;
 		conflict->theirs = theirs;
 
+		if ((error = checkout_conflict_detect_submodule(conflict)) < 0 ||
+		    (error = checkout_conflict_detect_binary(data->repo, conflict)) < 0)
+		{
+			git__free(conflict);
+			goto done;
+		}
+
 		git_vector_insert(&data->conflicts, conflict);
 	}
 
@@ -846,7 +964,7 @@ static int checkout_conflicts_coalesce_renames(
 
 	/* Juggle entries based on renames */
 	names = git_index_name_entrycount(data->index);
-	
+
 	for (i = 0; i < names; i++) {
 		name_entry = git_index_name_get_byindex(data->index, i);
 
@@ -882,7 +1000,8 @@ static int checkout_conflicts_coalesce_renames(
 			ancestor_conflict->one_to_two = 1;
 	}
 
-	git_vector_remove_matching(&data->conflicts, checkout_conflictdata_empty);
+	git_vector_remove_matching(
+		&data->conflicts, checkout_conflictdata_empty, NULL);
 
 done:
 	return error;
@@ -965,7 +1084,7 @@ static int checkout_get_actions(
 	checkout_data *data,
 	git_iterator *workdir)
 {
-	int error = 0;
+	int error = 0, act;
 	const git_index_entry *wditem;
 	git_vector pathspec = GIT_VECTOR_INIT, *deltas;
 	git_pool pathpool = GIT_POOL_INIT_STRINGPOOL;
@@ -992,12 +1111,9 @@ static int checkout_get_actions(
 	}
 
 	git_vector_foreach(deltas, i, delta) {
-		int act = checkout_action(data, delta, workdir, &wditem, &pathspec);
-
-		if (act < 0) {
-			error = act;
+		error = checkout_action(&act, data, delta, workdir, &wditem, &pathspec);
+		if (error != 0)
 			goto fail;
-		}
 
 		actions[i] = act;
 
@@ -1012,7 +1128,7 @@ static int checkout_get_actions(
 	}
 
 	error = checkout_remaining_wd_items(data, workdir, wditem, &pathspec);
-	if (error < 0)
+	if (error)
 		goto fail;
 
 	counts[CHECKOUT_ACTION__REMOVE] += data->removes.length;
@@ -1020,8 +1136,10 @@ static int checkout_get_actions(
 	if (counts[CHECKOUT_ACTION__CONFLICT] > 0 &&
 		(data->strategy & GIT_CHECKOUT_ALLOW_CONFLICTS) == 0)
 	{
-		giterr_set(GITERR_CHECKOUT, "%d conflicts prevent checkout",
-			(int)counts[CHECKOUT_ACTION__CONFLICT]);
+		giterr_set(GITERR_CHECKOUT, "%d %s checkout",
+			(int)counts[CHECKOUT_ACTION__CONFLICT],
+			counts[CHECKOUT_ACTION__CONFLICT] == 1 ?
+			"conflict prevents" : "conflicts prevent");
 		error = GIT_EMERGECONFLICT;
 		goto fail;
 	}
@@ -1082,7 +1200,7 @@ static int blob_content_to_file(
 	const char *path,
 	const char * hint_path,
 	mode_t entry_filemode,
-	git_checkout_opts *opts)
+	git_checkout_options *opts)
 {
 	int error = 0;
 	mode_t file_mode = opts->file_mode ? opts->file_mode : entry_filemode;
@@ -1094,7 +1212,8 @@ static int blob_content_to_file(
 
 	if (!opts->disable_filters)
 		error = git_filter_list_load(
-			&fl, git_blob_owner(blob), blob, hint_path, GIT_FILTER_TO_WORKTREE);
+			&fl, git_blob_owner(blob), blob, hint_path,
+			GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT);
 
 	if (!error)
 		error = git_filter_list_apply_to_blob(&out, fl, blob);
@@ -1160,9 +1279,8 @@ static int checkout_update_index(
 
 	memset(&entry, 0, sizeof(entry));
 	entry.path = (char *)file->path; /* cast to prevent warning */
-	git_index_entry__init_from_stat(
-		&entry, st, !(git_index_caps(data->index) & GIT_INDEXCAP_NO_FILEMODE));
-	git_oid_cpy(&entry.oid, &file->oid);
+	git_index_entry__init_from_stat(&entry, st, true);
+	git_oid_cpy(&entry.id, &file->id);
 
 	return git_index_add(data->index, &entry);
 }
@@ -1197,7 +1315,6 @@ static int checkout_submodule(
 	const git_diff_file *file)
 {
 	int error = 0;
-	git_submodule *sm;
 
 	/* Until submodules are supported, UPDATE_ONLY means do nothing here */
 	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0)
@@ -1208,7 +1325,7 @@ static int checkout_submodule(
 			data->opts.dir_mode, GIT_MKDIR_PATH)) < 0)
 		return error;
 
-	if ((error = git_submodule_lookup(&sm, data->repo, file->path)) < 0) {
+	if ((error = git_submodule_lookup(NULL, data->repo, file->path)) < 0) {
 		/* I've observed repos with submodules in the tree that do not
 		 * have a .gitmodules - core Git just makes an empty directory
 		 */
@@ -1319,7 +1436,7 @@ static int checkout_blob(
 	}
 
 	error = checkout_write_content(
-		data, &file->oid, git_buf_cstr(&data->path), NULL, file->mode, &st);
+		data, &file->id, git_buf_cstr(&data->path), NULL, file->mode, &st);
 
 	/* update the index unless prevented */
 	if (!error && (data->strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0)
@@ -1449,7 +1566,7 @@ static int checkout_create_submodules(
 
 	/* initial reload of submodules if .gitmodules was changed */
 	if (data->reload_submodules &&
-		(error = git_submodule_reload_all(data->repo)) < 0)
+		(error = git_submodule_reload_all(data->repo, 1)) < 0)
 		return error;
 
 	git_vector_foreach(&data->diff->deltas, i, delta) {
@@ -1473,7 +1590,7 @@ static int checkout_create_submodules(
 	}
 
 	/* final reload once submodules have been updated */
-	return git_submodule_reload_all(data->repo);
+	return git_submodule_reload_all(data->repo, 1);
 }
 
 static int checkout_lookup_head_tree(git_tree **out, git_repository *repo)
@@ -1572,7 +1689,7 @@ static int checkout_write_entry(
 		return error;
 
 	return checkout_write_content(data,
-		&side->oid, git_buf_cstr(&data->path), hint_path, side->mode, &st);
+		&side->id, git_buf_cstr(&data->path), hint_path, side->mode, &st);
 }
 
 static int checkout_write_entries(
@@ -1620,25 +1737,20 @@ static int checkout_write_merge(
 {
 	git_buf our_label = GIT_BUF_INIT, their_label = GIT_BUF_INIT,
 		path_suffixed = GIT_BUF_INIT, path_workdir = GIT_BUF_INIT;
-	git_merge_file_input ancestor = GIT_MERGE_FILE_INPUT_INIT,
-		ours = GIT_MERGE_FILE_INPUT_INIT,
-		theirs = GIT_MERGE_FILE_INPUT_INIT;
-	git_merge_file_result result = GIT_MERGE_FILE_RESULT_INIT;
+	git_merge_file_options opts = GIT_MERGE_FILE_OPTIONS_INIT;
+	git_merge_file_result result = {0};
 	git_filebuf output = GIT_FILEBUF_INIT;
 	int error = 0;
 
-	if ((conflict->ancestor &&
-		(error = git_merge_file_input_from_index_entry(
-		&ancestor, data->repo, conflict->ancestor)) < 0) ||
-		(error = git_merge_file_input_from_index_entry(
-		&ours, data->repo, conflict->ours)) < 0 ||
-		(error = git_merge_file_input_from_index_entry(
-		&theirs, data->repo, conflict->theirs)) < 0)
-		goto done;
+	if (data->opts.checkout_strategy & GIT_CHECKOUT_CONFLICT_STYLE_DIFF3)
+		opts.flags |= GIT_MERGE_FILE_STYLE_DIFF3;
 
-	ancestor.label = NULL;
-	ours.label = data->opts.our_label ? data->opts.our_label : "ours";
-	theirs.label = data->opts.their_label ? data->opts.their_label : "theirs";
+	opts.ancestor_label = data->opts.ancestor_label ?
+		data->opts.ancestor_label : "ancestor";
+	opts.our_label = data->opts.our_label ?
+		data->opts.our_label : "ours";
+	opts.their_label = data->opts.their_label ?
+		data->opts.their_label : "theirs";
 
 	/* If all the paths are identical, decorate the diff3 file with the branch
 	 * names.  Otherwise, append branch_name:path.
@@ -1647,16 +1759,17 @@ static int checkout_write_merge(
 		strcmp(conflict->ours->path, conflict->theirs->path) != 0) {
 
 		if ((error = conflict_entry_name(
-			&our_label, ours.label, conflict->ours->path)) < 0 ||
+			&our_label, opts.our_label, conflict->ours->path)) < 0 ||
 			(error = conflict_entry_name(
-			&their_label, theirs.label, conflict->theirs->path)) < 0)
+			&their_label, opts.their_label, conflict->theirs->path)) < 0)
 			goto done;
 
-		ours.label = git_buf_cstr(&our_label);
-		theirs.label = git_buf_cstr(&their_label);
+		opts.our_label = git_buf_cstr(&our_label);
+		opts.their_label = git_buf_cstr(&their_label);
 	}
 
-	if ((error = git_merge_files(&result, &ancestor, &ours, &theirs, 0)) < 0)
+	if ((error = git_merge_file_from_index(&result, data->repo,
+		conflict->ancestor, conflict->ours, conflict->theirs, &opts)) < 0)
 		goto done;
 
 	if (result.path == NULL || result.mode == 0) {
@@ -1674,7 +1787,7 @@ static int checkout_write_merge(
 
 	if ((error = git_futils_mkpath2file(path_workdir.ptr, 0755)) < 0 ||
 		(error = git_filebuf_open(&output, path_workdir.ptr, GIT_FILEBUF_DO_NOT_BUFFER, result.mode)) < 0 ||
-		(error = git_filebuf_write(&output, result.data, result.len)) < 0 ||
+		(error = git_filebuf_write(&output, result.ptr, result.len)) < 0 ||
 		(error = git_filebuf_commit(&output)) < 0)
 		goto done;
 
@@ -1682,9 +1795,6 @@ done:
 	git_buf_free(&our_label);
 	git_buf_free(&their_label);
 
-	git_merge_file_input_free(&ancestor);
-	git_merge_file_input_free(&ours);
-	git_merge_file_input_free(&theirs);
 	git_merge_file_result_free(&result);
 	git_buf_free(&path_workdir);
 	git_buf_free(&path_suffixed);
@@ -1699,6 +1809,7 @@ static int checkout_create_conflicts(checkout_data *data)
 	int error = 0;
 
 	git_vector_foreach(&data->conflicts, i, conflict) {
+
 		/* Both deleted: nothing to do */
 		if (conflict->ours == NULL && conflict->theirs == NULL)
 			error = 0;
@@ -1742,7 +1853,15 @@ static int checkout_create_conflicts(checkout_data *data)
 		else if (S_ISLNK(conflict->theirs->mode))
 			error = checkout_write_entry(data, conflict, conflict->ours);
 
-		else
+		/* If any side is a gitlink, do nothing. */
+		else if (conflict->submodule)
+			error = 0;
+
+		/* If any side is binary, write the ours side */
+		else if (conflict->binary)
+			error = checkout_write_entry(data, conflict, conflict->ours);
+
+		else if (!error)
 			error = checkout_write_merge(data, conflict);
 
 		if (error)
@@ -1760,9 +1879,6 @@ static int checkout_create_conflicts(checkout_data *data)
 
 static void checkout_data_clear(checkout_data *data)
 {
-	checkout_conflictdata *conflict;
-	size_t i;
-
 	if (data->opts_free_baseline) {
 		git_tree_free(data->opts.baseline);
 		data->opts.baseline = NULL;
@@ -1771,15 +1887,13 @@ static void checkout_data_clear(checkout_data *data)
 	git_vector_free(&data->removes);
 	git_pool_clear(&data->pool);
 
-	git_vector_foreach(&data->conflicts, i, conflict)
-		git__free(conflict);
-
-	git_vector_free(&data->conflicts);
+	git_vector_free_deep(&data->conflicts);
 
 	git__free(data->pfx);
 	data->pfx = NULL;
 
 	git_buf_free(&data->path);
+	git_buf_free(&data->tmp);
 
 	git_index_free(data->index);
 	data->index = NULL;
@@ -1788,7 +1902,7 @@ static void checkout_data_clear(checkout_data *data)
 static int checkout_data_init(
 	checkout_data *data,
 	git_iterator *target,
-	const git_checkout_opts *proposed)
+	const git_checkout_options *proposed)
 {
 	int error = 0;
 	git_repository *repo = git_iterator_owner(target);
@@ -1807,12 +1921,12 @@ static int checkout_data_init(
 	data->repo = repo;
 
 	GITERR_CHECK_VERSION(
-		proposed, GIT_CHECKOUT_OPTS_VERSION, "git_checkout_opts");
+		proposed, GIT_CHECKOUT_OPTIONS_VERSION, "git_checkout_options");
 
 	if (!proposed)
-		GIT_INIT_STRUCTURE(&data->opts, GIT_CHECKOUT_OPTS_VERSION);
+		GIT_INIT_STRUCTURE(&data->opts, GIT_CHECKOUT_OPTIONS_VERSION);
 	else
-		memmove(&data->opts, proposed, sizeof(git_checkout_opts));
+		memmove(&data->opts, proposed, sizeof(git_checkout_options));
 
 	if (!data->opts.target_directory)
 		data->opts.target_directory = git_repository_workdir(repo);
@@ -1891,6 +2005,29 @@ static int checkout_data_init(
 			goto cleanup;
 	}
 
+	if ((data->opts.checkout_strategy &
+		(GIT_CHECKOUT_CONFLICT_STYLE_MERGE | GIT_CHECKOUT_CONFLICT_STYLE_DIFF3)) == 0) {
+		const char *conflict_style;
+		git_config *cfg = NULL;
+
+		if ((error = git_repository_config__weakptr(&cfg, repo)) < 0 ||
+			(error = git_config_get_string(&conflict_style, cfg, "merge.conflictstyle")) < 0 ||
+			error == GIT_ENOTFOUND)
+			;
+		else if (error)
+			goto cleanup;
+		else if (strcmp(conflict_style, "merge") == 0)
+			data->opts.checkout_strategy |= GIT_CHECKOUT_CONFLICT_STYLE_MERGE;
+		else if (strcmp(conflict_style, "diff3") == 0)
+			data->opts.checkout_strategy |= GIT_CHECKOUT_CONFLICT_STYLE_DIFF3;
+		else {
+			giterr_set(GITERR_CHECKOUT, "unknown style '%s' given for 'merge.conflictstyle'",
+				conflict_style);
+			error = -1;
+			goto cleanup;
+		}
+	}
+
 	if ((error = git_vector_init(&data->removes, 0, git__strcmp_cb)) < 0 ||
 		(error = git_vector_init(&data->conflicts, 0, NULL)) < 0 ||
 		(error = git_pool_init(&data->pool, 1, 0)) < 0 ||
@@ -1909,7 +2046,7 @@ cleanup:
 
 int git_checkout_iterator(
 	git_iterator *target,
-	const git_checkout_opts *opts)
+	const git_checkout_options *opts)
 {
 	int error = 0;
 	git_iterator *baseline = NULL, *workdir = NULL;
@@ -1966,7 +2103,7 @@ int git_checkout_iterator(
 	 * actions to be taken, plus look for conflicts and send notifications,
 	 * then loop through conflicts.
 	 */
-	if ((error = checkout_get_actions(&actions, &counts, &data, workdir)) < 0)
+	if ((error = checkout_get_actions(&actions, &counts, &data, workdir)) != 0)
 		goto cleanup;
 
 	data.total_steps = counts[CHECKOUT_ACTION__REMOVE] +
@@ -1998,9 +2135,6 @@ int git_checkout_iterator(
 	assert(data.completed_steps == data.total_steps);
 
 cleanup:
-	if (error == GIT_EUSER)
-		giterr_clear();
-
 	if (!error && data.index != NULL &&
 		(data.strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0)
 		error = git_index_write(data.index);
@@ -2018,7 +2152,7 @@ cleanup:
 int git_checkout_index(
 	git_repository *repo,
 	git_index *index,
-	const git_checkout_opts *opts)
+	const git_checkout_options *opts)
 {
 	int error;
 	git_iterator *index_i;
@@ -2053,7 +2187,7 @@ int git_checkout_index(
 int git_checkout_tree(
 	git_repository *repo,
 	const git_object *treeish,
-	const git_checkout_opts *opts)
+	const git_checkout_options *opts)
 {
 	int error;
 	git_tree *tree = NULL;
@@ -2101,8 +2235,15 @@ int git_checkout_tree(
 
 int git_checkout_head(
 	git_repository *repo,
-	const git_checkout_opts *opts)
+	const git_checkout_options *opts)
 {
 	assert(repo);
 	return git_checkout_tree(repo, NULL, opts);
 }
+
+int git_checkout_init_options(git_checkout_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_checkout_options, GIT_CHECKOUT_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/checkout.h b/src/checkout.h
index 6d7186860..f1fe69628 100644
--- a/src/checkout.h
+++ b/src/checkout.h
@@ -19,6 +19,6 @@
  */
 extern int git_checkout_iterator(
 	git_iterator *target,
-	const git_checkout_opts *opts);
+	const git_checkout_options *opts);
 
 #endif
diff --git a/src/cherrypick.c b/src/cherrypick.c
new file mode 100644
index 000000000..e02348a03
--- /dev/null
+++ b/src/cherrypick.c
@@ -0,0 +1,226 @@
+/*
+* Copyright (C) the libgit2 contributors. All rights reserved.
+*
+* This file is part of libgit2, distributed under the GNU GPL v2 with
+* a Linking Exception. For full terms see the included COPYING file.
+*/
+
+#include "common.h"
+#include "repository.h"
+#include "filebuf.h"
+#include "merge.h"
+#include "vector.h"
+
+#include "git2/types.h"
+#include "git2/merge.h"
+#include "git2/cherrypick.h"
+#include "git2/commit.h"
+#include "git2/sys/commit.h"
+
+#define GIT_CHERRY_PICK_FILE_MODE		0666
+
+static int write_cherry_pick_head(
+	git_repository *repo,
+	const char *commit_oidstr)
+{
+	git_filebuf file = GIT_FILEBUF_INIT;
+	git_buf file_path = GIT_BUF_INIT;
+	int error = 0;
+
+	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_CHERRY_PICK_HEAD_FILE)) >= 0 &&
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_CHERRY_PICK_FILE_MODE)) >= 0 &&
+		(error = git_filebuf_printf(&file, "%s\n", commit_oidstr)) >= 0)
+		error = git_filebuf_commit(&file);
+
+	if (error < 0)
+		git_filebuf_cleanup(&file);
+
+	git_buf_free(&file_path);
+
+	return error;
+}
+
+static int write_merge_msg(
+	git_repository *repo,
+	const char *commit_msg)
+{
+	git_filebuf file = GIT_FILEBUF_INIT;
+	git_buf file_path = GIT_BUF_INIT;
+	int error = 0;
+
+	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_MSG_FILE)) < 0 ||
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_CHERRY_PICK_FILE_MODE)) < 0 ||
+		(error = git_filebuf_printf(&file, "%s", commit_msg)) < 0)
+		goto cleanup;
+
+	error = git_filebuf_commit(&file);
+
+cleanup:
+	if (error < 0)
+		git_filebuf_cleanup(&file);
+
+	git_buf_free(&file_path);
+
+	return error;
+}
+
+static int cherry_pick_normalize_opts(
+	git_repository *repo,
+	git_cherry_pick_options *opts,
+	const git_cherry_pick_options *given,
+	const char *their_label)
+{
+	int error = 0;
+	unsigned int default_checkout_strategy = GIT_CHECKOUT_SAFE_CREATE |
+		GIT_CHECKOUT_ALLOW_CONFLICTS;
+
+	GIT_UNUSED(repo);
+
+	if (given != NULL)
+		memcpy(opts, given, sizeof(git_cherry_pick_options));
+	else {
+		git_cherry_pick_options default_opts = GIT_CHERRY_PICK_OPTIONS_INIT;
+		memcpy(opts, &default_opts, sizeof(git_cherry_pick_options));
+	}
+
+	if (!opts->checkout_opts.checkout_strategy)
+		opts->checkout_opts.checkout_strategy = default_checkout_strategy;
+
+	if (!opts->checkout_opts.our_label)
+		opts->checkout_opts.our_label = "HEAD";
+
+	if (!opts->checkout_opts.their_label)
+		opts->checkout_opts.their_label = their_label;
+
+	return error;
+}
+
+static int cherry_pick_state_cleanup(git_repository *repo)
+{
+	const char *state_files[] = { GIT_CHERRY_PICK_HEAD_FILE, GIT_MERGE_MSG_FILE };
+
+	return git_repository__cleanup_files(repo, state_files, ARRAY_SIZE(state_files));
+}
+
+static int cherry_pick_seterr(git_commit *commit, const char *fmt)
+{
+	char commit_oidstr[GIT_OID_HEXSZ + 1];
+
+	giterr_set(GITERR_CHERRYPICK, fmt,
+		git_oid_tostr(commit_oidstr, GIT_OID_HEXSZ + 1, git_commit_id(commit)));
+
+	return -1;
+}
+
+int git_cherry_pick_commit(
+	git_index **out,
+	git_repository *repo,
+	git_commit *cherry_pick_commit,
+	git_commit *our_commit,
+	unsigned int mainline,
+	const git_merge_options *merge_opts)
+{
+	git_commit *parent_commit = NULL;
+	git_tree *parent_tree = NULL, *our_tree = NULL, *cherry_pick_tree = NULL;
+	int parent = 0, error = 0;
+
+	assert(out && repo && cherry_pick_commit && our_commit);
+
+	if (git_commit_parentcount(cherry_pick_commit) > 1) {
+		if (!mainline)
+			return cherry_pick_seterr(cherry_pick_commit,
+				"Mainline branch is not specified but %s is a merge commit");
+
+		parent = mainline;
+	} else {
+		if (mainline)
+			return cherry_pick_seterr(cherry_pick_commit,
+				"Mainline branch specified but %s is not a merge commit");
+
+		parent = git_commit_parentcount(cherry_pick_commit);
+	}
+
+	if (parent &&
+		((error = git_commit_parent(&parent_commit, cherry_pick_commit, (parent - 1))) < 0 ||
+		(error = git_commit_tree(&parent_tree, parent_commit)) < 0))
+		goto done;
+
+	if ((error = git_commit_tree(&cherry_pick_tree, cherry_pick_commit)) < 0 ||
+		(error = git_commit_tree(&our_tree, our_commit)) < 0)
+		goto done;
+
+	error = git_merge_trees(out, repo, parent_tree, our_tree, cherry_pick_tree, merge_opts);
+
+done:
+	git_tree_free(parent_tree);
+	git_tree_free(our_tree);
+	git_tree_free(cherry_pick_tree);
+	git_commit_free(parent_commit);
+
+	return error;
+}
+
+int git_cherry_pick(
+	git_repository *repo,
+	git_commit *commit,
+	const git_cherry_pick_options *given_opts)
+{
+	git_cherry_pick_options opts;
+	git_reference *our_ref = NULL;
+	git_commit *our_commit = NULL;
+	char commit_oidstr[GIT_OID_HEXSZ + 1];
+	const char *commit_msg, *commit_summary;
+	git_buf their_label = GIT_BUF_INIT;
+	git_index *index_new = NULL, *index_repo = NULL;
+	int error = 0;
+
+	assert(repo && commit);
+
+	GITERR_CHECK_VERSION(given_opts, GIT_CHERRY_PICK_OPTIONS_VERSION, "git_cherry_pick_options");
+
+	if ((error = git_repository__ensure_not_bare(repo, "cherry-pick")) < 0)
+		return error;
+
+	if ((commit_msg = git_commit_message(commit)) == NULL ||
+		(commit_summary = git_commit_summary(commit)) == NULL) {
+		error = -1;
+		goto on_error;
+	}
+
+	git_oid_nfmt(commit_oidstr, sizeof(commit_oidstr), git_commit_id(commit));
+
+	if ((error = write_merge_msg(repo, commit_msg)) < 0 ||
+		(error = git_buf_printf(&their_label, "%.7s... %s", commit_oidstr, commit_summary)) < 0 ||
+		(error = cherry_pick_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0 ||
+		(error = write_cherry_pick_head(repo, commit_oidstr)) < 0 ||
+		(error = git_repository_head(&our_ref, repo)) < 0 ||
+		(error = git_reference_peel((git_object **)&our_commit, our_ref, GIT_OBJ_COMMIT)) < 0 ||
+		(error = git_cherry_pick_commit(&index_new, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 ||
+		(error = git_merge__indexes(repo, index_new)) < 0 ||
+		(error = git_repository_index(&index_repo, repo)) < 0 ||
+		(error = git_merge__append_conflicts_to_merge_msg(repo, index_repo)) < 0 ||
+		(error = git_checkout_index(repo, index_repo, &opts.checkout_opts)) < 0)
+		goto on_error;
+
+	goto done;
+
+on_error:
+	cherry_pick_state_cleanup(repo);
+
+done:
+	git_index_free(index_new);
+	git_index_free(index_repo);
+	git_commit_free(our_commit);
+	git_reference_free(our_ref);
+	git_buf_free(&their_label);
+
+	return error;
+}
+
+int git_cherry_pick_init_options(
+	git_cherry_pick_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_cherry_pick_options, GIT_CHERRY_PICK_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/clone.c b/src/clone.c
index 23aacd478..6c4fb6727 100644
--- a/src/clone.c
+++ b/src/clone.c
@@ -22,12 +22,15 @@
 #include "refs.h"
 #include "path.h"
 #include "repository.h"
+#include "odb.h"
 
 static int create_branch(
 	git_reference **branch,
 	git_repository *repo,
 	const git_oid *target,
-	const char *name)
+	const char *name,
+	const git_signature *signature,
+	const char *log_message)
 {
 	git_commit *head_obj = NULL;
 	git_reference *branch_ref = NULL;
@@ -38,7 +41,7 @@ static int create_branch(
 		return error;
 
 	/* Create the new branch */
-	error = git_branch_create(&branch_ref, repo, name, head_obj, 0);
+	error = git_branch_create(&branch_ref, repo, name, head_obj, 0, signature, log_message);
 
 	git_commit_free(head_obj);
 
@@ -87,11 +90,13 @@ static int create_tracking_branch(
 	git_reference **branch,
 	git_repository *repo,
 	const git_oid *target,
-	const char *branch_name)
+	const char *branch_name,
+	const git_signature *signature,
+	const char *log_message)
 {
 	int error;
 
-	if ((error = create_branch(branch, repo, target, branch_name)) < 0)
+	if ((error = create_branch(branch, repo, target, branch_name, signature, log_message)) < 0)
 		return error;
 
 	return setup_tracking_config(
@@ -101,168 +106,106 @@ static int create_tracking_branch(
 		git_reference_name(*branch));
 }
 
-struct head_info {
-	git_repository *repo;
-	git_oid remote_head_oid;
-	git_buf branchname;
-	const git_refspec *refspec;
-	bool found;
-};
-
-static int reference_matches_remote_head(
-	const char *reference_name,
-	void *payload)
-{
-	struct head_info *head_info = (struct head_info *)payload;
-	git_oid oid;
-
-	/* TODO: Should we guard against references
-	 * which name doesn't start with refs/heads/ ?
-	 */
-
-	/* Stop looking if we've already found a match */
-	if (head_info->found)
-		return 0;
-
-	if (git_reference_name_to_id(
-		&oid,
-		head_info->repo,
-		reference_name) < 0) {
-			/* If the reference doesn't exists, it obviously cannot match the expected oid. */
-			giterr_clear();
-			return 0;
-	}
-
-	if (git_oid__cmp(&head_info->remote_head_oid, &oid) == 0) {
-		/* Determine the local reference name from the remote tracking one */
-		if (git_refspec_transform_l(
-			&head_info->branchname,
-			head_info->refspec,
-			reference_name) < 0)
-				return -1;
-
-		if (git_buf_len(&head_info->branchname) > 0) {
-			if (git_buf_sets(
-				&head_info->branchname,
-				git_buf_cstr(&head_info->branchname) + strlen(GIT_REFS_HEADS_DIR)) < 0)
-					return -1;
-
-			head_info->found = 1;
-		}
-	}
-
-	return 0;
-}
-
 static int update_head_to_new_branch(
 	git_repository *repo,
 	const git_oid *target,
-	const char *name)
+	const char *name,
+	const git_signature *signature,
+	const char *reflog_message)
 {
 	git_reference *tracking_branch = NULL;
 	int error;
 
-	if ((error = create_tracking_branch(
-		&tracking_branch,
-		repo,
-		target,
-		name)) < 0)
-			return error;
+	if (!git__prefixcmp(name, GIT_REFS_HEADS_DIR))
+		name += strlen(GIT_REFS_HEADS_DIR);
 
-	error = git_repository_set_head(repo, git_reference_name(tracking_branch));
+	error = create_tracking_branch(&tracking_branch, repo, target, name,
+			signature, reflog_message);
+
+	if (!error)
+		error = git_repository_set_head(
+			repo, git_reference_name(tracking_branch),
+			signature, reflog_message);
 
 	git_reference_free(tracking_branch);
 
+	/* if it already existed, then the user's refspec created it for us, ignore it' */
+	if (error == GIT_EEXISTS)
+		error = 0;
+
 	return error;
 }
 
-static int update_head_to_remote(git_repository *repo, git_remote *remote)
+static int update_head_to_remote(
+		git_repository *repo,
+		git_remote *remote,
+		const git_signature *signature,
+		const char *reflog_message)
 {
-	int retcode = -1;
+	int error = 0, found_branch = 0;
 	size_t refs_len;
-	git_refspec dummy_spec;
+	git_refspec dummy_spec, *refspec;
 	const git_remote_head *remote_head, **refs;
-	struct head_info head_info;
+	const git_oid *remote_head_id;
 	git_buf remote_master_name = GIT_BUF_INIT;
+	git_buf branch = GIT_BUF_INIT;
 
-	if (git_remote_ls(&refs, &refs_len, remote) < 0)
-		return -1;
+	if ((error = git_remote_ls(&refs, &refs_len, remote)) < 0)
+		return error;
 
 	/* Did we just clone an empty repository? */
-	if (refs_len == 0) {
+	if (refs_len == 0)
 		return setup_tracking_config(
-			repo,
-			"master",
-			GIT_REMOTE_ORIGIN,
-			GIT_REFS_HEADS_MASTER_FILE);
+			repo, "master", GIT_REMOTE_ORIGIN, GIT_REFS_HEADS_MASTER_FILE);
+
+	error = git_remote_default_branch(&branch, remote);
+	if (error == GIT_ENOTFOUND) {
+		git_buf_puts(&branch, GIT_REFS_HEADS_MASTER_FILE);
+	} else {
+		found_branch = 1;
 	}
 
 	/* Get the remote's HEAD. This is always the first ref in the list. */
 	remote_head = refs[0];
 	assert(remote_head);
 
-	git_oid_cpy(&head_info.remote_head_oid, &remote_head->oid);
-	git_buf_init(&head_info.branchname, 16);
-	head_info.repo = repo;
-	head_info.refspec = git_remote__matching_refspec(remote, GIT_REFS_HEADS_MASTER_FILE);
-	head_info.found = 0;
+	remote_head_id = &remote_head->oid;
+	refspec = git_remote__matching_refspec(remote, git_buf_cstr(&branch));
 
-	if (head_info.refspec == NULL) {
+	if (refspec == NULL) {
 		memset(&dummy_spec, 0, sizeof(git_refspec));
-		head_info.refspec = &dummy_spec;
+		refspec = &dummy_spec;
 	}
 
 	/* Determine the remote tracking reference name from the local master */
-	if (git_refspec_transform_r(
+	if ((error = git_refspec_transform(
 		&remote_master_name,
-		head_info.refspec,
-		GIT_REFS_HEADS_MASTER_FILE) < 0)
-			return -1;
-
-	/* Check to see if the remote HEAD points to the remote master */
-	if (reference_matches_remote_head(git_buf_cstr(&remote_master_name), &head_info) < 0)
-		goto cleanup;
-
-	if (head_info.found) {
-		retcode = update_head_to_new_branch(
-			repo,
-			&head_info.remote_head_oid,
-			git_buf_cstr(&head_info.branchname));
-
-		goto cleanup;
-	}
-
-	/* Not master. Check all the other refs. */
-	if (git_reference_foreach_name(
-		repo,
-		reference_matches_remote_head,
-		&head_info) < 0)
-			goto cleanup;
+		refspec,
+		git_buf_cstr(&branch))) < 0)
+		return error;
 
-	if (head_info.found) {
-		retcode = update_head_to_new_branch(
+	if (found_branch) {
+		error = update_head_to_new_branch(
 			repo,
-			&head_info.remote_head_oid,
-			git_buf_cstr(&head_info.branchname));
-
-		goto cleanup;
+			remote_head_id,
+			git_buf_cstr(&branch),
+			signature, reflog_message);
 	} else {
-		retcode = git_repository_set_head_detached(
-			repo,
-			&head_info.remote_head_oid);
-		goto cleanup;
+		error = git_repository_set_head_detached(
+			repo, remote_head_id, signature, reflog_message);
 	}
 
-cleanup:
 	git_buf_free(&remote_master_name);
-	git_buf_free(&head_info.branchname);
-	return retcode;
+	git_buf_free(&branch);
+	return error;
 }
 
 static int update_head_to_branch(
 		git_repository *repo,
 		const char *remote_name,
-		const char *branch)
+		const char *branch,
+		const git_signature *signature,
+		const char *reflog_message)
 {
 	int retcode;
 	git_buf remote_branch_name = GIT_BUF_INIT;
@@ -277,7 +220,8 @@ static int update_head_to_branch(
 	if ((retcode = git_reference_lookup(&remote_ref, repo, git_buf_cstr(&remote_branch_name))) < 0)
 		goto cleanup;
 
-	retcode = update_head_to_new_branch(repo, git_reference_target(remote_ref), branch);
+	retcode = update_head_to_new_branch(repo, git_reference_target(remote_ref), branch,
+			signature, reflog_message);
 
 cleanup:
 	git_reference_free(remote_ref);
@@ -298,6 +242,15 @@ static int create_and_configure_origin(
 	int error;
 	git_remote *origin = NULL;
 	const char *name;
+	char buf[GIT_PATH_MAX];
+
+	/* If the path exists and is a dir, the url should be the absolute path */
+	if (git_path_root(url) < 0 && git_path_exists(url) && git_path_isdir(url)) {
+		if (p_realpath(url, buf) == NULL)
+			return -1;
+
+		url = buf;
+	}
 
 	name = options->remote_name ? options->remote_name : "origin";
 	if ((error = git_remote_create(&origin, repo, name, url)) < 0)
@@ -323,7 +276,7 @@ on_error:
 static bool should_checkout(
 	git_repository *repo,
 	bool is_bare,
-	const git_checkout_opts *opts)
+	const git_checkout_options *opts)
 {
 	if (is_bare)
 		return false;
@@ -337,53 +290,86 @@ static bool should_checkout(
 	return !git_repository_head_unborn(repo);
 }
 
-int git_clone_into(git_repository *repo, git_remote *remote, const git_checkout_opts *co_opts, const char *branch)
+static int checkout_branch(git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch, const git_signature *signature, const char *reflog_message)
 {
-	int error = 0, old_fetchhead;
-	git_strarray refspecs;
+	int error;
 
-	assert(repo && remote);
+	if (branch)
+		error = update_head_to_branch(repo, git_remote_name(remote), branch,
+				signature, reflog_message);
+	/* Point HEAD to the same ref as the remote's head */
+	else
+		error = update_head_to_remote(repo, remote, signature, reflog_message);
+
+	if (!error && should_checkout(repo, git_repository_is_bare(repo), co_opts))
+		error = git_checkout_head(repo, co_opts);
+
+	return error;
+}
+
+int git_clone_into(git_repository *repo, git_remote *_remote, const git_checkout_options *co_opts, const char *branch, const git_signature *signature)
+{
+	int error;
+	git_buf reflog_message = GIT_BUF_INIT;
+	git_remote *remote;
+	const git_remote_callbacks *callbacks;
+
+	assert(repo && _remote);
 
 	if (!git_repository_is_empty(repo)) {
 		giterr_set(GITERR_INVALID, "the repository is not empty");
 		return -1;
 	}
 
-
-	if ((error = git_remote_get_fetch_refspecs(&refspecs, remote)) < 0)
+	if ((error = git_remote_dup(&remote, _remote)) < 0)
 		return error;
 
+	callbacks = git_remote_get_callbacks(_remote);
+	if (!giterr__check_version(callbacks, 1, "git_remote_callbacks") &&
+	    (error = git_remote_set_callbacks(remote, callbacks)) < 0)
+		goto cleanup;
+
 	if ((error = git_remote_add_fetch(remote, "refs/tags/*:refs/tags/*")) < 0)
-		return error;
+		goto cleanup;
 
-	old_fetchhead = git_remote_update_fetchhead(remote);
 	git_remote_set_update_fetchhead(remote, 0);
+	git_buf_printf(&reflog_message, "clone: from %s", git_remote_url(remote));
 
-	if ((error = git_remote_fetch(remote)) < 0)
+	if ((error = git_remote_fetch(remote, signature, git_buf_cstr(&reflog_message))) != 0)
 		goto cleanup;
 
-	if (branch)
-		error = update_head_to_branch(repo, git_remote_name(remote), branch);
-	/* Point HEAD to the same ref as the remote's head */
-	else
-		error = update_head_to_remote(repo, remote);
-
-	if (!error && should_checkout(repo, git_repository_is_bare(repo), co_opts))
-		error = git_checkout_head(repo, co_opts);
+	error = checkout_branch(repo, remote, co_opts, branch, signature, git_buf_cstr(&reflog_message));
 
 cleanup:
-	git_remote_set_update_fetchhead(remote, old_fetchhead);
-	/* Go back to the original refspecs */
-	if (git_remote_set_fetch_refspecs(remote, &refspecs) < 0) {
-		git_strarray_free(&refspecs);
-		return -1;
-	}
-
-	git_strarray_free(&refspecs);
+	git_remote_free(remote);
+	git_buf_free(&reflog_message);
 
 	return error;
 }
 
+int git_clone__should_clone_local(const char *url, git_clone_local_t local)
+{
+	const char *path;
+	int is_url;
+
+	if (local == GIT_CLONE_NO_LOCAL)
+		return false;
+
+	is_url = !git__prefixcmp(url, "file://");
+
+	if (is_url && local != GIT_CLONE_LOCAL && local != GIT_CLONE_LOCAL_NO_LINKS )
+		return false;
+
+	path = url;
+	if (is_url)
+		path = url + strlen("file://");
+
+	if ((git_path_exists(path) && git_path_isdir(path)) && local != GIT_CLONE_NO_LOCAL)
+		return true;
+
+	return false;
+}
+
 int git_clone(
 	git_repository **out,
 	const char *url,
@@ -418,18 +404,127 @@ int git_clone(
 		return error;
 
 	if (!(error = create_and_configure_origin(&origin, repo, url, &options))) {
-		error = git_clone_into(
-			repo, origin, &options.checkout_opts, options.checkout_branch);
+		if (git_clone__should_clone_local(url, options.local)) {
+			int link = options.local != GIT_CLONE_LOCAL_NO_LINKS;
+			error = git_clone_local_into(
+				repo, origin, &options.checkout_opts,
+				options.checkout_branch, link, options.signature);
+		} else {
+			error = git_clone_into(
+				repo, origin, &options.checkout_opts,
+				options.checkout_branch, options.signature);
+		}
 
 		git_remote_free(origin);
 	}
 
-	if (error < 0) {
+	if (error != 0) {
+		git_error_state last_error = {0};
+		giterr_capture(&last_error, error);
+
 		git_repository_free(repo);
 		repo = NULL;
+
 		(void)git_futils_rmdir_r(local_path, NULL, rmdir_flags);
+
+		giterr_restore(&last_error);
 	}
 
 	*out = repo;
 	return error;
 }
+
+int git_clone_init_options(git_clone_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_clone_options, GIT_CLONE_OPTIONS_INIT);
+	return 0;
+}
+
+static const char *repository_base(git_repository *repo)
+{
+	if (git_repository_is_bare(repo))
+		return git_repository_path(repo);
+
+	return git_repository_workdir(repo);
+}
+
+static bool can_link(const char *src, const char *dst, int link)
+{
+#ifdef GIT_WIN32
+	return false;
+#else
+
+	struct stat st_src, st_dst;
+
+	if (!link)
+		return false;
+
+	if (p_stat(src, &st_src) < 0)
+		return false;
+
+	if (p_stat(dst, &st_dst) < 0)
+		return false;
+
+	return st_src.st_dev == st_dst.st_dev;
+#endif
+}
+
+int git_clone_local_into(git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch, int link, const git_signature *signature)
+{
+	int error, flags;
+	git_repository *src;
+	git_buf src_odb = GIT_BUF_INIT, dst_odb = GIT_BUF_INIT, src_path = GIT_BUF_INIT;
+	git_buf reflog_message = GIT_BUF_INIT;
+
+	assert(repo && remote);
+
+	if (!git_repository_is_empty(repo)) {
+		giterr_set(GITERR_INVALID, "the repository is not empty");
+		return -1;
+	}
+
+	/*
+	 * Let's figure out what path we should use for the source
+	 * repo, if it's not rooted, the path should be relative to
+	 * the repository's worktree/gitdir.
+	 */
+	if ((error = git_path_from_url_or_path(&src_path, git_remote_url(remote))) < 0)
+		return error;
+
+	/* Copy .git/objects/ from the source to the target */
+	if ((error = git_repository_open(&src, git_buf_cstr(&src_path))) < 0) {
+		git_buf_free(&src_path);
+		return error;
+	}
+
+	git_buf_joinpath(&src_odb, git_repository_path(src), GIT_OBJECTS_DIR);
+	git_buf_joinpath(&dst_odb, git_repository_path(repo), GIT_OBJECTS_DIR);
+	if (git_buf_oom(&src_odb) || git_buf_oom(&dst_odb)) {
+		error = -1;
+		goto cleanup;
+	}
+
+	flags = 0;
+	if (can_link(git_repository_path(src), git_repository_path(repo), link))
+		flags |= GIT_CPDIR_LINK_FILES;
+
+	if ((error = git_futils_cp_r(git_buf_cstr(&src_odb), git_buf_cstr(&dst_odb),
+				     flags, GIT_OBJECT_DIR_MODE)) < 0)
+		goto cleanup;
+
+	git_buf_printf(&reflog_message, "clone: from %s", git_remote_url(remote));
+
+	if ((error = git_remote_fetch(remote, signature, git_buf_cstr(&reflog_message))) != 0)
+		goto cleanup;
+
+	error = checkout_branch(repo, remote, co_opts, branch, signature, git_buf_cstr(&reflog_message));
+
+cleanup:
+	git_buf_free(&reflog_message);
+	git_buf_free(&src_path);
+	git_buf_free(&src_odb);
+	git_buf_free(&dst_odb);
+	git_repository_free(src);
+	return error;
+}
diff --git a/src/compress.h b/src/clone.h
index 49e6f4749..14ca5d44c 100644
--- a/src/compress.h
+++ b/src/clone.h
@@ -4,13 +4,9 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
-#ifndef INCLUDE_compress_h__
-#define INCLUDE_compress_h__
+#ifndef INCLUDE_clone_h__
+#define INCLUDE_clone_h__
 
-#include "common.h"
+extern int git_clone__should_clone_local(const char *url, git_clone_local_t local);
 
-#include "buffer.h"
-
-int git__compress(git_buf *buf, const void *buff, size_t len);
-
-#endif /* INCLUDE_compress_h__ */
+#endif
diff --git a/src/commit.c b/src/commit.c
index 91b60bbb2..227d5c4a5 100644
--- a/src/commit.c
+++ b/src/commit.c
@@ -17,8 +17,6 @@
 #include "signature.h"
 #include "message.h"
 
-#include <stdarg.h>
-
 void git_commit__free(void *_commit)
 {
 	git_commit *commit = _commit;
@@ -31,45 +29,42 @@ void git_commit__free(void *_commit)
 	git__free(commit->raw_header);
 	git__free(commit->raw_message);
 	git__free(commit->message_encoding);
+	git__free(commit->summary);
 
 	git__free(commit);
 }
 
-int git_commit_create_v(
-	git_oid *oid,
-	git_repository *repo,
-	const char *update_ref,
-	const git_signature *author,
-	const git_signature *committer,
-	const char *message_encoding,
-	const char *message,
-	const git_tree *tree,
-	int parent_count,
-	...)
+static int update_ref_for_commit(git_repository *repo, git_reference *ref, const char *update_ref, const git_oid *id, const git_signature *committer)
 {
-	va_list ap;
-	int i, res;
-	const git_commit **parents;
-
-	parents = git__malloc(parent_count * sizeof(git_commit *));
-	GITERR_CHECK_ALLOC(parents);
+	git_reference *ref2 = NULL;
+	int error;
+	git_commit *c;
+	const char *shortmsg;
+	git_buf reflog_msg = GIT_BUF_INIT;
 
-	va_start(ap, parent_count);
-	for (i = 0; i < parent_count; ++i)
-		parents[i] = va_arg(ap, const git_commit *);
-	va_end(ap);
+	if ((error = git_commit_lookup(&c, repo, id)) < 0) {
+		return error;
+	}
 
-	res = git_commit_create(
-		oid, repo, update_ref, author, committer,
-		message_encoding, message,
-		tree, parent_count, parents);
+	shortmsg = git_commit_summary(c);
+	git_buf_printf(&reflog_msg, "commit%s: %s",
+		       git_commit_parentcount(c) == 0 ? " (initial)" : "",
+		       shortmsg);
+	git_commit_free(c);
+
+	if (ref) {
+		error = git_reference_set_target(&ref2, ref, id, committer, git_buf_cstr(&reflog_msg));
+		git_reference_free(ref2);
+	} else {
+		error = git_reference__update_terminal(repo, update_ref, id, committer, git_buf_cstr(&reflog_msg));
+	}
 
-	git__free((void *)parents);
-	return res;
+	git_buf_free(&reflog_msg);
+	return error;
 }
 
-int git_commit_create_from_oids(
-	git_oid *oid,
+int git_commit_create_from_callback(
+	git_oid *id,
 	git_repository *repo,
 	const char *update_ref,
 	const git_signature *author,
@@ -77,19 +72,44 @@ int git_commit_create_from_oids(
 	const char *message_encoding,
 	const char *message,
 	const git_oid *tree,
-	int parent_count,
-	const git_oid *parents[])
+	git_commit_parent_callback parent_cb,
+	void *parent_payload)
 {
+	git_reference *ref = NULL;
+	int error = 0, matched_parent = 0;
+	const git_oid *current_id = NULL;
 	git_buf commit = GIT_BUF_INIT;
-	int i;
+	size_t i = 0;
 	git_odb *odb;
+	const git_oid *parent;
 
-	assert(oid && repo && tree && parent_count >= 0);
+	assert(id && repo && tree && parent_cb);
+
+	if (update_ref) {
+		error = git_reference_lookup_resolved(&ref, repo, update_ref, 10);
+		if (error < 0 && error != GIT_ENOTFOUND)
+			return error;
+	}
+	giterr_clear();
+
+	if (ref)
+		current_id = git_reference_target(ref);
 
 	git_oid__writebuf(&commit, "tree ", tree);
 
-	for (i = 0; i < parent_count; ++i)
-		git_oid__writebuf(&commit, "parent ", parents[i]);
+	while ((parent = parent_cb(i, parent_payload)) != NULL) {
+		git_oid__writebuf(&commit, "parent ", parent);
+		if (i == 0 && current_id && git_oid_equal(current_id, parent))
+			matched_parent = 1;
+		i++;
+	}
+
+	if (ref && !matched_parent) {
+		git_reference_free(ref);
+		git_buf_free(&commit);
+		giterr_set(GITERR_OBJECT, "failed to create commit: current tip is not the first parent");
+		return GIT_EMODIFIED;
+	}
 
 	git_signature__writebuf(&commit, "author ", author);
 	git_signature__writebuf(&commit, "committer ", committer);
@@ -105,13 +125,16 @@ int git_commit_create_from_oids(
 	if (git_repository_odb__weakptr(&odb, repo) < 0)
 		goto on_error;
 
-	if (git_odb_write(oid, odb, commit.ptr, commit.size, GIT_OBJ_COMMIT) < 0)
+	if (git_odb_write(id, odb, commit.ptr, commit.size, GIT_OBJ_COMMIT) < 0)
 		goto on_error;
 
 	git_buf_free(&commit);
 
-	if (update_ref != NULL)
-		return git_reference__update_terminal(repo, update_ref, oid);
+	if (update_ref != NULL) {
+		error = update_ref_for_commit(repo, ref, update_ref, id, committer);
+		git_reference_free(ref);
+		return error;
+	}
 
 	return 0;
 
@@ -121,8 +144,101 @@ on_error:
 	return -1;
 }
 
+typedef struct {
+	size_t total;
+	va_list args;
+} commit_parent_varargs;
+
+static const git_oid *commit_parent_from_varargs(size_t curr, void *payload)
+{
+	commit_parent_varargs *data = payload;
+	const git_commit *commit;
+	if (curr >= data->total)
+		return NULL;
+	commit = va_arg(data->args, const git_commit *);
+	return commit ? git_commit_id(commit) : NULL;
+}
+
+int git_commit_create_v(
+	git_oid *id,
+	git_repository *repo,
+	const char *update_ref,
+	const git_signature *author,
+	const git_signature *committer,
+	const char *message_encoding,
+	const char *message,
+	const git_tree *tree,
+	size_t parent_count,
+	...)
+{
+	int error = 0;
+	commit_parent_varargs data;
+
+	assert(tree && git_tree_owner(tree) == repo);
+
+	data.total = parent_count;
+	va_start(data.args, parent_count);
+
+	error = git_commit_create_from_callback(
+		id, repo, update_ref, author, committer,
+		message_encoding, message, git_tree_id(tree),
+		commit_parent_from_varargs, &data);
+
+	va_end(data.args);
+	return error;
+}
+
+typedef struct {
+	size_t total;
+	const git_oid **parents;
+} commit_parent_oids;
+
+static const git_oid *commit_parent_from_ids(size_t curr, void *payload)
+{
+	commit_parent_oids *data = payload;
+	return (curr < data->total) ? data->parents[curr] : NULL;
+}
+
+int git_commit_create_from_ids(
+	git_oid *id,
+	git_repository *repo,
+	const char *update_ref,
+	const git_signature *author,
+	const git_signature *committer,
+	const char *message_encoding,
+	const char *message,
+	const git_oid *tree,
+	size_t parent_count,
+	const git_oid *parents[])
+{
+	commit_parent_oids data = { parent_count, parents };
+
+	return git_commit_create_from_callback(
+		id, repo, update_ref, author, committer,
+		message_encoding, message, tree,
+		commit_parent_from_ids, &data);
+}
+
+typedef struct {
+	size_t total;
+	const git_commit **parents;
+	git_repository *repo;
+} commit_parent_data;
+
+static const git_oid *commit_parent_from_array(size_t curr, void *payload)
+{
+	commit_parent_data *data = payload;
+	const git_commit *commit;
+	if (curr >= data->total)
+		return NULL;
+	commit = data->parents[curr];
+	if (git_commit_owner(commit) != data->repo)
+		return NULL;
+	return git_commit_id(commit);
+}
+
 int git_commit_create(
-	git_oid *oid,
+	git_oid *id,
 	git_repository *repo,
 	const char *update_ref,
 	const git_signature *author,
@@ -130,31 +246,86 @@ int git_commit_create(
 	const char *message_encoding,
 	const char *message,
 	const git_tree *tree,
-	int parent_count,
+	size_t parent_count,
 	const git_commit *parents[])
 {
-	int retval, i;
-	const git_oid **parent_oids;
+	commit_parent_data data = { parent_count, parents, repo };
+
+	assert(tree && git_tree_owner(tree) == repo);
+
+	return git_commit_create_from_callback(
+		id, repo, update_ref, author, committer,
+		message_encoding, message, git_tree_id(tree),
+		commit_parent_from_array, &data);
+}
+
+static const git_oid *commit_parent_for_amend(size_t curr, void *payload)
+{
+	const git_commit *commit_to_amend = payload;
+	if (curr >= git_array_size(commit_to_amend->parent_ids))
+		return NULL;
+	return git_array_get(commit_to_amend->parent_ids, curr);
+}
+
+int git_commit_amend(
+	git_oid *id,
+	const git_commit *commit_to_amend,
+	const char *update_ref,
+	const git_signature *author,
+	const git_signature *committer,
+	const char *message_encoding,
+	const char *message,
+	const git_tree *tree)
+{
+	git_repository *repo;
+	git_oid tree_id;
+	git_reference *ref;
+	int error;
 
-	assert(parent_count >= 0);
-	assert(git_object_owner((const git_object *)tree) == repo);
+	assert(id && commit_to_amend);
+
+	repo = git_commit_owner(commit_to_amend);
+
+	if (!author)
+		author = git_commit_author(commit_to_amend);
+	if (!committer)
+		committer = git_commit_committer(commit_to_amend);
+	if (!message_encoding)
+		message_encoding = git_commit_message_encoding(commit_to_amend);
+	if (!message)
+		message = git_commit_message(commit_to_amend);
+
+	if (!tree) {
+		git_tree *old_tree;
+		GITERR_CHECK_ERROR( git_commit_tree(&old_tree, commit_to_amend) );
+		git_oid_cpy(&tree_id, git_tree_id(old_tree));
+		git_tree_free(old_tree);
+	} else {
+		assert(git_tree_owner(tree) == repo);
+		git_oid_cpy(&tree_id, git_tree_id(tree));
+	}
 
-	parent_oids = git__malloc(parent_count * sizeof(git_oid *));
-	GITERR_CHECK_ALLOC(parent_oids);
+	if (update_ref) {
+		if ((error = git_reference_lookup_resolved(&ref, repo, update_ref, 5)) < 0)
+			return error;
 
-	for (i = 0; i < parent_count; ++i) {
-		assert(git_object_owner((const git_object *)parents[i]) == repo);
-		parent_oids[i] = git_object_id((const git_object *)parents[i]);
+		if (git_oid_cmp(git_commit_id(commit_to_amend), git_reference_target(ref))) {
+			git_reference_free(ref);
+			giterr_set(GITERR_REFERENCE, "commit to amend is not the tip of the given branch");
+			return -1;
+		}
 	}
 
-	retval = git_commit_create_from_oids(
-		oid, repo, update_ref, author, committer,
-		message_encoding, message,
-		git_object_id((const git_object *)tree), parent_count, parent_oids);
+	error = git_commit_create_from_callback(
+		id, repo, NULL, author, committer, message_encoding, message,
+		&tree_id, commit_parent_for_amend, (void *)commit_to_amend);
 
-	git__free((void *)parent_oids);
+	if (!error && update_ref) {
+		error = update_ref_for_commit(repo, ref, NULL, id, committer);
+		git_reference_free(ref);
+	}
 
-	return retval;
+	return error;
 }
 
 int git_commit__parse(void *_commit, git_odb_object *odb_obj)
@@ -163,33 +334,15 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 	const char *buffer_start = git_odb_object_data(odb_obj), *buffer;
 	const char *buffer_end = buffer_start + git_odb_object_size(odb_obj);
 	git_oid parent_id;
-	uint32_t parent_count = 0;
 	size_t header_len;
 
-	/* find end-of-header (counting parents as we go) */
-	for (buffer = buffer_start; buffer < buffer_end; ++buffer) {
-		if (!strncmp("\n\n", buffer, 2)) {
-			++buffer;
-			break;
-		}
-		if (!strncmp("\nparent ", buffer, strlen("\nparent ")))
-			++parent_count;
-	}
-
-	header_len = buffer - buffer_start;
-	commit->raw_header = git__strndup(buffer_start, header_len);
-	GITERR_CHECK_ALLOC(commit->raw_header);
+	buffer = buffer_start;
 
-	/* point "buffer" to header data */
-	buffer = commit->raw_header;
-	buffer_end = commit->raw_header + header_len;
-
-	if (parent_count < 1)
-		parent_count = 1;
-
-	git_array_init_to_size(commit->parent_ids, parent_count);
+	/* Allocate for one, which will allow not to realloc 90% of the time  */
+	git_array_init_to_size(commit->parent_ids, 1);
 	GITERR_CHECK_ARRAY(commit->parent_ids);
 
+	/* The tree is always the first field */
 	if (git_oid__parse(&commit->tree_id, &buffer, buffer_end, "tree ") < 0)
 		goto bad_buffer;
 
@@ -220,6 +373,9 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 	/* Parse add'l header entries */
 	while (buffer < buffer_end) {
 		const char *eoln = buffer;
+		if (buffer[-1] == '\n' && buffer[0] == '\n')
+			break;
+
 		while (eoln < buffer_end && *eoln != '\n')
 			++eoln;
 
@@ -235,13 +391,12 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 		buffer = eoln;
 	}
 
-	/* point "buffer" to data after header */
-	buffer = git_odb_object_data(odb_obj);
-	buffer_end = buffer + git_odb_object_size(odb_obj);
+	header_len = buffer - buffer_start;
+	commit->raw_header = git__strndup(buffer_start, header_len);
+	GITERR_CHECK_ALLOC(commit->raw_header);
 
-	buffer += header_len;
-	if (*buffer == '\n')
-		++buffer;
+	/* point "buffer" to data after header, +1 for the final LF */
+	buffer = buffer_start + header_len + 1;
 
 	/* extract commit message */
 	if (buffer <= buffer_end) {
@@ -275,10 +430,12 @@ GIT_COMMIT_GETTER(const git_oid *, tree_id, &commit->tree_id);
 
 const char *git_commit_message(const git_commit *commit)
 {
-	const char *message = commit->raw_message;
+	const char *message;
 
 	assert(commit);
 
+	message = commit->raw_message;
+
 	/* trim leading newlines from raw message */
 	while (*message && *message == '\n')
 		++message;
@@ -286,6 +443,36 @@ const char *git_commit_message(const git_commit *commit)
 	return message;
 }
 
+const char *git_commit_summary(git_commit *commit)
+{
+	git_buf summary = GIT_BUF_INIT;
+	const char *msg, *space;
+
+	assert(commit);
+
+	if (!commit->summary) {
+		for (msg = git_commit_message(commit), space = NULL; *msg; ++msg) {
+			if (msg[0] == '\n' && (!msg[1] || msg[1] == '\n'))
+				break;
+			else if (msg[0] == '\n')
+				git_buf_putc(&summary, ' ');
+			else if (git__isspace(msg[0]))
+				space = space ? space : msg;
+			else if (space) {
+				git_buf_put(&summary, space, (msg - space) + 1);
+				space = NULL;
+			} else
+				git_buf_putc(&summary, *msg);
+		}
+
+		commit->summary = git_buf_detach(&summary);
+		if (!commit->summary)
+			commit->summary = git__strdup("");
+	}
+
+	return commit->summary;
+}
+
 int git_commit_tree(git_tree **tree_out, const git_commit *commit)
 {
 	assert(commit);
@@ -325,19 +512,18 @@ int git_commit_nth_gen_ancestor(
 
 	assert(ancestor && commit);
 
-	current = (git_commit *)commit;
+	if (git_object_dup((git_object **) &current, (git_object *) commit) < 0)
+		return -1;
 
-	if (n == 0)
-		return git_commit_lookup(
-			ancestor,
-			commit->object.repo,
-			git_object_id((const git_object *)commit));
+	if (n == 0) {
+		*ancestor = current;
+		return 0;
+	}
 
 	while (n--) {
-		error = git_commit_parent(&parent, (git_commit *)current, 0);
+		error = git_commit_parent(&parent, current, 0);
 
-		if (current != commit)
-			git_commit_free(current);
+		git_commit_free(current);
 
 		if (error < 0)
 			return error;
diff --git a/src/commit.h b/src/commit.h
index d452e2975..efb080b50 100644
--- a/src/commit.h
+++ b/src/commit.h
@@ -26,6 +26,8 @@ struct git_commit {
 	char *message_encoding;
 	char *raw_message;
 	char *raw_header;
+
+	char *summary;
 };
 
 void git_commit__free(void *commit);
diff --git a/src/commit_list.c b/src/commit_list.c
index 64416e54d..9db3f5633 100644
--- a/src/commit_list.c
+++ b/src/commit_list.c
@@ -11,10 +11,10 @@
 #include "pool.h"
 #include "odb.h"
 
-int git_commit_list_time_cmp(void *a, void *b)
+int git_commit_list_time_cmp(const void *a, const void *b)
 {
-	git_commit_list_node *commit_a = (git_commit_list_node *)a;
-	git_commit_list_node *commit_b = (git_commit_list_node *)b;
+	const git_commit_list_node *commit_a = a;
+	const git_commit_list_node *commit_b = b;
 
 	return (commit_a->time < commit_b->time);
 }
diff --git a/src/commit_list.h b/src/commit_list.h
index d2f54b3ca..490d841be 100644
--- a/src/commit_list.h
+++ b/src/commit_list.h
@@ -39,7 +39,7 @@ typedef struct git_commit_list {
 } git_commit_list;
 
 git_commit_list_node *git_commit_list_alloc_node(git_revwalk *walk);
-int git_commit_list_time_cmp(void *a, void *b);
+int git_commit_list_time_cmp(const void *a, const void *b);
 void git_commit_list_free(git_commit_list **list_p);
 git_commit_list *git_commit_list_insert(git_commit_list_node *item, git_commit_list **list_p);
 git_commit_list *git_commit_list_insert_by_date(git_commit_list_node *item, git_commit_list **list_p);
diff --git a/src/common.h b/src/common.h
index 159d31b2e..807e5fa39 100644
--- a/src/common.h
+++ b/src/common.h
@@ -10,6 +10,13 @@
 #include "git2/common.h"
 #include "cc-compat.h"
 
+/** Declare a function as always inlined. */
+#if defined(_MSC_VER)
+# define GIT_INLINE(type) static __inline type
+#else
+# define GIT_INLINE(type) static inline type
+#endif
+
 #include <assert.h>
 #include <errno.h>
 #include <limits.h>
@@ -37,11 +44,15 @@
 #else
 
 # include <unistd.h>
+# include <strings.h>
 # ifdef GIT_THREADS
 #	include <pthread.h>
+#	include <sched.h>
 # endif
 #define GIT_STDLIB_CALL
 
+# include <arpa/inet.h>
+
 #endif
 
 #include "git2/types.h"
@@ -60,7 +71,7 @@
  * Check a return value and propogate result if non-zero.
  */
 #define GITERR_CHECK_ERROR(code) \
-	do { int _err = (code); if (_err < 0) return _err; } while (0)
+	do { int _err = (code); if (_err) return _err; } while (0)
 
 /**
  * Set the error message for this thread, formatting as needed.
@@ -74,28 +85,61 @@ void giterr_set(int error_class, const char *string, ...);
 int giterr_set_regex(const regex_t *regex, int error_code);
 
 /**
- * Gets the system error code for this thread.
+ * Set error message for user callback if needed.
+ *
+ * If the error code in non-zero and no error message is set, this
+ * sets a generic error message.
+ *
+ * @return This always returns the `error_code` parameter.
  */
-GIT_INLINE(int) giterr_system_last(void)
+GIT_INLINE(int) giterr_set_after_callback_function(
+	int error_code, const char *action)
 {
+	if (error_code) {
+		const git_error *e = giterr_last();
+		if (!e || !e->message)
+			giterr_set(e ? e->klass : GITERR_CALLBACK,
+				"%s callback returned %d", action, error_code);
+	}
+	return error_code;
+}
+
 #ifdef GIT_WIN32
-	return GetLastError();
+#define giterr_set_after_callback(code) \
+	giterr_set_after_callback_function((code), __FUNCTION__)
 #else
-	return errno;
+#define giterr_set_after_callback(code) \
+	giterr_set_after_callback_function((code), __func__)
 #endif
-}
+
+/**
+ * Gets the system error code for this thread.
+ */
+int giterr_system_last(void);
 
 /**
  * Sets the system error code for this thread.
  */
-GIT_INLINE(void) giterr_system_set(int code)
-{
-#ifdef GIT_WIN32
-	SetLastError(code);
-#else
-	errno = code;
-#endif
-}
+void giterr_system_set(int code);
+
+/**
+ * Structure to preserve libgit2 error state
+ */
+typedef struct {
+	int       error_code;
+	git_error error_msg;
+} git_error_state;
+
+/**
+ * Capture current error state to restore later, returning error code.
+ * If `error_code` is zero, this does nothing and returns zero.
+ */
+int giterr_capture(git_error_state *state, int error_code);
+
+/**
+ * Restore error state to a previous value, returning saved error code.
+ */
+int giterr_restore(git_error_state *state);
 
 /**
  * Check a versioned structure for validity
@@ -126,6 +170,11 @@ GIT_INLINE(void) git__init_structure(void *structure, size_t len, unsigned int v
 }
 #define GIT_INIT_STRUCTURE(S,V) git__init_structure(S, sizeof(*S), V)
 
+#define GIT_INIT_STRUCTURE_FROM_TEMPLATE(PTR,VERSION,TYPE,TPL) do { \
+	TYPE _tmpl = TPL; \
+	GITERR_CHECK_VERSION(&(VERSION), _tmpl.version, #TYPE);	\
+	memcpy((PTR), &_tmpl, sizeof(_tmpl)); } while (0)
+
 /* NOTE: other giterr functions are in the public errors.h header file */
 
 #include "util.h"
diff --git a/src/compress.c b/src/compress.c
deleted file mode 100644
index 14b79404c..000000000
--- a/src/compress.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (C) the libgit2 contributors. All rights reserved.
- *
- * This file is part of libgit2, distributed under the GNU GPL v2 with
- * a Linking Exception. For full terms see the included COPYING file.
- */
-
-#include "compress.h"
-
-#include <zlib.h>
-
-#define BUFFER_SIZE (1024 * 1024)
-
-int git__compress(git_buf *buf, const void *buff, size_t len)
-{
-	z_stream zs;
-	char *zb;
-	size_t have;
-
-	memset(&zs, 0, sizeof(zs));
-	if (deflateInit(&zs, Z_DEFAULT_COMPRESSION) != Z_OK)
-		return -1;
-
-	zb = git__malloc(BUFFER_SIZE);
-	GITERR_CHECK_ALLOC(zb);
-
-	zs.next_in = (void *)buff;
-	zs.avail_in = (uInt)len;
-
-	do {
-		zs.next_out = (unsigned char *)zb;
-		zs.avail_out = BUFFER_SIZE;
-
-		if (deflate(&zs, Z_FINISH) == Z_STREAM_ERROR) {
-			git__free(zb);
-			return -1;
-		}
-
-		have = BUFFER_SIZE - (size_t)zs.avail_out;
-
-		if (git_buf_put(buf, zb, have) < 0) {
-			git__free(zb);
-			return -1;
-		}
-
-	} while (zs.avail_out == 0);
-
-	assert(zs.avail_in == 0);
-
-	deflateEnd(&zs);
-	git__free(zb);
-	return 0;
-}
diff --git a/src/config.c b/src/config.c
index 0d9471383..8a0fb653c 100644
--- a/src/config.c
+++ b/src/config.c
@@ -6,7 +6,7 @@
  */
 
 #include "common.h"
-#include "fileops.h"
+#include "sysdir.h"
 #include "config.h"
 #include "git2/config.h"
 #include "git2/sys/config.h"
@@ -137,6 +137,38 @@ int git_config_open_ondisk(git_config **out, const char *path)
 	return error;
 }
 
+int git_config_snapshot(git_config **out, git_config *in)
+{
+	int error = 0;
+	size_t i;
+	file_internal *internal;
+	git_config *config;
+
+	*out = NULL;
+
+	if (git_config_new(&config) < 0)
+		return -1;
+
+	git_vector_foreach(&in->files, i, internal) {
+		git_config_backend *b;
+
+		if ((error = internal->file->snapshot(&b, internal->file)) < 0)
+			break;
+
+		if ((error = git_config_add_backend(config, b, internal->level, 0)) < 0) {
+			b->free(b);
+			break;
+		}
+	}
+
+	if (error < 0)
+		git_config_free(config);
+	else
+		*out = config;
+
+	return error;
+}
+
 static int find_internal_file_by_level(
 	file_internal **internal_out,
 	const git_config *cfg,
@@ -458,6 +490,7 @@ int git_config_iterator_glob_new(git_config_iterator **out, const git_config *cf
 	if ((result = regcomp(&iter->regex, regexp, REG_EXTENDED)) < 0) {
 		giterr_set_regex(&iter->regex, result);
 		regfree(&iter->regex);
+		git__free(iter);
 		return -1;
 	}
 
@@ -480,47 +513,45 @@ int git_config_foreach(
 int git_config_backend_foreach_match(
 	git_config_backend *backend,
 	const char *regexp,
-	int (*fn)(const git_config_entry *, void *),
-	void *data)
+	git_config_foreach_cb cb,
+	void *payload)
 {
 	git_config_entry *entry;
 	git_config_iterator* iter;
 	regex_t regex;
-	int result = 0;
+	int error = 0;
 
 	if (regexp != NULL) {
-		if ((result = regcomp(&regex, regexp, REG_EXTENDED)) < 0) {
-			giterr_set_regex(&regex, result);
+		if ((error = regcomp(&regex, regexp, REG_EXTENDED)) < 0) {
+			giterr_set_regex(&regex, error);
 			regfree(&regex);
 			return -1;
 		}
 	}
 
-	if ((result = backend->iterator(&iter, backend)) < 0) {
+	if ((error = backend->iterator(&iter, backend)) < 0) {
 		iter = NULL;
 		return -1;
 	}
 
-	while(!(iter->next(&entry, iter) < 0)) {
+	while (!(iter->next(&entry, iter) < 0)) {
 		/* skip non-matching keys if regexp was provided */
 		if (regexp && regexec(&regex, entry->name, 0, NULL, 0) != 0)
 			continue;
 
 		/* abort iterator on non-zero return value */
-		if (fn(entry, data)) {
-			giterr_clear();
-			result = GIT_EUSER;
-			goto cleanup;
+		if ((error = cb(entry, payload)) != 0) {
+			giterr_set_after_callback(error);
+			break;
 		}
 	}
 
-cleanup:
 	if (regexp != NULL)
 		regfree(&regex);
 
 	iter->free(iter);
 
-	return result;
+	return error;
 }
 
 int git_config_foreach_match(
@@ -536,10 +567,9 @@ int git_config_foreach_match(
 	if ((error = git_config_iterator_glob_new(&iter, cfg, regexp)) < 0)
 		return error;
 
-	while ((error = git_config_next(&entry, iter)) == 0) {
-		if(cb(entry, payload)) {
-			giterr_clear();
-			error = GIT_EUSER;
+	while (!(error = git_config_next(&entry, iter))) {
+		if ((error = cb(entry, payload)) != 0) {
+			giterr_set_after_callback(error);
 			break;
 		}
 	}
@@ -617,9 +647,112 @@ int git_config_set_string(git_config *cfg, const char *name, const char *value)
 	return error;
 }
 
+int git_config__update_entry(
+	git_config *config,
+	const char *key,
+	const char *value,
+	bool overwrite_existing,
+	bool only_if_existing)
+{
+	int error = 0;
+	const git_config_entry *ce = NULL;
+
+	if ((error = git_config__lookup_entry(&ce, config, key, false)) < 0)
+		return error;
+
+	if (!ce && only_if_existing) /* entry doesn't exist */
+		return 0;
+	if (ce && !overwrite_existing) /* entry would be overwritten */
+		return 0;
+	if (value && ce && ce->value && !strcmp(ce->value, value)) /* no change */
+		return 0;
+	if (!value && (!ce || !ce->value)) /* asked to delete absent entry */
+		return 0;
+
+	if (!value)
+		error = git_config_delete_entry(config, key);
+	else
+		error = git_config_set_string(config, key, value);
+
+	return error;
+}
+
 /***********
  * Getters
  ***********/
+
+static int config_error_notfound(const char *name)
+{
+	giterr_set(GITERR_CONFIG, "Config value '%s' was not found", name);
+	return GIT_ENOTFOUND;
+}
+
+enum {
+	GET_ALL_ERRORS = 0,
+	GET_NO_MISSING = 1,
+	GET_NO_ERRORS  = 2
+};
+
+static int get_entry(
+	const git_config_entry **out,
+	const git_config *cfg,
+	const char *name,
+	bool normalize_name,
+	int want_errors)
+{
+	int res = GIT_ENOTFOUND;
+	const char *key = name;
+	char *normalized = NULL;
+	size_t i;
+	file_internal *internal;
+
+	*out = NULL;
+
+	if (normalize_name) {
+		if ((res = git_config__normalize_name(name, &normalized)) < 0)
+			goto cleanup;
+		key = normalized;
+	}
+
+	res = GIT_ENOTFOUND;
+	git_vector_foreach(&cfg->files, i, internal) {
+		if (!internal || !internal->file)
+			continue;
+
+		res = internal->file->get(internal->file, key, out);
+		if (res != GIT_ENOTFOUND)
+			break;
+	}
+
+	git__free(normalized);
+
+cleanup:
+	if (res == GIT_ENOTFOUND)
+		res = (want_errors > GET_ALL_ERRORS) ? 0 : config_error_notfound(name);
+	else if (res && (want_errors == GET_NO_ERRORS)) {
+		giterr_clear();
+		res = 0;
+	}
+
+	return res;
+}
+
+int git_config_get_entry(
+	const git_config_entry **out, const git_config *cfg, const char *name)
+{
+	return get_entry(out, cfg, name, true, GET_ALL_ERRORS);
+}
+
+int git_config__lookup_entry(
+	const git_config_entry **out,
+	const git_config *cfg,
+	const char *key,
+	bool no_errors)
+{
+	return get_entry(
+		out, cfg, key, false, no_errors ? GET_NO_ERRORS : GET_NO_MISSING);
+}
+
 int git_config_get_mapped(
 	int *out,
 	const git_config *cfg,
@@ -627,116 +760,91 @@ int git_config_get_mapped(
 	const git_cvar_map *maps,
 	size_t map_n)
 {
-	const char *value;
+	const git_config_entry *entry;
 	int ret;
 
-	if ((ret = git_config_get_string(&value, cfg, name)) < 0)
+	if ((ret = get_entry(&entry, cfg, name, true, GET_ALL_ERRORS)) < 0)
 		return ret;
 
-	return git_config_lookup_map_value(out, maps, map_n, value);
+	return git_config_lookup_map_value(out, maps, map_n, entry->value);
 }
 
 int git_config_get_int64(int64_t *out, const git_config *cfg, const char *name)
 {
-	const char *value;
+	const git_config_entry *entry;
 	int ret;
 
-	if ((ret = git_config_get_string(&value, cfg, name)) < 0)
+	if ((ret = get_entry(&entry, cfg, name, true, GET_ALL_ERRORS)) < 0)
 		return ret;
 
-	return git_config_parse_int64(out, value);
+	return git_config_parse_int64(out, entry->value);
 }
 
 int git_config_get_int32(int32_t *out, const git_config *cfg, const char *name)
 {
-	const char *value;
+	const git_config_entry *entry;
 	int ret;
 
-	if ((ret = git_config_get_string(&value, cfg, name)) < 0)
+	if ((ret = get_entry(&entry, cfg, name, true, GET_ALL_ERRORS)) < 0)
 		return ret;
 
-	return git_config_parse_int32(out, value);
+	return git_config_parse_int32(out, entry->value);
 }
 
-static int get_string_at_file(const char **out, const git_config_backend *file, const char *name)
+int git_config_get_bool(int *out, const git_config *cfg, const char *name)
 {
 	const git_config_entry *entry;
-	int res;
+	int ret;
 
-	res = file->get(file, name, &entry);
-	if (!res)
-		*out = entry->value;
+	if ((ret = get_entry(&entry, cfg, name, true, GET_ALL_ERRORS)) < 0)
+		return ret;
 
-	return res;
+	return git_config_parse_bool(out, entry->value);
 }
 
-static int config_error_notfound(const char *name)
+int git_config_get_string(
+	const char **out, const git_config *cfg, const char *name)
 {
-	giterr_set(GITERR_CONFIG, "Config value '%s' was not found", name);
-	return GIT_ENOTFOUND;
+	const git_config_entry *entry;
+	int ret = get_entry(&entry, cfg, name, true, GET_ALL_ERRORS);
+	*out = !ret ? (entry->value ? entry->value : "") : NULL;
+	return ret;
 }
 
-static int get_string(const char **out, const git_config *cfg, const char *name)
+const char *git_config__get_string_force(
+	const git_config *cfg, const char *key, const char *fallback_value)
 {
-	file_internal *internal;
-	unsigned int i;
-	int res;
-
-	git_vector_foreach(&cfg->files, i, internal) {
-		if (!internal || !internal->file)
-			continue;
-
-		res = get_string_at_file(out, internal->file, name);
-		if (res != GIT_ENOTFOUND)
-			return res;
-	}
-
-	return config_error_notfound(name);
+	const git_config_entry *entry;
+	get_entry(&entry, cfg, key, false, GET_NO_ERRORS);
+	return (entry && entry->value) ? entry->value : fallback_value;
 }
 
-int git_config_get_bool(int *out, const git_config *cfg, const char *name)
+int git_config__get_bool_force(
+	const git_config *cfg, const char *key, int fallback_value)
 {
-	const char *value = NULL;
-	int ret;
-
-	if ((ret = get_string(&value, cfg, name)) < 0)
-		return ret;
-
-	return git_config_parse_bool(out, value);
-}
+	int val = fallback_value;
+	const git_config_entry *entry;
 
-int git_config_get_string(const char **out, const git_config *cfg, const char *name)
-{
-	int ret;
-	const char *str = NULL;
+	get_entry(&entry, cfg, key, false, GET_NO_ERRORS);
 
-	if ((ret = get_string(&str, cfg, name)) < 0)
-		return ret;
+	if (entry && git_config_parse_bool(&val, entry->value) < 0)
+		giterr_clear();
 
-	*out = str == NULL ? "" : str;
-	return 0;
+	return val;
 }
 
-int git_config_get_entry(const git_config_entry **out, const git_config *cfg, const char *name)
+int git_config__get_int_force(
+	const git_config *cfg, const char *key, int fallback_value)
 {
-	file_internal *internal;
-	unsigned int i;
-	git_config_backend *file;
-	int ret;
+	int32_t val = (int32_t)fallback_value;
+	const git_config_entry *entry;
 
-	*out = NULL;
+	get_entry(&entry, cfg, key, false, GET_NO_ERRORS);
 
-	git_vector_foreach(&cfg->files, i, internal) {
-		if (!internal || !internal->file)
-			continue;
-		file = internal->file;
+	if (entry && git_config_parse_int32(&val, entry->value) < 0)
+		giterr_clear();
 
-		ret = file->get(file, name, out);
-		if (ret != GIT_ENOTFOUND)
-			return ret;
-	}
-
-	return config_error_notfound(name);
+	return (int)val;
 }
 
 int git_config_get_multivar_foreach(
@@ -753,9 +861,10 @@ int git_config_get_multivar_foreach(
 	found = 0;
 	while ((err = iter->next(&entry, iter)) == 0) {
 		found = 1;
-		if(cb(entry, payload)) {
-			iter->free(iter);
-			return GIT_EUSER;
+
+		if ((err = cb(entry, payload)) != 0) {
+			giterr_set_after_callback(err);
+			break;
 		}
 	}
 
@@ -882,86 +991,50 @@ int git_config_next(git_config_entry **entry, git_config_iterator *iter)
 
 void git_config_iterator_free(git_config_iterator *iter)
 {
-	iter->free(iter);
-}
-
-static int git_config__find_file_to_path(
-	char *out, size_t outlen, int (*find)(git_buf *buf))
-{
-	int error = 0;
-	git_buf path = GIT_BUF_INIT;
-
-	if ((error = find(&path)) < 0)
-		goto done;
-
-	if (path.size >= outlen) {
-		giterr_set(GITERR_NOMEMORY, "Buffer is too short for the path");
-		error = GIT_EBUFS;
-		goto done;
-	}
-
-	git_buf_copy_cstr(out, outlen, &path);
-
-done:
-	git_buf_free(&path);
-	return error;
-}
-
-int git_config_find_global_r(git_buf *path)
-{
-	return git_futils_find_global_file(path, GIT_CONFIG_FILENAME_GLOBAL);
-}
-
-int git_config_find_global(char *global_config_path, size_t length)
-{
-	return git_config__find_file_to_path(
-		global_config_path, length, git_config_find_global_r);
-}
+	if (iter == NULL)
+		return;
 
-int git_config_find_xdg_r(git_buf *path)
-{
-	return git_futils_find_xdg_file(path, GIT_CONFIG_FILENAME_XDG);
+	iter->free(iter);
 }
 
-int git_config_find_xdg(char *xdg_config_path, size_t length)
+int git_config_find_global(git_buf *path)
 {
-	return git_config__find_file_to_path(
-		xdg_config_path, length, git_config_find_xdg_r);
+	git_buf_sanitize(path);
+	return git_sysdir_find_global_file(path, GIT_CONFIG_FILENAME_GLOBAL);
 }
 
-int git_config_find_system_r(git_buf *path)
+int git_config_find_xdg(git_buf *path)
 {
-	return git_futils_find_system_file(path, GIT_CONFIG_FILENAME_SYSTEM);
+	git_buf_sanitize(path);
+	return git_sysdir_find_xdg_file(path, GIT_CONFIG_FILENAME_XDG);
 }
 
-int git_config_find_system(char *system_config_path, size_t length)
+int git_config_find_system(git_buf *path)
 {
-	return git_config__find_file_to_path(
-		system_config_path, length, git_config_find_system_r);
+	git_buf_sanitize(path);
+	return git_sysdir_find_system_file(path, GIT_CONFIG_FILENAME_SYSTEM);
 }
 
 int git_config__global_location(git_buf *buf)
 {
 	const git_buf *paths;
 	const char *sep, *start;
-	size_t len;
 
-	if (git_futils_dirs_get(&paths, GIT_FUTILS_DIR_GLOBAL) < 0)
+	if (git_sysdir_get(&paths, GIT_SYSDIR_GLOBAL) < 0)
 		return -1;
 
 	/* no paths, so give up */
-	if (git_buf_len(paths) == 0)
+	if (!paths || !git_buf_len(paths))
 		return -1;
 
-	start = git_buf_cstr(paths);
-	sep = strchr(start, GIT_PATH_LIST_SEPARATOR);
-
-	if (sep)
-		len = sep - start;
-	else
-		len = paths->size;
+	/* find unescaped separator or end of string */
+	for (sep = start = git_buf_cstr(paths); *sep; ++sep) {
+		if (*sep == GIT_PATH_LIST_SEPARATOR &&
+			(sep <= start || sep[-1] != '\\'))
+			break;
+	}
 
-	if (git_buf_set(buf, start, len) < 0)
+	if (git_buf_set(buf, start, (size_t)(sep - start)) < 0)
 		return -1;
 
 	return git_buf_joinpath(buf, buf->ptr, GIT_CONFIG_FILENAME_GLOBAL);
@@ -976,16 +1049,16 @@ int git_config_open_default(git_config **out)
 	if ((error = git_config_new(&cfg)) < 0)
 		return error;
 
-	if (!git_config_find_global_r(&buf) || !git_config__global_location(&buf)) {
+	if (!git_config_find_global(&buf) || !git_config__global_location(&buf)) {
 		error = git_config_add_file_ondisk(cfg, buf.ptr,
 			GIT_CONFIG_LEVEL_GLOBAL, 0);
 	}
 
-	if (!error && !git_config_find_xdg_r(&buf))
+	if (!error && !git_config_find_xdg(&buf))
 		error = git_config_add_file_ondisk(cfg, buf.ptr,
 			GIT_CONFIG_LEVEL_XDG, 0);
 
-	if (!error && !git_config_find_system_r(&buf))
+	if (!error && !git_config_find_system(&buf))
 		error = git_config_add_file_ondisk(cfg, buf.ptr,
 			GIT_CONFIG_LEVEL_SYSTEM, 0);
 
@@ -1070,7 +1143,7 @@ int git_config_parse_int64(int64_t *out, const char *value)
 	const char *num_end;
 	int64_t num;
 
-	if (git__strtol64(&num, value, &num_end, 0) < 0)
+	if (!value || git__strtol64(&num, value, &num_end, 0) < 0)
 		goto fail_parse;
 
 	switch (*num_end) {
@@ -1104,7 +1177,7 @@ int git_config_parse_int64(int64_t *out, const char *value)
 	}
 
 fail_parse:
-	giterr_set(GITERR_CONFIG, "Failed to parse '%s' as an integer", value);
+	giterr_set(GITERR_CONFIG, "Failed to parse '%s' as an integer", value ? value : "(null)");
 	return -1;
 }
 
@@ -1124,7 +1197,7 @@ int git_config_parse_int32(int32_t *out, const char *value)
 	return 0;
 
 fail_parse:
-	giterr_set(GITERR_CONFIG, "Failed to parse '%s' as a 32-bit integer", value);
+	giterr_set(GITERR_CONFIG, "Failed to parse '%s' as a 32-bit integer", value ? value : "(null)");
 	return -1;
 }
 
@@ -1167,7 +1240,6 @@ struct rename_data {
 	git_config *config;
 	git_buf *name;
 	size_t old_len;
-	int actual_error;
 };
 
 static int rename_config_entries_cb(
@@ -1190,8 +1262,6 @@ static int rename_config_entries_cb(
 	if (!error)
 		error = git_config_delete_entry(data->config, entry->name);
 
-	data->actual_error = error; /* preserve actual error code */
-
 	return error;
 }
 
@@ -1216,7 +1286,6 @@ int git_config_rename_section(
 	data.config  = config;
 	data.name    = &replace;
 	data.old_len = strlen(old_section_name) + 1;
-	data.actual_error = 0;
 
 	if ((error = git_buf_join(&replace, '.', new_section_name, "")) < 0)
 		goto cleanup;
@@ -1233,12 +1302,16 @@ int git_config_rename_section(
 	error = git_config_foreach_match(
 		config, git_buf_cstr(&pattern), rename_config_entries_cb, &data);
 
-	if (error == GIT_EUSER)
-		error = data.actual_error;
-
 cleanup:
 	git_buf_free(&pattern);
 	git_buf_free(&replace);
 
 	return error;
 }
+
+int git_config_init_backend(git_config_backend *backend, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		backend, version, git_config_backend, GIT_CONFIG_BACKEND_INIT);
+	return 0;
+}
diff --git a/src/config.h b/src/config.h
index 01e8465cc..b0dcb49ac 100644
--- a/src/config.h
+++ b/src/config.h
@@ -24,11 +24,6 @@ struct git_config {
 	git_vector files;
 };
 
-extern int git_config_find_global_r(git_buf *global_config_path);
-extern int git_config_find_xdg_r(git_buf *system_config_path);
-extern int git_config_find_system_r(git_buf *system_config_path);
-
-
 extern int git_config__global_location(git_buf *buf);
 
 extern int git_config_rename_section(
@@ -51,5 +46,40 @@ extern int git_config_file__ondisk(git_config_backend **out, const char *path);
 
 extern int git_config__normalize_name(const char *in, char **out);
 
+/* internal only: does not normalize key and sets out to NULL if not found */
+extern int git_config__lookup_entry(
+	const git_config_entry **out,
+	const git_config *cfg,
+	const char *key,
+	bool no_errors);
+
+/* internal only: update and/or delete entry string with constraints */
+extern int git_config__update_entry(
+	git_config *cfg,
+	const char *key,
+	const char *value,
+	bool overwrite_existing,
+	bool only_if_existing);
+
+/*
+ * Lookup functions that cannot fail.  These functions look up a config
+ * value and return a fallback value if the value is missing or if any
+ * failures occur while trying to access the value.
+ */
+
+extern const char *git_config__get_string_force(
+	const git_config *cfg, const char *key, const char *fallback_value);
+
+extern int git_config__get_bool_force(
+	const git_config *cfg, const char *key, int fallback_value);
+
+extern int git_config__get_int_force(
+	const git_config *cfg, const char *key, int fallback_value);
+
+/* API for repository cvar-style lookups from config - not cached, but
+ * uses cvar value maps and fallbacks
+ */
+extern int git_config__cvar(
+	int *out, git_config *config, git_cvar_cached cvar);
 
 #endif
diff --git a/src/config_cache.c b/src/config_cache.c
index 6808521a3..45c39ce17 100644
--- a/src/config_cache.c
+++ b/src/config_cache.c
@@ -7,11 +7,11 @@
 
 #include "common.h"
 #include "fileops.h"
+#include "repository.h"
 #include "config.h"
 #include "git2/config.h"
 #include "vector.h"
 #include "filter.h"
-#include "repository.h"
 
 struct map_data {
 	const char *cvar_name;
@@ -51,6 +51,12 @@ static git_cvar_map _cvar_map_autocrlf[] = {
 	{GIT_CVAR_STRING, "input", GIT_AUTO_CRLF_INPUT}
 };
 
+static git_cvar_map _cvar_map_safecrlf[] = {
+	{GIT_CVAR_FALSE, NULL, GIT_SAFE_CRLF_FALSE},
+	{GIT_CVAR_TRUE, NULL, GIT_SAFE_CRLF_FAIL},
+	{GIT_CVAR_STRING, "warn", GIT_SAFE_CRLF_WARN}
+};
+
 /*
  * Generic map for integer values
  */
@@ -68,32 +74,39 @@ static struct map_data _cvar_maps[] = {
 	{"core.trustctime", NULL, 0, GIT_TRUSTCTIME_DEFAULT },
 	{"core.abbrev", _cvar_map_int, 1, GIT_ABBREV_DEFAULT },
 	{"core.precomposeunicode", NULL, 0, GIT_PRECOMPOSE_DEFAULT },
+	{"core.safecrlf", _cvar_map_safecrlf, ARRAY_SIZE(_cvar_map_safecrlf), GIT_SAFE_CRLF_DEFAULT},
+	{"core.logallrefupdates", NULL, 0, GIT_LOGALLREFUPDATES_DEFAULT },
 };
 
+int git_config__cvar(int *out, git_config *config, git_cvar_cached cvar)
+{
+	int error = 0;
+	struct map_data *data = &_cvar_maps[(int)cvar];
+	const git_config_entry *entry;
+
+	git_config__lookup_entry(&entry, config, data->cvar_name, false);
+
+	if (!entry)
+		*out = data->default_value;
+	else if (data->maps)
+		error = git_config_lookup_map_value(
+			out, data->maps, data->map_count, entry->value);
+	else
+		error = git_config_parse_bool(out, entry->value);
+
+	return error;
+}
+
 int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar)
 {
 	*out = repo->cvar_cache[(int)cvar];
 
 	if (*out == GIT_CVAR_NOT_CACHED) {
-		struct map_data *data = &_cvar_maps[(int)cvar];
-		git_config *config;
 		int error;
+		git_config *config;
 
-		error = git_repository_config__weakptr(&config, repo);
-		if (error < 0)
-			return error;
-
-		if (data->maps)
-			error = git_config_get_mapped(
-				out, config, data->cvar_name, data->maps, data->map_count);
-		else
-			error = git_config_get_bool(out, config, data->cvar_name);
-
-		if (error == GIT_ENOTFOUND) {
-			giterr_clear();
-			*out = data->default_value;
-		}
-		else if (error < 0)
+		if ((error = git_repository_config__weakptr(&config, repo)) < 0 ||
+			(error = git_config__cvar(out, config, cvar)) < 0)
 			return error;
 
 		repo->cvar_cache[(int)cvar] = *out;
diff --git a/src/config_file.c b/src/config_file.c
index 15c8de49c..56271144b 100644
--- a/src/config_file.c
+++ b/src/config_file.c
@@ -7,8 +7,8 @@
 
 #include "common.h"
 #include "config.h"
-#include "fileops.h"
 #include "filebuf.h"
+#include "sysdir.h"
 #include "buffer.h"
 #include "buf_text.h"
 #include "git2/config.h"
@@ -26,7 +26,7 @@ GIT__USE_STRMAP;
 typedef struct cvar_t {
 	struct cvar_t *next;
 	git_config_entry *entry;
-	int included; /* whether this is part of [include] */
+	bool included; /* whether this is part of [include] */
 } cvar_t;
 
 typedef struct git_config_file_iter {
@@ -87,28 +87,54 @@ struct reader {
 };
 
 typedef struct {
+	git_atomic refcount;
+	git_strmap *values;
+} refcounted_strmap;
+
+typedef struct {
 	git_config_backend parent;
+	/* mutex to coordinate accessing the values */
+	git_mutex values_mutex;
+	refcounted_strmap *values;
+	int readonly;
+} diskfile_header;
 
-	git_strmap *values;
+typedef struct {
+	diskfile_header header;
+
+	git_config_level_t level;
 
 	git_array_t(struct reader) readers;
 
 	char  *file_path;
-
-	git_config_level_t level;
 } diskfile_backend;
 
-static int config_parse(diskfile_backend *cfg_file, struct reader *reader, git_config_level_t level, int depth);
+typedef struct {
+	diskfile_header header;
+
+	diskfile_backend *snapshot_from;
+} diskfile_readonly_backend;
+
+static int config_parse(git_strmap *values, diskfile_backend *cfg_file, struct reader *reader, git_config_level_t level, int depth);
 static int parse_variable(struct reader *reader, char **var_name, char **var_value);
 static int config_write(diskfile_backend *cfg, const char *key, const regex_t *preg, const char *value);
 static char *escape_value(const char *ptr);
 
+int git_config_file__snapshot(git_config_backend **out, diskfile_backend *in);
+static int config_snapshot(git_config_backend **out, git_config_backend *in);
+
 static void set_parse_error(struct reader *reader, int col, const char *error_str)
 {
 	giterr_set(GITERR_CONFIG, "Failed to parse config file: %s (in %s:%d, column %d)",
 		error_str, reader->file_path, reader->line_number, col);
 }
 
+static int config_error_readonly(void)
+{
+	giterr_set(GITERR_CONFIG, "this backend is read-only");
+	return -1;
+}
+
 static void cvar_free(cvar_t *var)
 {
 	if (var == NULL)
@@ -120,18 +146,6 @@ static void cvar_free(cvar_t *var)
 	git__free(var);
 }
 
-static int cvar_length(cvar_t *var)
-{
-	int length = 0;
-
-	while (var) {
-		length++;
-		var = var->next;
-	}
-
-	return length;
-}
-
 int git_config_file_normalize_section(char *start, char *end)
 {
 	char *scan;
@@ -155,6 +169,30 @@ int git_config_file_normalize_section(char *start, char *end)
 	return 0;
 }
 
+/* Add or append the new config option */
+static int append_entry(git_strmap *values, cvar_t *var)
+{
+	git_strmap_iter pos;
+	cvar_t *existing;
+	int error = 0;
+
+	pos = git_strmap_lookup_index(values, var->entry->name);
+	if (!git_strmap_valid_index(values, pos)) {
+		git_strmap_insert(values, var->entry->name, var, error);
+	} else {
+		existing = git_strmap_value_at(values, pos);
+		while (existing->next != NULL) {
+			existing = existing->next;
+		}
+		existing->next = var;
+	}
+
+	if (error > 0)
+		error = 0;
+
+	return error;
+}
+
 static void free_vars(git_strmap *values)
 {
 	cvar_t *var = NULL;
@@ -172,6 +210,55 @@ static void free_vars(git_strmap *values)
 	git_strmap_free(values);
 }
 
+static void refcounted_strmap_free(refcounted_strmap *map)
+{
+	if (!map)
+		return;
+
+	if (git_atomic_dec(&map->refcount) != 0)
+		return;
+
+	free_vars(map->values);
+	git__free(map);
+}
+
+/**
+ * Take the current values map from the backend and increase its
+ * refcount. This is its own function to make sure we use the mutex to
+ * avoid the map pointer from changing under us.
+ */
+static refcounted_strmap *refcounted_strmap_take(diskfile_header *h)
+{
+	refcounted_strmap *map;
+
+	git_mutex_lock(&h->values_mutex);
+
+	map = h->values;
+	git_atomic_inc(&map->refcount);
+
+	git_mutex_unlock(&h->values_mutex);
+
+	return map;
+}
+
+static int refcounted_strmap_alloc(refcounted_strmap **out)
+{
+	refcounted_strmap *map;
+	int error;
+
+	map = git__calloc(1, sizeof(refcounted_strmap));
+	GITERR_CHECK_ALLOC(map);
+
+	git_atomic_set(&map->refcount, 1);
+
+	if ((error = git_strmap_alloc(&map->values)) < 0)
+		git__free(map);
+	else
+		*out = map;
+
+	return error;
+}
+
 static int config_open(git_config_backend *cfg, git_config_level_t level)
 {
 	int res;
@@ -180,11 +267,15 @@ static int config_open(git_config_backend *cfg, git_config_level_t level)
 
 	b->level = level;
 
-	b->values = git_strmap_alloc();
-	GITERR_CHECK_ALLOC(b->values);
+	if ((res = refcounted_strmap_alloc(&b->header.values)) < 0)
+		return res;
 
 	git_array_init(b->readers);
 	reader = git_array_alloc(b->readers);
+	if (!reader) {
+		refcounted_strmap_free(b->header.values);
+		return -1;
+	}
 	memset(reader, 0, sizeof(struct reader));
 
 	reader->file_path = git__strdup(b->file_path);
@@ -198,53 +289,73 @@ static int config_open(git_config_backend *cfg, git_config_level_t level)
 	if (res == GIT_ENOTFOUND)
 		return 0;
 
-	if (res < 0 || (res = config_parse(b, reader, level, 0)) < 0) {
-		free_vars(b->values);
-		b->values = NULL;
+	if (res < 0 || (res = config_parse(b->header.values->values, b, reader, level, 0)) < 0) {
+		refcounted_strmap_free(b->header.values);
+		b->header.values = NULL;
 	}
 
 	reader = git_array_get(b->readers, 0);
 	git_buf_free(&reader->buffer);
+
 	return res;
 }
 
+/* The meat of the refresh, as we want to use it in different places */
+static int config__refresh(git_config_backend *cfg)
+{
+	refcounted_strmap *values = NULL, *tmp;
+	diskfile_backend *b = (diskfile_backend *)cfg;
+	struct reader *reader = NULL;
+	int error = 0;
+
+	if ((error = refcounted_strmap_alloc(&values)) < 0)
+		goto out;
+
+	reader = git_array_get(b->readers, git_array_size(b->readers) - 1);
+	GITERR_CHECK_ALLOC(reader);
+
+	if ((error = config_parse(values->values, b, reader, b->level, 0)) < 0)
+		goto out;
+
+	git_mutex_lock(&b->header.values_mutex);
+
+	tmp = b->header.values;
+	b->header.values = values;
+	values = tmp;
+
+	git_mutex_unlock(&b->header.values_mutex);
+
+out:
+	refcounted_strmap_free(values);
+	if (reader)
+		git_buf_free(&reader->buffer);
+	return error;
+}
+
 static int config_refresh(git_config_backend *cfg)
 {
-	int res = 0, updated = 0, any_updated = 0;
+	int error = 0, updated = 0, any_updated = 0;
 	diskfile_backend *b = (diskfile_backend *)cfg;
-	git_strmap *old_values;
-	struct reader *reader;
+	struct reader *reader = NULL;
 	uint32_t i;
 
 	for (i = 0; i < git_array_size(b->readers); i++) {
 		reader = git_array_get(b->readers, i);
-		res = git_futils_readbuffer_updated(
-			&reader->buffer, reader->file_path, &reader->file_mtime, &reader->file_size, &updated);
+		error = git_futils_readbuffer_updated(
+			&reader->buffer, reader->file_path,
+			&reader->file_mtime, &reader->file_size, &updated);
 
-		if (res < 0)
-			return (res == GIT_ENOTFOUND) ? 0 : res;
+		if (error < 0 && error != GIT_ENOTFOUND)
+			return error;
 
 		if (updated)
 			any_updated = 1;
 	}
 
 	if (!any_updated)
-		return (res == GIT_ENOTFOUND) ? 0 : res;
-
-	/* need to reload - store old values and prep for reload */
-	old_values = b->values;
-	b->values = git_strmap_alloc();
-	GITERR_CHECK_ALLOC(b->values);
-
-	if ((res = config_parse(b, reader, b->level, 0)) < 0) {
-		free_vars(b->values);
-		b->values = old_values;
-	} else {
-		free_vars(old_values);
-	}
+		return (error == GIT_ENOTFOUND) ? 0 : error;
 
-	git_buf_free(&reader->buffer);
-	return res;
+	return config__refresh(cfg);
 }
 
 static void backend_free(git_config_backend *_backend)
@@ -262,13 +373,15 @@ static void backend_free(git_config_backend *_backend)
 	git_array_clear(backend->readers);
 
 	git__free(backend->file_path);
-	free_vars(backend->values);
+	refcounted_strmap_free(backend->header.values);
+	git_mutex_free(&backend->header.values_mutex);
 	git__free(backend);
 }
 
 static void config_iterator_free(
 	git_config_iterator* iter)
 {
+	iter->backend->free(iter->backend);
 	git__free(iter);
 }
 
@@ -277,12 +390,13 @@ static int config_iterator_next(
 	git_config_iterator *iter)
 {
 	git_config_file_iter *it = (git_config_file_iter *) iter;
-	diskfile_backend *b = (diskfile_backend *) it->parent.backend;
+	diskfile_header *h = (diskfile_header *) it->parent.backend;
+	git_strmap *values = h->values->values;
 	int err = 0;
 	cvar_t * var;
 
 	if (it->next_var == NULL) {
-		err = git_strmap_next((void**) &var, &(it->iter), b->values);
+		err = git_strmap_next((void**) &var, &(it->iter), values);
 	} else {
 		var = it->next_var;
 	}
@@ -302,15 +416,28 @@ static int config_iterator_new(
 	git_config_iterator **iter,
 	struct git_config_backend* backend)
 {
-	diskfile_backend *b = (diskfile_backend *)backend;
-	git_config_file_iter *it = git__calloc(1, sizeof(git_config_file_iter));
+	diskfile_header *h;
+	git_config_file_iter *it;
+	git_config_backend *snapshot;
+	diskfile_backend *b = (diskfile_backend *) backend;
+	int error;
 
-	GIT_UNUSED(b);
+	if ((error = config_snapshot(&snapshot, backend)) < 0)
+		return error;
+
+	if ((error = snapshot->open(snapshot, b->level)) < 0)
+		return error;
 
+	it = git__calloc(1, sizeof(git_config_file_iter));
 	GITERR_CHECK_ALLOC(it);
 
-	it->parent.backend = backend;
-	it->iter = git_strmap_begin(b->values);
+	h = (diskfile_header *)snapshot;
+
+	/* strmap_begin() is currently a macro returning 0 */
+	GIT_UNUSED(h);
+
+	it->parent.backend = snapshot;
+	it->iter = git_strmap_begin(h->values);
 	it->next_var = NULL;
 
 	it->parent.next = config_iterator_next;
@@ -322,8 +449,9 @@ static int config_iterator_new(
 
 static int config_set(git_config_backend *cfg, const char *name, const char *value)
 {
-	cvar_t *var = NULL, *old_var = NULL;
 	diskfile_backend *b = (diskfile_backend *)cfg;
+	refcounted_strmap *map;
+	git_strmap *values;
 	char *key, *esc_value = NULL;
 	khiter_t pos;
 	int rval, ret;
@@ -331,112 +459,92 @@ static int config_set(git_config_backend *cfg, const char *name, const char *val
 	if ((rval = git_config__normalize_name(name, &key)) < 0)
 		return rval;
 
+	map = refcounted_strmap_take(&b->header);
+	values = map->values;
+
 	/*
 	 * Try to find it in the existing values and update it if it
 	 * only has one value.
 	 */
-	pos = git_strmap_lookup_index(b->values, key);
-	if (git_strmap_valid_index(b->values, pos)) {
-		cvar_t *existing = git_strmap_value_at(b->values, pos);
-		char *tmp = NULL;
-
-		git__free(key);
+	pos = git_strmap_lookup_index(values, key);
+	if (git_strmap_valid_index(values, pos)) {
+		cvar_t *existing = git_strmap_value_at(values, pos);
 
 		if (existing->next != NULL) {
 			giterr_set(GITERR_CONFIG, "Multivar incompatible with simple set");
-			return -1;
+			ret = -1;
+			goto out;
 		}
 
 		/* don't update if old and new values already match */
 		if ((!existing->entry->value && !value) ||
-			(existing->entry->value && value && !strcmp(existing->entry->value, value)))
-			return 0;
-
-		if (value) {
-			tmp = git__strdup(value);
-			GITERR_CHECK_ALLOC(tmp);
-			esc_value = escape_value(value);
-			GITERR_CHECK_ALLOC(esc_value);
+			(existing->entry->value && value &&
+			 !strcmp(existing->entry->value, value))) {
+			ret = 0;
+			goto out;
 		}
-
-		git__free((void *)existing->entry->value);
-		existing->entry->value = tmp;
-
-		ret = config_write(b, existing->entry->name, NULL, esc_value);
-
-		git__free(esc_value);
-		return ret;
 	}
 
-	var = git__malloc(sizeof(cvar_t));
-	GITERR_CHECK_ALLOC(var);
-	memset(var, 0x0, sizeof(cvar_t));
-	var->entry = git__malloc(sizeof(git_config_entry));
-	GITERR_CHECK_ALLOC(var->entry);
-	memset(var->entry, 0x0, sizeof(git_config_entry));
-
-	var->entry->name = key;
-	var->entry->value = NULL;
+	/* No early returns due to sanity checks, let's write it out and refresh */
 
 	if (value) {
-		var->entry->value = git__strdup(value);
-		GITERR_CHECK_ALLOC(var->entry->value);
 		esc_value = escape_value(value);
 		GITERR_CHECK_ALLOC(esc_value);
 	}
 
-	if ((ret = config_write(b, key, NULL, esc_value)) < 0) {
-		git__free(esc_value);
-		cvar_free(var);
-		return ret;
-	}
+	if ((ret = config_write(b, key, NULL, esc_value)) < 0)
+		goto out;
 
-	git__free(esc_value);
-	git_strmap_insert2(b->values, key, var, old_var, rval);
-	if (rval < 0)
-		return -1;
-	if (old_var != NULL)
-		cvar_free(old_var);
+	ret = config_refresh(cfg);
 
-	return 0;
+out:
+	refcounted_strmap_free(map);
+	git__free(esc_value);
+	git__free(key);
+	return ret;
 }
 
 /*
  * Internal function that actually gets the value in string form
  */
-static int config_get(const git_config_backend *cfg, const char *name, const git_config_entry **out)
+static int config_get(git_config_backend *cfg, const char *key, const git_config_entry **out)
 {
-	diskfile_backend *b = (diskfile_backend *)cfg;
-	char *key;
+	diskfile_header *h = (diskfile_header *)cfg;
+	refcounted_strmap *map;
+	git_strmap *values;
 	khiter_t pos;
-	int error;
 	cvar_t *var;
+	int error;
 
-	if ((error = git_config__normalize_name(name, &key)) < 0)
+	if (!h->readonly && ((error = config_refresh(cfg)) < 0))
 		return error;
 
-	pos = git_strmap_lookup_index(b->values, key);
-	git__free(key);
+	map = refcounted_strmap_take(h);
+	values = map->values;
+
+	pos = git_strmap_lookup_index(values, key);
 
 	/* no error message; the config system will write one */
-	if (!git_strmap_valid_index(b->values, pos))
+	if (!git_strmap_valid_index(values, pos)) {
+		refcounted_strmap_free(map);
 		return GIT_ENOTFOUND;
+	}
 
-	var = git_strmap_value_at(b->values, pos);
+	var = git_strmap_value_at(values, pos);
 	while (var->next)
 		var = var->next;
 
+	refcounted_strmap_free(map);
 	*out = var->entry;
-
 	return 0;
 }
 
 static int config_set_multivar(
 	git_config_backend *cfg, const char *name, const char *regexp, const char *value)
 {
-	int replaced = 0;
-	cvar_t *var, *newvar;
 	diskfile_backend *b = (diskfile_backend *)cfg;
+	refcounted_strmap *map;
+	git_strmap *values;
 	char *key;
 	regex_t preg;
 	int result;
@@ -447,62 +555,33 @@ static int config_set_multivar(
 	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
-	pos = git_strmap_lookup_index(b->values, key);
-	if (!git_strmap_valid_index(b->values, pos)) {
+	map = refcounted_strmap_take(&b->header);
+	values = b->header.values->values;
+
+	pos = git_strmap_lookup_index(values, key);
+	if (!git_strmap_valid_index(values, pos)) {
 		/* If we don't have it, behave like a normal set */
 		result = config_set(cfg, name, value);
+		refcounted_strmap_free(map);
 		git__free(key);
 		return result;
 	}
 
-	var = git_strmap_value_at(b->values, pos);
-
 	result = regcomp(&preg, regexp, REG_EXTENDED);
 	if (result < 0) {
-		git__free(key);
 		giterr_set_regex(&preg, result);
-		regfree(&preg);
-		return -1;
-	}
-
-	for (;;) {
-		if (regexec(&preg, var->entry->value, 0, NULL, 0) == 0) {
-			char *tmp = git__strdup(value);
-			GITERR_CHECK_ALLOC(tmp);
-
-			git__free((void *)var->entry->value);
-			var->entry->value = tmp;
-			replaced = 1;
-		}
-
-		if (var->next == NULL)
-			break;
-
-		var = var->next;
+		result = -1;
+		goto out;
 	}
 
-	/* If we've reached the end of the variables and we haven't found it yet, we need to append it */
-	if (!replaced) {
-		newvar = git__malloc(sizeof(cvar_t));
-		GITERR_CHECK_ALLOC(newvar);
-		memset(newvar, 0x0, sizeof(cvar_t));
-		newvar->entry = git__malloc(sizeof(git_config_entry));
-		GITERR_CHECK_ALLOC(newvar->entry);
-		memset(newvar->entry, 0x0, sizeof(git_config_entry));
+	/* If we do have it, set call config_write() and reload */
+	if ((result = config_write(b, key, &preg, value)) < 0)
+		goto out;
 
-		newvar->entry->name = git__strdup(var->entry->name);
-		GITERR_CHECK_ALLOC(newvar->entry->name);
-
-		newvar->entry->value = git__strdup(value);
-		GITERR_CHECK_ALLOC(newvar->entry->value);
-
-		newvar->entry->level = var->entry->level;
-
-		var->next = newvar;
-	}
-
-	result = config_write(b, key, &preg, value);
+	result = config_refresh(cfg);
 
+out:
+	refcounted_strmap_free(map);
 	git__free(key);
 	regfree(&preg);
 
@@ -513,6 +592,7 @@ static int config_delete(git_config_backend *cfg, const char *name)
 {
 	cvar_t *var;
 	diskfile_backend *b = (diskfile_backend *)cfg;
+	refcounted_strmap *map;	git_strmap *values;
 	char *key;
 	int result;
 	khiter_t pos;
@@ -520,35 +600,37 @@ static int config_delete(git_config_backend *cfg, const char *name)
 	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
-	pos = git_strmap_lookup_index(b->values, key);
+	map = refcounted_strmap_take(&b->header);
+	values = b->header.values->values;
+
+	pos = git_strmap_lookup_index(values, key);
 	git__free(key);
 
-	if (!git_strmap_valid_index(b->values, pos)) {
+	if (!git_strmap_valid_index(values, pos)) {
+		refcounted_strmap_free(map);
 		giterr_set(GITERR_CONFIG, "Could not find key '%s' to delete", name);
 		return GIT_ENOTFOUND;
 	}
 
-	var = git_strmap_value_at(b->values, pos);
+	var = git_strmap_value_at(values, pos);
+	refcounted_strmap_free(map);
 
 	if (var->next != NULL) {
 		giterr_set(GITERR_CONFIG, "Cannot delete multivar with a single delete");
 		return -1;
 	}
 
-	git_strmap_delete_at(b->values, pos);
-
-	result = config_write(b, var->entry->name, NULL, NULL);
+	if ((result = config_write(b, var->entry->name, NULL, NULL)) < 0)
+		return result;
 
-	cvar_free(var);
-	return result;
+	return config_refresh(cfg);
 }
 
 static int config_delete_multivar(git_config_backend *cfg, const char *name, const char *regexp)
 {
-	cvar_t *var, *prev = NULL, *new_head = NULL;
-	cvar_t **to_delete;
-	int to_delete_idx;
 	diskfile_backend *b = (diskfile_backend *)cfg;
+	refcounted_strmap *map;
+	git_strmap *values;
 	char *key;
 	regex_t preg;
 	int result;
@@ -557,66 +639,45 @@ static int config_delete_multivar(git_config_backend *cfg, const char *name, con
 	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
-	pos = git_strmap_lookup_index(b->values, key);
+	map = refcounted_strmap_take(&b->header);
+	values = b->header.values->values;
 
-	if (!git_strmap_valid_index(b->values, pos)) {
-		giterr_set(GITERR_CONFIG, "Could not find key '%s' to delete", name);
+	pos = git_strmap_lookup_index(values, key);
+
+	if (!git_strmap_valid_index(values, pos)) {
+		refcounted_strmap_free(map);
 		git__free(key);
+		giterr_set(GITERR_CONFIG, "Could not find key '%s' to delete", name);
 		return GIT_ENOTFOUND;
 	}
 
-	var = git_strmap_value_at(b->values, pos);
+	refcounted_strmap_free(map);
 
 	result = regcomp(&preg, regexp, REG_EXTENDED);
 	if (result < 0) {
-		git__free(key);
 		giterr_set_regex(&preg, result);
-		regfree(&preg);
-		return -1;
-	}
-
-	to_delete = git__calloc(cvar_length(var), sizeof(cvar_t *));
-	GITERR_CHECK_ALLOC(to_delete);
-	to_delete_idx = 0;
-
-	while (var != NULL) {
-		cvar_t *next = var->next;
-
-		if (regexec(&preg, var->entry->value, 0, NULL, 0) == 0) {
-			// If we are past the head, reattach previous node to next one,
-			// otherwise set the new head for the strmap.
-			if (prev != NULL) {
-				prev->next = next;
-			} else {
-				new_head = next;
-			}
-
-			to_delete[to_delete_idx++] = var;
-		} else {
-			prev = var;
-		}
-
-		var = next;
+		result = -1;
+		goto out;
 	}
 
-	if (new_head != NULL) {
-		git_strmap_set_value_at(b->values, pos, new_head);
-	} else {
-		git_strmap_delete_at(b->values, pos);
-	}
+	if ((result = config_write(b, key, &preg, NULL)) < 0)
+		goto out;
 
-	if (to_delete_idx > 0)
-		result = config_write(b, key, &preg, NULL);
-
-	while (to_delete_idx-- > 0)
-		cvar_free(to_delete[to_delete_idx]);
+	result = config_refresh(cfg);
 
+out:
 	git__free(key);
-	git__free(to_delete);
 	regfree(&preg);
 	return result;
 }
 
+static int config_snapshot(git_config_backend **out, git_config_backend *in)
+{
+	diskfile_backend *b = (diskfile_backend *) in;
+
+	return git_config_file__snapshot(out, b);
+}
+
 int git_config_file__ondisk(git_config_backend **out, const char *path)
 {
 	diskfile_backend *backend;
@@ -624,20 +685,122 @@ int git_config_file__ondisk(git_config_backend **out, const char *path)
 	backend = git__calloc(1, sizeof(diskfile_backend));
 	GITERR_CHECK_ALLOC(backend);
 
-	backend->parent.version = GIT_CONFIG_BACKEND_VERSION;
+	backend->header.parent.version = GIT_CONFIG_BACKEND_VERSION;
+	git_mutex_init(&backend->header.values_mutex);
 
 	backend->file_path = git__strdup(path);
 	GITERR_CHECK_ALLOC(backend->file_path);
 
-	backend->parent.open = config_open;
-	backend->parent.get = config_get;
-	backend->parent.set = config_set;
-	backend->parent.set_multivar = config_set_multivar;
-	backend->parent.del = config_delete;
-	backend->parent.del_multivar = config_delete_multivar;
-	backend->parent.iterator = config_iterator_new;
-	backend->parent.refresh = config_refresh;
-	backend->parent.free = backend_free;
+	backend->header.parent.open = config_open;
+	backend->header.parent.get = config_get;
+	backend->header.parent.set = config_set;
+	backend->header.parent.set_multivar = config_set_multivar;
+	backend->header.parent.del = config_delete;
+	backend->header.parent.del_multivar = config_delete_multivar;
+	backend->header.parent.iterator = config_iterator_new;
+	backend->header.parent.refresh = config_refresh;
+	backend->header.parent.snapshot = config_snapshot;
+	backend->header.parent.free = backend_free;
+
+	*out = (git_config_backend *)backend;
+
+	return 0;
+}
+
+static int config_set_readonly(git_config_backend *cfg, const char *name, const char *value)
+{
+	GIT_UNUSED(cfg);
+	GIT_UNUSED(name);
+	GIT_UNUSED(value);
+
+	return config_error_readonly();
+}
+
+static int config_set_multivar_readonly(
+	git_config_backend *cfg, const char *name, const char *regexp, const char *value)
+{
+	GIT_UNUSED(cfg);
+	GIT_UNUSED(name);
+	GIT_UNUSED(regexp);
+	GIT_UNUSED(value);
+
+	return config_error_readonly();
+}
+
+static int config_delete_multivar_readonly(git_config_backend *cfg, const char *name, const char *regexp)
+{
+	GIT_UNUSED(cfg);
+	GIT_UNUSED(name);
+	GIT_UNUSED(regexp);
+
+	return config_error_readonly();
+}
+
+static int config_delete_readonly(git_config_backend *cfg, const char *name)
+{
+	GIT_UNUSED(cfg);
+	GIT_UNUSED(name);
+
+	return config_error_readonly();
+}
+
+static int config_refresh_readonly(git_config_backend *cfg)
+{
+	GIT_UNUSED(cfg);
+
+	return config_error_readonly();
+}
+
+static void backend_readonly_free(git_config_backend *_backend)
+{
+	diskfile_backend *backend = (diskfile_backend *)_backend;
+
+	if (backend == NULL)
+		return;
+
+	refcounted_strmap_free(backend->header.values);
+	git_mutex_free(&backend->header.values_mutex);
+	git__free(backend);
+}
+
+static int config_readonly_open(git_config_backend *cfg, git_config_level_t level)
+{
+	diskfile_readonly_backend *b = (diskfile_readonly_backend *) cfg;
+	diskfile_backend *src = b->snapshot_from;
+	refcounted_strmap *src_map;
+
+	/* We're just copying data, don't care about the level */
+	GIT_UNUSED(level);
+
+	src_map = refcounted_strmap_take(&src->header);
+	b->header.values = src_map;
+
+	return 0;
+}
+
+int git_config_file__snapshot(git_config_backend **out, diskfile_backend *in)
+{
+	diskfile_readonly_backend *backend;
+
+	backend = git__calloc(1, sizeof(diskfile_readonly_backend));
+	GITERR_CHECK_ALLOC(backend);
+
+	backend->header.parent.version = GIT_CONFIG_BACKEND_VERSION;
+	git_mutex_init(&backend->header.values_mutex);
+
+	backend->snapshot_from = in;
+
+	backend->header.readonly = 1;
+	backend->header.parent.version = GIT_CONFIG_BACKEND_VERSION;
+	backend->header.parent.open = config_readonly_open;
+	backend->header.parent.get = config_get;
+	backend->header.parent.set = config_set_readonly;
+	backend->header.parent.set_multivar = config_set_multivar_readonly;
+	backend->header.parent.del = config_delete_readonly;
+	backend->header.parent.del_multivar = config_delete_multivar_readonly;
+	backend->header.parent.iterator = config_iterator_new;
+	backend->header.parent.refresh = config_refresh_readonly;
+	backend->header.parent.free = backend_readonly_free;
 
 	*out = (git_config_backend *)backend;
 
@@ -1012,21 +1175,20 @@ static int included_path(git_buf *out, const char *dir, const char *path)
 {
 	/* From the user's home */
 	if (path[0] == '~' && path[1] == '/')
-		return git_futils_find_global_file(out, &path[1]);
+		return git_sysdir_find_global_file(out, &path[1]);
 
 	return git_path_join_unrooted(out, path, dir, NULL);
 }
 
-static int config_parse(diskfile_backend *cfg_file, struct reader *reader, git_config_level_t level, int depth)
+static int config_parse(git_strmap *values, diskfile_backend *cfg_file, struct reader *reader, git_config_level_t level, int depth)
 {
 	int c;
 	char *current_section = NULL;
 	char *var_name;
 	char *var_value;
-	cvar_t *var, *existing;
+	cvar_t *var;
 	git_buf buf = GIT_BUF_INIT;
 	int result = 0;
-	khiter_t pos;
 	uint32_t reader_idx;
 
 	if (depth >= MAX_INCLUDE_DEPTH) {
@@ -1081,29 +1243,23 @@ static int config_parse(diskfile_backend *cfg_file, struct reader *reader, git_c
 			git_buf_printf(&buf, "%s.%s", current_section, var_name);
 			git__free(var_name);
 
-			if (git_buf_oom(&buf))
+			if (git_buf_oom(&buf)) {
+				git__free(var_value);
 				return -1;
+			}
 
 			var->entry->name = git_buf_detach(&buf);
 			var->entry->value = var_value;
 			var->entry->level = level;
 			var->included = !!depth;
 
-			/* Add or append the new config option */
-			pos = git_strmap_lookup_index(cfg_file->values, var->entry->name);
-			if (!git_strmap_valid_index(cfg_file->values, pos)) {
-				git_strmap_insert(cfg_file->values, var->entry->name, var, result);
-				if (result < 0)
-					break;
+
+			if ((result = append_entry(values, var)) < 0)
+				break;
+			else
 				result = 0;
-			} else {
-				existing = git_strmap_value_at(cfg_file->values, pos);
-				while (existing->next != NULL) {
-					existing = existing->next;
-				}
-				existing->next = var;
-			}
 
+			/* Add or append the new config option */
 			if (!git__strcmp(var->entry->name, "include.path")) {
 				struct reader *r;
 				git_buf path = GIT_BUF_INIT;
@@ -1132,7 +1288,7 @@ static int config_parse(diskfile_backend *cfg_file, struct reader *reader, git_c
 									    &r->file_size, NULL)) < 0)
 					break;
 
-				result = config_parse(cfg_file, r, level, depth+1);
+				result = config_parse(values, cfg_file, r, level, depth+1);
 				r = git_array_get(cfg_file->readers, index);
 				git_buf_free(&r->buffer);
 
@@ -1375,7 +1531,7 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 			 * this, but instead we'll handle it gracefully with an error. */
 			if (value == NULL) {
 				giterr_set(GITERR_CONFIG,
-					"Race condition when writing a config file (a cvar has been removed)");
+					"race condition when writing a config file (a cvar has been removed)");
 				goto rewrite_fail;
 			}
 
diff --git a/src/config_file.h b/src/config_file.h
index d4a1a4061..fcccbd5cc 100644
--- a/src/config_file.h
+++ b/src/config_file.h
@@ -16,7 +16,8 @@ GIT_INLINE(int) git_config_file_open(git_config_backend *cfg, unsigned int level
 
 GIT_INLINE(void) git_config_file_free(git_config_backend *cfg)
 {
-	cfg->free(cfg);
+	if (cfg)
+		cfg->free(cfg);
 }
 
 GIT_INLINE(int) git_config_file_get_string(
diff --git a/src/crlf.c b/src/crlf.c
index b25c02cce..821e04eb2 100644
--- a/src/crlf.c
+++ b/src/crlf.c
@@ -21,6 +21,7 @@ struct crlf_attrs {
 	int crlf_action;
 	int eol;
 	int auto_crlf;
+	int safe_crlf;
 };
 
 struct crlf_filter {
@@ -101,7 +102,7 @@ static int has_cr_in_index(const git_filter_source *src)
 	if (!S_ISREG(entry->mode)) /* don't crlf filter non-blobs */
 		return true;
 
-	if (git_blob_lookup(&blob, repo, &entry->oid) < 0)
+	if (git_blob_lookup(&blob, repo, &entry->id) < 0)
 		return false;
 
 	blobcontent = git_blob_rawcontent(blob);
@@ -137,6 +138,26 @@ static int crlf_apply_to_odb(
 		if (git_buf_text_gather_stats(&stats, from, false))
 			return GIT_PASSTHROUGH;
 
+		/* If there are no CR characters to filter out, then just pass */
+		if (!stats.cr)
+			return GIT_PASSTHROUGH;
+
+		/* If safecrlf is enabled, sanity-check the result. */
+		if (stats.cr != stats.crlf || stats.lf != stats.crlf) {
+			switch (ca->safe_crlf) {
+			case GIT_SAFE_CRLF_FAIL:
+				giterr_set(
+					GITERR_FILTER, "LF would be replaced by CRLF in '%s'",
+					git_filter_source_path(src));
+				return -1;
+			case GIT_SAFE_CRLF_WARN:
+				/* TODO: issue warning when warning API is available */;
+				break;
+			default:
+				break;
+			}
+		}
+
 		/*
 		 * We're currently not going to even try to convert stuff
 		 * that has bare CR characters. Does anybody do that crazy
@@ -218,24 +239,11 @@ static int crlf_apply_to_workdir(
 	if (!workdir_ending)
 		return -1;
 
-	if (!strcmp("\n", workdir_ending)) {
-		if (ca->crlf_action == GIT_CRLF_GUESS && ca->auto_crlf)
-			return GIT_PASSTHROUGH;
-
-		if (git_buf_find(from, '\r') < 0)
-			return GIT_PASSTHROUGH;
-
-		if (git_buf_text_crlf_to_lf(to, from) < 0)
-			return -1;
-	} else {
-		/* only other supported option is lf->crlf conversion */
-		assert(!strcmp("\r\n", workdir_ending));
-
-		if (git_buf_text_lf_to_crlf(to, from) < 0)
-			return -1;
-	}
+	/* only LF->CRLF conversion is supported, do nothing on LF platforms */
+	if (strcmp(workdir_ending, "\r\n") != 0)
+		return GIT_PASSTHROUGH;
 
-	return 0;
+	return git_buf_text_lf_to_crlf(to, from);
 }
 
 static int crlf_check(
@@ -269,7 +277,10 @@ static int crlf_check(
 	if (ca.crlf_action == GIT_CRLF_BINARY)
 		return GIT_PASSTHROUGH;
 
-	if (ca.crlf_action == GIT_CRLF_GUESS) {
+	if (ca.crlf_action == GIT_CRLF_GUESS ||
+		(ca.crlf_action == GIT_CRLF_AUTO &&
+		git_filter_source_mode(src) == GIT_FILTER_SMUDGE)) {
+
 		error = git_repository__cvar(
 			&ca.auto_crlf, git_filter_source_repo(src), GIT_CVAR_AUTO_CRLF);
 		if (error < 0)
@@ -277,6 +288,22 @@ static int crlf_check(
 
 		if (ca.auto_crlf == GIT_AUTO_CRLF_FALSE)
 			return GIT_PASSTHROUGH;
+
+		if (ca.auto_crlf == GIT_AUTO_CRLF_INPUT &&
+			git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
+			return GIT_PASSTHROUGH;
+	}
+
+	if (git_filter_source_mode(src) == GIT_FILTER_CLEAN) {
+		error = git_repository__cvar(
+			&ca.safe_crlf, git_filter_source_repo(src), GIT_CVAR_SAFE_CRLF);
+		if (error < 0)
+			return error;
+
+		/* downgrade FAIL to WARN if ALLOW_UNSAFE option is used */
+		if ((git_filter_source_options(src) & GIT_FILTER_OPT_ALLOW_UNSAFE) &&
+			ca.safe_crlf == GIT_SAFE_CRLF_FAIL)
+			ca.safe_crlf = GIT_SAFE_CRLF_WARN;
 	}
 
 	*payload = git__malloc(sizeof(ca));
diff --git a/src/date.c b/src/date.c
index 7849c2f02..0e1b31aee 100644
--- a/src/date.c
+++ b/src/date.c
@@ -874,3 +874,31 @@ int git__date_parse(git_time_t *out, const char *date)
 	*out = approxidate_str(date, time_sec, &error_ret);
    return error_ret;
 }
+
+int git__date_rfc2822_fmt(char *out, size_t len, const git_time *date)
+{
+	int written;
+	struct tm gmt;
+	time_t t;
+
+	assert(out && date);
+
+	t = (time_t) (date->time + date->offset * 60);
+
+	if (p_gmtime_r (&t, &gmt) == NULL)
+		return -1;
+
+	written = p_snprintf(out, len, "%.3s, %u %.3s %.4u %02u:%02u:%02u %+03d%02d",
+		weekday_names[gmt.tm_wday],
+		gmt.tm_mday,
+		month_names[gmt.tm_mon],
+		gmt.tm_year + 1900,
+		gmt.tm_hour, gmt.tm_min, gmt.tm_sec,
+		date->offset / 60, date->offset % 60);
+
+	if (written < 0 || (written > (int) len - 1))
+		return -1;
+
+	return 0;
+}
+
diff --git a/src/delta.c b/src/delta.c
index b3435ba87..8375a2c4d 100644
--- a/src/delta.c
+++ b/src/delta.c
@@ -144,7 +144,7 @@ git_delta_create_index(const void *buf, unsigned long bufsize)
 		entries = 0xfffffffeU / RABIN_WINDOW;
 	}
 	hsize = entries / 4;
-	for (i = 4; (1u << i) < hsize && i < 31; i++);
+	for (i = 4; i < 31 && (1u << i) < hsize; i++);
 	hsize = 1 << i;
 	hmask = hsize - 1;
 
diff --git a/src/diff.c b/src/diff.c
index 4c33a0213..325c599e0 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -49,16 +49,29 @@ static git_diff_delta *diff_delta__alloc(
 	return delta;
 }
 
-static int diff_notify(
-	const git_diff *diff,
-	const git_diff_delta *delta,
-	const char *matched_pathspec)
+static int diff_insert_delta(
+	git_diff *diff, git_diff_delta *delta, const char *matched_pathspec)
 {
-	if (!diff->opts.notify_cb)
-		return 0;
+	int error = 0;
+
+	if (diff->opts.notify_cb) {
+		error = diff->opts.notify_cb(
+			diff, delta, matched_pathspec, diff->opts.notify_payload);
+
+		if (error) {
+			git__free(delta);
+
+			if (error > 0)	/* positive value means to skip this delta */
+				return 0;
+			else			/* negative value means to cancel diff */
+				return giterr_set_after_callback_function(error, "git_diff");
+		}
+	}
+
+	if ((error = git_vector_insert(&diff->deltas, delta)) < 0)
+		git__free(delta);
 
-	return diff->opts.notify_cb(
-		diff, delta, matched_pathspec, diff->opts.notify_payload);
+	return error;
 }
 
 static int diff_delta__from_one(
@@ -68,7 +81,6 @@ static int diff_delta__from_one(
 {
 	git_diff_delta *delta;
 	const char *matched_pathspec;
-	int notify_res;
 
 	if ((entry->flags & GIT_IDXENTRY_VALID) != 0)
 		return 0;
@@ -98,34 +110,25 @@ static int diff_delta__from_one(
 	if (delta->status == GIT_DELTA_DELETED) {
 		delta->old_file.mode = entry->mode;
 		delta->old_file.size = entry->file_size;
-		git_oid_cpy(&delta->old_file.oid, &entry->oid);
+		git_oid_cpy(&delta->old_file.id, &entry->id);
 	} else /* ADDED, IGNORED, UNTRACKED */ {
 		delta->new_file.mode = entry->mode;
 		delta->new_file.size = entry->file_size;
-		git_oid_cpy(&delta->new_file.oid, &entry->oid);
+		git_oid_cpy(&delta->new_file.id, &entry->id);
 	}
 
-	delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+	delta->old_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
 	if (delta->status == GIT_DELTA_DELETED ||
-		!git_oid_iszero(&delta->new_file.oid))
-		delta->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
-
-	notify_res = diff_notify(diff, delta, matched_pathspec);
+		!git_oid_iszero(&delta->new_file.id))
+		delta->new_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
-	if (notify_res)
-		git__free(delta);
-	else if (git_vector_insert(&diff->deltas, delta) < 0) {
-		git__free(delta);
-		return -1;
-	}
-
-	return notify_res < 0 ? GIT_EUSER : 0;
+	return diff_insert_delta(diff, delta, matched_pathspec);
 }
 
 static int diff_delta__from_two(
 	git_diff *diff,
-	git_delta_t   status,
+	git_delta_t status,
 	const git_index_entry *old_entry,
 	uint32_t old_mode,
 	const git_index_entry *new_entry,
@@ -134,7 +137,6 @@ static int diff_delta__from_two(
 	const char *matched_pathspec)
 {
 	git_diff_delta *delta;
-	int notify_res;
 	const char *canonical_path = old_entry->path;
 
 	if (status == GIT_DELTA_UNMODIFIED &&
@@ -154,35 +156,26 @@ static int diff_delta__from_two(
 	GITERR_CHECK_ALLOC(delta);
 	delta->nfiles = 2;
 
-	git_oid_cpy(&delta->old_file.oid, &old_entry->oid);
+	git_oid_cpy(&delta->old_file.id, &old_entry->id);
 	delta->old_file.size = old_entry->file_size;
 	delta->old_file.mode = old_mode;
-	delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+	delta->old_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
-	git_oid_cpy(&delta->new_file.oid, &new_entry->oid);
+	git_oid_cpy(&delta->new_file.id, &new_entry->id);
 	delta->new_file.size = new_entry->file_size;
 	delta->new_file.mode = new_mode;
 
 	if (new_oid) {
 		if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE))
-			git_oid_cpy(&delta->old_file.oid, new_oid);
+			git_oid_cpy(&delta->old_file.id, new_oid);
 		else
-			git_oid_cpy(&delta->new_file.oid, new_oid);
+			git_oid_cpy(&delta->new_file.id, new_oid);
 	}
 
-	if (new_oid || !git_oid_iszero(&new_entry->oid))
-		delta->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
-
-	notify_res = diff_notify(diff, delta, matched_pathspec);
+	if (new_oid || !git_oid_iszero(&new_entry->id))
+		delta->new_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
-	if (notify_res)
-		git__free(delta);
-	else if (git_vector_insert(&diff->deltas, delta) < 0) {
-		git__free(delta);
-		return -1;
-	}
-
-	return notify_res < 0 ? GIT_EUSER : 0;
+	return diff_insert_delta(diff, delta, matched_pathspec);
 }
 
 static git_diff_delta *diff_delta__last_for_item(
@@ -196,21 +189,21 @@ static git_diff_delta *diff_delta__last_for_item(
 	switch (delta->status) {
 	case GIT_DELTA_UNMODIFIED:
 	case GIT_DELTA_DELETED:
-		if (git_oid__cmp(&delta->old_file.oid, &item->oid) == 0)
+		if (git_oid__cmp(&delta->old_file.id, &item->id) == 0)
 			return delta;
 		break;
 	case GIT_DELTA_ADDED:
-		if (git_oid__cmp(&delta->new_file.oid, &item->oid) == 0)
+		if (git_oid__cmp(&delta->new_file.id, &item->id) == 0)
 			return delta;
 		break;
 	case GIT_DELTA_UNTRACKED:
 		if (diff->strcomp(delta->new_file.path, item->path) == 0 &&
-			git_oid__cmp(&delta->new_file.oid, &item->oid) == 0)
+			git_oid__cmp(&delta->new_file.id, &item->id) == 0)
 			return delta;
 		break;
 	case GIT_DELTA_MODIFIED:
-		if (git_oid__cmp(&delta->old_file.oid, &item->oid) == 0 ||
-			git_oid__cmp(&delta->new_file.oid, &item->oid) == 0)
+		if (git_oid__cmp(&delta->old_file.id, &item->id) == 0 ||
+			git_oid__cmp(&delta->new_file.id, &item->id) == 0)
 			return delta;
 		break;
 	default:
@@ -304,26 +297,6 @@ bool git_diff_delta__should_skip(
 }
 
 
-static int config_bool(git_config *cfg, const char *name, int defvalue)
-{
-	int val = defvalue;
-
-	if (git_config_get_bool(&val, cfg, name) < 0)
-		giterr_clear();
-
-	return val;
-}
-
-static int config_int(git_config *cfg, const char *name, int defvalue)
-{
-	int val = defvalue;
-
-	if (git_config_get_int32(&val, cfg, name) < 0)
-		giterr_clear();
-
-	return val;
-}
-
 static const char *diff_mnemonic_prefix(
 	git_iterator_type_t type, bool left_side)
 {
@@ -345,6 +318,31 @@ static const char *diff_mnemonic_prefix(
 	return pfx;
 }
 
+static void diff_set_ignore_case(git_diff *diff, bool ignore_case)
+{
+	if (!ignore_case) {
+		diff->opts.flags &= ~GIT_DIFF_IGNORE_CASE;
+
+		diff->strcomp    = git__strcmp;
+		diff->strncomp   = git__strncmp;
+		diff->pfxcomp    = git__prefixcmp;
+		diff->entrycomp  = git_index_entry_cmp;
+
+		git_vector_set_cmp(&diff->deltas, git_diff_delta__cmp);
+	} else {
+		diff->opts.flags |= GIT_DIFF_IGNORE_CASE;
+
+		diff->strcomp    = git__strcasecmp;
+		diff->strncomp   = git__strncasecmp;
+		diff->pfxcomp    = git__prefixcmp_icase;
+		diff->entrycomp  = git_index_entry_icmp;
+
+		git_vector_set_cmp(&diff->deltas, git_diff_delta__casecmp);
+	}
+
+	git_vector_sort(&diff->deltas);
+}
+
 static git_diff *diff_list_alloc(
 	git_repository *repo,
 	git_iterator *old_iter,
@@ -371,24 +369,10 @@ static git_diff *diff_list_alloc(
 
 	/* Use case-insensitive compare if either iterator has
 	 * the ignore_case bit set */
-	if (!git_iterator_ignore_case(old_iter) &&
-		!git_iterator_ignore_case(new_iter)) {
-		diff->opts.flags &= ~GIT_DIFF_IGNORE_CASE;
-
-		diff->strcomp    = git__strcmp;
-		diff->strncomp   = git__strncmp;
-		diff->pfxcomp    = git__prefixcmp;
-		diff->entrycomp  = git_index_entry__cmp;
-	} else {
-		diff->opts.flags |= GIT_DIFF_IGNORE_CASE;
-
-		diff->strcomp    = git__strcasecmp;
-		diff->strncomp   = git__strncasecmp;
-		diff->pfxcomp    = git__prefixcmp_icase;
-		diff->entrycomp  = git_index_entry__cmp_icase;
-
-		git_vector_set_cmp(&diff->deltas, git_diff_delta__casecmp);
-	}
+	diff_set_ignore_case(
+		diff,
+		git_iterator_ignore_case(old_iter) ||
+		git_iterator_ignore_case(new_iter));
 
 	return diff;
 }
@@ -397,7 +381,7 @@ static int diff_list_apply_options(
 	git_diff *diff,
 	const git_diff_options *opts)
 {
-	git_config *cfg;
+	git_config *cfg = NULL;
 	git_repository *repo = diff->repo;
 	git_pool *pool = &diff->pool;
 	int val;
@@ -422,20 +406,20 @@ static int diff_list_apply_options(
 		diff->opts.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
 
 	/* load config values that affect diff behavior */
-	if (git_repository_config__weakptr(&cfg, repo) < 0)
-		return -1;
+	if ((val = git_repository_config_snapshot(&cfg, repo)) < 0)
+		return val;
 
-	if (!git_repository__cvar(&val, repo, GIT_CVAR_SYMLINKS) && val)
+	if (!git_config__cvar(&val, cfg, GIT_CVAR_SYMLINKS) && val)
 		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_HAS_SYMLINKS;
 
-	if (!git_repository__cvar(&val, repo, GIT_CVAR_IGNORESTAT) && val)
+	if (!git_config__cvar(&val, cfg, GIT_CVAR_IGNORESTAT) && val)
 		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_IGNORE_STAT;
 
 	if ((diff->opts.flags & GIT_DIFF_IGNORE_FILEMODE) == 0 &&
-		!git_repository__cvar(&val, repo, GIT_CVAR_FILEMODE) && val)
+		!git_config__cvar(&val, cfg, GIT_CVAR_FILEMODE) && val)
 		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_TRUST_MODE_BITS;
 
-	if (!git_repository__cvar(&val, repo, GIT_CVAR_TRUSTCTIME) && val)
+	if (!git_config__cvar(&val, cfg, GIT_CVAR_TRUSTCTIME) && val)
 		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_TRUST_CTIME;
 
 	/* Don't set GIT_DIFFCAPS_USE_DEV - compile time option in core git */
@@ -445,7 +429,7 @@ static int diff_list_apply_options(
 
 	/* If not given explicit `opts`, check `diff.xyz` configs */
 	if (!opts) {
-		int context = config_int(cfg, "diff.context", 3);
+		int context = git_config__get_int_force(cfg, "diff.context", 3);
 		diff->opts.context_lines = context >= 0 ? (uint16_t)context : 3;
 
 		/* add other defaults here */
@@ -458,14 +442,21 @@ static int diff_list_apply_options(
 		diff->new_src = tmp_src;
 	}
 
+	/* Unset UPDATE_INDEX unless diffing workdir and index */
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_UPDATE_INDEX) &&
+		(!(diff->old_src == GIT_ITERATOR_TYPE_WORKDIR ||
+		   diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ||
+		 !(diff->old_src == GIT_ITERATOR_TYPE_INDEX ||
+		   diff->new_src == GIT_ITERATOR_TYPE_INDEX)))
+		diff->opts.flags &= ~GIT_DIFF_UPDATE_INDEX;
+
 	/* if ignore_submodules not explicitly set, check diff config */
 	if (diff->opts.ignore_submodules <= 0) {
-		const char *str;
+		const git_config_entry *entry;
+		git_config__lookup_entry(&entry, cfg, "diff.ignoresubmodules", true);
 
-		if (git_config_get_string(&str , cfg, "diff.ignoreSubmodules") < 0)
-			giterr_clear();
-		else if (str != NULL &&
-			git_submodule_parse_ignore(&diff->opts.ignore_submodules, str) < 0)
+		if (entry && git_submodule_parse_ignore(
+				&diff->opts.ignore_submodules, entry->value) < 0)
 			giterr_clear();
 	}
 
@@ -474,9 +465,9 @@ static int diff_list_apply_options(
 		const char *use_old = DIFF_OLD_PREFIX_DEFAULT;
 		const char *use_new = DIFF_NEW_PREFIX_DEFAULT;
 
-		if (config_bool(cfg, "diff.noprefix", 0)) {
+		if (git_config__get_bool_force(cfg, "diff.noprefix", 0))
 			use_old = use_new = "";
-		} else if (config_bool(cfg, "diff.mnemonicprefix", 0)) {
+		else if (git_config__get_bool_force(cfg, "diff.mnemonicprefix", 0)) {
 			use_old = diff_mnemonic_prefix(diff->old_src, true);
 			use_new = diff_mnemonic_prefix(diff->new_src, false);
 		}
@@ -490,8 +481,6 @@ static int diff_list_apply_options(
 	/* strdup prefix from pool so we're not dependent on external data */
 	diff->opts.old_prefix = diff_strdup_prefix(pool, diff->opts.old_prefix);
 	diff->opts.new_prefix = diff_strdup_prefix(pool, diff->opts.new_prefix);
-	if (!diff->opts.old_prefix || !diff->opts.new_prefix)
-		return -1;
 
 	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
 		const char *tmp_prefix = diff->opts.old_prefix;
@@ -499,19 +488,15 @@ static int diff_list_apply_options(
 		diff->opts.new_prefix  = tmp_prefix;
 	}
 
-	return 0;
+	git_config_free(cfg);
+
+	/* check strdup results for error */
+	return (!diff->opts.old_prefix || !diff->opts.new_prefix) ? -1 : 0;
 }
 
 static void diff_list_free(git_diff *diff)
 {
-	git_diff_delta *delta;
-	unsigned int i;
-
-	git_vector_foreach(&diff->deltas, i, delta) {
-		git__free(delta);
-		diff->deltas.contents[i] = NULL;
-	}
-	git_vector_free(&diff->deltas);
+	git_vector_free_deep(&diff->deltas);
 
 	git_pathspec__vfree(&diff->pathspec);
 	git_pool_clear(&diff->pool);
@@ -534,73 +519,106 @@ void git_diff_addref(git_diff *diff)
 }
 
 int git_diff__oid_for_file(
-	git_repository *repo,
+	git_oid *out,
+	git_diff *diff,
 	const char *path,
 	uint16_t  mode,
-	git_off_t size,
-	git_oid *oid)
+	git_off_t size)
 {
-	int result = 0;
+	git_index_entry entry;
+
+	memset(&entry, 0, sizeof(entry));
+	entry.mode = mode;
+	entry.file_size = size;
+	entry.path = (char *)path;
+
+	return git_diff__oid_for_entry(out, diff, &entry, NULL);
+}
+
+int git_diff__oid_for_entry(
+	git_oid *out,
+	git_diff *diff,
+	const git_index_entry *src,
+	const git_oid *update_match)
+{
+	int error = 0;
 	git_buf full_path = GIT_BUF_INIT;
+	git_index_entry entry = *src;
+	git_filter_list *fl = NULL;
+
+	memset(out, 0, sizeof(*out));
 
 	if (git_buf_joinpath(
-		&full_path, git_repository_workdir(repo), path) < 0)
+		&full_path, git_repository_workdir(diff->repo), entry.path) < 0)
 		return -1;
 
-	if (!mode) {
+	if (!entry.mode) {
 		struct stat st;
 
-		if (p_stat(path, &st) < 0) {
-			giterr_set(GITERR_OS, "Could not stat '%s'", path);
-			result = -1;
-			goto cleanup;
+		diff->perf.stat_calls++;
+
+		if (p_stat(full_path.ptr, &st) < 0) {
+			error = git_path_set_error(errno, entry.path, "stat");
+			git_buf_free(&full_path);
+			return error;
 		}
 
-		mode = st.st_mode;
-		size = st.st_size;
+		git_index_entry__init_from_stat(
+			&entry, &st, (diff->diffcaps & GIT_DIFFCAPS_TRUST_MODE_BITS) != 0);
 	}
 
 	/* calculate OID for file if possible */
-	if (S_ISGITLINK(mode)) {
+	if (S_ISGITLINK(entry.mode)) {
 		git_submodule *sm;
-		const git_oid *sm_oid;
 
-		if (!git_submodule_lookup(&sm, repo, path) &&
-			(sm_oid = git_submodule_wd_id(sm)) != NULL)
-			git_oid_cpy(oid, sm_oid);
-		else {
+		if (!git_submodule_lookup(&sm, diff->repo, entry.path)) {
+			const git_oid *sm_oid = git_submodule_wd_id(sm);
+			if (sm_oid)
+				git_oid_cpy(out, sm_oid);
+			git_submodule_free(sm);
+		} else {
 			/* if submodule lookup failed probably just in an intermediate
 			 * state where some init hasn't happened, so ignore the error
 			 */
 			giterr_clear();
-			memset(oid, 0, sizeof(*oid));
 		}
-	} else if (S_ISLNK(mode)) {
-		result = git_odb__hashlink(oid, full_path.ptr);
-	} else if (!git__is_sizet(size)) {
-		giterr_set(GITERR_OS, "File size overflow (for 32-bits) on '%s'", path);
-		result = -1;
-	} else {
-		git_filter_list *fl = NULL;
-
-		result = git_filter_list_load(&fl, repo, NULL, path, GIT_FILTER_TO_ODB);
-		if (!result) {
-			int fd = git_futils_open_ro(full_path.ptr);
-			if (fd < 0)
-				result = fd;
-			else {
-				result = git_odb__hashfd_filtered(
-					oid, fd, (size_t)size, GIT_OBJ_BLOB, fl);
-				p_close(fd);
-			}
-
-			git_filter_list_free(fl);
+	} else if (S_ISLNK(entry.mode)) {
+		error = git_odb__hashlink(out, full_path.ptr);
+		diff->perf.oid_calculations++;
+	} else if (!git__is_sizet(entry.file_size)) {
+		giterr_set(GITERR_OS, "File size overflow (for 32-bits) on '%s'",
+			entry.path);
+		error = -1;
+	} else if (!(error = git_filter_list_load(
+		&fl, diff->repo, NULL, entry.path,
+		GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE)))
+	{
+		int fd = git_futils_open_ro(full_path.ptr);
+		if (fd < 0)
+			error = fd;
+		else {
+			error = git_odb__hashfd_filtered(
+				out, fd, (size_t)entry.file_size, GIT_OBJ_BLOB, fl);
+			p_close(fd);
+			diff->perf.oid_calculations++;
 		}
+
+		git_filter_list_free(fl);
 	}
 
-cleanup:
+	/* update index for entry if requested */
+	if (!error && update_match && git_oid_equal(out, update_match)) {
+		git_index *idx;
+
+		if (!(error = git_repository_index(&idx, diff->repo))) {
+			memcpy(&entry.id, out, sizeof(entry.id));
+			error = git_index_add(idx, &entry);
+			git_index_free(idx);
+		}
+ 	}
+
 	git_buf_free(&full_path);
-	return result;
+	return error;
 }
 
 static bool diff_time_eq(
@@ -616,7 +634,6 @@ typedef struct {
 	git_iterator *new_iter;
 	const git_index_entry *oitem;
 	const git_index_entry *nitem;
-	git_buf ignore_prefix;
 } diff_in_progress;
 
 #define MODE_BITS_MASK 0000777
@@ -650,24 +667,24 @@ static int maybe_modified_submodule(
 	}
 
 	if (ign <= 0 && git_submodule_ignore(sub) == GIT_SUBMODULE_IGNORE_ALL)
-		return 0;
-
-	if ((error = git_submodule__status(
+		/* ignore it */;
+	else if ((error = git_submodule__status(
 			&sm_status, NULL, NULL, found_oid, sub, ign)) < 0)
-		return error;
+		/* return error below */;
 
 	/* check IS_WD_UNMODIFIED because this case is only used
 	 * when the new side of the diff is the working directory
 	 */
-	if (!GIT_SUBMODULE_STATUS_IS_WD_UNMODIFIED(sm_status))
+	else if (!GIT_SUBMODULE_STATUS_IS_WD_UNMODIFIED(sm_status))
 		*status = GIT_DELTA_MODIFIED;
 
 	/* now that we have a HEAD OID, check if HEAD moved */
-	if ((sm_status & GIT_SUBMODULE_STATUS_IN_WD) != 0 &&
-		!git_oid_equal(&info->oitem->oid, found_oid))
+	else if ((sm_status & GIT_SUBMODULE_STATUS_IN_WD) != 0 &&
+		!git_oid_equal(&info->oitem->id, found_oid))
 		*status = GIT_DELTA_MODIFIED;
 
-	return 0;
+	git_submodule_free(sub);
+	return error;
 }
 
 static int maybe_modified(
@@ -681,7 +698,9 @@ static int maybe_modified(
 	unsigned int omode = oitem->mode;
 	unsigned int nmode = nitem->mode;
 	bool new_is_workdir = (info->new_iter->type == GIT_ITERATOR_TYPE_WORKDIR);
+	bool modified_uncertain = false;
 	const char *matched_pathspec;
+	int error = 0;
 
 	if (!git_pathspec__match(
 			&diff->pathspec, oitem->path,
@@ -716,23 +735,22 @@ static int maybe_modified(
 		if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE))
 			status = GIT_DELTA_TYPECHANGE;
 		else {
-			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0 ||
-				diff_delta__from_one(diff, GIT_DELTA_ADDED, nitem) < 0)
-				return -1;
-			return 0;
+			if (!(error = diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem)))
+				error = diff_delta__from_one(diff, GIT_DELTA_ADDED, nitem);
+			return error;
 		}
 	}
 
 	/* if oids and modes match (and are valid), then file is unmodified */
-	else if (git_oid_equal(&oitem->oid, &nitem->oid) &&
+	else if (git_oid_equal(&oitem->id, &nitem->id) &&
 			 omode == nmode &&
-			 !git_oid_iszero(&oitem->oid))
+			 !git_oid_iszero(&oitem->id))
 		status = GIT_DELTA_UNMODIFIED;
 
 	/* if we have an unknown OID and a workdir iterator, then check some
 	 * circumstances that can accelerate things or need special handling
 	 */
-	else if (git_oid_iszero(&nitem->oid) && new_is_workdir) {
+	else if (git_oid_iszero(&nitem->id) && new_is_workdir) {
 		bool use_ctime = ((diff->diffcaps & GIT_DIFFCAPS_TRUST_CTIME) != 0);
 		bool use_nanos = ((diff->diffcaps & GIT_DIFFCAPS_TRUST_NANOSECS) != 0);
 
@@ -741,22 +759,28 @@ static int maybe_modified(
 		/* TODO: add check against index file st_mtime to avoid racy-git */
 
 		if (S_ISGITLINK(nmode)) {
-			if (maybe_modified_submodule(&status, &noid, diff, info) < 0)
-				return -1;
+			if ((error = maybe_modified_submodule(&status, &noid, diff, info)) < 0)
+				return error;
 		}
 
 		/* if the stat data looks different, then mark modified - this just
 		 * means that the OID will be recalculated below to confirm change
 		 */
-		else if (omode != nmode ||
-			oitem->file_size != nitem->file_size ||
-			!diff_time_eq(&oitem->mtime, &nitem->mtime, use_nanos) ||
+		else if (omode != nmode || oitem->file_size != nitem->file_size) {
+			status = GIT_DELTA_MODIFIED;
+			modified_uncertain =
+				(oitem->file_size <= 0 && nitem->file_size > 0);
+		}
+		else if (!diff_time_eq(&oitem->mtime, &nitem->mtime, use_nanos) ||
 			(use_ctime &&
 			 !diff_time_eq(&oitem->ctime, &nitem->ctime, use_nanos)) ||
 			oitem->ino != nitem->ino ||
 			oitem->uid != nitem->uid ||
 			oitem->gid != nitem->gid)
+		{
 			status = GIT_DELTA_MODIFIED;
+			modified_uncertain = true;
+		}
 	}
 
 	/* if mode is GITLINK and submodules are ignored, then skip */
@@ -767,11 +791,15 @@ static int maybe_modified(
 	/* if we got here and decided that the files are modified, but we
 	 * haven't calculated the OID of the new item, then calculate it now
 	 */
-	if (status == GIT_DELTA_MODIFIED && git_oid_iszero(&nitem->oid)) {
+	if (modified_uncertain && git_oid_iszero(&nitem->id)) {
 		if (git_oid_iszero(&noid)) {
-			if (git_diff__oid_for_file(diff->repo,
-					nitem->path, nitem->mode, nitem->file_size, &noid) < 0)
-				return -1;
+			const git_oid *update_check =
+				DIFF_FLAG_IS_SET(diff, GIT_DIFF_UPDATE_INDEX) ?
+				&oitem->id : NULL;
+
+			if ((error = git_diff__oid_for_entry(
+					&noid, diff, nitem, update_check)) < 0)
+				return error;
 		}
 
 		/* if oid matches, then mark unmodified (except submodules, where
@@ -779,7 +807,7 @@ static int maybe_modified(
 		 * matches between the index and the workdir HEAD)
 		 */
 		if (omode == nmode && !S_ISGITLINK(omode) &&
-			git_oid_equal(&oitem->oid, &noid))
+			git_oid_equal(&oitem->id, &noid))
 			status = GIT_DELTA_UNMODIFIED;
 	}
 
@@ -805,72 +833,6 @@ static bool entry_is_prefixed(
 			item->path[pathlen] == '/');
 }
 
-static int diff_scan_inside_untracked_dir(
-	git_diff *diff, diff_in_progress *info, git_delta_t *delta_type)
-{
-	int error = 0;
-	git_buf base = GIT_BUF_INIT;
-	bool is_ignored;
-
-	*delta_type = GIT_DELTA_IGNORED;
-	git_buf_sets(&base, info->nitem->path);
-
-	/* advance into untracked directory */
-	if ((error = git_iterator_advance_into(&info->nitem, info->new_iter)) < 0) {
-
-		/* skip ahead if empty */
-		if (error == GIT_ENOTFOUND) {
-			giterr_clear();
-			error = git_iterator_advance(&info->nitem, info->new_iter);
-		}
-
-		goto done;
-	}
-
-	/* look for actual untracked file */
-	while (info->nitem != NULL &&
-		   !diff->pfxcomp(info->nitem->path, git_buf_cstr(&base))) {
-		is_ignored = git_iterator_current_is_ignored(info->new_iter);
-
-		/* need to recurse into non-ignored directories */
-		if (!is_ignored && S_ISDIR(info->nitem->mode)) {
-			error = git_iterator_advance_into(&info->nitem, info->new_iter);
-
-			if (!error)
-				continue;
-			else if (error == GIT_ENOTFOUND) {
-				error = 0;
-				is_ignored = true; /* treat empty as ignored */
-			} else
-				break; /* real error, must stop */
-		}
-
-		/* found a non-ignored item - treat parent dir as untracked */
-		if (!is_ignored) {
-			*delta_type = GIT_DELTA_UNTRACKED;
-			break;
-		}
-
-		if ((error = git_iterator_advance(&info->nitem, info->new_iter)) < 0)
-			break;
-	}
-
-	/* finish off scan */
-	while (info->nitem != NULL &&
-		   !diff->pfxcomp(info->nitem->path, git_buf_cstr(&base))) {
-		if ((error = git_iterator_advance(&info->nitem, info->new_iter)) < 0)
-			break;
-	}
-
-done:
-	git_buf_free(&base);
-
-	if (error == GIT_ITEROVER)
-		error = 0;
-
-	return error;
-}
-
 static int handle_unmatched_new_item(
 	git_diff *diff, diff_in_progress *info)
 {
@@ -882,24 +844,13 @@ static int handle_unmatched_new_item(
 	/* check if this is a prefix of the other side */
 	contains_oitem = entry_is_prefixed(diff, info->oitem, nitem);
 
-	/* check if this is contained in an ignored parent directory */
-	if (git_buf_len(&info->ignore_prefix)) {
-		if (diff->pfxcomp(nitem->path, git_buf_cstr(&info->ignore_prefix)) == 0)
-			delta_type = GIT_DELTA_IGNORED;
-		else
-			git_buf_clear(&info->ignore_prefix);
-	}
+	/* update delta_type if this item is ignored */
+	if (git_iterator_current_is_ignored(info->new_iter))
+		delta_type = GIT_DELTA_IGNORED;
 
 	if (nitem->mode == GIT_FILEMODE_TREE) {
 		bool recurse_into_dir = contains_oitem;
 
-		/* if not already inside an ignored dir, check if this is ignored */
-		if (delta_type != GIT_DELTA_IGNORED &&
-			git_iterator_current_is_ignored(info->new_iter)) {
-			delta_type = GIT_DELTA_IGNORED;
-			git_buf_sets(&info->ignore_prefix, nitem->path);
-		}
-
 		/* check if user requests recursion into this type of dir */
 		recurse_into_dir = contains_oitem ||
 			(delta_type == GIT_DELTA_UNTRACKED &&
@@ -908,12 +859,14 @@ static int handle_unmatched_new_item(
 			 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS));
 
 		/* do not advance into directories that contain a .git file */
-		if (recurse_into_dir) {
+		if (recurse_into_dir && !contains_oitem) {
 			git_buf *full = NULL;
 			if (git_iterator_current_workdir_path(&full, info->new_iter) < 0)
 				return -1;
-			if (full && git_path_contains_dir(full, DOT_GIT))
+			if (full && git_path_contains(full, DOT_GIT)) {
+				/* TODO: warning if not a valid git repository */
 				recurse_into_dir = false;
+			}
 		}
 
 		/* still have to look into untracked directories to match core git -
@@ -924,9 +877,10 @@ static int handle_unmatched_new_item(
 			DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_ENABLE_FAST_UNTRACKED_DIRS))
 		{
 			git_diff_delta *last;
+			git_iterator_status_t untracked_state;
 
 			/* attempt to insert record for this directory */
-			if ((error = diff_delta__from_one(diff, delta_type, nitem)) < 0)
+			if ((error = diff_delta__from_one(diff, delta_type, nitem)) != 0)
 				return error;
 
 			/* if delta wasn't created (because of rules), just skip ahead */
@@ -935,11 +889,14 @@ static int handle_unmatched_new_item(
 				return git_iterator_advance(&info->nitem, info->new_iter);
 
 			/* iterate into dir looking for an actual untracked file */
-			if (diff_scan_inside_untracked_dir(diff, info, &delta_type) < 0)
-				return -1;
+			if ((error = git_iterator_advance_over_with_status(
+					&info->nitem, &untracked_state, info->new_iter)) < 0 &&
+				error != GIT_ITEROVER)
+				return error;
 
-			/* it iteration changed delta type, the update the record */
-			if (delta_type == GIT_DELTA_IGNORED) {
+			/* if we found nothing or just ignored items, update the record */
+			if (untracked_state == GIT_ITERATOR_STATUS_IGNORED ||
+				untracked_state == GIT_ITERATOR_STATUS_EMPTY) {
 				last->status = GIT_DELTA_IGNORED;
 
 				/* remove the record if we don't want ignored records */
@@ -970,42 +927,35 @@ static int handle_unmatched_new_item(
 		}
 	}
 
-	/* In core git, the next two checks are effectively reversed --
-	 * i.e. when an file contained in an ignored directory is explicitly
-	 * ignored, it shows up as an ignored file in the diff list, even though
-	 * other untracked files in the same directory are skipped completely.
-	 *
-	 * To me, this seems odd.  If the directory is ignored and the file is
-	 * untracked, we should skip it consistently, regardless of whether it
-	 * happens to match a pattern in the ignore file.
-	 *
-	 * To match the core git behavior, reverse the following two if checks
-	 * so that individual file ignores are checked before container
-	 * directory exclusions are used to skip the file.
-	 */
 	else if (delta_type == GIT_DELTA_IGNORED &&
-		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS))
+		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS) &&
+		git_iterator_current_tree_is_ignored(info->new_iter))
 		/* item contained in ignored directory, so skip over it */
 		return git_iterator_advance(&info->nitem, info->new_iter);
 
-	else if (git_iterator_current_is_ignored(info->new_iter))
-		delta_type = GIT_DELTA_IGNORED;
-
 	else if (info->new_iter->type != GIT_ITERATOR_TYPE_WORKDIR)
 		delta_type = GIT_DELTA_ADDED;
 
 	else if (nitem->mode == GIT_FILEMODE_COMMIT) {
-		git_submodule *sm;
-
 		/* ignore things that are not actual submodules */
-		if (git_submodule_lookup(&sm, info->repo, nitem->path) != 0) {
+		if (git_submodule_lookup(NULL, info->repo, nitem->path) != 0) {
 			giterr_clear();
 			delta_type = GIT_DELTA_IGNORED;
+
+			/* if this contains a tracked item, treat as normal TREE */
+			if (contains_oitem) {
+				error = git_iterator_advance_into(&info->nitem, info->new_iter);
+				if (error != GIT_ENOTFOUND)
+					return error;
+
+				giterr_clear();
+				return git_iterator_advance(&info->nitem, info->new_iter);
+			}
 		}
 	}
 
 	/* Actually create the record for this item if necessary */
-	if ((error = diff_delta__from_one(diff, delta_type, nitem)) < 0)
+	if ((error = diff_delta__from_one(diff, delta_type, nitem)) != 0)
 		return error;
 
 	/* If user requested TYPECHANGE records, then check for that instead of
@@ -1030,7 +980,7 @@ static int handle_unmatched_old_item(
 	git_diff *diff, diff_in_progress *info)
 {
 	int error = diff_delta__from_one(diff, GIT_DELTA_DELETED, info->oitem);
-	if (error < 0)
+	if (error != 0)
 		return error;
 
 	/* if we are generating TYPECHANGE records then check for that
@@ -1092,7 +1042,6 @@ int git_diff__from_iterators(
 	info.repo = repo;
 	info.old_iter = old_iter;
 	info.new_iter = new_iter;
-	git_buf_init(&info.ignore_prefix, 0);
 
 	/* make iterators have matching icase behavior */
 	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_CASE)) {
@@ -1139,14 +1088,14 @@ int git_diff__from_iterators(
 			error = 0;
 	}
 
+	diff->perf.stat_calls += old_iter->stat_calls + new_iter->stat_calls;
+
 cleanup:
 	if (!error)
 		*diff_ptr = diff;
 	else
 		git_diff_free(diff);
 
-	git_buf_free(&info.ignore_prefix);
-
 	return error;
 }
 
@@ -1205,39 +1154,25 @@ int git_diff_tree_to_index(
 	const git_diff_options *opts)
 {
 	int error = 0;
-	bool reset_index_ignore_case = false;
+	bool index_ignore_case = false;
 
 	assert(diff && repo);
 
 	if (!index && (error = diff_load_index(&index, repo)) < 0)
 		return error;
 
-	if (index->ignore_case) {
-		git_index__set_ignore_case(index, false);
-		reset_index_ignore_case = true;
-	}
+	index_ignore_case = index->ignore_case;
 
 	DIFF_FROM_ITERATORS(
-		git_iterator_for_tree(&a, old_tree, 0, pfx, pfx),
-		git_iterator_for_index(&b, index, 0, pfx, pfx)
+		git_iterator_for_tree(
+			&a, old_tree, GIT_ITERATOR_DONT_IGNORE_CASE, pfx, pfx),
+		git_iterator_for_index(
+			&b, index, GIT_ITERATOR_DONT_IGNORE_CASE, pfx, pfx)
 	);
 
-	if (reset_index_ignore_case) {
-		git_index__set_ignore_case(index, true);
-
-		if (!error) {
-			git_diff *d = *diff;
-
-			d->opts.flags |= GIT_DIFF_IGNORE_CASE;
-			d->strcomp    = git__strcasecmp;
-			d->strncomp   = git__strncasecmp;
-			d->pfxcomp    = git__prefixcmp_icase;
-			d->entrycomp  = git_index_entry__cmp_icase;
-
-			git_vector_set_cmp(&d->deltas, git_diff_delta__casecmp);
-			git_vector_sort(&d->deltas);
-		}
-	}
+	/* if index is in case-insensitive order, re-sort deltas to match */
+	if (!error && index_ignore_case)
+		diff_set_ignore_case(*diff, true);
 
 	return error;
 }
@@ -1261,6 +1196,9 @@ int git_diff_index_to_workdir(
 			&b, repo, GIT_ITERATOR_DONT_AUTOEXPAND, pfx, pfx)
 	);
 
+	if (!error && DIFF_FLAG_IS_SET(*diff, GIT_DIFF_UPDATE_INDEX))
+		error = git_index_write(index);
+
 	return error;
 }
 
@@ -1313,20 +1251,6 @@ int git_diff_tree_to_workdir_with_index(
 	return error;
 }
 
-int git_diff_options_init(git_diff_options *options, unsigned int version)
-{
-	git_diff_options template = GIT_DIFF_OPTIONS_INIT;
-
-	if (version != template.version) {
-		giterr_set(GITERR_INVALID,
-			"Invalid version %d for git_diff_options", (int)version);
-		return -1;
-	}
-
-	memcpy(options, &template, sizeof(*options));
-	return 0;
-}
-
 size_t git_diff_num_deltas(const git_diff *diff)
 {
 	assert(diff);
@@ -1358,13 +1282,22 @@ int git_diff_is_sorted_icase(const git_diff *diff)
 	return (diff->opts.flags & GIT_DIFF_IGNORE_CASE) != 0;
 }
 
+int git_diff_get_perfdata(git_diff_perfdata *out, const git_diff *diff)
+{
+	assert(out);
+	GITERR_CHECK_VERSION(out, GIT_DIFF_PERFDATA_VERSION, "git_diff_perfdata");
+	out->stat_calls = diff->perf.stat_calls;
+	out->oid_calculations = diff->perf.oid_calculations;
+	return 0;
+}
+
 int git_diff__paired_foreach(
 	git_diff *head2idx,
 	git_diff *idx2wd,
 	int (*cb)(git_diff_delta *h2i, git_diff_delta *i2w, void *payload),
 	void *payload)
 {
-	int cmp;
+	int cmp, error = 0;
 	git_diff_delta *h2i, *i2w;
 	size_t i, j, i_max, j_max;
 	int (*strcomp)(const char *, const char *) = git__strcmp;
@@ -1420,18 +1353,17 @@ int git_diff__paired_foreach(
 			strcomp(h2i->new_file.path, i2w->old_file.path);
 
 		if (cmp < 0) {
-			if (cb(h2i, NULL, payload))
-				return GIT_EUSER;
-			i++;
+			i++; i2w = NULL;
 		} else if (cmp > 0) {
-			if (cb(NULL, i2w, payload))
-				return GIT_EUSER;
-			j++;
+			j++; h2i = NULL;
 		} else {
-			if (cb(h2i, i2w, payload))
-				return GIT_EUSER;
 			i++; j++;
 		}
+
+		if ((error = cb(h2i, i2w, payload)) != 0) {
+			giterr_set_after_callback(error);
+			break;
+		}
 	}
 
 	/* restore case-insensitive delta sort */
@@ -1447,5 +1379,240 @@ int git_diff__paired_foreach(
 		git_vector_sort(&idx2wd->deltas);
 	}
 
+	return error;
+}
+
+int git_diff__commit(
+	git_diff **diff,
+	git_repository *repo,
+	const git_commit *commit,
+	const git_diff_options *opts)
+{
+	git_commit *parent = NULL;
+	git_diff *commit_diff = NULL;
+	git_tree *old_tree = NULL, *new_tree = NULL;
+	size_t parents;
+	int error = 0;
+
+	if ((parents = git_commit_parentcount(commit)) > 1) {
+		char commit_oidstr[GIT_OID_HEXSZ + 1];
+
+		error = -1;
+		giterr_set(GITERR_INVALID, "Commit %s is a merge commit",
+			git_oid_tostr(commit_oidstr, GIT_OID_HEXSZ + 1, git_commit_id(commit)));
+		goto on_error;
+	}
+
+	if (parents > 0)
+		if ((error = git_commit_parent(&parent, commit, 0)) < 0 ||
+			(error = git_commit_tree(&old_tree, parent)) < 0)
+				goto on_error;
+
+	if ((error = git_commit_tree(&new_tree, commit)) < 0 ||
+		(error = git_diff_tree_to_tree(&commit_diff, repo, old_tree, new_tree, opts)) < 0)
+			goto on_error;
+
+	*diff = commit_diff;
+
+on_error:
+	git_tree_free(new_tree);
+	git_tree_free(old_tree);
+	git_commit_free(parent);
+
+	return error;
+}
+
+int git_diff_format_email__append_header_tobuf(
+	git_buf *out,
+	const git_oid *id,
+	const git_signature *author,
+	const char *summary,
+	size_t patch_no,
+	size_t total_patches,
+	bool exclude_patchno_marker)
+{
+	char idstr[GIT_OID_HEXSZ + 1];
+	char date_str[GIT_DATE_RFC2822_SZ];
+	int error = 0;
+
+	git_oid_fmt(idstr, id);
+	idstr[GIT_OID_HEXSZ] = '\0';
+
+	if ((error = git__date_rfc2822_fmt(date_str, sizeof(date_str), &author->when)) < 0)
+		return error;
+
+	error = git_buf_printf(out,
+				"From %s Mon Sep 17 00:00:00 2001\n" \
+				"From: %s <%s>\n" \
+				"Date: %s\n" \
+				"Subject: ",
+				idstr,
+				author->name, author->email,
+				date_str);
+
+	if (error < 0)
+		return error;
+
+	if (!exclude_patchno_marker) {
+		if (total_patches == 1) {
+			error = git_buf_puts(out, "[PATCH] ");
+		} else {
+			error = git_buf_printf(out, "[PATCH %"PRIuZ"/%"PRIuZ"] ", patch_no, total_patches);
+		}
+
+		if (error < 0)
+			return error;
+	}
+
+	error = git_buf_printf(out, "%s\n\n", summary);
+
+	return error;
+}
+
+int git_diff_format_email__append_patches_tobuf(
+	git_buf *out,
+	git_diff *diff)
+{
+	size_t i, deltas;
+	int error = 0;
+
+	deltas = git_diff_num_deltas(diff);
+
+	for (i = 0; i < deltas; ++i) {
+		git_patch *patch = NULL;
+
+		if ((error = git_patch_from_diff(&patch, diff, i)) >= 0)
+			error = git_patch_to_buf(out, patch);
+
+		git_patch_free(patch);
+
+		if (error < 0)
+			break;
+	}
+
+	return error;
+}
+
+int git_diff_format_email(
+	git_buf *out,
+	git_diff *diff,
+	const git_diff_format_email_options *opts)
+{
+	git_diff_stats *stats = NULL;
+	char *summary = NULL, *loc = NULL;
+	bool ignore_marker;
+	unsigned int format_flags = 0;
+	int error;
+
+	assert(out && diff && opts);
+	assert(opts->summary && opts->id && opts->author);
+
+	GITERR_CHECK_VERSION(opts, GIT_DIFF_FORMAT_EMAIL_OPTIONS_VERSION, "git_format_email_options");
+
+	if ((ignore_marker = opts->flags & GIT_DIFF_FORMAT_EMAIL_EXCLUDE_SUBJECT_PATCH_MARKER) == false) {
+		if (opts->patch_no > opts->total_patches) {
+			giterr_set(GITERR_INVALID, "patch %"PRIuZ" out of range. max %"PRIuZ, opts->patch_no, opts->total_patches);
+			return -1;
+		}
+
+		if (opts->patch_no == 0) {
+			giterr_set(GITERR_INVALID, "invalid patch no %"PRIuZ". should be >0", opts->patch_no);
+			return -1;
+		}
+	}
+
+	/* the summary we receive may not be clean.
+	 * it could potentially contain new line characters
+	 * or not be set, sanitize, */
+	if ((loc = strpbrk(opts->summary, "\r\n")) != NULL) {
+		size_t offset = 0;
+
+		if ((offset = (loc - opts->summary)) == 0) {
+			giterr_set(GITERR_INVALID, "summary is empty");
+			error = -1;
+		}
+
+		summary = git__calloc(offset + 1, sizeof(char));
+		GITERR_CHECK_ALLOC(summary);
+		strncpy(summary, opts->summary, offset);
+	}
+
+	error = git_diff_format_email__append_header_tobuf(out,
+				opts->id, opts->author, summary == NULL ? opts->summary : summary,
+				opts->patch_no, opts->total_patches, ignore_marker);
+
+	if (error < 0)
+		goto on_error;
+
+	format_flags = GIT_DIFF_STATS_FULL | GIT_DIFF_STATS_INCLUDE_SUMMARY;
+
+	if ((error = git_buf_puts(out, "---\n")) < 0 ||
+		(error = git_diff_get_stats(&stats, diff)) < 0 ||
+		(error = git_diff_stats_to_buf(out, stats, format_flags, 0)) < 0 ||
+		(error = git_buf_putc(out, '\n')) < 0 ||
+		(error = git_diff_format_email__append_patches_tobuf(out, diff)) < 0)
+			goto on_error;
+
+	error = git_buf_puts(out, "--\nlibgit2 " LIBGIT2_VERSION "\n\n");
+
+on_error:
+	git__free(summary);
+	git_diff_stats_free(stats);
+
+	return error;
+}
+
+int git_diff_commit_as_email(
+	git_buf *out,
+	git_repository *repo,
+	git_commit *commit,
+	size_t patch_no,
+	size_t total_patches,
+	git_diff_format_email_flags_t flags,
+	const git_diff_options *diff_opts)
+{
+	git_diff *diff = NULL;
+	git_diff_format_email_options opts = GIT_DIFF_FORMAT_EMAIL_OPTIONS_INIT;
+	int error;
+
+	assert (out && repo && commit);
+
+	opts.flags = flags;
+	opts.patch_no = patch_no;
+	opts.total_patches = total_patches;
+	opts.id = git_commit_id(commit);
+	opts.summary = git_commit_summary(commit);
+	opts.author = git_commit_author(commit);
+
+	if ((error = git_diff__commit(&diff, repo, commit, diff_opts)) < 0)
+		return error;
+
+	error = git_diff_format_email(out, diff, &opts);
+
+	git_diff_free(diff);
+	return error;
+}
+
+int git_diff_init_options(git_diff_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_diff_options, GIT_DIFF_OPTIONS_INIT);
+	return 0;
+}
+
+int git_diff_find_init_options(
+	git_diff_find_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_diff_find_options, GIT_DIFF_FIND_OPTIONS_INIT);
+	return 0;
+}
+
+int git_diff_format_email_init_options(
+	git_diff_format_email_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_diff_format_email_options,
+		GIT_DIFF_FORMAT_EMAIL_OPTIONS_INIT);
 	return 0;
 }
diff --git a/src/diff.h b/src/diff.h
index 2c9298a5f..3305238d0 100644
--- a/src/diff.h
+++ b/src/diff.h
@@ -8,6 +8,7 @@
 #define INCLUDE_diff_h__
 
 #include "git2/diff.h"
+#include "git2/sys/diff.h"
 #include "git2/oid.h"
 
 #include <stdio.h>
@@ -62,6 +63,7 @@ struct git_diff {
 	git_iterator_type_t old_src;
 	git_iterator_type_t new_src;
 	uint32_t diffcaps;
+	git_diff_perfdata perf;
 
 	int (*strcomp)(const char *, const char *);
 	int (*strncomp)(const char *, const char *, size_t);
@@ -90,7 +92,9 @@ extern int git_diff_delta__format_file_header(
 	int oid_strlen);
 
 extern int git_diff__oid_for_file(
-	git_repository *, const char *, uint16_t, git_off_t, git_oid *);
+	git_oid *out, git_diff *, const char *, uint16_t, git_off_t);
+extern int git_diff__oid_for_entry(
+	git_oid *out, git_diff *, const git_index_entry *, const git_oid *update);
 
 extern int git_diff__from_iterators(
 	git_diff **diff_ptr,
@@ -116,6 +120,9 @@ extern void git_diff_find_similar__hashsig_free(void *sig, void *payload);
 extern int git_diff_find_similar__calc_similarity(
 	int *score, void *siga, void *sigb, void *payload);
 
+extern int git_diff__commit(
+	git_diff **diff, git_repository *repo, const git_commit *commit, const git_diff_options *opts);
+
 /*
  * Sometimes a git_diff_file will have a zero size; this attempts to
  * fill in the size without loading the blob if possible.  If that is
@@ -134,7 +141,7 @@ GIT_INLINE(int) git_diff_file__resolve_zero_size(
 		return error;
 
 	error = git_odb__read_header_or_object(
-		odb_obj, &len, &type, odb, &file->oid);
+		odb_obj, &len, &type, odb, &file->id);
 
 	git_odb_free(odb);
 
diff --git a/src/diff_driver.c b/src/diff_driver.c
index bd5a8fbd9..c3c5f365b 100644
--- a/src/diff_driver.c
+++ b/src/diff_driver.c
@@ -14,6 +14,7 @@
 #include "strmap.h"
 #include "map.h"
 #include "buf_text.h"
+#include "config.h"
 #include "repository.h"
 
 GIT__USE_STRMAP;
@@ -25,10 +26,13 @@ typedef enum {
 	DIFF_DRIVER_PATTERNLIST = 3,
 } git_diff_driver_t;
 
+typedef struct {
+	regex_t re;
+	int flags;
+} git_diff_driver_pattern;
+
 enum {
-	DIFF_CONTEXT_FIND_NORMAL = 0,
-	DIFF_CONTEXT_FIND_ICASE = (1 << 0),
-	DIFF_CONTEXT_FIND_EXT = (1 << 1),
+	REG_NEGATE = (1 << 15) /* get out of the way of existing flags */
 };
 
 /* data for finding function context for a given file type */
@@ -36,11 +40,13 @@ struct git_diff_driver {
 	git_diff_driver_t type;
 	uint32_t binary_flags;
 	uint32_t other_flags;
-	git_array_t(regex_t) fn_patterns;
+	git_array_t(git_diff_driver_pattern) fn_patterns;
 	regex_t  word_pattern;
 	char name[GIT_FLEX_ARRAY];
 };
 
+#include "userdiff.h"
+
 struct git_diff_driver_registry {
 	git_strmap *drivers;
 };
@@ -60,7 +66,7 @@ git_diff_driver_registry *git_diff_driver_registry_new()
 	if (!reg)
 		return NULL;
 
-	if ((reg->drivers = git_strmap_alloc()) == NULL) {
+	if (git_strmap_alloc(&reg->drivers) < 0) {
 		git_diff_driver_registry_free(reg);
 		return NULL;
 	}
@@ -80,34 +86,59 @@ void git_diff_driver_registry_free(git_diff_driver_registry *reg)
 	git__free(reg);
 }
 
-static int diff_driver_add_funcname(
-	git_diff_driver *drv, const char *name, int regex_flags)
+static int diff_driver_add_patterns(
+	git_diff_driver *drv, const char *regex_str, int regex_flags)
 {
-	int error;
-	regex_t re, *re_ptr;
-
-	if ((error = regcomp(&re, name, regex_flags)) != 0) {
-		/* TODO: warning about bad regex instead of failure */
-		error = giterr_set_regex(&re, error);
-		regfree(&re);
-		return error;
+	int error = 0;
+	const char *scan, *end;
+	git_diff_driver_pattern *pat = NULL;
+	git_buf buf = GIT_BUF_INIT;
+
+	for (scan = regex_str; scan; scan = end) {
+		/* get pattern to fill in */
+		if ((pat = git_array_alloc(drv->fn_patterns)) == NULL) {
+			error = -1;
+			break;
+		}
+
+		pat->flags = regex_flags;
+		if (*scan == '!') {
+			pat->flags |= REG_NEGATE;
+			++scan;
+		}
+
+		if ((end = strchr(scan, '\n')) != NULL) {
+			error = git_buf_set(&buf, scan, end - scan);
+			end++;
+		} else {
+			error = git_buf_sets(&buf, scan);
+		}
+		if (error < 0)
+			break;
+
+		if ((error = regcomp(&pat->re, buf.ptr, regex_flags)) < 0) {
+			/* if regex fails to compile, warn? fail? */
+			error = giterr_set_regex(&pat->re, error);
+			regfree(&pat->re);
+			break;
+		}
 	}
 
-	re_ptr = git_array_alloc(drv->fn_patterns);
-	GITERR_CHECK_ALLOC(re_ptr);
+	if (error && pat != NULL)
+		(void)git_array_pop(drv->fn_patterns); /* release last item */
+	git_buf_free(&buf);
 
-	memcpy(re_ptr, &re, sizeof(re));
-	return 0;
+	return error;
 }
 
 static int diff_driver_xfuncname(const git_config_entry *entry, void *payload)
 {
-	return diff_driver_add_funcname(payload, entry->value, REG_EXTENDED);
+	return diff_driver_add_patterns(payload, entry->value, REG_EXTENDED);
 }
 
 static int diff_driver_funcname(const git_config_entry *entry, void *payload)
 {
-	return diff_driver_add_funcname(payload, entry->value, 0);
+	return diff_driver_add_patterns(payload, entry->value, 0);
 }
 
 static git_diff_driver_registry *git_repository_driver_registry(
@@ -127,34 +158,79 @@ static git_diff_driver_registry *git_repository_driver_registry(
 	return repo->diff_drivers;
 }
 
+static int git_diff_driver_builtin(
+	git_diff_driver **out,
+	git_diff_driver_registry *reg,
+	const char *driver_name)
+{
+	int error = 0;
+	git_diff_driver_definition *ddef = NULL;
+	git_diff_driver *drv = NULL;
+	size_t namelen, idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(builtin_defs); ++idx) {
+		if (!strcasecmp(driver_name, builtin_defs[idx].name)) {
+			ddef = &builtin_defs[idx];
+			break;
+		}
+	}
+	if (!ddef)
+		goto done;
+
+	namelen = strlen(ddef->name);
+
+	drv = git__calloc(1, sizeof(git_diff_driver) + namelen + 1);
+	GITERR_CHECK_ALLOC(drv);
+
+	drv->type = DIFF_DRIVER_PATTERNLIST;
+	memcpy(drv->name, ddef->name, namelen);
+
+	if (ddef->fns &&
+		(error = diff_driver_add_patterns(
+			drv, ddef->fns, ddef->flags | REG_EXTENDED)) < 0)
+		goto done;
+
+	if (ddef->words &&
+		(error = regcomp(
+			&drv->word_pattern, ddef->words, ddef->flags | REG_EXTENDED)))
+	{
+		error = giterr_set_regex(&drv->word_pattern, error);
+		goto done;
+	}
+
+	git_strmap_insert(reg->drivers, drv->name, drv, error);
+	if (error > 0)
+		error = 0;
+
+done:
+	if (error && drv)
+		git_diff_driver_free(drv);
+	else
+		*out = drv;
+
+	return error;
+}
+
 static int git_diff_driver_load(
 	git_diff_driver **out, git_repository *repo, const char *driver_name)
 {
-	int error = 0, bval;
+	int error = 0;
 	git_diff_driver_registry *reg;
-	git_diff_driver *drv;
+	git_diff_driver *drv = NULL;
 	size_t namelen = strlen(driver_name);
 	khiter_t pos;
 	git_config *cfg;
 	git_buf name = GIT_BUF_INIT;
-	const char *val;
+	const git_config_entry *ce;
 	bool found_driver = false;
 
-	reg = git_repository_driver_registry(repo);
-	if (!reg)
+	if ((reg = git_repository_driver_registry(repo)) == NULL)
 		return -1;
-	else {
-		pos = git_strmap_lookup_index(reg->drivers, driver_name);
-		if (git_strmap_valid_index(reg->drivers, pos)) {
-			*out = git_strmap_value_at(reg->drivers, pos);
-			return 0;
-		}
-	}
 
-	/* if you can't read config for repo, just use default driver */
-	if (git_repository_config__weakptr(&cfg, repo) < 0) {
-		giterr_clear();
-		return GIT_ENOTFOUND;
+	pos = git_strmap_lookup_index(reg->drivers, driver_name);
+	if (git_strmap_valid_index(reg->drivers, pos)) {
+		*out = git_strmap_value_at(reg->drivers, pos);
+		return 0;
 	}
 
 	drv = git__calloc(1, sizeof(git_diff_driver) + namelen + 1);
@@ -162,25 +238,29 @@ static int git_diff_driver_load(
 	drv->type = DIFF_DRIVER_AUTO;
 	memcpy(drv->name, driver_name, namelen);
 
+	/* if you can't read config for repo, just use default driver */
+	if (git_repository_config_snapshot(&cfg, repo) < 0) {
+		giterr_clear();
+		goto done;
+	}
+
 	if ((error = git_buf_printf(&name, "diff.%s.binary", driver_name)) < 0)
 		goto done;
-	if ((error = git_config_get_string(&val, cfg, name.ptr)) < 0) {
-		if (error != GIT_ENOTFOUND)
-			goto done;
-		/* diff.<driver>.binary unspecified, so just continue */
-		giterr_clear();
-	} else if (git_config_parse_bool(&bval, val) < 0) {
-		/* TODO: warn that diff.<driver>.binary has invalid value */
-		giterr_clear();
-	} else if (bval) {
+
+	switch (git_config__get_bool_force(cfg, name.ptr, -1)) {
+	case true:
 		/* if diff.<driver>.binary is true, just return the binary driver */
 		*out = &global_drivers[DIFF_DRIVER_BINARY];
 		goto done;
-	} else {
+	case false:
 		/* if diff.<driver>.binary is false, force binary checks off */
 		/* but still may have custom function context patterns, etc. */
 		drv->binary_flags = GIT_DIFF_FORCE_TEXT;
 		found_driver = true;
+		break;
+	default:
+		/* diff.<driver>.binary unspecified or "auto", so just continue */
+		break;
 	}
 
 	/* TODO: warn if diff.<name>.command or diff.<name>.textconv are set */
@@ -211,16 +291,16 @@ static int git_diff_driver_load(
 
 	git_buf_truncate(&name, namelen + strlen("diff.."));
 	git_buf_put(&name, "wordregex", strlen("wordregex"));
-	if ((error = git_config_get_string(&val, cfg, name.ptr)) < 0) {
-		if (error != GIT_ENOTFOUND)
-			goto done;
-		giterr_clear(); /* no diff.<driver>.wordregex, so just continue */
-	} else if ((error = regcomp(&drv->word_pattern, val, REG_EXTENDED)) != 0) {
-		/* TODO: warning about bad regex instead of failure */
-		error = giterr_set_regex(&drv->word_pattern, error);
+	if ((error = git_config__lookup_entry(&ce, cfg, name.ptr, false)) < 0)
 		goto done;
-	} else {
+	if (!ce || !ce->value)
+		/* no diff.<driver>.wordregex, so just continue */;
+	else if (!(error = regcomp(&drv->word_pattern, ce->value, REG_EXTENDED)))
 		found_driver = true;
+	else {
+		/* TODO: warn about bad regex instead of failure */
+		error = giterr_set_regex(&drv->word_pattern, error);
+		goto done;
 	}
 
 	/* TODO: look up diff.<driver>.algorithm to turn on minimal / patience
@@ -235,14 +315,19 @@ static int git_diff_driver_load(
 	git_strmap_insert(reg->drivers, drv->name, drv, error);
 	if (error < 0)
 		goto done;
+	error = 0;
 
 	*out = drv;
 
 done:
 	git_buf_free(&name);
+	git_config_free(cfg);
 
-	if (!*out)
-		*out = &global_drivers[DIFF_DRIVER_AUTO];
+	if (!*out) {
+		int error2 = git_diff_driver_builtin(out, reg, driver_name);
+		if (!error)
+			error = error2;
+	}
 
 	if (drv && drv != *out)
 		git_diff_driver_free(drv);
@@ -257,14 +342,13 @@ int git_diff_driver_lookup(
 	const char *value;
 
 	assert(out);
+	*out = NULL;
 
 	if (!repo || !path || !strlen(path))
-		goto use_auto;
-
-	if ((error = git_attr_get(&value, repo, 0, path, "diff")) < 0)
-		return error;
-
-	if (GIT_ATTR_UNSPECIFIED(value))
+		/* just use the auto value */;
+	else if ((error = git_attr_get(&value, repo, 0, path, "diff")) < 0)
+		/* return error below */;
+	else if (GIT_ATTR_UNSPECIFIED(value))
 		/* just use the auto value */;
 	else if (GIT_ATTR_FALSE(value))
 		*out = &global_drivers[DIFF_DRIVER_BINARY];
@@ -273,17 +357,16 @@ int git_diff_driver_lookup(
 
 	/* otherwise look for driver information in config and build driver */
 	else if ((error = git_diff_driver_load(out, repo, value)) < 0) {
-		if (error != GIT_ENOTFOUND)
-			return error;
-		else
+		if (error == GIT_ENOTFOUND) {
+			error = 0;
 			giterr_clear();
+		}
 	}
 
-use_auto:
 	if (!*out)
 		*out = &global_drivers[DIFF_DRIVER_AUTO];
 
-	return 0;
+	return error;
 }
 
 void git_diff_driver_free(git_diff_driver *driver)
@@ -294,7 +377,7 @@ void git_diff_driver_free(git_diff_driver *driver)
 		return;
 
 	for (i = 0; i < git_array_size(driver->fn_patterns); ++i)
-		regfree(git_array_get(driver->fn_patterns, i));
+		regfree(& git_array_get(driver->fn_patterns, i)->re);
 	git_array_clear(driver->fn_patterns);
 
 	regfree(&driver->word_pattern);
@@ -314,7 +397,11 @@ void git_diff_driver_update_options(
 int git_diff_driver_content_is_binary(
 	git_diff_driver *driver, const char *content, size_t content_len)
 {
-	const git_buf search = { (char *)content, 0, min(content_len, 4000) };
+	git_buf search;
+
+	search.ptr   = (char *)content;
+	search.size  = min(content_len, GIT_FILTER_BYTES_TO_CHECK_NUL);
+	search.asize = 0;
 
 	GIT_UNUSED(driver);
 
@@ -331,23 +418,34 @@ int git_diff_driver_content_is_binary(
 }
 
 static int diff_context_line__simple(
-	git_diff_driver *driver, const char *line, size_t line_len)
+	git_diff_driver *driver, git_buf *line)
 {
+	char firstch = line->ptr[0];
 	GIT_UNUSED(driver);
-	GIT_UNUSED(line_len);
-	return (git__isalpha(*line) || *line == '_' || *line == '$');
+	return (git__isalpha(firstch) || firstch == '_' || firstch == '$');
 }
 
 static int diff_context_line__pattern_match(
-	git_diff_driver *driver, const char *line, size_t line_len)
+	git_diff_driver *driver, git_buf *line)
 {
-	size_t i;
+	size_t i, maxi = git_array_size(driver->fn_patterns);
+	regmatch_t pmatch[2];
+
+	for (i = 0; i < maxi; ++i) {
+		git_diff_driver_pattern *pat = git_array_get(driver->fn_patterns, i);
 
-	GIT_UNUSED(line_len);
+		if (!regexec(&pat->re, line->ptr, 2, pmatch, 0)) {
+			if (pat->flags & REG_NEGATE)
+				return false;
+
+			/* use pmatch data to trim line data */
+			i = (pmatch[1].rm_so >= 0) ? 1 : 0;
+			git_buf_consume(line, git_buf_cstr(line) + pmatch[i].rm_so);
+			git_buf_truncate(line, pmatch[i].rm_eo - pmatch[i].rm_so);
+			git_buf_rtrim(line);
 
-	for (i = 0; i < git_array_size(driver->fn_patterns); ++i) {
-		if (!regexec(git_array_get(driver->fn_patterns, i), line, 0, NULL, 0))
 			return true;
+		}
 	}
 
 	return false;
@@ -369,8 +467,7 @@ static long diff_context_find(
 	if (!ctxt->line.size)
 		return -1;
 
-	if (!ctxt->match_line ||
-		!ctxt->match_line(ctxt->driver, ctxt->line.ptr, ctxt->line.size))
+	if (!ctxt->match_line || !ctxt->match_line(ctxt->driver, &ctxt->line))
 		return -1;
 
 	if (out_size > (long)ctxt->line.size)
diff --git a/src/diff_driver.h b/src/diff_driver.h
index 9d3f18660..0706dcfc5 100644
--- a/src/diff_driver.h
+++ b/src/diff_driver.h
@@ -31,7 +31,7 @@ typedef long (*git_diff_find_context_fn)(
 	const char *, long, char *, long, void *);
 
 typedef int (*git_diff_find_context_line)(
-	git_diff_driver *, const char *, size_t);
+	git_diff_driver *, git_buf *);
 
 typedef struct {
 	git_diff_driver *driver;
diff --git a/src/diff_file.c b/src/diff_file.c
index a4c8641bc..f2a1d5099 100644
--- a/src/diff_file.c
+++ b/src/diff_file.c
@@ -127,57 +127,38 @@ int git_diff_file_content__init_from_diff(
 	return diff_file_content_init_common(fc, &diff->opts);
 }
 
-int git_diff_file_content__init_from_blob(
+int git_diff_file_content__init_from_src(
 	git_diff_file_content *fc,
 	git_repository *repo,
 	const git_diff_options *opts,
-	const git_blob *blob,
+	const git_diff_file_content_src *src,
 	git_diff_file *as_file)
 {
 	memset(fc, 0, sizeof(*fc));
 	fc->repo = repo;
 	fc->file = as_file;
-	fc->blob = blob;
+	fc->blob = src->blob;
 
-	if (!blob) {
+	if (!src->blob && !src->buf) {
 		fc->flags |= GIT_DIFF_FLAG__NO_DATA;
 	} else {
 		fc->flags |= GIT_DIFF_FLAG__LOADED;
-		fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
-		fc->file->size = git_blob_rawsize(blob);
+		fc->file->flags |= GIT_DIFF_FLAG_VALID_ID;
 		fc->file->mode = GIT_FILEMODE_BLOB;
-		git_oid_cpy(&fc->file->oid, git_blob_id(blob));
 
-		fc->map.len  = (size_t)fc->file->size;
-		fc->map.data = (char *)git_blob_rawcontent(blob);
-	}
-
-	return diff_file_content_init_common(fc, opts);
-}
-
-int git_diff_file_content__init_from_raw(
-	git_diff_file_content *fc,
-	git_repository *repo,
-	const git_diff_options *opts,
-	const char *buf,
-	size_t buflen,
-	git_diff_file *as_file)
-{
-	memset(fc, 0, sizeof(*fc));
-	fc->repo = repo;
-	fc->file = as_file;
+		if (src->blob) {
+			fc->file->size = git_blob_rawsize(src->blob);
+			git_oid_cpy(&fc->file->id, git_blob_id(src->blob));
 
-	if (!buf) {
-		fc->flags |= GIT_DIFF_FLAG__NO_DATA;
-	} else {
-		fc->flags |= GIT_DIFF_FLAG__LOADED;
-		fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
-		fc->file->size = buflen;
-		fc->file->mode = GIT_FILEMODE_BLOB;
-		git_odb_hash(&fc->file->oid, buf, buflen, GIT_OBJ_BLOB);
+			fc->map.len  = (size_t)fc->file->size;
+			fc->map.data = (char *)git_blob_rawcontent(src->blob);
+		} else {
+			fc->file->size = src->buflen;
+			git_odb_hash(&fc->file->id, src->buf, src->buflen, GIT_OBJ_BLOB);
 
-		fc->map.len  = buflen;
-		fc->map.data = (char *)buf;
+			fc->map.len  = src->buflen;
+			fc->map.data = (char *)src->buf;
+		}
 	}
 
 	return diff_file_content_init_common(fc, opts);
@@ -196,28 +177,36 @@ static int diff_file_content_commit_to_str(
 		unsigned int sm_status = 0;
 		const git_oid *sm_head;
 
-		if ((error = git_submodule_lookup(&sm, fc->repo, fc->file->path)) < 0 ||
-			(error = git_submodule_status(&sm_status, sm)) < 0) {
+		if ((error = git_submodule_lookup(&sm, fc->repo, fc->file->path)) < 0) {
 			/* GIT_EEXISTS means a "submodule" that has not been git added */
-			if (error == GIT_EEXISTS)
+			if (error == GIT_EEXISTS) {
+				giterr_clear();
 				error = 0;
+			}
+			return error;
+		}
+
+		if ((error = git_submodule_status(&sm_status, sm)) < 0) {
+			git_submodule_free(sm);
 			return error;
 		}
 
 		/* update OID if we didn't have it previously */
-		if ((fc->file->flags & GIT_DIFF_FLAG_VALID_OID) == 0 &&
+		if ((fc->file->flags & GIT_DIFF_FLAG_VALID_ID) == 0 &&
 			((sm_head = git_submodule_wd_id(sm)) != NULL ||
 			 (sm_head = git_submodule_head_id(sm)) != NULL))
 		{
-			git_oid_cpy(&fc->file->oid, sm_head);
-			fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
+			git_oid_cpy(&fc->file->id, sm_head);
+			fc->file->flags |= GIT_DIFF_FLAG_VALID_ID;
 		}
 
 		if (GIT_SUBMODULE_STATUS_IS_WD_DIRTY(sm_status))
 			status = "-dirty";
+
+		git_submodule_free(sm);
 	}
 
-	git_oid_tostr(oid, sizeof(oid), &fc->file->oid);
+	git_oid_tostr(oid, sizeof(oid), &fc->file->id);
 	if (git_buf_printf(&content, "Subproject commit %s%s\n", oid, status) < 0)
 		return -1;
 
@@ -233,7 +222,7 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 	int error = 0;
 	git_odb_object *odb_obj = NULL;
 
-	if (git_oid_iszero(&fc->file->oid))
+	if (git_oid_iszero(&fc->file->id))
 		return 0;
 
 	if (fc->file->mode == GIT_FILEMODE_COMMIT)
@@ -255,7 +244,7 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 		git_odb_object_free(odb_obj);
 	} else {
 		error = git_blob_lookup(
-			(git_blob **)&fc->blob, fc->repo, &fc->file->oid);
+			(git_blob **)&fc->blob, fc->repo, &fc->file->id);
 	}
 
 	if (!error) {
@@ -311,7 +300,8 @@ static int diff_file_content_load_workdir_file(
 		goto cleanup;
 
 	if ((error = git_filter_list_load(
-			&fl, fc->repo, NULL, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
+			&fl, fc->repo, NULL, fc->file->path,
+			GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE)) < 0)
 		goto cleanup;
 
 	/* if there are no filters, try to mmap the file */
@@ -331,7 +321,8 @@ static int diff_file_content_load_workdir_file(
 
 		error = git_filter_list_apply_to_data(&out, fl, &raw);
 
-		git_buf_free(&raw);
+		if (out.ptr != raw.ptr)
+			git_buf_free(&raw);
 
 		if (!error) {
 			fc->map.len  = out.size;
@@ -368,10 +359,10 @@ static int diff_file_content_load_workdir(git_diff_file_content *fc)
 		error = diff_file_content_load_workdir_file(fc, &path);
 
 	/* once data is loaded, update OID if we didn't have it previously */
-	if (!error && (fc->file->flags & GIT_DIFF_FLAG_VALID_OID) == 0) {
+	if (!error && (fc->file->flags & GIT_DIFF_FLAG_VALID_ID) == 0) {
 		error = git_odb_hash(
-			&fc->file->oid, fc->map.data, fc->map.len, GIT_OBJ_BLOB);
-		fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
+			&fc->file->id, fc->map.data, fc->map.len, GIT_OBJ_BLOB);
+		fc->file->flags |= GIT_DIFF_FLAG_VALID_ID;
 	}
 
 	git_buf_free(&path);
diff --git a/src/diff_file.h b/src/diff_file.h
index 84bf255aa..4d290ad43 100644
--- a/src/diff_file.h
+++ b/src/diff_file.h
@@ -31,19 +31,21 @@ extern int git_diff_file_content__init_from_diff(
 	size_t delta_index,
 	bool use_old);
 
-extern int git_diff_file_content__init_from_blob(
-	git_diff_file_content *fc,
-	git_repository *repo,
-	const git_diff_options *opts,
-	const git_blob *blob,
-	git_diff_file *as_file);
+typedef struct {
+	const git_blob *blob;
+	const void *buf;
+	size_t buflen;
+	const char *as_path;
+} git_diff_file_content_src;
+
+#define GIT_DIFF_FILE_CONTENT_SRC__BLOB(BLOB,PATH) { (BLOB),NULL,0,(PATH) }
+#define GIT_DIFF_FILE_CONTENT_SRC__BUF(BUF,LEN,PATH) { NULL,(BUF),(LEN),(PATH) }
 
-extern int git_diff_file_content__init_from_raw(
+extern int git_diff_file_content__init_from_src(
 	git_diff_file_content *fc,
 	git_repository *repo,
 	const git_diff_options *opts,
-	const char *buf,
-	size_t buflen,
+	const git_diff_file_content_src *src,
 	git_diff_file *as_file);
 
 /* this loads the blob/file-on-disk as needed */
diff --git a/src/diff_patch.c b/src/diff_patch.c
index cc49d68eb..38d5f4257 100644
--- a/src/diff_patch.c
+++ b/src/diff_patch.c
@@ -5,6 +5,7 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 #include "common.h"
+#include "git2/blob.h"
 #include "diff.h"
 #include "diff_file.h"
 #include "diff_driver.h"
@@ -133,9 +134,9 @@ static int diff_patch_load(git_patch *patch, git_diff_output *output)
 
 	incomplete_data =
 		(((patch->ofile.flags & GIT_DIFF_FLAG__NO_DATA) != 0 ||
-		  (patch->ofile.file->flags & GIT_DIFF_FLAG_VALID_OID) != 0) &&
+		  (patch->ofile.file->flags & GIT_DIFF_FLAG_VALID_ID) != 0) &&
 		 ((patch->nfile.flags & GIT_DIFF_FLAG__NO_DATA) != 0 ||
-		  (patch->nfile.file->flags & GIT_DIFF_FLAG_VALID_OID) != 0));
+		  (patch->nfile.file->flags & GIT_DIFF_FLAG_VALID_ID) != 0));
 
 	/* always try to load workdir content first because filtering may
 	 * need 2x data size and this minimizes peak memory footprint
@@ -169,7 +170,7 @@ static int diff_patch_load(git_patch *patch, git_diff_output *output)
 	if (incomplete_data &&
 		patch->ofile.file->mode == patch->nfile.file->mode &&
 		patch->ofile.file->mode != GIT_FILEMODE_COMMIT &&
-		git_oid_equal(&patch->ofile.file->oid, &patch->nfile.file->oid) &&
+		git_oid_equal(&patch->ofile.file->id, &patch->nfile.file->id) &&
 		patch->delta->status == GIT_DELTA_MODIFIED) /* not RENAMED/COPIED! */
 		patch->delta->status = GIT_DELTA_UNMODIFIED;
 
@@ -184,7 +185,7 @@ cleanup:
 			patch->delta->status != GIT_DELTA_UNMODIFIED &&
 			(patch->ofile.map.len || patch->nfile.map.len) &&
 			(patch->ofile.map.len != patch->nfile.map.len ||
-			 !git_oid_equal(&patch->ofile.file->oid, &patch->nfile.file->oid)))
+			 !git_oid_equal(&patch->ofile.file->id, &patch->nfile.file->id)))
 			patch->flags |= GIT_DIFF_PATCH_DIFFABLE;
 
 		patch->flags |= GIT_DIFF_PATCH_LOADED;
@@ -193,21 +194,18 @@ cleanup:
 	return error;
 }
 
-static int diff_patch_file_callback(
+static int diff_patch_invoke_file_callback(
 	git_patch *patch, git_diff_output *output)
 {
-	float progress;
+	float progress = patch->diff ?
+		((float)patch->delta_index / patch->diff->deltas.length) : 1.0f;
 
 	if (!output->file_cb)
 		return 0;
 
-	progress = patch->diff ?
-		((float)patch->delta_index / patch->diff->deltas.length) : 1.0f;
-
-	if (output->file_cb(patch->delta, progress, output->payload) != 0)
-		output->error = GIT_EUSER;
-
-	return output->error;
+	return giterr_set_after_callback_function(
+		output->file_cb(patch->delta, progress, output->payload),
+		"git_patch");
 }
 
 static int diff_patch_generate(git_patch *patch, git_diff_output *output)
@@ -229,7 +227,7 @@ static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 		return 0;
 
 	if (output->diff_cb != NULL &&
-		!(error = output->diff_cb(output, patch)))
+		(error = output->diff_cb(output, patch)) < 0)
 		patch->flags |= GIT_DIFF_PATCH_DIFFED;
 
 	return error;
@@ -272,9 +270,10 @@ int git_diff_foreach(
 	size_t idx;
 	git_patch patch;
 
-	if (diff_required(diff, "git_diff_foreach") < 0)
-		return -1;
+	if ((error = diff_required(diff, "git_diff_foreach")) < 0)
+		return error;
 
+	memset(&xo, 0, sizeof(xo));
 	diff_output_init(
 		&xo.output, &diff->opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, &diff->opts);
@@ -285,22 +284,18 @@ int git_diff_foreach(
 		if (git_diff_delta__should_skip(&diff->opts, patch.delta))
 			continue;
 
-		if (!(error = diff_patch_init_from_diff(&patch, diff, idx))) {
-
-			error = diff_patch_file_callback(&patch, &xo.output);
+		if ((error = diff_patch_init_from_diff(&patch, diff, idx)) < 0)
+			break;
 
-			if (!error)
-				error = diff_patch_generate(&patch, &xo.output);
+		if (!(error = diff_patch_invoke_file_callback(&patch, &xo.output)))
+			error = diff_patch_generate(&patch, &xo.output);
 
-			git_patch_free(&patch);
-		}
+		git_patch_free(&patch);
 
-		if (error < 0)
+		if (error)
 			break;
 	}
 
-	if (error == GIT_EUSER)
-		giterr_clear(); /* don't leave error message set invalidly */
 	return error;
 }
 
@@ -321,7 +316,7 @@ static int diff_single_generate(diff_patch_with_delta *pd, git_xdiff_output *xo)
 		(has_old ? GIT_DELTA_MODIFIED : GIT_DELTA_ADDED) :
 		(has_old ? GIT_DELTA_DELETED : GIT_DELTA_UNTRACKED);
 
-	if (git_oid_equal(&patch->nfile.file->oid, &patch->ofile.file->oid))
+	if (git_oid_equal(&patch->nfile.file->id, &patch->ofile.file->id))
 		pd->delta.status = GIT_DELTA_UNMODIFIED;
 
 	patch->delta = &pd->delta;
@@ -332,49 +327,53 @@ static int diff_single_generate(diff_patch_with_delta *pd, git_xdiff_output *xo)
 		!(patch->ofile.opts_flags & GIT_DIFF_INCLUDE_UNMODIFIED))
 		return error;
 
-	error = diff_patch_file_callback(patch, (git_diff_output *)xo);
+	error = diff_patch_invoke_file_callback(patch, (git_diff_output *)xo);
 
 	if (!error)
 		error = diff_patch_generate(patch, (git_diff_output *)xo);
 
-	if (error == GIT_EUSER)
-		giterr_clear(); /* don't leave error message set invalidly */
-
 	return error;
 }
 
-static int diff_patch_from_blobs(
+static int diff_patch_from_sources(
 	diff_patch_with_delta *pd,
 	git_xdiff_output *xo,
-	const git_blob *old_blob,
-	const char *old_path,
-	const git_blob *new_blob,
-	const char *new_path,
+	git_diff_file_content_src *oldsrc,
+	git_diff_file_content_src *newsrc,
 	const git_diff_options *opts)
 {
 	int error = 0;
 	git_repository *repo =
-		new_blob ? git_object_owner((const git_object *)new_blob) :
-		old_blob ? git_object_owner((const git_object *)old_blob) : NULL;
+		oldsrc->blob ? git_blob_owner(oldsrc->blob) :
+		newsrc->blob ? git_blob_owner(newsrc->blob) : NULL;
+	git_diff_file *lfile = &pd->delta.old_file, *rfile = &pd->delta.new_file;
+	git_diff_file_content *ldata = &pd->patch.ofile, *rdata = &pd->patch.nfile;
 
 	GITERR_CHECK_VERSION(opts, GIT_DIFF_OPTIONS_VERSION, "git_diff_options");
 
 	if (opts && (opts->flags & GIT_DIFF_REVERSE) != 0) {
-		const git_blob *tmp_blob;
-		const char *tmp_path;
-		tmp_blob = old_blob; old_blob = new_blob; new_blob = tmp_blob;
-		tmp_path = old_path; old_path = new_path; new_path = tmp_path;
+		void *tmp = lfile; lfile = rfile; rfile = tmp;
+		tmp = ldata; ldata = rdata; rdata = tmp;
 	}
 
 	pd->patch.delta = &pd->delta;
 
-	pd->delta.old_file.path = old_path;
-	pd->delta.new_file.path = new_path;
+	if (!oldsrc->as_path) {
+		if (newsrc->as_path)
+			oldsrc->as_path = newsrc->as_path;
+		else
+			oldsrc->as_path = newsrc->as_path = "file";
+	}
+	else if (!newsrc->as_path)
+		newsrc->as_path = oldsrc->as_path;
+
+	lfile->path = oldsrc->as_path;
+	rfile->path = newsrc->as_path;
 
-	if ((error = git_diff_file_content__init_from_blob(
-			&pd->patch.ofile, repo, opts, old_blob, &pd->delta.old_file)) < 0 ||
-		(error = git_diff_file_content__init_from_blob(
-			&pd->patch.nfile, repo, opts, new_blob, &pd->delta.new_file)) < 0)
+	if ((error = git_diff_file_content__init_from_src(
+			ldata, repo, opts, oldsrc, lfile)) < 0 ||
+		(error = git_diff_file_content__init_from_src(
+			rdata, repo, opts, newsrc, rfile)) < 0)
 		return error;
 
 	return diff_single_generate(pd, xo);
@@ -409,11 +408,9 @@ static int diff_patch_with_delta_alloc(
 	return 0;
 }
 
-int git_diff_blobs(
-	const git_blob *old_blob,
-	const char *old_path,
-	const git_blob *new_blob,
-	const char *new_path,
+static int diff_from_sources(
+	git_diff_file_content_src *oldsrc,
+	git_diff_file_content_src *newsrc,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
@@ -424,32 +421,24 @@ int git_diff_blobs(
 	diff_patch_with_delta pd;
 	git_xdiff_output xo;
 
-	memset(&pd, 0, sizeof(pd));
 	memset(&xo, 0, sizeof(xo));
-
 	diff_output_init(
 		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
-	if (!old_path && new_path)
-		old_path = new_path;
-	else if (!new_path && old_path)
-		new_path = old_path;
+	memset(&pd, 0, sizeof(pd));
 
-	error = diff_patch_from_blobs(
-		&pd, &xo, old_blob, old_path, new_blob, new_path, opts);
+	error = diff_patch_from_sources(&pd, &xo, oldsrc, newsrc, opts);
 
 	git_patch_free(&pd.patch);
 
 	return error;
 }
 
-int git_patch_from_blobs(
+static int patch_from_sources(
 	git_patch **out,
-	const git_blob *old_blob,
-	const char *old_path,
-	const git_blob *new_blob,
-	const char *new_path,
+	git_diff_file_content_src *oldsrc,
+	git_diff_file_content_src *newsrc,
 	const git_diff_options *opts)
 {
 	int error = 0;
@@ -459,18 +448,15 @@ int git_patch_from_blobs(
 	assert(out);
 	*out = NULL;
 
-	if (diff_patch_with_delta_alloc(&pd, &old_path, &new_path) < 0)
-		return -1;
+	if ((error = diff_patch_with_delta_alloc(
+			&pd, &oldsrc->as_path, &newsrc->as_path)) < 0)
+		return error;
 
 	memset(&xo, 0, sizeof(xo));
-
 	diff_output_to_patch(&xo.output, &pd->patch);
 	git_xdiff_init(&xo, opts);
 
-	error = diff_patch_from_blobs(
-		pd, &xo, old_blob, old_path, new_blob, new_path, opts);
-
-	if (!error)
+	if (!(error = diff_patch_from_sources(pd, &xo, oldsrc, newsrc, opts)))
 		*out = (git_patch *)pd;
 	else
 		git_patch_free((git_patch *)pd);
@@ -478,46 +464,38 @@ int git_patch_from_blobs(
 	return error;
 }
 
-static int diff_patch_from_blob_and_buffer(
-	diff_patch_with_delta *pd,
-	git_xdiff_output *xo,
+int git_diff_blobs(
 	const git_blob *old_blob,
 	const char *old_path,
-	const char *buf,
-	size_t buflen,
-	const char *buf_path,
-	const git_diff_options *opts)
+	const git_blob *new_blob,
+	const char *new_path,
+	const git_diff_options *opts,
+	git_diff_file_cb file_cb,
+	git_diff_hunk_cb hunk_cb,
+	git_diff_line_cb data_cb,
+	void *payload)
 {
-	int error = 0;
-	git_repository *repo =
-		old_blob ? git_object_owner((const git_object *)old_blob) : NULL;
-
-	GITERR_CHECK_VERSION(opts, GIT_DIFF_OPTIONS_VERSION, "git_diff_options");
-
-	pd->patch.delta = &pd->delta;
-
-	if (opts && (opts->flags & GIT_DIFF_REVERSE) != 0) {
-		pd->delta.old_file.path = buf_path;
-		pd->delta.new_file.path = old_path;
-
-		if (!(error = git_diff_file_content__init_from_raw(
-				&pd->patch.ofile, repo, opts, buf, buflen, &pd->delta.old_file)))
-			error = git_diff_file_content__init_from_blob(
-				&pd->patch.nfile, repo, opts, old_blob, &pd->delta.new_file);
-	} else {
-		pd->delta.old_file.path = old_path;
-		pd->delta.new_file.path = buf_path;
-
-		if (!(error = git_diff_file_content__init_from_blob(
-				&pd->patch.ofile, repo, opts, old_blob, &pd->delta.old_file)))
-			error = git_diff_file_content__init_from_raw(
-				&pd->patch.nfile, repo, opts, buf, buflen, &pd->delta.new_file);
-	}
-
-	if (error < 0)
-		return error;
+	git_diff_file_content_src osrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BLOB(old_blob, old_path);
+	git_diff_file_content_src nsrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BLOB(new_blob, new_path);
+	return diff_from_sources(
+		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+}
 
-	return diff_single_generate(pd, xo);
+int git_patch_from_blobs(
+	git_patch **out,
+	const git_blob *old_blob,
+	const char *old_path,
+	const git_blob *new_blob,
+	const char *new_path,
+	const git_diff_options *opts)
+{
+	git_diff_file_content_src osrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BLOB(old_blob, old_path);
+	git_diff_file_content_src nsrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BLOB(new_blob, new_path);
+	return patch_from_sources(out, &osrc, &nsrc, opts);
 }
 
 int git_diff_blob_to_buffer(
@@ -532,28 +510,12 @@ int git_diff_blob_to_buffer(
 	git_diff_line_cb data_cb,
 	void *payload)
 {
-	int error = 0;
-	diff_patch_with_delta pd;
-	git_xdiff_output xo;
-
-	memset(&pd, 0, sizeof(pd));
-	memset(&xo, 0, sizeof(xo));
-
-	diff_output_init(
-		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
-	git_xdiff_init(&xo, opts);
-
-	if (!old_path && buf_path)
-		old_path = buf_path;
-	else if (!buf_path && old_path)
-		buf_path = old_path;
-
-	error = diff_patch_from_blob_and_buffer(
-		&pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts);
-
-	git_patch_free(&pd.patch);
-
-	return error;
+	git_diff_file_content_src osrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BLOB(old_blob, old_path);
+	git_diff_file_content_src nsrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BUF(buf, buflen, buf_path);
+	return diff_from_sources(
+		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_blob_and_buffer(
@@ -565,30 +527,49 @@ int git_patch_from_blob_and_buffer(
 	const char *buf_path,
 	const git_diff_options *opts)
 {
-	int error = 0;
-	diff_patch_with_delta *pd;
-	git_xdiff_output xo;
-
-	assert(out);
-	*out = NULL;
-
-	if (diff_patch_with_delta_alloc(&pd, &old_path, &buf_path) < 0)
-		return -1;
-
-	memset(&xo, 0, sizeof(xo));
-
-	diff_output_to_patch(&xo.output, &pd->patch);
-	git_xdiff_init(&xo, opts);
-
-	error = diff_patch_from_blob_and_buffer(
-		pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts);
+	git_diff_file_content_src osrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BLOB(old_blob, old_path);
+	git_diff_file_content_src nsrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BUF(buf, buflen, buf_path);
+	return patch_from_sources(out, &osrc, &nsrc, opts);
+}
 
-	if (!error)
-		*out = (git_patch *)pd;
-	else
-		git_patch_free((git_patch *)pd);
+int git_diff_buffers(
+	const void *old_buf,
+	size_t old_len,
+	const char *old_path,
+	const void *new_buf,
+	size_t new_len,
+	const char *new_path,
+	const git_diff_options *opts,
+	git_diff_file_cb file_cb,
+	git_diff_hunk_cb hunk_cb,
+	git_diff_line_cb data_cb,
+	void *payload)
+{
+	git_diff_file_content_src osrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BUF(old_buf, old_len, old_path);
+	git_diff_file_content_src nsrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BUF(new_buf, new_len, new_path);
+	return diff_from_sources(
+		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+}
 
-	return error;
+int git_patch_from_buffers(
+	git_patch **out,
+	const void *old_buf,
+	size_t old_len,
+	const char *old_path,
+	const char *new_buf,
+	size_t new_len,
+	const char *new_path,
+	const git_diff_options *opts)
+{
+	git_diff_file_content_src osrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BUF(old_buf, old_len, old_path);
+	git_diff_file_content_src nsrc =
+		GIT_DIFF_FILE_CONTENT_SRC__BUF(new_buf, new_len, new_path);
+	return patch_from_sources(out, &osrc, &nsrc, opts);
 }
 
 int git_patch_from_diff(
@@ -622,17 +603,18 @@ int git_patch_from_diff(
 	if ((error = diff_patch_alloc_from_diff(&patch, diff, idx)) < 0)
 		return error;
 
+	memset(&xo, 0, sizeof(xo));
 	diff_output_to_patch(&xo.output, patch);
 	git_xdiff_init(&xo, &diff->opts);
 
-	error = diff_patch_file_callback(patch, &xo.output);
+	error = diff_patch_invoke_file_callback(patch, &xo.output);
 
 	if (!error)
 		error = diff_patch_generate(patch, &xo.output);
 
 	if (!error) {
-		/* if cumulative diff size is < 0.5 total size, flatten the patch */
-		/* unload the file content */
+		/* TODO: if cumulative diff size is < 0.5 total size, flatten patch */
+		/* TODO: and unload the file content */
 	}
 
 	if (error || !patch_ptr)
@@ -640,8 +622,6 @@ int git_patch_from_diff(
 	else
 		*patch_ptr = patch;
 
-	if (error == GIT_EUSER)
-		giterr_clear(); /* don't leave error message set invalidly */
 	return error;
 }
 
@@ -651,13 +631,13 @@ void git_patch_free(git_patch *patch)
 		GIT_REFCOUNT_DEC(patch, diff_patch_free);
 }
 
-const git_diff_delta *git_patch_get_delta(git_patch *patch)
+const git_diff_delta *git_patch_get_delta(const git_patch *patch)
 {
 	assert(patch);
 	return patch->delta;
 }
 
-size_t git_patch_num_hunks(git_patch *patch)
+size_t git_patch_num_hunks(const git_patch *patch)
 {
 	assert(patch);
 	return git_array_size(patch->hunks);
@@ -728,7 +708,7 @@ int git_patch_get_hunk(
 	return 0;
 }
 
-int git_patch_num_lines_in_hunk(git_patch *patch, size_t hunk_idx)
+int git_patch_num_lines_in_hunk(const git_patch *patch, size_t hunk_idx)
 {
 	diff_patch_hunk *hunk;
 	assert(patch);
@@ -905,7 +885,7 @@ static int diff_patch_line_cb(
 	GIT_UNUSED(hunk_);
 
 	hunk = git_array_last(patch->hunks);
-	GITERR_CHECK_ALLOC(hunk);
+	assert(hunk); /* programmer error if no hunk is available */
 
 	line = git_array_alloc(patch->lines);
 	GITERR_CHECK_ALLOC(line);
diff --git a/src/diff_print.c b/src/diff_print.c
index b04b11515..bb925ef98 100644
--- a/src/diff_print.c
+++ b/src/diff_print.c
@@ -8,6 +8,10 @@
 #include "diff.h"
 #include "diff_patch.h"
 #include "fileops.h"
+#include "zstream.h"
+#include "blob.h"
+#include "delta.h"
+#include "git2/sys/diff.h"
 
 typedef struct {
 	git_diff *diff;
@@ -36,9 +40,11 @@ static int diff_print_info_init(
 
 	if (diff)
 		pi->flags = diff->opts.flags;
+	else
+		pi->flags = 0;
 
-	if (diff && diff->opts.oid_abbrev != 0)
-		pi->oid_strlen = diff->opts.oid_abbrev;
+	if (diff && diff->opts.id_abbrev != 0)
+		pi->oid_strlen = diff->opts.id_abbrev;
 	else if (!diff || !diff->repo)
 		pi->oid_strlen = GIT_ABBREV_DEFAULT;
 	else if (git_repository__cvar(
@@ -89,12 +95,6 @@ char git_diff_status_char(git_delta_t status)
 	return code;
 }
 
-static int callback_error(void)
-{
-	giterr_clear();
-	return GIT_EUSER;
-}
-
 static int diff_print_one_name_only(
 	const git_diff_delta *delta, float progress, void *data)
 {
@@ -108,19 +108,16 @@ static int diff_print_one_name_only(
 		return 0;
 
 	git_buf_clear(out);
-
-	if (git_buf_puts(out, delta->new_file.path) < 0 ||
-		git_buf_putc(out, '\n'))
+	git_buf_puts(out, delta->new_file.path);
+	git_buf_putc(out, '\n');
+	if (git_buf_oom(out))
 		return -1;
 
 	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
 	pi->line.content     = git_buf_cstr(out);
 	pi->line.content_len = git_buf_len(out);
 
-	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
-		return callback_error();
-
-	return 0;
+	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
 }
 
 static int diff_print_one_name_status(
@@ -154,7 +151,6 @@ static int diff_print_one_name_status(
 		git_buf_printf(out, "%c\t%s%c\n", code, delta->old_file.path, old_suffix);
 	else
 		git_buf_printf(out, "%c\t%s\n", code, delta->old_file.path);
-
 	if (git_buf_oom(out))
 		return -1;
 
@@ -162,10 +158,7 @@ static int diff_print_one_name_status(
 	pi->line.content     = git_buf_cstr(out);
 	pi->line.content_len = git_buf_len(out);
 
-	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
-		return callback_error();
-
-	return 0;
+	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
 }
 
 static int diff_print_one_raw(
@@ -183,11 +176,12 @@ static int diff_print_one_raw(
 
 	git_buf_clear(out);
 
-	git_oid_tostr(start_oid, pi->oid_strlen, &delta->old_file.oid);
-	git_oid_tostr(end_oid, pi->oid_strlen, &delta->new_file.oid);
+	git_oid_tostr(start_oid, pi->oid_strlen, &delta->old_file.id);
+	git_oid_tostr(end_oid, pi->oid_strlen, &delta->new_file.id);
 
 	git_buf_printf(
-		out, ":%06o %06o %s... %s... %c",
+		out, (pi->oid_strlen <= GIT_OID_HEXSZ) ?
+			":%06o %06o %s... %s... %c" : ":%06o %06o %s %s %c",
 		delta->old_file.mode, delta->new_file.mode, start_oid, end_oid, code);
 
 	if (delta->similarity > 0)
@@ -208,10 +202,7 @@ static int diff_print_one_raw(
 	pi->line.content     = git_buf_cstr(out);
 	pi->line.content_len = git_buf_len(out);
 
-	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
-		return callback_error();
-
-	return 0;
+	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
 }
 
 static int diff_print_oid_range(
@@ -219,8 +210,8 @@ static int diff_print_oid_range(
 {
 	char start_oid[GIT_OID_HEXSZ+1], end_oid[GIT_OID_HEXSZ+1];
 
-	git_oid_tostr(start_oid, oid_strlen, &delta->old_file.oid);
-	git_oid_tostr(end_oid, oid_strlen, &delta->new_file.oid);
+	git_oid_tostr(start_oid, oid_strlen, &delta->old_file.id);
+	git_oid_tostr(end_oid, oid_strlen, &delta->new_file.id);
 
 	/* TODO: Match git diff more closely */
 	if (delta->old_file.mode == delta->new_file.mode) {
@@ -238,10 +229,7 @@ static int diff_print_oid_range(
 		git_buf_printf(out, "index %s..%s\n", start_oid, end_oid);
 	}
 
-	if (git_buf_oom(out))
-		return -1;
-
-	return 0;
+	return git_buf_oom(out) ? -1 : 0;
 }
 
 static int diff_delta_format_with_paths(
@@ -254,11 +242,11 @@ static int diff_delta_format_with_paths(
 	const char *oldpath = delta->old_file.path;
 	const char *newpath = delta->new_file.path;
 
-	if (git_oid_iszero(&delta->old_file.oid)) {
+	if (git_oid_iszero(&delta->old_file.id)) {
 		oldpfx = "";
 		oldpath = "/dev/null";
 	}
-	if (git_oid_iszero(&delta->new_file.oid)) {
+	if (git_oid_iszero(&delta->new_file.id)) {
 		newpfx = "";
 		newpath = "/dev/null";
 	}
@@ -285,8 +273,7 @@ int git_diff_delta__format_file_header(
 	git_buf_printf(out, "diff --git %s%s %s%s\n",
 		oldpfx, delta->old_file.path, newpfx, delta->new_file.path);
 
-	if (diff_print_oid_range(out, delta, oid_strlen) < 0)
-		return -1;
+	GITERR_CHECK_ERROR(diff_print_oid_range(out, delta, oid_strlen));
 
 	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
 		diff_delta_format_with_paths(
@@ -295,15 +282,160 @@ int git_diff_delta__format_file_header(
 	return git_buf_oom(out) ? -1 : 0;
 }
 
+static int print_binary_hunk(diff_print_info *pi, git_blob *old, git_blob *new)
+{
+	git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT, *out = NULL;
+	const void *old_data, *new_data;
+	git_off_t old_data_len, new_data_len;
+	unsigned long delta_data_len, inflated_len;
+	const char *out_type = "literal";
+	char *scan, *end;
+	int error;
+
+	old_data = old ? git_blob_rawcontent(old) : NULL;
+	new_data = new ? git_blob_rawcontent(new) : NULL;
+
+	old_data_len = old ? git_blob_rawsize(old) : 0;
+	new_data_len = new ? git_blob_rawsize(new) : 0;
+
+	/* The git_delta function accepts unsigned long only */
+	if (!git__is_ulong(old_data_len) || !git__is_ulong(new_data_len))
+		return GIT_EBUFS;
+
+	out = &deflate;
+	inflated_len = (unsigned long)new_data_len;
+
+	if ((error = git_zstream_deflatebuf(
+			out, new_data, (size_t)new_data_len)) < 0)
+		goto done;
+
+	/* The git_delta function accepts unsigned long only */
+	if (!git__is_ulong((git_off_t)deflate.size)) {
+		error = GIT_EBUFS;
+		goto done;
+	}
+
+	if (old && new) {
+		void *delta_data = git_delta(
+			old_data, (unsigned long)old_data_len,
+			new_data, (unsigned long)new_data_len,
+			&delta_data_len, (unsigned long)deflate.size);
+
+		if (delta_data) {
+			error = git_zstream_deflatebuf(
+				&delta, delta_data, (size_t)delta_data_len);
+
+			git__free(delta_data);
+
+			if (error < 0)
+				goto done;
+
+			if (delta.size < deflate.size) {
+				out = &delta;
+				out_type = "delta";
+				inflated_len = delta_data_len;
+			}
+		}
+	}
+
+	git_buf_printf(pi->buf, "%s %lu\n", out_type, inflated_len);
+	pi->line.num_lines++;
+
+	for (scan = out->ptr, end = out->ptr + out->size; scan < end; ) {
+		size_t chunk_len = end - scan;
+		if (chunk_len > 52)
+			chunk_len = 52;
+
+		if (chunk_len <= 26)
+			git_buf_putc(pi->buf, (char)chunk_len + 'A' - 1);
+		else
+			git_buf_putc(pi->buf, (char)chunk_len - 26 + 'a' - 1);
+
+		git_buf_put_base85(pi->buf, scan, chunk_len);
+		git_buf_putc(pi->buf, '\n');
+
+		if (git_buf_oom(pi->buf)) {
+			error = -1;
+			goto done;
+		}
+
+		scan += chunk_len;
+		pi->line.num_lines++;
+	}
+
+done:
+	git_buf_free(&deflate);
+	git_buf_free(&delta);
+
+	return error;
+}
+
+/* git diff --binary 8d7523f~2 8d7523f~1 */
+static int diff_print_patch_file_binary(
+	diff_print_info *pi, const git_diff_delta *delta,
+	const char *oldpfx, const char *newpfx)
+{
+	git_blob *old = NULL, *new = NULL;
+	const git_oid *old_id, *new_id;
+	int error;
+	size_t pre_binary_size;
+
+	if ((pi->flags & GIT_DIFF_SHOW_BINARY) == 0)
+		goto noshow;
+
+	pre_binary_size = pi->buf->size;
+	git_buf_printf(pi->buf, "GIT binary patch\n");
+	pi->line.num_lines++;
+
+	old_id = (delta->status != GIT_DELTA_ADDED) ? &delta->old_file.id : NULL;
+	new_id = (delta->status != GIT_DELTA_DELETED) ? &delta->new_file.id : NULL;
+
+	if (old_id && (error = git_blob_lookup(&old, pi->diff->repo, old_id)) < 0)
+		goto done;
+	if (new_id && (error = git_blob_lookup(&new, pi->diff->repo,new_id)) < 0)
+		goto done;
+
+	if ((error = print_binary_hunk(pi, old, new)) < 0 ||
+		(error = git_buf_putc(pi->buf, '\n')) < 0 ||
+		(error = print_binary_hunk(pi, new, old)) < 0)
+	{
+		if (error == GIT_EBUFS) {
+			giterr_clear();
+			git_buf_truncate(pi->buf, pre_binary_size);
+			goto noshow;
+		}
+	}
+
+	pi->line.num_lines++;
+
+done:
+	git_blob_free(old);
+	git_blob_free(new);
+
+	return error;
+
+noshow:
+	pi->line.num_lines = 1;
+	return diff_delta_format_with_paths(
+		pi->buf, delta, oldpfx, newpfx,
+		"Binary files %s%s and %s%s differ\n");
+}
+
 static int diff_print_patch_file(
 	const git_diff_delta *delta, float progress, void *data)
 {
+	int error;
 	diff_print_info *pi = data;
 	const char *oldpfx =
 		pi->diff ? pi->diff->opts.old_prefix : DIFF_OLD_PREFIX_DEFAULT;
 	const char *newpfx =
 		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
 
+	bool binary = !!(delta->flags & GIT_DIFF_FLAG_BINARY);
+	bool show_binary = !!(pi->flags & GIT_DIFF_SHOW_BINARY);
+	int oid_strlen = binary && show_binary ?
+		GIT_OID_HEXSZ + 1 : pi->oid_strlen;
+
 	GIT_UNUSED(progress);
 
 	if (S_ISDIR(delta->new_file.mode) ||
@@ -313,36 +445,30 @@ static int diff_print_patch_file(
 		 (pi->flags & GIT_DIFF_SHOW_UNTRACKED_CONTENT) == 0))
 		return 0;
 
-	if (git_diff_delta__format_file_header(
-			pi->buf, delta, oldpfx, newpfx, pi->oid_strlen) < 0)
-		return -1;
+	if ((error = git_diff_delta__format_file_header(
+			pi->buf, delta, oldpfx, newpfx, oid_strlen)) < 0)
+		return error;
 
 	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
 	pi->line.content     = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
 
-	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
-		return callback_error();
+	if ((error = pi->print_cb(delta, NULL, &pi->line, pi->payload)) != 0)
+		return error;
 
-	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
+	if (!binary)
 		return 0;
 
 	git_buf_clear(pi->buf);
 
-	if (diff_delta_format_with_paths(
-			pi->buf, delta, oldpfx, newpfx,
-			"Binary files %s%s and %s%s differ\n") < 0)
-		return -1;
+	if ((error = diff_print_patch_file_binary(pi, delta, oldpfx, newpfx)) < 0)
+		return error;
 
 	pi->line.origin      = GIT_DIFF_LINE_BINARY;
 	pi->line.content     = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
-	pi->line.num_lines   = 1;
 
-	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
-		return callback_error();
-
-	return 0;
+	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
 }
 
 static int diff_print_patch_hunk(
@@ -359,10 +485,7 @@ static int diff_print_patch_hunk(
 	pi->line.content     = h->header;
 	pi->line.content_len = h->header_len;
 
-	if (pi->print_cb(d, h, &pi->line, pi->payload))
-		return callback_error();
-
-	return 0;
+	return pi->print_cb(d, h, &pi->line, pi->payload);
 }
 
 static int diff_print_patch_line(
@@ -376,10 +499,7 @@ static int diff_print_patch_line(
 	if (S_ISDIR(delta->new_file.mode))
 		return 0;
 
-	if (pi->print_cb(delta, hunk, line, pi->payload))
-		return callback_error();
-
-	return 0;
+	return pi->print_cb(delta, hunk, line, pi->payload);
 }
 
 /* print a git_diff to an output callback */
@@ -421,9 +541,14 @@ int git_diff_print(
 
 	if (!(error = diff_print_info_init(
 			&pi, &buf, diff, format, print_cb, payload)))
+	{
 		error = git_diff_foreach(
 			diff, print_file, print_hunk, print_line, &pi);
 
+		if (error) /* make sure error message is set */
+			giterr_set_after_callback_function(error, "git_diff_print");
+	}
+
 	git_buf_free(&buf);
 
 	return error;
@@ -444,16 +569,21 @@ int git_patch_print(
 	if (!(error = diff_print_info_init(
 			&pi, &temp, git_patch__diff(patch),
 			GIT_DIFF_FORMAT_PATCH, print_cb, payload)))
+	{
 		error = git_patch__invoke_callbacks(
 			patch, diff_print_patch_file, diff_print_patch_hunk,
 			diff_print_patch_line, &pi);
 
+		if (error) /* make sure error message is set */
+			giterr_set_after_callback_function(error, "git_patch_print");
+	}
+
 	git_buf_free(&temp);
 
 	return error;
 }
 
-static int diff_print_to_buffer_cb(
+int git_diff_print_callback__to_buf(
 	const git_diff_delta *delta,
 	const git_diff_hunk *hunk,
 	const git_diff_line *line,
@@ -462,6 +592,11 @@ static int diff_print_to_buffer_cb(
 	git_buf *output = payload;
 	GIT_UNUSED(delta); GIT_UNUSED(hunk);
 
+	if (!output) {
+		giterr_set(GITERR_INVALID, "Buffer pointer must be provided");
+		return -1;
+	}
+
 	if (line->origin == GIT_DIFF_LINE_ADDITION ||
 		line->origin == GIT_DIFF_LINE_DELETION ||
 		line->origin == GIT_DIFF_LINE_CONTEXT)
@@ -470,23 +605,28 @@ static int diff_print_to_buffer_cb(
 	return git_buf_put(output, line->content, line->content_len);
 }
 
-/* print a git_patch to a string buffer */
-int git_patch_to_str(
-	char **string,
-	git_patch *patch)
+int git_diff_print_callback__to_file_handle(
+	const git_diff_delta *delta,
+	const git_diff_hunk *hunk,
+	const git_diff_line *line,
+	void *payload)
 {
-	int error;
-	git_buf output = GIT_BUF_INIT;
-
-	error = git_patch_print(patch, diff_print_to_buffer_cb, &output);
+	FILE *fp = payload ? payload : stdout;
 
-	/* GIT_EUSER means git_buf_put in print_to_buffer_cb returned -1,
-	 * meaning a memory allocation failure, so just map to -1...
-	 */
-	if (error == GIT_EUSER)
-		error = -1;
+	GIT_UNUSED(delta); GIT_UNUSED(hunk);
 
-	*string = git_buf_detach(&output);
+	if (line->origin == GIT_DIFF_LINE_CONTEXT ||
+		line->origin == GIT_DIFF_LINE_ADDITION ||
+		line->origin == GIT_DIFF_LINE_DELETION)
+		fputc(line->origin, fp);
+	fwrite(line->content, 1, line->content_len, fp);
+	return 0;
+}
 
-	return error;
+/* print a git_patch to a git_buf */
+int git_patch_to_buf(git_buf *out, git_patch *patch)
+{
+	assert(out && patch);
+	git_buf_sanitize(out);
+	return git_patch_print(patch, git_diff_print_callback__to_buf, out);
 }
diff --git a/src/diff_stats.c b/src/diff_stats.c
new file mode 100644
index 000000000..42ccbfb87
--- /dev/null
+++ b/src/diff_stats.c
@@ -0,0 +1,336 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "vector.h"
+#include "diff.h"
+#include "diff_patch.h"
+
+#define DIFF_RENAME_FILE_SEPARATOR " => "
+#define STATS_FULL_MIN_SCALE 7
+
+typedef struct {
+	size_t insertions;
+	size_t deletions;
+} diff_file_stats;
+
+struct git_diff_stats {
+	git_diff *diff;
+	diff_file_stats *filestats;
+
+	size_t files_changed;
+	size_t insertions;
+	size_t deletions;
+	size_t renames;
+
+	size_t max_name;
+	size_t max_filestat;
+	int max_digits;
+};
+
+static int digits_for_value(size_t val)
+{
+	int count = 1;
+	size_t placevalue = 10;
+
+	while (val >= placevalue) {
+		++count;
+		placevalue *= 10;
+	}
+
+	return count;
+}
+
+int git_diff_file_stats__full_to_buf(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const diff_file_stats *filestat,
+	const git_diff_stats *stats,
+	size_t width)
+{
+	const char *old_path = NULL, *new_path = NULL;
+	size_t padding, old_size, new_size;
+
+	old_path = delta->old_file.path;
+	new_path = delta->new_file.path;
+	old_size = delta->old_file.size;
+	new_size = delta->new_file.size;
+
+	if (git_buf_printf(out, " %s", old_path) < 0)
+		goto on_error;
+
+	if (strcmp(old_path, new_path) != 0) {
+		padding = stats->max_name - strlen(old_path) - strlen(new_path);
+
+		if (git_buf_printf(out, DIFF_RENAME_FILE_SEPARATOR "%s", new_path) < 0)
+			goto on_error;
+	} else {
+		padding = stats->max_name - strlen(old_path);
+
+		if (stats->renames > 0)
+			padding += strlen(DIFF_RENAME_FILE_SEPARATOR);
+	}
+
+	if (git_buf_putcn(out, ' ', padding) < 0 ||
+		git_buf_puts(out, " | ") < 0)
+		goto on_error;
+
+	if (delta->flags & GIT_DIFF_FLAG_BINARY) {
+		if (git_buf_printf(out,
+				"Bin %" PRIuZ " -> %" PRIuZ " bytes", old_size, new_size) < 0)
+			goto on_error;
+	}
+	else {
+		if (git_buf_printf(out,
+				"%*" PRIuZ, stats->max_digits,
+				filestat->insertions + filestat->deletions) < 0)
+			goto on_error;
+
+		if (filestat->insertions || filestat->deletions) {
+			if (git_buf_putc(out, ' ') < 0)
+				goto on_error;
+
+			if (!width) {
+				if (git_buf_putcn(out, '+', filestat->insertions) < 0 ||
+					git_buf_putcn(out, '-', filestat->deletions) < 0)
+					goto on_error;
+			} else {
+				size_t total = filestat->insertions + filestat->deletions;
+				size_t full = (total * width + stats->max_filestat / 2) /
+					stats->max_filestat;
+				size_t plus = full * filestat->insertions / total;
+				size_t minus = full - plus;
+
+				if (git_buf_putcn(out, '+', max(plus,  1)) < 0 ||
+					git_buf_putcn(out, '-', max(minus, 1)) < 0)
+					goto on_error;
+			}
+		}
+	}
+
+	git_buf_putc(out, '\n');
+
+on_error:
+	return (git_buf_oom(out) ? -1 : 0);
+}
+
+int git_diff_file_stats__number_to_buf(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const diff_file_stats *filestats)
+{
+	int error;
+	const char *path = delta->new_file.path;
+
+	if (delta->flags & GIT_DIFF_FLAG_BINARY)
+		error = git_buf_printf(out, "%-8c" "%-8c" "%s\n", '-', '-', path);
+	else
+		error = git_buf_printf(out, "%-8" PRIuZ "%-8" PRIuZ "%s\n",
+			filestats->insertions, filestats->deletions, path);
+
+	return error;
+}
+
+int git_diff_file_stats__summary_to_buf(
+	git_buf *out,
+	const git_diff_delta *delta)
+{
+	if (delta->old_file.mode != delta->new_file.mode) {
+		if (delta->old_file.mode == 0) {
+			git_buf_printf(out, " create mode %06o %s\n",
+				delta->new_file.mode, delta->new_file.path);
+		}
+		else if (delta->new_file.mode == 0) {
+			git_buf_printf(out, " delete mode %06o %s\n",
+				delta->old_file.mode, delta->old_file.path);
+		}
+		else {
+			git_buf_printf(out, " mode change %06o => %06o %s\n",
+				delta->old_file.mode, delta->new_file.mode, delta->new_file.path);
+		}
+	}
+
+	return 0;
+}
+
+int git_diff_get_stats(
+	git_diff_stats **out,
+	git_diff *diff)
+{
+	size_t i, deltas;
+	size_t total_insertions = 0, total_deletions = 0;
+	git_diff_stats *stats = NULL;
+	int error = 0;
+
+	assert(out && diff);
+
+	stats = git__calloc(1, sizeof(git_diff_stats));
+	GITERR_CHECK_ALLOC(stats);
+
+	deltas = git_diff_num_deltas(diff);
+
+	stats->filestats = git__calloc(deltas, sizeof(diff_file_stats));
+	if (!stats->filestats) {
+		git__free(stats);
+		return -1;
+	}
+
+	stats->diff = diff;
+	GIT_REFCOUNT_INC(diff);
+
+	for (i = 0; i < deltas && !error; ++i) {
+		git_patch *patch = NULL;
+		size_t add = 0, remove = 0, namelen;
+		const git_diff_delta *delta;
+
+		if ((error = git_patch_from_diff(&patch, diff, i)) < 0)
+			break;
+
+		/* keep a count of renames because it will affect formatting */
+		delta = git_patch_get_delta(patch);
+
+		namelen = strlen(delta->new_file.path);
+		if (strcmp(delta->old_file.path, delta->new_file.path) != 0) {
+			namelen += strlen(delta->old_file.path);
+			stats->renames++;
+		}
+
+		/* and, of course, count the line stats */
+		error = git_patch_line_stats(NULL, &add, &remove, patch);
+
+		git_patch_free(patch);
+
+		stats->filestats[i].insertions = add;
+		stats->filestats[i].deletions = remove;
+
+		total_insertions += add;
+		total_deletions += remove;
+
+		if (stats->max_name < namelen)
+			stats->max_name = namelen;
+		if (stats->max_filestat < add + remove)
+			stats->max_filestat = add + remove;
+	}
+
+	stats->files_changed = deltas;
+	stats->insertions = total_insertions;
+	stats->deletions = total_deletions;
+	stats->max_digits = digits_for_value(stats->max_filestat + 1);
+
+	if (error < 0) {
+		git_diff_stats_free(stats);
+		stats = NULL;
+	}
+
+	*out = stats;
+	return error;
+}
+
+size_t git_diff_stats_files_changed(
+	const git_diff_stats *stats)
+{
+	assert(stats);
+
+	return stats->files_changed;
+}
+
+size_t git_diff_stats_insertions(
+	const git_diff_stats *stats)
+{
+	assert(stats);
+
+	return stats->insertions;
+}
+
+size_t git_diff_stats_deletions(
+	const git_diff_stats *stats)
+{
+	assert(stats);
+
+	return stats->deletions;
+}
+
+int git_diff_stats_to_buf(
+	git_buf *out,
+	const git_diff_stats *stats,
+	git_diff_stats_format_t format,
+	size_t width)
+{
+	int error = 0;
+	size_t i;
+	const git_diff_delta *delta;
+
+	assert(out && stats);
+
+	if (format & GIT_DIFF_STATS_NUMBER) {
+		for (i = 0; i < stats->files_changed; ++i) {
+			if ((delta = git_diff_get_delta(stats->diff, i)) == NULL)
+				continue;
+
+			error = git_diff_file_stats__number_to_buf(
+				out, delta, &stats->filestats[i]);
+			if (error < 0)
+				return error;
+		}
+	}
+
+	if (format & GIT_DIFF_STATS_FULL) {
+		if (width > 0) {
+			if (width > stats->max_name + stats->max_digits + 5)
+				width -= (stats->max_name + stats->max_digits + 5);
+			if (width < STATS_FULL_MIN_SCALE)
+				width = STATS_FULL_MIN_SCALE;
+		}
+		if (width > stats->max_filestat)
+			width = 0;
+
+		for (i = 0; i < stats->files_changed; ++i) {
+			if ((delta = git_diff_get_delta(stats->diff, i)) == NULL)
+				continue;
+
+			error = git_diff_file_stats__full_to_buf(
+				out, delta, &stats->filestats[i], stats, width);
+			if (error < 0)
+				return error;
+		}
+	}
+
+	if (format & GIT_DIFF_STATS_FULL || format & GIT_DIFF_STATS_SHORT) {
+		error = git_buf_printf(
+			out, " %" PRIuZ " file%s changed, %" PRIuZ
+			" insertion%s(+), %" PRIuZ " deletion%s(-)\n",
+			stats->files_changed, stats->files_changed != 1 ? "s" : "",
+			stats->insertions, stats->insertions != 1 ? "s" : "",
+			stats->deletions, stats->deletions != 1 ? "s" : "");
+
+		if (error < 0)
+			return error;
+	}
+
+	if (format & GIT_DIFF_STATS_INCLUDE_SUMMARY) {
+		for (i = 0; i < stats->files_changed; ++i) {
+			if ((delta = git_diff_get_delta(stats->diff, i)) == NULL)
+				continue;
+
+			error = git_diff_file_stats__summary_to_buf(out, delta);
+			if (error < 0)
+				return error;
+		}
+	}
+
+	return error;
+}
+
+void git_diff_stats_free(git_diff_stats *stats)
+{
+	if (stats == NULL)
+		return;
+
+	git_diff_free(stats->diff); /* bumped refcount in constructor */
+	git__free(stats->filestats);
+	git__free(stats);
+}
+
diff --git a/src/diff_tform.c b/src/diff_tform.c
index 28a9cc70d..a2dab0ae2 100644
--- a/src/diff_tform.c
+++ b/src/diff_tform.c
@@ -13,6 +13,7 @@
 #include "hashsig.h"
 #include "path.h"
 #include "fileops.h"
+#include "config.h"
 
 static git_diff_delta *diff_delta__dup(
 	const git_diff_delta *d, git_pool *pool)
@@ -90,7 +91,7 @@ static git_diff_delta *diff_delta__merge_like_cgit(
 		dup->status = a->status;
 	}
 
-	git_oid_cpy(&dup->old_file.oid, &a->old_file.oid);
+	git_oid_cpy(&dup->old_file.id, &a->old_file.id);
 	dup->old_file.mode  = a->old_file.mode;
 	dup->old_file.size  = a->old_file.size;
 	dup->old_file.flags = a->old_file.flags;
@@ -123,7 +124,7 @@ static git_diff_delta *diff_delta__merge_like_cgit_reversed(
 		dup->status = b->status;
 	}
 
-	git_oid_cpy(&dup->old_file.oid, &b->old_file.oid);
+	git_oid_cpy(&dup->old_file.id, &b->old_file.id);
 	dup->old_file.mode  = b->old_file.mode;
 	dup->old_file.size  = b->old_file.size;
 	dup->old_file.flags = b->old_file.flags;
@@ -208,9 +209,7 @@ int git_diff_merge(git_diff *onto, const git_diff *from)
 			git_pool_strdup_safe(&onto->pool, onto->opts.new_prefix);
 	}
 
-	git_vector_foreach(&onto_new, i, delta)
-		git__free(delta);
-	git_vector_free(&onto_new);
+	git_vector_free_deep(&onto_new);
 	git_pool_clear(&onto_pool);
 
 	return error;
@@ -219,7 +218,7 @@ int git_diff_merge(git_diff *onto, const git_diff *from)
 int git_diff_find_similar__hashsig_for_file(
 	void **out, const git_diff_file *f, const char *path, void *p)
 {
-	git_hashsig_option_t opt = (git_hashsig_option_t)p;
+	git_hashsig_option_t opt = (git_hashsig_option_t)(intptr_t)p;
 	int error = 0;
 
 	GIT_UNUSED(f);
@@ -236,7 +235,7 @@ int git_diff_find_similar__hashsig_for_file(
 int git_diff_find_similar__hashsig_for_buf(
 	void **out, const git_diff_file *f, const char *buf, size_t len, void *p)
 {
-	git_hashsig_option_t opt = (git_hashsig_option_t)p;
+	git_hashsig_option_t opt = (git_hashsig_option_t)(intptr_t)p;
 	int error = 0;
 
 	GIT_UNUSED(f);
@@ -275,28 +274,30 @@ static int normalize_find_opts(
 {
 	git_config *cfg = NULL;
 
+	GITERR_CHECK_VERSION(given, GIT_DIFF_FIND_OPTIONS_VERSION, "git_diff_find_options");
+
 	if (diff->repo != NULL &&
 		git_repository_config__weakptr(&cfg, diff->repo) < 0)
 		return -1;
 
-	if (given != NULL)
+	if (given)
 		memcpy(opts, given, sizeof(*opts));
-	else {
-		const char *val = NULL;
-
-		GIT_INIT_STRUCTURE(opts, GIT_DIFF_FIND_OPTIONS_VERSION);
 
-		opts->flags = GIT_DIFF_FIND_RENAMES;
-
-		if (git_config_get_string(&val, cfg, "diff.renames") < 0)
-			giterr_clear();
-		else if (val &&
-			(!strcasecmp(val, "copies") || !strcasecmp(val, "copy")))
-			opts->flags = GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES;
+	if (!given ||
+		 (given->flags & GIT_DIFF_FIND_ALL) == GIT_DIFF_FIND_BY_CONFIG)
+	{
+		const char *rule =
+			git_config__get_string_force(cfg, "diff.renames", "true");
+		int boolval;
+
+		if (!git__parse_bool(&boolval, rule) && !boolval)
+			/* don't set FIND_RENAMES if bool value is false */;
+		else if (!strcasecmp(rule, "copies") || !strcasecmp(rule, "copy"))
+			opts->flags |= GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES;
+		else
+			opts->flags |= GIT_DIFF_FIND_RENAMES;
 	}
 
-	GITERR_CHECK_VERSION(opts, GIT_DIFF_FIND_OPTIONS_VERSION, "git_diff_find_options");
-
 	/* some flags imply others */
 
 	if (opts->flags & GIT_DIFF_FIND_EXACT_MATCH_ONLY) {
@@ -335,14 +336,11 @@ static int normalize_find_opts(
 #undef USE_DEFAULT
 
 	if (!opts->rename_limit) {
-		int32_t limit = 0;
+		opts->rename_limit = git_config__get_int_force(
+			cfg, "diff.renamelimit", DEFAULT_RENAME_LIMIT);
 
-		opts->rename_limit = DEFAULT_RENAME_LIMIT;
-
-		if (git_config_get_int32(&limit, cfg, "diff.renameLimit") < 0)
-			giterr_clear();
-		else if (limit > 0)
-			opts->rename_limit = limit;
+		if (opts->rename_limit <= 0)
+			opts->rename_limit = DEFAULT_RENAME_LIMIT;
 	}
 
 	/* assign the internal metric with whitespace flag as payload */
@@ -366,12 +364,28 @@ static int normalize_find_opts(
 	return 0;
 }
 
+static int insert_delete_side_of_split(
+	git_diff *diff, git_vector *onto, const git_diff_delta *delta)
+{
+	/* make new record for DELETED side of split */
+	git_diff_delta *deleted = diff_delta__dup(delta, &diff->pool);
+	GITERR_CHECK_ALLOC(deleted);
+
+	deleted->status = GIT_DELTA_DELETED;
+	deleted->nfiles = 1;
+	memset(&deleted->new_file, 0, sizeof(deleted->new_file));
+	deleted->new_file.path = deleted->old_file.path;
+	deleted->new_file.flags |= GIT_DIFF_FLAG_VALID_ID;
+
+	return git_vector_insert(onto, deleted);
+}
+
 static int apply_splits_and_deletes(
 	git_diff *diff, size_t expected_size, bool actually_split)
 {
 	git_vector onto = GIT_VECTOR_INIT;
 	size_t i;
-	git_diff_delta *delta, *deleted;
+	git_diff_delta *delta;
 
 	if (git_vector_init(&onto, expected_size, git_diff_delta__cmp) < 0)
 		return -1;
@@ -384,17 +398,7 @@ static int apply_splits_and_deletes(
 		if ((delta->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0 && actually_split) {
 			delta->similarity = 0;
 
-			/* make new record for DELETED side of split */
-			if (!(deleted = diff_delta__dup(delta, &diff->pool)))
-				goto on_error;
-
-			deleted->status = GIT_DELTA_DELETED;
-			deleted->nfiles = 1;
-			memset(&deleted->new_file, 0, sizeof(deleted->new_file));
-			deleted->new_file.path = deleted->old_file.path;
-			deleted->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
-
-			if (git_vector_insert(&onto, deleted) < 0)
+			if (insert_delete_side_of_split(diff, &onto, delta) < 0)
 				goto on_error;
 
 			if (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR)
@@ -404,7 +408,7 @@ static int apply_splits_and_deletes(
 			delta->nfiles = 1;
 			memset(&delta->old_file, 0, sizeof(delta->old_file));
 			delta->old_file.path = delta->new_file.path;
-			delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+			delta->old_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 		}
 
 		/* clean up delta before inserting into new list */
@@ -436,9 +440,7 @@ static int apply_splits_and_deletes(
 	return 0;
 
 on_error:
-	git_vector_foreach(&onto, i, delta)
-		git__free(delta);
-	git_vector_free(&onto);
+	git_vector_free_deep(&onto);
 
 	return -1;
 }
@@ -470,7 +472,7 @@ static int similarity_init(
 	info->blob = NULL;
 	git_buf_init(&info->data, 0);
 
-	if (info->file->size > 0)
+	if (info->file->size > 0 || info->src == GIT_ITERATOR_TYPE_WORKDIR)
 		return 0;
 
 	return git_diff_file__resolve_zero_size(
@@ -508,7 +510,7 @@ static int similarity_sig(
 				(git_object **)&info->blob, info->repo,
 				info->odb_obj, GIT_OBJ_BLOB);
 		else
-			error = git_blob_lookup(&info->blob, info->repo, &file->oid);
+			error = git_blob_lookup(&info->blob, info->repo, &file->id);
 
 		if (error < 0) {
 			/* if lookup fails, just skip this item in similarity calc */
@@ -570,21 +572,21 @@ static int similarity_measure(
 
 	/* if exact match is requested, force calculation of missing OIDs now */
 	if (exact_match) {
-		if (git_oid_iszero(&a_file->oid) &&
+		if (git_oid_iszero(&a_file->id) &&
 			diff->old_src == GIT_ITERATOR_TYPE_WORKDIR &&
-			!git_diff__oid_for_file(diff->repo, a_file->path,
-				a_file->mode, a_file->size, &a_file->oid))
-			a_file->flags |= GIT_DIFF_FLAG_VALID_OID;
+			!git_diff__oid_for_file(&a_file->id,
+				diff, a_file->path, a_file->mode, a_file->size))
+			a_file->flags |= GIT_DIFF_FLAG_VALID_ID;
 
-		if (git_oid_iszero(&b_file->oid) &&
+		if (git_oid_iszero(&b_file->id) &&
 			diff->new_src == GIT_ITERATOR_TYPE_WORKDIR &&
-			!git_diff__oid_for_file(diff->repo, b_file->path,
-				b_file->mode, b_file->size, &b_file->oid))
-			b_file->flags |= GIT_DIFF_FLAG_VALID_OID;
+			!git_diff__oid_for_file(&b_file->id,
+				diff, b_file->path, b_file->mode, b_file->size))
+			b_file->flags |= GIT_DIFF_FLAG_VALID_ID;
 	}
 
 	/* check OID match as a quick test */
-	if (git_oid__cmp(&a_file->oid, &b_file->oid) == 0) {
+	if (git_oid__cmp(&a_file->id, &b_file->id) == 0) {
 		*score = 100;
 		return 0;
 	}
@@ -740,6 +742,8 @@ static bool is_rename_source(
 	case GIT_DELTA_UNMODIFIED:
 		if (!FLAG_SET(opts, GIT_DIFF_FIND_COPIES_FROM_UNMODIFIED))
 			return false;
+		if (FLAG_SET(opts, GIT_DIFF_FIND_REMOVE_UNMODIFIED))
+			delta->flags |= GIT_DIFF_FLAG__TO_DELETE;
 		break;
 
 	default: /* MODIFIED, RENAMED, COPIED */
@@ -808,11 +812,11 @@ int git_diff_find_similar(
 	int error = 0, result;
 	uint16_t similarity;
 	git_diff_delta *src, *tgt;
-	git_diff_find_options opts;
+	git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT;
 	size_t num_deltas, num_srcs = 0, num_tgts = 0;
 	size_t tried_srcs = 0, tried_tgts = 0;
 	size_t num_rewrites = 0, num_updates = 0, num_bumped = 0;
-	void **sigcache; /* cache of similarity metric file signatures */
+	void **sigcache = NULL; /* cache of similarity metric file signatures */
 	diff_find_match *tgt2src = NULL;
 	diff_find_match *src2tgt = NULL;
 	diff_find_match *tgt2src_copy = NULL;
@@ -826,7 +830,11 @@ int git_diff_find_similar(
 
 	/* TODO: maybe abort if deltas.length > rename_limit ??? */
 	if (!git__is_uint32(num_deltas))
-		return 0;
+		goto cleanup;
+
+	/* No flags set; nothing to do */
+	if ((opts.flags & GIT_DIFF_FIND_ALL) == 0)
+		goto cleanup;
 
 	sigcache = git__calloc(num_deltas * 2, sizeof(void *));
 	GITERR_CHECK_ALLOC(sigcache);
@@ -991,7 +999,7 @@ find_best_matches:
 				memcpy(&src->old_file, &swap, sizeof(src->old_file));
 				memset(&src->new_file, 0, sizeof(src->new_file));
 				src->new_file.path = src->old_file.path;
-				src->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+				src->new_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
 				num_updates++;
 
@@ -1016,7 +1024,7 @@ find_best_matches:
 				src->nfiles = 1;
 				memset(&src->old_file, 0, sizeof(src->old_file));
 				src->old_file.path = src->new_file.path;
-				src->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+				src->old_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
 				src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
 				num_rewrites--;
@@ -1058,10 +1066,7 @@ find_best_matches:
 			}
 		}
 
-		else if (delta_is_new_only(tgt)) {
-			if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES))
-				continue;
-
+		else if (FLAG_SET(&opts, GIT_DIFF_FIND_COPIES)) {
 			if (tgt2src_copy[t].similarity < opts.copy_threshold)
 				continue;
 
@@ -1069,10 +1074,21 @@ find_best_matches:
 			best_match = &tgt2src_copy[t];
 			src = GIT_VECTOR_GET(&diff->deltas, best_match->idx);
 
+			if (delta_is_split(tgt)) {
+				error = insert_delete_side_of_split(diff, &diff->deltas, tgt);
+				if (error < 0)
+					goto cleanup;
+				num_rewrites--;
+			}
+
+			if (!delta_is_split(tgt) && !delta_is_new_only(tgt))
+				continue;
+
 			tgt->status     = GIT_DELTA_COPIED;
 			tgt->similarity = best_match->similarity;
 			tgt->nfiles     = 2;
 			memcpy(&tgt->old_file, &src->old_file, sizeof(tgt->old_file));
+			tgt->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
 
 			num_updates++;
 		}
@@ -1093,11 +1109,13 @@ cleanup:
 	git__free(src2tgt);
 	git__free(tgt2src_copy);
 
-	for (t = 0; t < num_deltas * 2; ++t) {
-		if (sigcache[t] != NULL)
-			opts.metric->free_signature(sigcache[t], opts.metric->payload);
+	if (sigcache) {
+		for (t = 0; t < num_deltas * 2; ++t) {
+			if (sigcache[t] != NULL)
+				opts.metric->free_signature(sigcache[t], opts.metric->payload);
+		}
+		git__free(sigcache);
 	}
-	git__free(sigcache);
 
 	if (!given_opts || !given_opts->metric)
 		git__free(opts.metric);
diff --git a/src/diff_xdiff.c b/src/diff_xdiff.c
index e0bc11f7f..e5984f1c9 100644
--- a/src/diff_xdiff.c
+++ b/src/diff_xdiff.c
@@ -28,25 +28,29 @@ static int git_xdiff_parse_hunk(git_diff_hunk *hunk, const char *header)
 {
 	/* expect something of the form "@@ -%d[,%d] +%d[,%d] @@" */
 	if (*header != '@')
-		return -1;
+		goto fail;
 	if (git_xdiff_scan_int(&header, &hunk->old_start) < 0)
-		return -1;
+		goto fail;
 	if (*header == ',') {
 		if (git_xdiff_scan_int(&header, &hunk->old_lines) < 0)
-			return -1;
+			goto fail;
 	} else
 		hunk->old_lines = 1;
 	if (git_xdiff_scan_int(&header, &hunk->new_start) < 0)
-		return -1;
+		goto fail;
 	if (*header == ',') {
 		if (git_xdiff_scan_int(&header, &hunk->new_lines) < 0)
-			return -1;
+			goto fail;
 	} else
 		hunk->new_lines = 1;
 	if (hunk->old_start < 0 || hunk->new_start < 0)
-		return -1;
+		goto fail;
 
 	return 0;
+
+fail:
+	giterr_set(GITERR_INVALID, "Malformed hunk header from xdiff");
+	return -1;
 }
 
 typedef struct {
@@ -122,8 +126,9 @@ static int git_xdiff_cb(void *priv, mmbuffer_t *bufs, int len)
 		info->hunk.header[info->hunk.header_len] = '\0';
 
 		if (output->hunk_cb != NULL &&
-			output->hunk_cb(delta, &info->hunk, output->payload))
-			output->error = GIT_EUSER;
+			(output->error = output->hunk_cb(
+				delta, &info->hunk, output->payload)))
+			return output->error;
 
 		info->old_lineno = info->hunk.old_start;
 		info->new_lineno = info->hunk.new_start;
@@ -146,10 +151,9 @@ static int git_xdiff_cb(void *priv, mmbuffer_t *bufs, int len)
 		output->error = diff_update_lines(
 			info, &line, bufs[1].ptr, bufs[1].size);
 
-		if (!output->error &&
-			output->data_cb != NULL &&
-			output->data_cb(delta, &info->hunk, &line, output->payload))
-			output->error = GIT_EUSER;
+		if (!output->error && output->data_cb != NULL)
+			output->error = output->data_cb(
+				delta, &info->hunk, &line, output->payload);
 	}
 
 	if (len == 3 && !output->error) {
@@ -168,10 +172,9 @@ static int git_xdiff_cb(void *priv, mmbuffer_t *bufs, int len)
 		output->error = diff_update_lines(
 			info, &line, bufs[2].ptr, bufs[2].size);
 
-		if (!output->error &&
-			output->data_cb != NULL &&
-			output->data_cb(delta, &info->hunk, &line, output->payload))
-			output->error = GIT_EUSER;
+		if (!output->error && output->data_cb != NULL)
+			output->error = output->data_cb(
+				delta, &info->hunk, &line, output->payload);
 	}
 
 	return output->error;
@@ -219,11 +222,9 @@ void git_xdiff_init(git_xdiff_output *xo, const git_diff_options *opts)
 
 	xo->output.diff_cb = git_xdiff;
 
-	memset(&xo->config, 0, sizeof(xo->config));
 	xo->config.ctxlen = opts ? opts->context_lines : 3;
 	xo->config.interhunkctxlen = opts ? opts->interhunk_lines : 0;
 
-	memset(&xo->params, 0, sizeof(xo->params));
 	if (flags & GIT_DIFF_IGNORE_WHITESPACE)
 		xo->params.flags |= XDF_WHITESPACE_FLAGS;
 	if (flags & GIT_DIFF_IGNORE_WHITESPACE_CHANGE)
@@ -236,6 +237,5 @@ void git_xdiff_init(git_xdiff_output *xo, const git_diff_options *opts)
 	if (flags & GIT_DIFF_MINIMAL)
 		xo->params.flags |= XDF_NEED_MINIMAL;
 
-	memset(&xo->callback, 0, sizeof(xo->callback));
 	xo->callback.outf = git_xdiff_cb;
 }
diff --git a/src/errors.c b/src/errors.c
index d04da4ca9..393a7875f 100644
--- a/src/errors.c
+++ b/src/errors.c
@@ -8,7 +8,6 @@
 #include "global.h"
 #include "posix.h"
 #include "buffer.h"
-#include <stdarg.h>
 
 /********************************************
  * New error handling
@@ -23,7 +22,8 @@ static void set_error(int error_class, char *string)
 {
 	git_error *error = &GIT_GLOBAL->error_t;
 
-	git__free(error->message);
+	if (error->message != string)
+		git__free(error->message);
 
 	error->message = string;
 	error->klass = error_class;
@@ -103,8 +103,10 @@ int giterr_set_regex(const regex_t *regex, int error_code)
 
 void giterr_clear(void)
 {
-	set_error(0, NULL);
-	GIT_GLOBAL->last_error = NULL;
+	if (GIT_GLOBAL->last_error != NULL) {
+		set_error(0, NULL);
+		GIT_GLOBAL->last_error = NULL;
+	}
 
 	errno = 0;
 #ifdef GIT_WIN32
@@ -134,3 +136,39 @@ const git_error *giterr_last(void)
 {
 	return GIT_GLOBAL->last_error;
 }
+
+int giterr_capture(git_error_state *state, int error_code)
+{
+	state->error_code = error_code;
+	if (error_code)
+		giterr_detach(&state->error_msg);
+	return error_code;
+}
+
+int giterr_restore(git_error_state *state)
+{
+	if (state && state->error_code && state->error_msg.message)
+		set_error(state->error_msg.klass, state->error_msg.message);
+	else
+		giterr_clear();
+
+	return state ? state->error_code : 0;
+}
+
+int giterr_system_last(void)
+{
+#ifdef GIT_WIN32
+	return GetLastError();
+#else
+	return errno;
+#endif
+}
+
+void giterr_system_set(int code)
+{
+#ifdef GIT_WIN32
+	SetLastError(code);
+#else
+	errno = code;
+#endif
+}
diff --git a/src/fetch.c b/src/fetch.c
index 276591821..9ff95d935 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -42,8 +42,9 @@ static int maybe_want(git_remote *remote, git_remote_head *head, git_odb *odb, g
 		return 0;
 
 	/* If we have the object, mark it so we don't ask for it */
-	if (git_odb_exists(odb, &head->oid))
+	if (git_odb_exists(odb, &head->oid)) {
 		head->local = 1;
+	}
 	else
 		remote->need_pack = 1;
 
@@ -104,7 +105,9 @@ cleanup:
 int git_fetch_negotiate(git_remote *remote)
 {
 	git_transport *t = remote->transport;
-	
+
+    remote->need_pack = 0;
+
 	if (filter_wants(remote) < 0) {
 		giterr_set(GITERR_NET, "Failed to filter the reference list for wants");
 		return -1;
@@ -128,9 +131,9 @@ int git_fetch_download_pack(git_remote *remote)
 {
 	git_transport *t = remote->transport;
 
-	if(!remote->need_pack)
+	if (!remote->need_pack)
 		return 0;
 
 	return t->download_pack(t, remote->repo, &remote->stats,
-				remote->callbacks.transfer_progress, remote->callbacks.payload);
+			remote->callbacks.transfer_progress, remote->callbacks.payload);
 }
diff --git a/src/fetch.h b/src/fetch.h
index 9605da1b5..f66e44663 100644
--- a/src/fetch.h
+++ b/src/fetch.h
@@ -17,7 +17,7 @@ int git_fetch__download_pack(
 		git_transport *t,
 		git_repository *repo,
 		git_transfer_progress *stats,
-		git_transfer_progress_callback progress_cb,
+		git_transfer_progress_cb progress_cb,
 		void *progress_payload);
 
 int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
diff --git a/src/fetchhead.c b/src/fetchhead.c
index 67089d13d..a95ea4ca4 100644
--- a/src/fetchhead.c
+++ b/src/fetchhead.c
@@ -210,7 +210,7 @@ static int fetchhead_ref_parse(
 			name = desc + 1;
 
 		if (name) {
-			if ((desc = strchr(name, '\'')) == NULL ||
+			if ((desc = strstr(name, "' ")) == NULL ||
 				git__prefixcmp(desc, "' of ") != 0) {
 				giterr_set(GITERR_FETCHHEAD,
 					"Invalid description in FETCH_HEAD line %d", line_num);
@@ -260,8 +260,8 @@ int git_repository_fetchhead_foreach(git_repository *repo,
 	while ((line = git__strsep(&buffer, "\n")) != NULL) {
 		++line_num;
 
-		if ((error = fetchhead_ref_parse(&oid, &is_merge, &name, &remote_url,
-			line, line_num)) < 0)
+		if ((error = fetchhead_ref_parse(
+				&oid, &is_merge, &name, &remote_url, line, line_num)) < 0)
 			goto done;
 
 		if (git_buf_len(&name) > 0)
@@ -269,8 +269,9 @@ int git_repository_fetchhead_foreach(git_repository *repo,
 		else
 			ref_name = NULL;
 
-		if ((cb(ref_name, remote_url, &oid, is_merge, payload)) != 0) {
-			error = GIT_EUSER;
+		error = cb(ref_name, remote_url, &oid, is_merge, payload);
+		if (error) {
+			giterr_set_after_callback(error);
 			goto done;
 		}
 	}
diff --git a/src/filebuf.c b/src/filebuf.c
index 9c3dae811..d23bcc11c 100644
--- a/src/filebuf.c
+++ b/src/filebuf.c
@@ -4,8 +4,6 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
-#include <stdarg.h>
-
 #include "common.h"
 #include "filebuf.h"
 #include "fileops.h"
diff --git a/src/fileops.c b/src/fileops.c
index 5763b370b..bebbae4f9 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -132,6 +132,7 @@ int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len)
 
 	if (read_size != (ssize_t)len) {
 		giterr_set(GITERR_OS, "Failed to read descriptor");
+		git_buf_free(buf);
 		return -1;
 	}
 
@@ -403,7 +404,6 @@ typedef struct {
 	const char *base;
 	size_t baselen;
 	uint32_t flags;
-	int error;
 	int depth;
 } futils__rmdir_data;
 
@@ -447,8 +447,8 @@ static int futils__rm_first_parent(git_buf *path, const char *ceiling)
 
 static int futils__rmdir_recurs_foreach(void *opaque, git_buf *path)
 {
+	int error = 0;
 	futils__rmdir_data *data = opaque;
-	int error = data->error;
 	struct stat st;
 
 	if (data->depth > FUTILS_MAX_DEPTH)
@@ -474,13 +474,14 @@ static int futils__rmdir_recurs_foreach(void *opaque, git_buf *path)
 		data->depth++;
 
 		error = git_path_direach(path, 0, futils__rmdir_recurs_foreach, data);
-		if (error < 0)
-			return (error == GIT_EUSER) ? data->error : error;
 
 		data->depth--;
 
+		if (error < 0)
+			return error;
+
 		if (data->depth == 0 && (data->flags & GIT_RMDIR_SKIP_ROOT) != 0)
-			return data->error;
+			return error;
 
 		if ((error = p_rmdir(path->ptr)) < 0) {
 			if ((data->flags & GIT_RMDIR_SKIP_NONEMPTY) != 0 &&
@@ -499,28 +500,23 @@ static int futils__rmdir_recurs_foreach(void *opaque, git_buf *path)
 	else if ((data->flags & GIT_RMDIR_SKIP_NONEMPTY) == 0)
 		error = futils__error_cannot_rmdir(path->ptr, "still present");
 
-	data->error = error;
 	return error;
 }
 
 static int futils__rmdir_empty_parent(void *opaque, git_buf *path)
 {
 	futils__rmdir_data *data = opaque;
-	int error;
+	int error = 0;
 
 	if (git_buf_len(path) <= data->baselen)
-		return GIT_ITEROVER;
-
-	error = p_rmdir(git_buf_cstr(path));
+		error = GIT_ITEROVER;
 
-	if (error) {
+	else if (p_rmdir(git_buf_cstr(path)) < 0) {
 		int en = errno;
 
 		if (en == ENOENT || en == ENOTDIR) {
-			giterr_clear();
-			error = 0;
+			/* do nothing */
 		} else if (en == ENOTEMPTY || en == EEXIST || en == EBUSY) {
-			giterr_clear();
 			error = GIT_ITEROVER;
 		} else {
 			error = git_path_set_error(errno, git_buf_cstr(path), "rmdir");
@@ -535,12 +531,13 @@ int git_futils_rmdir_r(
 {
 	int error;
 	git_buf fullpath = GIT_BUF_INIT;
-	futils__rmdir_data data = { 0 };
+	futils__rmdir_data data;
 
 	/* build path and find "root" where we should start calling mkdir */
 	if (git_path_join_unrooted(&fullpath, path, base, NULL) < 0)
 		return -1;
 
+	memset(&data, 0, sizeof(data));
 	data.base    = base ? base : "";
 	data.baselen = base ? strlen(base) : 0;
 	data.flags   = flags;
@@ -548,12 +545,13 @@ int git_futils_rmdir_r(
 	error = futils__rmdir_recurs_foreach(&data, &fullpath);
 
 	/* remove now-empty parents if requested */
-	if (!error && (flags & GIT_RMDIR_EMPTY_PARENTS) != 0) {
+	if (!error && (flags & GIT_RMDIR_EMPTY_PARENTS) != 0)
 		error = git_path_walk_up(
 			&fullpath, base, futils__rmdir_empty_parent, &data);
 
-		if (error == GIT_ITEROVER)
-			error = 0;
+	if (error == GIT_ITEROVER) {
+		giterr_clear();
+		error = 0;
 	}
 
 	git_buf_free(&fullpath);
@@ -561,219 +559,6 @@ int git_futils_rmdir_r(
 	return error;
 }
 
-
-static int git_futils_guess_system_dirs(git_buf *out)
-{
-#ifdef GIT_WIN32
-	return git_win32__find_system_dirs(out, L"etc\\");
-#else
-	return git_buf_sets(out, "/etc");
-#endif
-}
-
-static int git_futils_guess_global_dirs(git_buf *out)
-{
-#ifdef GIT_WIN32
-	return git_win32__find_global_dirs(out);
-#else
-	return git_buf_sets(out, getenv("HOME"));
-#endif
-}
-
-static int git_futils_guess_xdg_dirs(git_buf *out)
-{
-#ifdef GIT_WIN32
-	return git_win32__find_xdg_dirs(out);
-#else
-	const char *env = NULL;
-
-	if ((env = getenv("XDG_CONFIG_HOME")) != NULL)
-		return git_buf_joinpath(out, env, "git");
-	else if ((env = getenv("HOME")) != NULL)
-		return git_buf_joinpath(out, env, ".config/git");
-
-	git_buf_clear(out);
-	return 0;
-#endif
-}
-
-static int git_futils_guess_template_dirs(git_buf *out)
-{
-#ifdef GIT_WIN32
-	return git_win32__find_system_dirs(out, L"share\\git-core\\templates");
-#else
-	return git_buf_sets(out, "/usr/share/git-core/templates");
-#endif
-}
-
-typedef int (*git_futils_dirs_guess_cb)(git_buf *out);
-
-static git_buf git_futils__dirs[GIT_FUTILS_DIR__MAX] =
-	{ GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT };
-
-static git_futils_dirs_guess_cb git_futils__dir_guess[GIT_FUTILS_DIR__MAX] = {
-	git_futils_guess_system_dirs,
-	git_futils_guess_global_dirs,
-	git_futils_guess_xdg_dirs,
-	git_futils_guess_template_dirs,
-};
-
-void git_futils_dirs_global_shutdown(void)
-{
-	int i;
-	for (i = 0; i < GIT_FUTILS_DIR__MAX; ++i)
-		git_buf_free(&git_futils__dirs[i]);
-}
-
-int git_futils_dirs_global_init(void)
-{
-	git_futils_dir_t i;
-	const git_buf *path;
-	int error = 0;
-
-	for (i = 0; !error && i < GIT_FUTILS_DIR__MAX; i++)
-		error = git_futils_dirs_get(&path, i);
-
-	git__on_shutdown(git_futils_dirs_global_shutdown);
-
-	return error;
-}
-
-static int git_futils_check_selector(git_futils_dir_t which)
-{
-	if (which < GIT_FUTILS_DIR__MAX)
-		return 0;
-	giterr_set(GITERR_INVALID, "config directory selector out of range");
-	return -1;
-}
-
-int git_futils_dirs_get(const git_buf **out, git_futils_dir_t which)
-{
-	assert(out);
-
-	*out = NULL;
-
-	GITERR_CHECK_ERROR(git_futils_check_selector(which));
-
-	if (!git_buf_len(&git_futils__dirs[which]))
-		GITERR_CHECK_ERROR(
-			git_futils__dir_guess[which](&git_futils__dirs[which]));
-
-	*out = &git_futils__dirs[which];
-	return 0;
-}
-
-int git_futils_dirs_get_str(char *out, size_t outlen, git_futils_dir_t which)
-{
-	const git_buf *path = NULL;
-
-	GITERR_CHECK_ERROR(git_futils_check_selector(which));
-	GITERR_CHECK_ERROR(git_futils_dirs_get(&path, which));
-
-	if (!out || path->size >= outlen) {
-		giterr_set(GITERR_NOMEMORY, "Buffer is too short for the path");
-		return GIT_EBUFS;
-	}
-
-	git_buf_copy_cstr(out, outlen, path);
-	return 0;
-}
-
-#define PATH_MAGIC "$PATH"
-
-int git_futils_dirs_set(git_futils_dir_t which, const char *search_path)
-{
-	const char *expand_path = NULL;
-	git_buf merge = GIT_BUF_INIT;
-
-	GITERR_CHECK_ERROR(git_futils_check_selector(which));
-
-	if (search_path != NULL)
-		expand_path = strstr(search_path, PATH_MAGIC);
-
-	/* init with default if not yet done and needed (ignoring error) */
-	if ((!search_path || expand_path) &&
-		!git_buf_len(&git_futils__dirs[which]))
-		git_futils__dir_guess[which](&git_futils__dirs[which]);
-
-	/* if $PATH is not referenced, then just set the path */
-	if (!expand_path)
-		return git_buf_sets(&git_futils__dirs[which], search_path);
-
-	/* otherwise set to join(before $PATH, old value, after $PATH) */
-	if (expand_path > search_path)
-		git_buf_set(&merge, search_path, expand_path - search_path);
-
-	if (git_buf_len(&git_futils__dirs[which]))
-		git_buf_join(&merge, GIT_PATH_LIST_SEPARATOR,
-			merge.ptr, git_futils__dirs[which].ptr);
-
-	expand_path += strlen(PATH_MAGIC);
-	if (*expand_path)
-		git_buf_join(&merge, GIT_PATH_LIST_SEPARATOR, merge.ptr, expand_path);
-
-	git_buf_swap(&git_futils__dirs[which], &merge);
-	git_buf_free(&merge);
-
-	return git_buf_oom(&git_futils__dirs[which]) ? -1 : 0;
-}
-
-static int git_futils_find_in_dirlist(
-	git_buf *path, const char *name, git_futils_dir_t which, const char *label)
-{
-	size_t len;
-	const char *scan, *next = NULL;
-	const git_buf *syspath;
-
-	GITERR_CHECK_ERROR(git_futils_dirs_get(&syspath, which));
-
-	for (scan = git_buf_cstr(syspath); scan; scan = next) {
-		for (next = strchr(scan, GIT_PATH_LIST_SEPARATOR);
-			 next && next > scan && next[-1] == '\\';
-			 next = strchr(next + 1, GIT_PATH_LIST_SEPARATOR))
-			/* find unescaped separator or end of string */;
-
-		len = next ? (size_t)(next++ - scan) : strlen(scan);
-		if (!len)
-			continue;
-
-		GITERR_CHECK_ERROR(git_buf_set(path, scan, len));
-		if (name)
-			GITERR_CHECK_ERROR(git_buf_joinpath(path, path->ptr, name));
-
-		if (git_path_exists(path->ptr))
-			return 0;
-	}
-
-	git_buf_clear(path);
-	giterr_set(GITERR_OS, "The %s file '%s' doesn't exist", label, name);
-	return GIT_ENOTFOUND;
-}
-
-int git_futils_find_system_file(git_buf *path, const char *filename)
-{
-	return git_futils_find_in_dirlist(
-		path, filename, GIT_FUTILS_DIR_SYSTEM, "system");
-}
-
-int git_futils_find_global_file(git_buf *path, const char *filename)
-{
-	return git_futils_find_in_dirlist(
-		path, filename, GIT_FUTILS_DIR_GLOBAL, "global");
-}
-
-int git_futils_find_xdg_file(git_buf *path, const char *filename)
-{
-	return git_futils_find_in_dirlist(
-		path, filename, GIT_FUTILS_DIR_XDG, "global/xdg");
-}
-
-int git_futils_find_template_dir(git_buf *path)
-{
-	return git_futils_find_in_dirlist(
-		path, NULL, GIT_FUTILS_DIR_TEMPLATE, "template");
-}
-
 int git_futils_fake_symlink(const char *old, const char *new)
 {
 	int retcode = GIT_ERROR;
@@ -858,7 +643,6 @@ typedef struct {
 	uint32_t flags;
 	uint32_t mkdir_flags;
 	mode_t dirmode;
-	int error;
 } cp_r_info;
 
 #define GIT_CPDIR__MKDIR_DONE_FOR_TO_ROOT (1u << 10)
@@ -896,23 +680,21 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		from->ptr[git_path_basename_offset(from)] == '.')
 		return 0;
 
-	if (git_buf_joinpath(
-			&info->to, info->to_root, from->ptr + info->from_prefix) < 0) {
-		error = -1;
-		goto exit;
-	}
+	if ((error = git_buf_joinpath(
+			&info->to, info->to_root, from->ptr + info->from_prefix)) < 0)
+		return error;
 
 	if (!(error = git_path_lstat(info->to.ptr, &to_st)))
 		exists = true;
 	else if (error != GIT_ENOTFOUND)
-		goto exit;
+		return error;
 	else {
 		giterr_clear();
 		error = 0;
 	}
 
 	if ((error = git_path_lstat(from->ptr, &from_st)) < 0)
-		goto exit;
+		return error;
 
 	if (S_ISDIR(from_st.st_mode)) {
 		mode_t oldmode = info->dirmode;
@@ -926,17 +708,13 @@ static int _cp_r_callback(void *ref, git_buf *from)
 			error = _cp_r_mkdir(info, from);
 
 		/* recurse onto target directory */
-		if (!error && (!exists || S_ISDIR(to_st.st_mode))) {
+		if (!error && (!exists || S_ISDIR(to_st.st_mode)))
 			error = git_path_direach(from, 0, _cp_r_callback, info);
 
-			if (error == GIT_EUSER)
-				error = info->error;
-		}
-
 		if (oldmode != 0)
 			info->dirmode = oldmode;
 
-		goto exit;
+		return error;
 	}
 
 	if (exists) {
@@ -946,8 +724,7 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		if (p_unlink(info->to.ptr) < 0) {
 			giterr_set(GITERR_OS, "Cannot overwrite existing file '%s'",
 				info->to.ptr);
-			error = -1;
-			goto exit;
+			return GIT_EEXISTS;
 		}
 	}
 
@@ -960,12 +737,14 @@ static int _cp_r_callback(void *ref, git_buf *from)
 	/* Make container directory on demand if needed */
 	if ((info->flags & GIT_CPDIR_CREATE_EMPTY_DIRS) == 0 &&
 		(error = _cp_r_mkdir(info, from)) < 0)
-		goto exit;
+		return error;
 
 	/* make symlink or regular file */
-	if (S_ISLNK(from_st.st_mode))
+	if (info->flags & GIT_CPDIR_LINK_FILES) {
+		error = p_link(from->ptr, info->to.ptr);
+	} else if (S_ISLNK(from_st.st_mode)) {
 		error = cp_link(from->ptr, info->to.ptr, (size_t)from_st.st_size);
-	else {
+	} else {
 		mode_t usemode = from_st.st_mode;
 
 		if ((info->flags & GIT_CPDIR_SIMPLE_TO_MODE) != 0)
@@ -974,8 +753,6 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		error = git_futils_cp(from->ptr, info->to.ptr, usemode);
 	}
 
-exit:
-	info->error = error;
 	return error;
 }
 
@@ -992,11 +769,11 @@ int git_futils_cp_r(
 	if (git_buf_joinpath(&path, from, "") < 0) /* ensure trailing slash */
 		return -1;
 
+	memset(&info, 0, sizeof(info));
 	info.to_root = to;
 	info.flags   = flags;
 	info.dirmode = dirmode;
 	info.from_prefix = path.size;
-	info.error = 0;
 	git_buf_init(&info.to, 0);
 
 	/* precalculate mkdir flags */
@@ -1018,9 +795,6 @@ int git_futils_cp_r(
 	git_buf_free(&path);
 	git_buf_free(&info.to);
 
-	if (error == GIT_EUSER)
-		error = info.error;
-
 	return error;
 }
 
@@ -1033,10 +807,8 @@ int git_futils_filestamp_check(
 	if (stamp == NULL)
 		return 1;
 
-	if (p_stat(path, &st) < 0) {
-		giterr_set(GITERR_OS, "Could not stat '%s'", path);
+	if (p_stat(path, &st) < 0)
 		return GIT_ENOTFOUND;
-	}
 
 	if (stamp->mtime == (git_time_t)st.st_mtime &&
 		stamp->size  == (git_off_t)st.st_size   &&
@@ -1060,3 +832,16 @@ void git_futils_filestamp_set(
 	else
 		memset(target, 0, sizeof(*target));
 }
+
+
+void git_futils_filestamp_set_from_stat(
+	git_futils_filestamp *stamp, struct stat *st)
+{
+	if (st) {
+		stamp->mtime = (git_time_t)st->st_mtime;
+		stamp->size  = (git_off_t)st->st_size;
+		stamp->ino   = (unsigned int)st->st_ino;
+	} else {
+		memset(stamp, 0, sizeof(*stamp));
+	}
+}
diff --git a/src/fileops.h b/src/fileops.h
index 636c9b67d..4f5700a99 100644
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -173,6 +173,7 @@ extern int git_futils_cp(
  * - GIT_CPDIR_SIMPLE_TO_MODE: default tries to replicate the mode of the
  *   source file to the target; with this flag, always use 0666 (or 0777 if
  *   source has exec bits set) for target.
+ * - GIT_CPDIR_LINK_FILES will try to use hardlinks for the files
  */
 typedef enum {
 	GIT_CPDIR_CREATE_EMPTY_DIRS = (1u << 0),
@@ -181,6 +182,7 @@ typedef enum {
 	GIT_CPDIR_OVERWRITE         = (1u << 3),
 	GIT_CPDIR_CHMOD_DIRS        = (1u << 4),
 	GIT_CPDIR_SIMPLE_TO_MODE    = (1u << 5),
+	GIT_CPDIR_LINK_FILES        = (1u << 6),
 } git_futils_cpdir_flags;
 
 /**
@@ -268,89 +270,6 @@ extern int git_futils_mmap_ro_file(
 extern void git_futils_mmap_free(git_map *map);
 
 /**
- * Find a "global" file (i.e. one in a user's home directory).
- *
- * @param path buffer to write the full path into
- * @param filename name of file to find in the home directory
- * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
- */
-extern int git_futils_find_global_file(git_buf *path, const char *filename);
-
-/**
- * Find an "XDG" file (i.e. one in user's XDG config path).
- *
- * @param path buffer to write the full path into
- * @param filename name of file to find in the home directory
- * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
- */
-extern int git_futils_find_xdg_file(git_buf *path, const char *filename);
-
-/**
- * Find a "system" file (i.e. one shared for all users of the system).
- *
- * @param path buffer to write the full path into
- * @param filename name of file to find in the home directory
- * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
- */
-extern int git_futils_find_system_file(git_buf *path, const char *filename);
-
-/**
- * Find template directory.
- *
- * @param path buffer to write the full path into
- * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
- */
-extern int git_futils_find_template_dir(git_buf *path);
-
-typedef enum {
-	GIT_FUTILS_DIR_SYSTEM = 0,
-	GIT_FUTILS_DIR_GLOBAL = 1,
-	GIT_FUTILS_DIR_XDG    = 2,
-	GIT_FUTILS_DIR_TEMPLATE = 3,
-	GIT_FUTILS_DIR__MAX   = 4,
-} git_futils_dir_t;
-
-/**
- * Configures global data for configuration file search paths.
- *
- * @return 0 on success, <0 on failure
- */
-extern int git_futils_dirs_global_init(void);
-
-/**
- * Get the search path for global/system/xdg files
- *
- * @param out pointer to git_buf containing search path
- * @param which which list of paths to return
- * @return 0 on success, <0 on failure
- */
-extern int git_futils_dirs_get(const git_buf **out, git_futils_dir_t which);
-
-/**
- * Get search path into a preallocated buffer
- *
- * @param out String buffer to write into
- * @param outlen Size of string buffer
- * @param which Which search path to return
- * @return 0 on success, GIT_EBUFS if out is too small, <0 on other failure
- */
-
-extern int git_futils_dirs_get_str(
-	char *out, size_t outlen, git_futils_dir_t which);
-
-/**
- * Set search paths for global/system/xdg files
- *
- * The first occurrence of the magic string "$PATH" in the new value will
- * be replaced with the old value of the search path.
- *
- * @param which Which search path to modify
- * @param paths New search path (separated by GIT_PATH_LIST_SEPARATOR)
- * @return 0 on success, <0 on failure (allocation error)
- */
-extern int git_futils_dirs_set(git_futils_dir_t which, const char *paths);
-
-/**
  * Create a "fake" symlink (text file containing the target path).
  *
  * @param new symlink file to be created
@@ -375,13 +294,14 @@ typedef struct {
  * Compare stat information for file with reference info.
  *
  * This function updates the file stamp to current data for the given path
- * and returns 0 if the file is up-to-date relative to the prior setting or
- * 1 if the file has been changed.  (This also may return GIT_ENOTFOUND if
- * the file doesn't exist.)
+ * and returns 0 if the file is up-to-date relative to the prior setting,
+ * 1 if the file has been changed, or GIT_ENOTFOUND if the file doesn't
+ * exist.  This will not call giterr_set, so you must set the error if you
+ * plan to return an error.
  *
  * @param stamp File stamp to be checked
  * @param path Path to stat and check if changed
- * @return 0 if up-to-date, 1 if out-of-date, <0 on error
+ * @return 0 if up-to-date, 1 if out-of-date, GIT_ENOTFOUND if cannot stat
  */
 extern int git_futils_filestamp_check(
 	git_futils_filestamp *stamp, const char *path);
@@ -400,8 +320,9 @@ extern void git_futils_filestamp_set(
 	git_futils_filestamp *tgt, const git_futils_filestamp *src);
 
 /**
- * Free the configuration file search paths.
+ * Set file stamp data from stat structure
  */
-extern void git_futils_dirs_global_shutdown(void);
+extern void git_futils_filestamp_set_from_stat(
+	git_futils_filestamp *stamp, struct stat *st);
 
 #endif /* INCLUDE_fileops_h__ */
diff --git a/src/filter.c b/src/filter.c
index 9f866fe88..b9e4f9ec8 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -23,6 +23,7 @@ struct git_filter_source {
 	git_oid         oid;  /* zero if unknown (which is likely) */
 	uint16_t        filemode; /* zero if unknown */
 	git_filter_mode_t mode;
+	uint32_t        options;
 };
 
 typedef struct {
@@ -69,7 +70,7 @@ static void filter_registry_shutdown(void)
 		return;
 
 	git_vector_foreach(&reg->filters, pos, fdef) {
-		if (fdef->initialized && fdef->filter && fdef->filter->shutdown) {
+		if (fdef->filter && fdef->filter->shutdown) {
 			fdef->filter->shutdown(fdef->filter);
 			fdef->initialized = false;
 		}
@@ -358,6 +359,11 @@ git_filter_mode_t git_filter_source_mode(const git_filter_source *src)
 	return src->mode;
 }
 
+uint32_t git_filter_source_options(const git_filter_source *src)
+{
+	return src->options;
+}
+
 static int filter_list_new(
 	git_filter_list **out, const git_filter_source *src)
 {
@@ -372,6 +378,7 @@ static int filter_list_new(
 	fl->source.repo = src->repo;
 	fl->source.path = fl->path;
 	fl->source.mode = src->mode;
+	fl->source.options = src->options;
 
 	*out = fl;
 	return 0;
@@ -419,12 +426,16 @@ static int filter_list_check_attributes(
 }
 
 int git_filter_list_new(
-	git_filter_list **out, git_repository *repo, git_filter_mode_t mode)
+	git_filter_list **out,
+	git_repository *repo,
+	git_filter_mode_t mode,
+	uint32_t options)
 {
 	git_filter_source src = { 0 };
 	src.repo = repo;
 	src.path = NULL;
 	src.mode = mode;
+	src.options = options;
 	return filter_list_new(out, &src);
 }
 
@@ -433,7 +444,8 @@ int git_filter_list_load(
 	git_repository *repo,
 	git_blob *blob, /* can be NULL */
 	const char *path,
-	git_filter_mode_t mode)
+	git_filter_mode_t mode,
+	uint32_t options)
 {
 	int error = 0;
 	git_filter_list *fl = NULL;
@@ -448,6 +460,7 @@ int git_filter_list_load(
 	src.repo = repo;
 	src.path = path;
 	src.mode = mode;
+	src.options = options;
 	if (blob)
 		git_oid_cpy(&src.oid, git_blob_id(blob));
 
@@ -578,6 +591,9 @@ int git_filter_list_apply_to_data(
 	git_buf *dbuffer[2], local = GIT_BUF_INIT;
 	unsigned int si = 0;
 
+	git_buf_sanitize(tgt);
+	git_buf_sanitize(src);
+
 	if (!fl)
 		return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size);
 
@@ -613,11 +629,11 @@ int git_filter_list_apply_to_data(
 			/* PASSTHROUGH means filter decided not to process the buffer */
 			error = 0;
 		} else if (!error) {
-			git_buf_shorten(dbuffer[di], 0); /* force NUL termination */
+			git_buf_sanitize(dbuffer[di]); /* force NUL termination */
 			si = di; /* swap buffers */
 		} else {
 			tgt->size = 0;
-			return error;
+			goto cleanup;
 		}
 	}
 
@@ -625,9 +641,10 @@ int git_filter_list_apply_to_data(
 	if (si != 1)
 		git_buf_swap(dbuffer[0], dbuffer[1]);
 
+cleanup:
 	git_buf_free(&local); /* don't leak if we allocated locally */
 
-	return 0;
+	return error;
 }
 
 int git_filter_list_apply_to_file(
diff --git a/src/filter.h b/src/filter.h
index d0ace0f9a..5a366108b 100644
--- a/src/filter.h
+++ b/src/filter.h
@@ -10,6 +10,10 @@
 #include "common.h"
 #include "git2/filter.h"
 
+/* Amount of file to examine for NUL byte when checking binary-ness */
+#define GIT_FILTER_BYTES_TO_CHECK_NUL 8000
+
+/* Possible CRLF values */
 typedef enum {
 	GIT_CRLF_GUESS = -1,
 	GIT_CRLF_BINARY = 0,
diff --git a/src/fnmatch.c b/src/fnmatch.c
index e3e47f37b..d7899e3e6 100644
--- a/src/fnmatch.c
+++ b/src/fnmatch.c
@@ -6,6 +6,40 @@
  */
 
 /*
+ * This file contains code originally derrived from OpenBSD fnmatch.c 
+ *
+ * Copyright (c) 1989, 1993, 1994
+ *      The Regents of the University of California.  All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * Guido van Rossum.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
  * Function fnmatch() as specified in POSIX 1003.2-1992, section B.6.
  * Compares a filename or pathname to a pattern.
  */
@@ -30,6 +64,7 @@ p_fnmatchx(const char *pattern, const char *string, int flags, size_t recurs)
 		const char *stringstart;
 		char *newp;
 		char c, test;
+		int recurs_flags = flags & ~FNM_PERIOD;
 
 		if (recurs-- == 0)
 				return FNM_NORES;
@@ -53,9 +88,17 @@ p_fnmatchx(const char *pattern, const char *string, int flags, size_t recurs)
 						break;
 				case '*':
 						c = *pattern;
-						/* Collapse multiple stars. */
-						while (c == '*')
+
+						/* Let '**' override PATHNAME match for this segment.
+						 * It will be restored if/when we recurse below.
+						 */
+						if (c == '*') {
+							flags &= ~FNM_PATHNAME;
+							while (c == '*')
 								c = *++pattern;
+							if (c == '/')
+								c = *++pattern;
+						}
 
 						if (*string == '.' && (flags & FNM_PERIOD) &&
 							(string == stringstart ||
@@ -80,7 +123,7 @@ p_fnmatchx(const char *pattern, const char *string, int flags, size_t recurs)
 						while ((test = *string) != EOS) {
 								int e;
 
-								e = p_fnmatchx(pattern, string, flags & ~FNM_PERIOD, recurs);
+								e = p_fnmatchx(pattern, string, recurs_flags, recurs);
 								if (e != FNM_NOMATCH)
 										return e;
 								if (test == '/' && (flags & FNM_PATHNAME))
diff --git a/src/fnmatch.h b/src/fnmatch.h
index 920e7de4d..88af45939 100644
--- a/src/fnmatch.h
+++ b/src/fnmatch.h
@@ -1,8 +1,29 @@
 /*
- * Copyright (C) the libgit2 contributors. All rights reserved.
+ * Copyright (C) 2008 The Android Open Source Project
+ * All rights reserved.
  *
- * This file is part of libgit2, distributed under the GNU GPL v2 with
- * a Linking Exception. For full terms see the included COPYING file.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
  */
 #ifndef INCLUDE_fnmatch__compat_h__
 #define INCLUDE_fnmatch__compat_h__
diff --git a/src/global.c b/src/global.c
index 7d39c6fa8..03a4bcedf 100644
--- a/src/global.c
+++ b/src/global.c
@@ -7,7 +7,7 @@
 #include "common.h"
 #include "global.h"
 #include "hash.h"
-#include "fileops.h"
+#include "sysdir.h"
 #include "git2/threads.h"
 #include "thread-utils.h"
 
@@ -16,13 +16,20 @@ git_mutex git__mwindow_mutex;
 
 #define MAX_SHUTDOWN_CB 8
 
-git_global_shutdown_fn git__shutdown_callbacks[MAX_SHUTDOWN_CB];
-git_atomic git__n_shutdown_callbacks;
+#ifdef GIT_SSL
+# include <openssl/ssl.h>
+SSL_CTX *git__ssl_ctx;
+static git_mutex *openssl_locks;
+#endif
+
+static git_global_shutdown_fn git__shutdown_callbacks[MAX_SHUTDOWN_CB];
+static git_atomic git__n_shutdown_callbacks;
+static git_atomic git__n_inits;
 
 void git__on_shutdown(git_global_shutdown_fn callback)
 {
 	int count = git_atomic_inc(&git__n_shutdown_callbacks);
-	assert(count <= MAX_SHUTDOWN_CB);
+	assert(count <= MAX_SHUTDOWN_CB && count > 0);
 	git__shutdown_callbacks[count - 1] = callback;
 }
 
@@ -30,10 +37,68 @@ static void git__shutdown(void)
 {
 	int pos;
 
-	while ((pos = git_atomic_dec(&git__n_shutdown_callbacks)) >= 0) {
-		if (git__shutdown_callbacks[pos])
-			git__shutdown_callbacks[pos]();
+	for (pos = git_atomic_get(&git__n_shutdown_callbacks); pos > 0; pos = git_atomic_dec(&git__n_shutdown_callbacks)) {
+		git_global_shutdown_fn cb = git__swap(git__shutdown_callbacks[pos - 1], NULL);
+		if (cb != NULL)
+			cb();
+	}
+
+}
+
+#if defined(GIT_THREADS) && defined(GIT_SSL)
+void openssl_locking_function(int mode, int n, const char *file, int line)
+{
+	int lock;
+
+	GIT_UNUSED(file);
+	GIT_UNUSED(line);
+
+	lock = mode & CRYPTO_LOCK;
+
+	if (lock) {
+		git_mutex_lock(&openssl_locks[n]);
+	} else {
+		git_mutex_unlock(&openssl_locks[n]);
+	}
+}
+#endif
+
+
+static void init_ssl(void)
+{
+#ifdef GIT_SSL
+	SSL_load_error_strings();
+	OpenSSL_add_ssl_algorithms();
+	git__ssl_ctx = SSL_CTX_new(SSLv23_method());
+	SSL_CTX_set_mode(git__ssl_ctx, SSL_MODE_AUTO_RETRY);
+	SSL_CTX_set_verify(git__ssl_ctx, SSL_VERIFY_NONE, NULL);
+	if (!SSL_CTX_set_default_verify_paths(git__ssl_ctx)) {
+		SSL_CTX_free(git__ssl_ctx);
+		git__ssl_ctx = NULL;
+	}
+
+# ifdef GIT_THREADS
+	{
+		int num_locks, i;
+
+		num_locks = CRYPTO_num_locks();
+		openssl_locks = git__calloc(num_locks, sizeof(git_mutex));
+		if (openssl_locks == NULL) {
+			SSL_CTX_free(git__ssl_ctx);
+			git__ssl_ctx = NULL;
+		}
+
+		for (i = 0; i < num_locks; i++) {
+			if (git_mutex_init(&openssl_locks[i]) != 0) {
+				SSL_CTX_free(git__ssl_ctx);
+				git__ssl_ctx = NULL;
+			}
+		}
+
+		CRYPTO_set_locking_callback(openssl_locking_function);
 	}
+# endif
+#endif
 }
 
 /**
@@ -73,10 +138,9 @@ static void git__shutdown(void)
 #if defined(GIT_THREADS) && defined(GIT_WIN32)
 
 static DWORD _tls_index;
-static DWORD _mutex = 0;
-static DWORD _n_inits = 0;
+static volatile LONG _mutex = 0;
 
-static int synchronized_threads_init()
+static int synchronized_threads_init(void)
 {
 	int error;
 
@@ -86,7 +150,7 @@ static int synchronized_threads_init()
 
 	/* Initialize any other subsystems that have global state */
 	if ((error = git_hash_global_init()) >= 0)
-		error = git_futils_dirs_global_init();
+		error = git_sysdir_global_init();
 
 	win32_pthread_initialize();
 
@@ -101,7 +165,7 @@ int git_threads_init(void)
 	while (InterlockedCompareExchange(&_mutex, 1, 0)) { Sleep(0); }
 
 	/* Only do work on a 0 -> 1 transition of the refcount */
-	if (1 == ++_n_inits)
+	if (1 == git_atomic_inc(&git__n_inits))
 		error = synchronized_threads_init();
 
 	/* Exit the lock */
@@ -110,7 +174,7 @@ int git_threads_init(void)
 	return error;
 }
 
-static void synchronized_threads_shutdown()
+static void synchronized_threads_shutdown(void)
 {
 	/* Shut down any subsystems that have global state */
 	git__shutdown();
@@ -124,7 +188,7 @@ void git_threads_shutdown(void)
 	while (InterlockedCompareExchange(&_mutex, 1, 0)) { Sleep(0); }
 
 	/* Only do work on a 1 -> 0 transition of the refcount */
-	if (0 == --_n_inits)
+	if (0 == git_atomic_dec(&git__n_inits))
 		synchronized_threads_shutdown();
 
 	/* Exit the lock */
@@ -135,7 +199,7 @@ git_global_st *git__global_state(void)
 {
 	void *ptr;
 
-	assert(_n_inits);
+	assert(git_atomic_get(&git__n_inits) > 0);
 
 	if ((ptr = TlsGetValue(_tls_index)) != NULL)
 		return ptr;
@@ -153,7 +217,6 @@ git_global_st *git__global_state(void)
 
 static pthread_key_t _tls_key;
 static pthread_once_t _once_init = PTHREAD_ONCE_INIT;
-static git_atomic git__n_inits;
 int init_error = 0;
 
 static void cb__free_status(void *st)
@@ -167,9 +230,13 @@ static void init_once(void)
 		return;
 	pthread_key_create(&_tls_key, &cb__free_status);
 
+
 	/* Initialize any other subsystems that have global state */
 	if ((init_error = git_hash_global_init()) >= 0)
-		init_error = git_futils_dirs_global_init();
+		init_error = git_sysdir_global_init();
+
+	/* OpenSSL needs to be initialized from the main thread */
+	init_ssl();
 
 	GIT_MEMORY_BARRIER;
 }
@@ -183,6 +250,7 @@ int git_threads_init(void)
 
 void git_threads_shutdown(void)
 {
+	void *ptr = NULL;
 	pthread_once_t new_once = PTHREAD_ONCE_INIT;
 
 	if (git_atomic_dec(&git__n_inits) > 0) return;
@@ -190,7 +258,7 @@ void git_threads_shutdown(void)
 	/* Shut down any subsystems that have global state */
 	git__shutdown();
 
-	void *ptr = pthread_getspecific(_tls_key);
+	ptr = pthread_getspecific(_tls_key);
 	pthread_setspecific(_tls_key, NULL);
 	git__free(ptr);
 
@@ -203,7 +271,7 @@ git_global_st *git__global_state(void)
 {
 	void *ptr;
 
-	assert(git__n_inits.val);
+	assert(git_atomic_get(&git__n_inits) > 0);
 
 	if ((ptr = pthread_getspecific(_tls_key)) != NULL)
 		return ptr;
@@ -223,14 +291,16 @@ static git_global_st __state;
 
 int git_threads_init(void)
 {
-	/* noop */
+	init_ssl();
+	git_atomic_inc(&git__n_inits);
 	return 0;
 }
 
 void git_threads_shutdown(void)
 {
 	/* Shut down any subsystems that have global state */
-	git__shutdown();
+	if (0 == git_atomic_dec(&git__n_inits))
+		git__shutdown();
 }
 
 git_global_st *git__global_state(void)
diff --git a/src/global.h b/src/global.h
index 778250376..745df3e4a 100644
--- a/src/global.h
+++ b/src/global.h
@@ -15,6 +15,11 @@ typedef struct {
 	git_error error_t;
 } git_global_st;
 
+#ifdef GIT_SSL
+# include <openssl/ssl.h>
+extern SSL_CTX *git__ssl_ctx;
+#endif
+
 git_global_st *git__global_state(void);
 
 extern git_mutex git__mwindow_mutex;
diff --git a/src/graph.c b/src/graph.c
index 277f588ca..1c264d997 100644
--- a/src/graph.c
+++ b/src/graph.c
@@ -1,4 +1,3 @@
-
 /*
  * Copyright (C) the libgit2 contributors. All rights reserved.
  *
@@ -13,9 +12,9 @@
 static int interesting(git_pqueue *list, git_commit_list *roots)
 {
 	unsigned int i;
-	/* element 0 isn't used - we need to start at 1 */
-	for (i = 1; i < list->size; i++) {
-		git_commit_list_node *commit = list->d[i];
+
+	for (i = 0; i < git_pqueue_size(list); i++) {
+		git_commit_list_node *commit = git_pqueue_get(list, i);
 		if ((commit->flags & STALE) == 0)
 			return 1;
 	}
@@ -42,7 +41,7 @@ static int mark_parents(git_revwalk *walk, git_commit_list_node *one,
 		return 0;
 	}
 
-	if (git_pqueue_init(&list, 2, git_commit_list_time_cmp) < 0)
+	if (git_pqueue_init(&list, 0, 2, git_commit_list_time_cmp) < 0)
 		return -1;
 
 	if (git_commit_list_parse(walk, one) < 0)
@@ -59,10 +58,9 @@ static int mark_parents(git_revwalk *walk, git_commit_list_node *one,
 
 	/* as long as there are non-STALE commits */
 	while (interesting(&list, roots)) {
-		git_commit_list_node *commit;
+		git_commit_list_node *commit = git_pqueue_pop(&list);
 		int flags;
 
-		commit = git_pqueue_pop(&list);
 		if (commit == NULL)
 			break;
 
@@ -110,16 +108,16 @@ static int ahead_behind(git_commit_list_node *one, git_commit_list_node *two,
 {
 	git_commit_list_node *commit;
 	git_pqueue pq;
-	int i;
+	int error = 0, i;
 	*ahead = 0;
 	*behind = 0;
 
-	if (git_pqueue_init(&pq, 2, git_commit_list_time_cmp) < 0)
+	if (git_pqueue_init(&pq, 0, 2, git_commit_list_time_cmp) < 0)
 		return -1;
-	if (git_pqueue_insert(&pq, one) < 0)
-		goto on_error;
-	if (git_pqueue_insert(&pq, two) < 0)
-		goto on_error;
+
+	if ((error = git_pqueue_insert(&pq, one)) < 0 ||
+		(error = git_pqueue_insert(&pq, two)) < 0)
+		goto done;
 
 	while ((commit = git_pqueue_pop(&pq)) != NULL) {
 		if (commit->flags & RESULT ||
@@ -132,18 +130,15 @@ static int ahead_behind(git_commit_list_node *one, git_commit_list_node *two,
 
 		for (i = 0; i < commit->out_degree; i++) {
 			git_commit_list_node *p = commit->parents[i];
-			if (git_pqueue_insert(&pq, p) < 0)
-				return -1;
+			if ((error = git_pqueue_insert(&pq, p)) < 0)
+				goto done;
 		}
 		commit->flags |= RESULT;
 	}
 
+done:
 	git_pqueue_free(&pq);
-	return 0;
-
-on_error:
-	git_pqueue_free(&pq);
-	return -1;
+	return error;
 }
 
 int git_graph_ahead_behind(size_t *ahead, size_t *behind, git_repository *repo,
@@ -176,3 +171,22 @@ on_error:
 	git_revwalk_free(walk);
 	return -1;
 }
+
+int git_graph_descendant_of(git_repository *repo, const git_oid *commit, const git_oid *ancestor)
+{
+	git_oid merge_base;
+	int error;
+
+	if (git_oid_equal(commit, ancestor))
+		return 0;
+
+	error = git_merge_base(&merge_base, repo, commit, ancestor);
+	/* No merge-base found, it's not a descendant */
+	if (error == GIT_ENOTFOUND)
+		return 0;
+
+	if (error < 0)
+		return error;
+
+	return git_oid_equal(&merge_base, ancestor);
+}
diff --git a/src/ignore.c b/src/ignore.c
index 27d7c7ec4..78f01ac44 100644
--- a/src/ignore.c
+++ b/src/ignore.c
@@ -1,7 +1,7 @@
 #include "git2/ignore.h"
 #include "common.h"
 #include "ignore.h"
-#include "attr.h"
+#include "attrcache.h"
 #include "path.h"
 #include "config.h"
 
@@ -10,38 +10,37 @@
 #define GIT_IGNORE_DEFAULT_RULES ".\n..\n.git\n"
 
 static int parse_ignore_file(
-	git_repository *repo, void *parsedata, const char *buffer, git_attr_file *ignores)
+	git_repository *repo, git_attr_file *attrs, const char *data)
 {
 	int error = 0;
-	git_attr_fnmatch *match = NULL;
-	const char *scan = NULL;
-	char *context = NULL;
 	int ignore_case = false;
+	const char *scan = data, *context = NULL;
+	git_attr_fnmatch *match = NULL;
 
-	/* Prefer to have the caller pass in a git_ignores as the parsedata
-	 * object.  If they did not, then look up the value of ignore_case */
-	if (parsedata != NULL)
-		ignore_case = ((git_ignores *)parsedata)->ignore_case;
-	else if (git_repository__cvar(&ignore_case, repo, GIT_CVAR_IGNORECASE) < 0)
-		return error;
+	if (git_repository__cvar(&ignore_case, repo, GIT_CVAR_IGNORECASE) < 0)
+		giterr_clear();
 
-	if (ignores->key && git__suffixcmp(ignores->key, "/" GIT_IGNORE_FILE) == 0) {
-		context = ignores->key + 2;
-		context[strlen(context) - strlen(GIT_IGNORE_FILE)] = '\0';
-	}
+	/* if subdir file path, convert context for file paths */
+	if (attrs->entry &&
+		git_path_root(attrs->entry->path) < 0 &&
+		!git__suffixcmp(attrs->entry->path, "/" GIT_IGNORE_FILE))
+		context = attrs->entry->path;
 
-	scan = buffer;
+	if (git_mutex_lock(&attrs->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock ignore file");
+		return -1;
+	}
 
 	while (!error && *scan) {
-		if (!match) {
-			match = git__calloc(1, sizeof(*match));
-			GITERR_CHECK_ALLOC(match);
+		if (!match && !(match = git__calloc(1, sizeof(*match)))) {
+			error = -1;
+			break;
 		}
 
 		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG;
 
 		if (!(error = git_attr_fnmatch__parse(
-			match, ignores->pool, context, &scan)))
+			match, &attrs->pool, context, &scan)))
 		{
 			match->flags |= GIT_ATTR_FNMATCH_IGNORE;
 
@@ -49,7 +48,7 @@ static int parse_ignore_file(
 				match->flags |= GIT_ATTR_FNMATCH_ICASE;
 
 			scan = git__next_line(scan);
-			error = git_vector_insert(&ignores->rules, match);
+			error = git_vector_insert(&attrs->rules, match);
 		}
 
 		if (error != 0) {
@@ -63,32 +62,55 @@ static int parse_ignore_file(
 		}
 	}
 
+	git_mutex_unlock(&attrs->lock);
 	git__free(match);
-	/* restore file path used for context */
-	if (context)
-		context[strlen(context)] = '.'; /* first char of GIT_IGNORE_FILE */
 
 	return error;
 }
 
-#define push_ignore_file(R,IGN,S,B,F) \
-	git_attr_cache__push_file((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,parse_ignore_file,(IGN),(S))
+static int push_ignore_file(
+	git_ignores *ignores,
+	git_vector *which_list,
+	const char *base,
+	const char *filename)
+{
+	int error = 0;
+	git_attr_file *file = NULL;
+
+	error = git_attr_cache__get(
+		&file, ignores->repo, GIT_ATTR_FILE__FROM_FILE,
+		base, filename, parse_ignore_file);
+	if (error < 0)
+		return error;
+
+	if (file != NULL) {
+		if ((error = git_vector_insert(which_list, file)) < 0)
+			git_attr_file__free(file);
+	}
 
-static int push_one_ignore(void *ref, git_buf *path)
+	return error;
+}
+
+static int push_one_ignore(void *payload, git_buf *path)
 {
-	git_ignores *ign = (git_ignores *)ref;
-	return push_ignore_file(ign->repo, ign, &ign->ign_path, path->ptr, GIT_IGNORE_FILE);
+	git_ignores *ign = payload;
+	ign->depth++;
+	return push_ignore_file(ign, &ign->ign_path, path->ptr, GIT_IGNORE_FILE);
 }
 
-static int get_internal_ignores(git_attr_file **ign, git_repository *repo)
+static int get_internal_ignores(git_attr_file **out, git_repository *repo)
 {
 	int error;
 
-	if (!(error = git_attr_cache__init(repo)))
-		error = git_attr_cache__internal_file(repo, GIT_IGNORE_INTERNAL, ign);
+	if ((error = git_attr_cache__init(repo)) < 0)
+		return error;
 
-	if (!error && !(*ign)->rules.length)
-		error = parse_ignore_file(repo, NULL, GIT_IGNORE_DEFAULT_RULES, *ign);
+	error = git_attr_cache__get(
+		out, repo, GIT_ATTR_FILE__IN_MEMORY, NULL, GIT_IGNORE_INTERNAL, NULL);
+
+	/* if internal rules list is empty, insert default rules */
+	if (!error && !(*out)->rules.length)
+		error = parse_ignore_file(repo, *out, GIT_IGNORE_DEFAULT_RULES);
 
 	return error;
 }
@@ -101,33 +123,32 @@ int git_ignore__for_path(
 	int error = 0;
 	const char *workdir = git_repository_workdir(repo);
 
-	assert(ignores);
+	assert(ignores && path);
 
+	memset(ignores, 0, sizeof(*ignores));
 	ignores->repo = repo;
-	git_buf_init(&ignores->dir, 0);
-	ignores->ign_internal = NULL;
 
 	/* Read the ignore_case flag */
 	if ((error = git_repository__cvar(
 			&ignores->ignore_case, repo, GIT_CVAR_IGNORECASE)) < 0)
 		goto cleanup;
 
-	if ((error = git_vector_init(&ignores->ign_path, 8, NULL)) < 0 ||
-		(error = git_vector_init(&ignores->ign_global, 2, NULL)) < 0 ||
-		(error = git_attr_cache__init(repo)) < 0)
+	if ((error = git_attr_cache__init(repo)) < 0)
 		goto cleanup;
 
 	/* given a unrooted path in a non-bare repo, resolve it */
 	if (workdir && git_path_root(path) < 0)
 		error = git_path_find_dir(&ignores->dir, path, workdir);
 	else
-		error = git_buf_sets(&ignores->dir, path);
+		error = git_buf_joinpath(&ignores->dir, path, "");
 	if (error < 0)
 		goto cleanup;
 
+	if (workdir && !git__prefixcmp(ignores->dir.ptr, workdir))
+		ignores->dir_root = strlen(workdir);
+
 	/* set up internals */
-	error = get_internal_ignores(&ignores->ign_internal, repo);
-	if (error < 0)
+	if ((error = get_internal_ignores(&ignores->ign_internal, repo)) < 0)
 		goto cleanup;
 
 	/* load .gitignore up the path */
@@ -139,14 +160,16 @@ int git_ignore__for_path(
 	}
 
 	/* load .git/info/exclude */
-	error = push_ignore_file(repo, ignores, &ignores->ign_global,
+	error = push_ignore_file(
+		ignores, &ignores->ign_global,
 		git_repository_path(repo), GIT_IGNORE_FILE_INREPO);
 	if (error < 0)
 		goto cleanup;
 
 	/* load core.excludesfile */
 	if (git_repository_attr_cache(repo)->cfg_excl_file != NULL)
-		error = push_ignore_file(repo, ignores, &ignores->ign_global, NULL,
+		error = push_ignore_file(
+			ignores, &ignores->ign_global, NULL,
 			git_repository_attr_cache(repo)->cfg_excl_file);
 
 cleanup:
@@ -161,57 +184,79 @@ int git_ignore__push_dir(git_ignores *ign, const char *dir)
 	if (git_buf_joinpath(&ign->dir, ign->dir.ptr, dir) < 0)
 		return -1;
 
+	ign->depth++;
+
 	return push_ignore_file(
-		ign->repo, ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
+		ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
 }
 
 int git_ignore__pop_dir(git_ignores *ign)
 {
 	if (ign->ign_path.length > 0) {
 		git_attr_file *file = git_vector_last(&ign->ign_path);
-		const char *start, *end, *scan;
-		size_t keylen;
+		const char *start = file->entry->path, *end;
 
-		/* - ign->dir looks something like "a/b" (or "a/b/c/d")
-		 * - file->key looks something like "0#a/b/.gitignore
+		/* - ign->dir looks something like "/home/user/a/b/" (or "a/b/c/d/")
+		 * - file->path looks something like "a/b/.gitignore
 		 *
-		 * We are popping the last directory off ign->dir.  We also want to
-		 * remove the file from the vector if the directory part of the key
-		 * matches the ign->dir path.  We need to test if the "a/b" part of
+		 * We are popping the last directory off ign->dir.  We also want
+		 * to remove the file from the vector if the popped directory
+		 * matches the ignore path.  We need to test if the "a/b" part of
 		 * the file key matches the path we are about to pop.
 		 */
 
-		for (start = end = scan = &file->key[2]; *scan; ++scan)
-			if (*scan == '/')
-				end = scan; /* point 'end' to last '/' in key */
-		keylen = (end - start) + 1;
+		if ((end = strrchr(start, '/')) != NULL) {
+			size_t dirlen = (end - start) + 1;
+			const char *relpath = ign->dir.ptr + ign->dir_root;
+			size_t pathlen = ign->dir.size - ign->dir_root;
 
-		if (ign->dir.size >= keylen &&
-			!memcmp(ign->dir.ptr + ign->dir.size - keylen, start, keylen))
-			git_vector_pop(&ign->ign_path);
+			if (pathlen == dirlen && !memcmp(relpath, start, dirlen)) {
+				git_vector_pop(&ign->ign_path);
+				git_attr_file__free(file);
+			}
+		}
+	}
 
+	if (--ign->depth > 0) {
 		git_buf_rtruncate_at_char(&ign->dir, '/');
+		git_path_to_dir(&ign->dir);
 	}
+
 	return 0;
 }
 
 void git_ignore__free(git_ignores *ignores)
 {
-	/* don't need to free ignores->ign_internal since it is in cache */
+	unsigned int i;
+	git_attr_file *file;
+
+	git_attr_file__free(ignores->ign_internal);
+
+	git_vector_foreach(&ignores->ign_path, i, file) {
+		git_attr_file__free(file);
+		ignores->ign_path.contents[i] = NULL;
+	}
 	git_vector_free(&ignores->ign_path);
+
+	git_vector_foreach(&ignores->ign_global, i, file) {
+		git_attr_file__free(file);
+		ignores->ign_global.contents[i] = NULL;
+	}
 	git_vector_free(&ignores->ign_global);
+
 	git_buf_free(&ignores->dir);
 }
 
 static bool ignore_lookup_in_rules(
-	git_vector *rules, git_attr_path *path, int *ignored)
+	int *ignored, git_attr_file *file, git_attr_path *path)
 {
 	size_t j;
 	git_attr_fnmatch *match;
 
-	git_vector_rforeach(rules, j, match) {
+	git_vector_rforeach(&file->rules, j, match) {
 		if (git_attr_fnmatch__match(match, path)) {
-			*ignored = ((match->flags & GIT_ATTR_FNMATCH_NEGATIVE) == 0);
+			*ignored = ((match->flags & GIT_ATTR_FNMATCH_NEGATIVE) == 0) ?
+				GIT_IGNORE_TRUE : GIT_IGNORE_FALSE;
 			return true;
 		}
 	}
@@ -220,66 +265,66 @@ static bool ignore_lookup_in_rules(
 }
 
 int git_ignore__lookup(
-	git_ignores *ignores, const char *pathname, int *ignored)
+	int *out, git_ignores *ignores, const char *pathname)
 {
 	unsigned int i;
 	git_attr_file *file;
 	git_attr_path path;
 
+	*out = GIT_IGNORE_NOTFOUND;
+
 	if (git_attr_path__init(
 		&path, pathname, git_repository_workdir(ignores->repo)) < 0)
 		return -1;
 
 	/* first process builtins - success means path was found */
-	if (ignore_lookup_in_rules(
-			&ignores->ign_internal->rules, &path, ignored))
+	if (ignore_lookup_in_rules(out, ignores->ign_internal, &path))
 		goto cleanup;
 
 	/* next process files in the path */
 	git_vector_foreach(&ignores->ign_path, i, file) {
-		if (ignore_lookup_in_rules(&file->rules, &path, ignored))
+		if (ignore_lookup_in_rules(out, file, &path))
 			goto cleanup;
 	}
 
 	/* last process global ignores */
 	git_vector_foreach(&ignores->ign_global, i, file) {
-		if (ignore_lookup_in_rules(&file->rules, &path, ignored))
+		if (ignore_lookup_in_rules(out, file, &path))
 			goto cleanup;
 	}
 
-	*ignored = 0;
-
 cleanup:
 	git_attr_path__free(&path);
 	return 0;
 }
 
-int git_ignore_add_rule(
-	git_repository *repo,
-	const char *rules)
+int git_ignore_add_rule(git_repository *repo, const char *rules)
 {
 	int error;
-	git_attr_file *ign_internal;
+	git_attr_file *ign_internal = NULL;
 
-	if (!(error = get_internal_ignores(&ign_internal, repo)))
-		error = parse_ignore_file(repo, NULL, rules, ign_internal);
+	if ((error = get_internal_ignores(&ign_internal, repo)) < 0)
+		return error;
+
+	error = parse_ignore_file(repo, ign_internal, rules);
+	git_attr_file__free(ign_internal);
 
 	return error;
 }
 
-int git_ignore_clear_internal_rules(
-	git_repository *repo)
+int git_ignore_clear_internal_rules(git_repository *repo)
 {
 	int error;
 	git_attr_file *ign_internal;
 
-	if (!(error = get_internal_ignores(&ign_internal, repo))) {
-		git_attr_file__clear_rules(ign_internal);
+	if ((error = get_internal_ignores(&ign_internal, repo)) < 0)
+		return error;
 
-		return parse_ignore_file(
-			repo, NULL, GIT_IGNORE_DEFAULT_RULES, ign_internal);
-	}
+	if (!(error = git_attr_file__clear_rules(ign_internal, true)))
+		error = parse_ignore_file(
+			repo, ign_internal, GIT_IGNORE_DEFAULT_RULES);
 
+	git_attr_file__free(ign_internal);
 	return error;
 }
 
@@ -291,8 +336,6 @@ int git_ignore_path_is_ignored(
 	int error;
 	const char *workdir;
 	git_attr_path path;
-	char *tail, *end;
-	bool full_is_dir;
 	git_ignores ignores;
 	unsigned int i;
 	git_attr_file *file;
@@ -301,56 +344,42 @@ int git_ignore_path_is_ignored(
 
 	workdir = repo ? git_repository_workdir(repo) : NULL;
 
-	if ((error = git_attr_path__init(&path, pathname, workdir)) < 0)
-		return error;
+	memset(&path, 0, sizeof(path));
+	memset(&ignores, 0, sizeof(ignores));
 
-	tail = path.path;
-	end  = &path.full.ptr[path.full.size];
-	full_is_dir = path.is_dir;
+	if ((error = git_attr_path__init(&path, pathname, workdir)) < 0 ||
+		(error = git_ignore__for_path(repo, path.path, &ignores)) < 0)
+		goto cleanup;
 
 	while (1) {
-		/* advance to next component of path */
-		path.basename = tail;
-
-		while (tail < end && *tail != '/') tail++;
-		*tail = '\0';
-
-		path.full.size = (tail - path.full.ptr);
-		path.is_dir = (tail == end) ? full_is_dir : true;
-
-		/* initialize ignores the first time through */
-		if (path.basename == path.path &&
-			(error = git_ignore__for_path(repo, path.path, &ignores)) < 0)
-			break;
-
 		/* first process builtins - success means path was found */
-		if (ignore_lookup_in_rules(
-				&ignores.ign_internal->rules, &path, ignored))
+		if (ignore_lookup_in_rules(ignored, ignores.ign_internal, &path))
 			goto cleanup;
 
 		/* next process files in the path */
 		git_vector_foreach(&ignores.ign_path, i, file) {
-			if (ignore_lookup_in_rules(&file->rules, &path, ignored))
+			if (ignore_lookup_in_rules(ignored, file, &path))
 				goto cleanup;
 		}
 
 		/* last process global ignores */
 		git_vector_foreach(&ignores.ign_global, i, file) {
-			if (ignore_lookup_in_rules(&file->rules, &path, ignored))
+			if (ignore_lookup_in_rules(ignored, file, &path))
 				goto cleanup;
 		}
 
-		/* if we found no rules before reaching the end, we're done */
-		if (tail == end)
+		/* move up one directory */
+		if (path.basename == path.path)
 			break;
-
-		/* now add this directory to list of ignores */
-		if ((error = git_ignore__push_dir(&ignores, path.path)) < 0)
+		path.basename[-1] = '\0';
+		while (path.basename > path.path && *path.basename != '/')
+			path.basename--;
+		if (path.basename > path.path)
+			path.basename++;
+		path.is_dir = 1;
+
+		if ((error = git_ignore__pop_dir(&ignores)) < 0)
 			break;
-
-		/* reinstate divider in path */
-		*tail = '/';
-		while (*tail == '/') tail++;
 	}
 
 	*ignored = 0;
@@ -361,7 +390,6 @@ cleanup:
 	return error;
 }
 
-
 int git_ignore__check_pathspec_for_exact_ignores(
 	git_repository *repo,
 	git_vector *vspec,
diff --git a/src/ignore.h b/src/ignore.h
index 851c824bf..77668c661 100644
--- a/src/ignore.h
+++ b/src/ignore.h
@@ -28,7 +28,9 @@ typedef struct {
 	git_attr_file *ign_internal;
 	git_vector ign_path;
 	git_vector ign_global;
+	size_t dir_root; /* offset in dir to repo root */
 	int ignore_case;
+	int depth;
 } git_ignores;
 
 extern int git_ignore__for_path(
@@ -40,7 +42,14 @@ extern int git_ignore__pop_dir(git_ignores *ign);
 
 extern void git_ignore__free(git_ignores *ign);
 
-extern int git_ignore__lookup(git_ignores *ign, const char *path, int *ignored);
+enum {
+	GIT_IGNORE_UNCHECKED = -2,
+	GIT_IGNORE_NOTFOUND = -1,
+	GIT_IGNORE_FALSE = 0,
+	GIT_IGNORE_TRUE = 1,
+};
+
+extern int git_ignore__lookup(int *out, git_ignores *ign, const char *path);
 
 /* command line Git sometimes generates an error message if given a
  * pathspec that contains an exact match to an ignored file (provided
diff --git a/src/index.c b/src/index.c
index 09e7b2346..b63a0bec6 100644
--- a/src/index.c
+++ b/src/index.c
@@ -90,12 +90,24 @@ struct entry_long {
 
 struct entry_srch_key {
 	const char *path;
+	size_t pathlen;
 	int stage;
 };
 
+struct entry_internal {
+	git_index_entry entry;
+	size_t pathlen;
+	char path[GIT_FLEX_ARRAY];
+};
+
+struct reuc_entry_internal {
+	git_index_reuc_entry entry;
+	size_t pathlen;
+	char path[GIT_FLEX_ARRAY];
+};
+
 /* local declarations */
 static size_t read_extension(git_index *index, const char *buffer, size_t buffer_size);
-static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size);
 static int read_header(struct index_header *dest, const void *buffer);
 
 static int parse_index(git_index *index, const char *buffer, size_t buffer_size);
@@ -105,59 +117,72 @@ static int write_index(git_index *index, git_filebuf *file);
 static void index_entry_free(git_index_entry *entry);
 static void index_entry_reuc_free(git_index_reuc_entry *reuc);
 
-static int index_srch(const void *key, const void *array_member)
+int git_index_entry_srch(const void *key, const void *array_member)
 {
 	const struct entry_srch_key *srch_key = key;
-	const git_index_entry *entry = array_member;
-	int ret;
-
-	ret = strcmp(srch_key->path, entry->path);
+	const struct entry_internal *entry = array_member;
+	int cmp;
+	size_t len1, len2, len;
+
+	len1 = srch_key->pathlen;
+	len2 = entry->pathlen;
+	len = len1 < len2 ? len1 : len2;
+
+	cmp = memcmp(srch_key->path, entry->path, len);
+	if (cmp)
+		return cmp;
+	if (len1 < len2)
+		return -1;
+	if (len1 > len2)
+		return 1;
 
-	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
-		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
+	if (srch_key->stage != GIT_INDEX_STAGE_ANY)
+		return srch_key->stage - GIT_IDXENTRY_STAGE(&entry->entry);
 
-	return ret;
+	return 0;
 }
 
-static int index_isrch(const void *key, const void *array_member)
+int git_index_entry_isrch(const void *key, const void *array_member)
 {
 	const struct entry_srch_key *srch_key = key;
-	const git_index_entry *entry = array_member;
-	int ret;
+	const struct entry_internal *entry = array_member;
+	int cmp;
+	size_t len1, len2, len;
 
-	ret = strcasecmp(srch_key->path, entry->path);
+	len1 = srch_key->pathlen;
+	len2 = entry->pathlen;
+	len = len1 < len2 ? len1 : len2;
 
-	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
-		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
+	cmp = strncasecmp(srch_key->path, entry->path, len);
 
-	return ret;
-}
+	if (cmp)
+		return cmp;
+	if (len1 < len2)
+		return -1;
+	if (len1 > len2)
+		return 1;
 
-static int index_cmp_path(const void *a, const void *b)
-{
-	return strcmp((const char *)a, (const char *)b);
-}
+	if (srch_key->stage != GIT_INDEX_STAGE_ANY)
+		return srch_key->stage - GIT_IDXENTRY_STAGE(&entry->entry);
 
-static int index_icmp_path(const void *a, const void *b)
-{
-	return strcasecmp((const char *)a, (const char *)b);
+	return 0;
 }
 
-static int index_srch_path(const void *path, const void *array_member)
+static int index_entry_srch_path(const void *path, const void *array_member)
 {
 	const git_index_entry *entry = array_member;
 
 	return strcmp((const char *)path, entry->path);
 }
 
-static int index_isrch_path(const void *path, const void *array_member)
+static int index_entry_isrch_path(const void *path, const void *array_member)
 {
 	const git_index_entry *entry = array_member;
 
 	return strcasecmp((const char *)path, entry->path);
 }
 
-static int index_cmp(const void *a, const void *b)
+int git_index_entry_cmp(const void *a, const void *b)
 {
 	int diff;
 	const git_index_entry *entry_a = a;
@@ -171,7 +196,7 @@ static int index_cmp(const void *a, const void *b)
 	return diff;
 }
 
-static int index_icmp(const void *a, const void *b)
+int git_index_entry_icmp(const void *a, const void *b)
 {
 	int diff;
 	const git_index_entry *entry_a = a;
@@ -262,21 +287,16 @@ static int reuc_icmp(const void *a, const void *b)
 
 static void index_entry_reuc_free(git_index_reuc_entry *reuc)
 {
-	if (!reuc)
-		return;
-	git__free(reuc->path);
 	git__free(reuc);
 }
 
 static void index_entry_free(git_index_entry *entry)
 {
-	if (!entry)
-		return;
-	git__free(entry->path);
+	memset(&entry->id, 0, sizeof(entry->id));
 	git__free(entry);
 }
 
-static unsigned int index_create_mode(unsigned int mode)
+unsigned int git_index__create_mode(unsigned int mode)
 {
 	if (S_ISLNK(mode))
 		return S_IFLNK;
@@ -296,23 +316,74 @@ static unsigned int index_merge_mode(
 
 	if (index->distrust_filemode && S_ISREG(mode))
 		return (existing && S_ISREG(existing->mode)) ?
-			existing->mode : index_create_mode(0666);
+			existing->mode : git_index__create_mode(0666);
 
-	return index_create_mode(mode);
+	return git_index__create_mode(mode);
 }
 
-void git_index__set_ignore_case(git_index *index, bool ignore_case)
+static int index_sort_if_needed(git_index *index, bool need_lock)
 {
-	index->ignore_case = ignore_case;
+	/* not truly threadsafe because between when this checks and/or
+	 * sorts the array another thread could come in and unsort it
+	 */
 
-	index->entries_cmp_path = ignore_case ? index_icmp_path : index_cmp_path;
-	index->entries_search = ignore_case ? index_isrch : index_srch;
-	index->entries_search_path = ignore_case ? index_isrch_path : index_srch_path;
+	if (git_vector_is_sorted(&index->entries))
+		return 0;
+
+	if (need_lock && git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to lock index");
+		return -1;
+	}
 
-	git_vector_set_cmp(&index->entries, ignore_case ? index_icmp : index_cmp);
 	git_vector_sort(&index->entries);
 
-	index->reuc_search = ignore_case ? reuc_isrch : reuc_srch;
+	if (need_lock)
+		git_mutex_unlock(&index->lock);
+
+	return 0;
+}
+
+GIT_INLINE(int) index_find_in_entries(
+	size_t *out, git_vector *entries, git_vector_cmp entry_srch,
+	const char *path, size_t path_len, int stage)
+{
+	struct entry_srch_key srch_key;
+	srch_key.path = path;
+	srch_key.pathlen = !path_len ? strlen(path) : path_len;
+	srch_key.stage = stage;
+	return git_vector_bsearch2(out, entries, entry_srch, &srch_key);
+}
+
+GIT_INLINE(int) index_find(
+	size_t *out, git_index *index,
+	const char *path, size_t path_len, int stage, bool need_lock)
+{
+	if (index_sort_if_needed(index, need_lock) < 0)
+		return -1;
+
+	return index_find_in_entries(
+		out, &index->entries, index->entries_search, path, path_len, stage);
+}
+
+void git_index__set_ignore_case(git_index *index, bool ignore_case)
+{
+	index->ignore_case = ignore_case;
+
+	if (ignore_case) {
+		index->entries_cmp_path    = git__strcasecmp_cb;
+		index->entries_search      = git_index_entry_isrch;
+		index->entries_search_path = index_entry_isrch_path;
+		index->reuc_search         = reuc_isrch;
+	} else {
+		index->entries_cmp_path    = git__strcmp_cb;
+		index->entries_search      = git_index_entry_srch;
+		index->entries_search_path = index_entry_srch_path;
+		index->reuc_search         = reuc_srch;
+	}
+
+	git_vector_set_cmp(&index->entries,
+		ignore_case ? git_index_entry_icmp : git_index_entry_cmp);
+	index_sort_if_needed(index, true);
 
 	git_vector_set_cmp(&index->reuc, ignore_case ? reuc_icmp : reuc_cmp);
 	git_vector_sort(&index->reuc);
@@ -321,41 +392,51 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case)
 int git_index_open(git_index **index_out, const char *index_path)
 {
 	git_index *index;
-	int error;
+	int error = -1;
 
 	assert(index_out);
 
 	index = git__calloc(1, sizeof(git_index));
 	GITERR_CHECK_ALLOC(index);
 
+	if (git_mutex_init(&index->lock)) {
+		giterr_set(GITERR_OS, "Failed to initialize lock");
+		git__free(index);
+		return -1;
+	}
+
 	if (index_path != NULL) {
 		index->index_file_path = git__strdup(index_path);
-		GITERR_CHECK_ALLOC(index->index_file_path);
+		if (!index->index_file_path)
+			goto fail;
 
 		/* Check if index file is stored on disk already */
 		if (git_path_exists(index->index_file_path) == true)
 			index->on_disk = 1;
 	}
 
-	if (git_vector_init(&index->entries, 32, index_cmp) < 0 ||
-		git_vector_init(&index->names, 32, conflict_name_cmp) < 0 ||
-		git_vector_init(&index->reuc, 32, reuc_cmp) < 0)
-		return -1;
+	if (git_vector_init(&index->entries, 32, git_index_entry_cmp) < 0 ||
+		git_vector_init(&index->names, 8, conflict_name_cmp) < 0 ||
+		git_vector_init(&index->reuc, 8, reuc_cmp) < 0 ||
+		git_vector_init(&index->deleted, 8, git_index_entry_cmp) < 0)
+		goto fail;
 
-	index->entries_cmp_path = index_cmp_path;
-	index->entries_search = index_srch;
-	index->entries_search_path = index_srch_path;
+	index->entries_cmp_path = git__strcmp_cb;
+	index->entries_search = git_index_entry_srch;
+	index->entries_search_path = index_entry_srch_path;
 	index->reuc_search = reuc_srch;
 
-	if ((index_path != NULL) && ((error = git_index_read(index, true)) < 0)) {
-		git_index_free(index);
-		return error;
-	}
+	if (index_path != NULL && (error = git_index_read(index, true)) < 0)
+		goto fail;
 
 	*index_out = index;
 	GIT_REFCOUNT_INC(index);
 
 	return 0;
+
+fail:
+	git_index_free(index);
+	return error;
 }
 
 int git_index_new(git_index **out)
@@ -365,12 +446,19 @@ int git_index_new(git_index **out)
 
 static void index_free(git_index *index)
 {
+	/* index iterators increment the refcount of the index, so if we
+	 * get here then there should be no outstanding iterators.
+	 */
+	assert(!git_atomic_get(&index->readers));
+
 	git_index_clear(index);
 	git_vector_free(&index->entries);
 	git_vector_free(&index->names);
 	git_vector_free(&index->reuc);
+	git_vector_free(&index->deleted);
 
 	git__free(index->index_file_path);
+	git_mutex_free(&index->lock);
 
 	git__memzero(index, sizeof(*index));
 	git__free(index);
@@ -384,28 +472,71 @@ void git_index_free(git_index *index)
 	GIT_REFCOUNT_DEC(index, index_free);
 }
 
-static void index_entries_free(git_vector *entries)
+/* call with locked index */
+static void index_free_deleted(git_index *index)
 {
+	int readers = (int)git_atomic_get(&index->readers);
 	size_t i;
 
-	for (i = 0; i < entries->length; ++i)
-		index_entry_free(git__swap(entries->contents[i], NULL));
+	if (readers > 0 || !index->deleted.length)
+		return;
+
+	for (i = 0; i < index->deleted.length; ++i) {
+		git_index_entry *ie = git__swap(index->deleted.contents[i], NULL);
+		index_entry_free(ie);
+	}
 
-	git_vector_clear(entries);
+	git_vector_clear(&index->deleted);
 }
 
-void git_index_clear(git_index *index)
+/* call with locked index */
+static int index_remove_entry(git_index *index, size_t pos)
 {
+	int error = 0;
+	git_index_entry *entry = git_vector_get(&index->entries, pos);
+
+	if (entry != NULL)
+		git_tree_cache_invalidate_path(index->tree, entry->path);
+
+	error = git_vector_remove(&index->entries, pos);
+
+	if (!error) {
+		if (git_atomic_get(&index->readers) > 0) {
+			error = git_vector_insert(&index->deleted, entry);
+		} else {
+			index_entry_free(entry);
+		}
+	}
+
+	return error;
+}
+
+int git_index_clear(git_index *index)
+{
+	int error = 0;
+
 	assert(index);
 
-	index_entries_free(&index->entries);
+	git_tree_cache_free(index->tree);
+	index->tree = NULL;
+
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
+	}
+
+	while (!error && index->entries.length > 0)
+		error = index_remove_entry(index, index->entries.length - 1);
+	index_free_deleted(index);
+
 	git_index_reuc_clear(index);
 	git_index_name_clear(index);
 
 	git_futils_filestamp_set(&index->stamp, NULL);
 
-	git_tree_cache_free(index->tree);
-	index->tree = NULL;
+	git_mutex_unlock(&index->lock);
+
+	return error;
 }
 
 static int create_index_error(int error, const char *msg)
@@ -414,7 +545,7 @@ static int create_index_error(int error, const char *msg)
 	return error;
 }
 
-int git_index_set_caps(git_index *index, unsigned int caps)
+int git_index_set_caps(git_index *index, int caps)
 {
 	unsigned int old_ignore_case;
 
@@ -450,7 +581,7 @@ int git_index_set_caps(git_index *index, unsigned int caps)
 	return 0;
 }
 
-unsigned int git_index_caps(const git_index *index)
+int git_index_caps(const git_index *index)
 {
 	return ((index->ignore_case ? GIT_INDEXCAP_IGNORE_CASE : 0) |
 			(index->distrust_filemode ? GIT_INDEXCAP_NO_FILEMODE : 0) |
@@ -471,20 +602,29 @@ int git_index_read(git_index *index, int force)
 
 	if (!index->on_disk) {
 		if (force)
-			git_index_clear(index);
+			return git_index_clear(index);
 		return 0;
 	}
 
 	updated = git_futils_filestamp_check(&stamp, index->index_file_path);
-	if (updated < 0 || (!updated && !force))
+	if (updated < 0) {
+		giterr_set(
+			GITERR_INDEX,
+			"Failed to read index: '%s' no longer exists",
+			index->index_file_path);
 		return updated;
+	}
+	if (!updated && !force)
+		return 0;
 
 	error = git_futils_readbuffer(&buffer, index->index_file_path);
 	if (error < 0)
 		return error;
 
-	git_index_clear(index);
-	error = parse_index(index, buffer.ptr, buffer.size);
+	error = git_index_clear(index);
+
+	if (!error)
+		error = parse_index(index, buffer.ptr, buffer.size);
 
 	if (!error)
 		git_futils_filestamp_set(&index->stamp, &stamp);
@@ -493,6 +633,18 @@ int git_index_read(git_index *index, int force)
 	return error;
 }
 
+int git_index__changed_relative_to(
+	git_index *index, const git_futils_filestamp *fs)
+{
+	/* attempt to update index (ignoring errors) */
+	if (git_index_read(index, false) < 0)
+		giterr_clear();
+
+	return (index->stamp.mtime != fs->mtime ||
+			index->stamp.size != fs->size ||
+			index->stamp.ino != fs->ino);
+}
+
 int git_index_write(git_index *index)
 {
 	git_filebuf file = GIT_FILEBUF_INIT;
@@ -502,13 +654,14 @@ int git_index_write(git_index *index)
 		return create_index_error(-1,
 			"Failed to read index: The index is in-memory only");
 
-	git_vector_sort(&index->entries);
+	if (index_sort_if_needed(index, true) < 0)
+		return -1;
 	git_vector_sort(&index->reuc);
 
 	if ((error = git_filebuf_open(
 		&file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS, GIT_INDEX_FILE_MODE)) < 0) {
 		if (error == GIT_ELOCKED)
-			giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrrent or crashed process");
+			giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrent or crashed process");
 
 		return error;
 	}
@@ -521,15 +674,15 @@ int git_index_write(git_index *index)
 	if ((error = git_filebuf_commit(&file)) < 0)
 		return error;
 
-	error = git_futils_filestamp_check(&index->stamp, index->index_file_path);
-	if (error < 0)
-		return error;
+	if (git_futils_filestamp_check(&index->stamp, index->index_file_path) < 0)
+		/* index could not be read from disk! */;
+	else
+		index->on_disk = 1;
 
-	index->on_disk = 1;
 	return 0;
 }
 
-const char * git_index_path(git_index *index)
+const char * git_index_path(const git_index *index)
 {
 	assert(index);
 	return index->index_file_path;
@@ -550,7 +703,8 @@ int git_index_write_tree(git_oid *oid, git_index *index)
 	return git_tree__write_index(oid, index, repo);
 }
 
-int git_index_write_tree_to(git_oid *oid, git_index *index, git_repository *repo)
+int git_index_write_tree_to(
+	git_oid *oid, git_index *index, git_repository *repo)
 {
 	assert(oid && index && repo);
 	return git_tree__write_index(oid, index, repo);
@@ -566,7 +720,8 @@ const git_index_entry *git_index_get_byindex(
 	git_index *index, size_t n)
 {
 	assert(index);
-	git_vector_sort(&index->entries);
+	if (index_sort_if_needed(index, true) < 0)
+		return NULL;
 	return git_vector_get(&index->entries, n);
 }
 
@@ -577,9 +732,7 @@ const git_index_entry *git_index_get_bypath(
 
 	assert(index);
 
-	git_vector_sort(&index->entries);
-
-	if (git_index__find(&pos, index, path, stage) < 0) {
+	if (index_find(&pos, index, path, 0, stage, true) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return NULL;
 	}
@@ -597,26 +750,25 @@ void git_index_entry__init_from_stat(
 	entry->dev  = st->st_rdev;
 	entry->ino  = st->st_ino;
 	entry->mode = (!trust_mode && S_ISREG(st->st_mode)) ?
-		index_create_mode(0666) : index_create_mode(st->st_mode);
+		git_index__create_mode(0666) : git_index__create_mode(st->st_mode);
 	entry->uid  = st->st_uid;
 	entry->gid  = st->st_gid;
 	entry->file_size = st->st_size;
 }
 
-int git_index_entry__cmp(const void *a, const void *b)
+static git_index_entry *index_entry_alloc(const char *path)
 {
-	const git_index_entry *entry_a = a;
-	const git_index_entry *entry_b = b;
-
-	return strcmp(entry_a->path, entry_b->path);
-}
+	size_t pathlen = strlen(path);
+	struct entry_internal *entry =
+		git__calloc(sizeof(struct entry_internal) + pathlen + 1, 1);
+	if (!entry)
+		return NULL;
 
-int git_index_entry__cmp_icase(const void *a, const void *b)
-{
-	const git_index_entry *entry_a = a;
-	const git_index_entry *entry_b = b;
+	entry->pathlen = pathlen;
+	memcpy(entry->path, path, pathlen);
+	entry->entry.path = entry->path;
 
-	return strcasecmp(entry_a->path, entry_b->path);
+	return (git_index_entry *)entry;
 }
 
 static int index_entry_init(
@@ -638,19 +790,31 @@ static int index_entry_init(
 	if (error < 0)
 		return error;
 
-	entry = git__calloc(1, sizeof(git_index_entry));
+	entry = index_entry_alloc(rel_path);
 	GITERR_CHECK_ALLOC(entry);
 
+	entry->id = oid;
 	git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode);
 
-	entry->oid = oid;
-	entry->path = git__strdup(rel_path);
-	GITERR_CHECK_ALLOC(entry->path);
-
-	*entry_out = entry;
+	*entry_out = (git_index_entry *)entry;
 	return 0;
 }
 
+static git_index_reuc_entry *reuc_entry_alloc(const char *path)
+{
+	size_t pathlen = strlen(path);
+	struct reuc_entry_internal *entry =
+		git__calloc(sizeof(struct reuc_entry_internal) + pathlen + 1, 1);
+	if (!entry)
+		return NULL;
+
+	entry->pathlen = pathlen;
+	memcpy(entry->path, path, pathlen);
+	entry->entry.path = entry->path;
+
+	return (git_index_reuc_entry *)entry;
+}
+
 static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 	const char *path,
 	int ancestor_mode, const git_oid *ancestor_oid,
@@ -661,15 +825,9 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 
 	assert(reuc_out && path);
 
-	*reuc_out = NULL;
-
-	reuc = git__calloc(1, sizeof(git_index_reuc_entry));
+	*reuc_out = reuc = reuc_entry_alloc(path);
 	GITERR_CHECK_ALLOC(reuc);
 
-	reuc->path = git__strdup(path);
-	if (reuc->path == NULL)
-		return -1;
-
 	if ((reuc->mode[0] = ancestor_mode) > 0)
 		git_oid_cpy(&reuc->oid[0], ancestor_oid);
 
@@ -679,37 +837,158 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 	if ((reuc->mode[2] = their_mode) > 0)
 		git_oid_cpy(&reuc->oid[2], their_oid);
 
-	*reuc_out = reuc;
 	return 0;
 }
 
-static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
+static void index_entry_cpy(git_index_entry *tgt, const git_index_entry *src)
+{
+	const char *tgt_path = tgt->path;
+	memcpy(tgt, src, sizeof(*tgt));
+	tgt->path = tgt_path; /* reset to existing path data */
+}
+
+static int index_entry_dup(git_index_entry **out, const git_index_entry *src)
 {
 	git_index_entry *entry;
 
-	entry = git__malloc(sizeof(git_index_entry));
-	if (!entry)
-		return NULL;
+	if (!src) {
+		*out = NULL;
+		return 0;
+	}
 
-	memcpy(entry, source_entry, sizeof(git_index_entry));
+	*out = entry = index_entry_alloc(src->path);
+	GITERR_CHECK_ALLOC(entry);
 
-	/* duplicate the path string so we own it */
-	entry->path = git__strdup(entry->path);
-	if (!entry->path)
-		return NULL;
+	index_entry_cpy(entry, src);
 
-	return entry;
+	return 0;
 }
 
-static int index_insert(git_index *index, git_index_entry *entry, int replace)
+static int has_file_name(git_index *index,
+	 const git_index_entry *entry, size_t pos, int ok_to_replace)
 {
+	int retval = 0;
+	size_t len = strlen(entry->path);
+	int stage = GIT_IDXENTRY_STAGE(entry);
+	const char *name = entry->path;
+
+	while (pos < index->entries.length) {
+		struct entry_internal *p = index->entries.contents[pos++];
+
+		if (len >= p->pathlen)
+			break;
+		if (memcmp(name, p->path, len))
+			break;
+		if (GIT_IDXENTRY_STAGE(&p->entry) != stage)
+			continue;
+		if (p->path[len] != '/')
+			continue;
+		retval = -1;
+		if (!ok_to_replace)
+			break;
+
+		if (index_remove_entry(index, --pos) < 0)
+			break;
+	}
+	return retval;
+}
+
+/*
+ * Do we have another file with a pathname that is a proper
+ * subset of the name we're trying to add?
+ */
+static int has_dir_name(git_index *index,
+		const git_index_entry *entry, int ok_to_replace)
+{
+	int retval = 0;
+	int stage = GIT_IDXENTRY_STAGE(entry);
+	const char *name = entry->path;
+	const char *slash = name + strlen(name);
+
+	for (;;) {
+		size_t len, pos;
+
+		for (;;) {
+			if (*--slash == '/')
+				break;
+			if (slash <= entry->path)
+				return retval;
+		}
+		len = slash - name;
+
+		if (!index_find(&pos, index, name, len, stage, false)) {
+			retval = -1;
+			if (!ok_to_replace)
+				break;
+
+			if (index_remove_entry(index, pos) < 0)
+				break;
+			continue;
+		}
+
+		/*
+		 * Trivial optimization: if we find an entry that
+		 * already matches the sub-directory, then we know
+		 * we're ok, and we can exit.
+		 */
+		for (; pos < index->entries.length; ++pos) {
+			struct entry_internal *p = index->entries.contents[pos];
+
+			if (p->pathlen <= len ||
+			    p->path[len] != '/' ||
+			    memcmp(p->path, name, len))
+				break; /* not our subdirectory */
+
+			if (GIT_IDXENTRY_STAGE(&p->entry) == stage)
+				return retval;
+		}
+	}
+
+	return retval;
+}
+
+static int check_file_directory_collision(git_index *index,
+		git_index_entry *entry, size_t pos, int ok_to_replace)
+{
+	int retval = has_file_name(index, entry, pos, ok_to_replace);
+	retval = retval + has_dir_name(index, entry, ok_to_replace);
+
+	if (retval) {
+		giterr_set(GITERR_INDEX,
+			"'%s' appears as both a file and a directory", entry->path);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int index_no_dups(void **old, void *new)
+{
+	const git_index_entry *entry = new;
+	GIT_UNUSED(old);
+	giterr_set(GITERR_INDEX, "'%s' appears multiple times at stage %d",
+		entry->path, GIT_IDXENTRY_STAGE(entry));
+	return GIT_EEXISTS;
+}
+
+/* index_insert takes ownership of the new entry - if it can't insert
+ * it, then it will return an error **and also free the entry**.  When
+ * it replaces an existing entry, it will update the entry_ptr with the
+ * actual entry in the index (and free the passed in one).
+ */
+static int index_insert(
+	git_index *index, git_index_entry **entry_ptr, int replace)
+{
+	int error = 0;
 	size_t path_length, position;
-	git_index_entry **existing = NULL;
+	git_index_entry *existing = NULL, *entry;
+
+	assert(index && entry_ptr);
 
-	assert(index && entry && entry->path != NULL);
+	entry = *entry_ptr;
 
 	/* make sure that the path length flag is correct */
-	path_length = strlen(entry->path);
+	path_length = ((struct entry_internal *)entry)->pathlen;
 
 	entry->flags &= ~GIT_IDXENTRY_NAMEMASK;
 
@@ -718,28 +997,51 @@ static int index_insert(git_index *index, git_index_entry *entry, int replace)
 	else
 		entry->flags |= GIT_IDXENTRY_NAMEMASK;
 
-	/* look if an entry with this path already exists */
-	if (!git_index__find(
-			&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
-		existing = (git_index_entry **)&index->entries.contents[position];
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire index lock");
+		return -1;
+	}
 
+	git_vector_sort(&index->entries);
+
+	/* look if an entry with this path already exists */
+	if (!index_find(
+			&position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry), false)) {
+		existing = index->entries.contents[position];
 		/* update filemode to existing values if stat is not trusted */
-		entry->mode = index_merge_mode(index, *existing, entry->mode);
+		entry->mode = index_merge_mode(index, existing, entry->mode);
 	}
 
-	/* if replacing is not requested or no existing entry exists, just
-	 * insert entry at the end; the index is no longer sorted
+	/* look for tree / blob name collisions, removing conflicts if requested */
+	error = check_file_directory_collision(index, entry, position, replace);
+	if (error < 0)
+		/* skip changes */;
+
+	/* if we are replacing an existing item, overwrite the existing entry
+	 * and return it in place of the passed in one.
 	 */
-	if (!replace || !existing)
-		return git_vector_insert(&index->entries, entry);
+	else if (existing) {
+		if (replace)
+			index_entry_cpy(existing, entry);
+		index_entry_free(entry);
+		*entry_ptr = entry = existing;
+	}
+	else {
+		/* if replace is not requested or no existing entry exists, insert
+		 * at the sorted position.  (Since we re-sort after each insert to
+		 * check for dups, this is actually cheaper in the long run.)
+		 */
+		error = git_vector_insert_sorted(&index->entries, entry, index_no_dups);
+	}
 
-	/* exists, replace it (preserving name from existing entry) */
-	git__free(entry->path);
-	entry->path = (*existing)->path;
-	git__free(*existing);
-	*existing = entry;
+	if (error < 0) {
+		index_entry_free(*entry_ptr);
+		*entry_ptr = NULL;
+	}
 
-	return 0;
+	git_mutex_unlock(&index->lock);
+
+	return error;
 }
 
 static int index_conflict_to_reuc(git_index *index, const char *path)
@@ -757,9 +1059,9 @@ static int index_conflict_to_reuc(git_index *index, const char *path)
 	our_mode = conflict_entries[1] == NULL ? 0 : conflict_entries[1]->mode;
 	their_mode = conflict_entries[2] == NULL ? 0 : conflict_entries[2]->mode;
 
-	ancestor_oid = conflict_entries[0] == NULL ? NULL : &conflict_entries[0]->oid;
-	our_oid = conflict_entries[1] == NULL ? NULL : &conflict_entries[1]->oid;
-	their_oid = conflict_entries[2] == NULL ? NULL : &conflict_entries[2]->oid;
+	ancestor_oid = conflict_entries[0] == NULL ? NULL : &conflict_entries[0]->id;
+	our_oid = conflict_entries[1] == NULL ? NULL : &conflict_entries[1]->id;
+	their_oid = conflict_entries[2] == NULL ? NULL : &conflict_entries[2]->id;
 
 	if ((ret = git_index_reuc_add(index, path, ancestor_mode, ancestor_oid,
 		our_mode, our_oid, their_mode, their_oid)) >= 0)
@@ -776,19 +1078,15 @@ int git_index_add_bypath(git_index *index, const char *path)
 	assert(index && path);
 
 	if ((ret = index_entry_init(&entry, index, path)) < 0 ||
-		(ret = index_insert(index, entry, 1)) < 0)
-		goto on_error;
+		(ret = index_insert(index, &entry, 1)) < 0)
+		return ret;
 
 	/* Adding implies conflict was resolved, move conflict entries to REUC */
 	if ((ret = index_conflict_to_reuc(index, path)) < 0 && ret != GIT_ENOTFOUND)
-		goto on_error;
+		return ret;
 
 	git_tree_cache_invalidate_path(index->tree, entry->path);
 	return 0;
-
-on_error:
-	index_entry_free(entry);
-	return ret;
 }
 
 int git_index_remove_bypath(git_index *index, const char *path)
@@ -806,19 +1104,30 @@ int git_index_remove_bypath(git_index *index, const char *path)
 	return 0;
 }
 
+static bool valid_filemode(const int filemode)
+{
+	return (filemode == GIT_FILEMODE_BLOB ||
+		filemode == GIT_FILEMODE_BLOB_EXECUTABLE ||
+		filemode == GIT_FILEMODE_LINK ||
+		filemode == GIT_FILEMODE_COMMIT);
+}
+
+
 int git_index_add(git_index *index, const git_index_entry *source_entry)
 {
 	git_index_entry *entry = NULL;
 	int ret;
 
-	entry = index_entry_dup(source_entry);
-	if (entry == NULL)
+	assert(index && source_entry && source_entry->path);
+
+	if (!valid_filemode(source_entry->mode)) {
+		giterr_set(GITERR_INDEX, "invalid filemode");
 		return -1;
+	}
 
-	if ((ret = index_insert(index, entry, 1)) < 0) {
-		index_entry_free(entry);
+	if ((ret = index_entry_dup(&entry, source_entry)) < 0 ||
+		(ret = index_insert(index, &entry, 1)) < 0)
 		return ret;
-	}
 
 	git_tree_cache_invalidate_path(index->tree, entry->path);
 	return 0;
@@ -826,27 +1135,23 @@ int git_index_add(git_index *index, const git_index_entry *source_entry)
 
 int git_index_remove(git_index *index, const char *path, int stage)
 {
-	size_t position;
 	int error;
-	git_index_entry *entry;
-
-	git_vector_sort(&index->entries);
+	size_t position;
 
-	if (git_index__find(&position, index, path, stage) < 0) {
-		giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d",
-			path, stage);
-		return GIT_ENOTFOUND;
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
 	}
 
-	entry = git_vector_get(&index->entries, position);
-	if (entry != NULL)
-		git_tree_cache_invalidate_path(index->tree, entry->path);
-
-	error = git_vector_remove(&index->entries, position);
-
-	if (!error)
-		index_entry_free(entry);
+	if (index_find(&position, index, path, 0, stage, false) < 0) {
+		giterr_set(
+			GITERR_INDEX, "Index does not contain %s at stage %d", path, stage);
+		error = GIT_ENOTFOUND;
+	} else {
+		error = index_remove_entry(index, position);
+	}
 
+	git_mutex_unlock(&index->lock);
 	return error;
 }
 
@@ -857,14 +1162,16 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 	size_t pos;
 	git_index_entry *entry;
 
-	if (git_buf_sets(&pfx, dir) < 0 || git_path_to_dir(&pfx) < 0)
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
 		return -1;
+	}
 
-	git_vector_sort(&index->entries);
-
-	pos = git_index__prefix_position(index, pfx.ptr);
+	if (!(error = git_buf_sets(&pfx, dir)) &&
+		!(error = git_path_to_dir(&pfx)))
+		index_find(&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY, false);
 
-	while (1) {
+	while (!error) {
 		entry = git_vector_get(&index->entries, pos);
 		if (!entry || git__prefixcmp(entry->path, pfx.ptr) != 0)
 			break;
@@ -874,32 +1181,22 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 			continue;
 		}
 
-		git_tree_cache_invalidate_path(index->tree, entry->path);
-
-		if ((error = git_vector_remove(&index->entries, pos)) < 0)
-			break;
-		index_entry_free(entry);
+		error = index_remove_entry(index, pos);
 
-		/* removed entry at 'pos' so we don't need to increment it */
+		/* removed entry at 'pos' so we don't need to increment */
 	}
 
+	git_mutex_unlock(&index->lock);
 	git_buf_free(&pfx);
 
 	return error;
 }
 
-int git_index__find(
-	size_t *at_pos, git_index *index, const char *path, int stage)
+int git_index__find_pos(
+	size_t *out, git_index *index, const char *path, size_t path_len, int stage)
 {
-	struct entry_srch_key srch_key;
-
-	assert(path);
-
-	srch_key.path = path;
-	srch_key.stage = stage;
-
-	return git_vector_bsearch2(
-		at_pos, &index->entries, index->entries_search, &srch_key);
+	assert(index && path);
+	return index_find(out, index, path, path_len, stage, true);
 }
 
 int git_index_find(size_t *at_pos, git_index *index, const char *path)
@@ -908,7 +1205,14 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path)
 
 	assert(index && path);
 
-	if (git_vector_bsearch2(&pos, &index->entries, index->entries_search_path, path) < 0) {
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
+	}
+
+	if (git_vector_bsearch2(
+			&pos, &index->entries, index->entries_search_path, path) < 0) {
+		git_mutex_unlock(&index->lock);
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return GIT_ENOTFOUND;
 	}
@@ -916,36 +1220,20 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path)
 	/* Since our binary search only looked at path, we may be in the
 	 * middle of a list of stages.
 	 */
-	while (pos > 0) {
-		const git_index_entry *prev = git_vector_get(&index->entries, pos-1);
+	for (; pos > 0; --pos) {
+		const git_index_entry *prev = git_vector_get(&index->entries, pos - 1);
 
 		if (index->entries_cmp_path(prev->path, path) != 0)
 			break;
-
-		--pos;
 	}
 
 	if (at_pos)
 		*at_pos = pos;
 
+	git_mutex_unlock(&index->lock);
 	return 0;
 }
 
-size_t git_index__prefix_position(git_index *index, const char *path)
-{
-	struct entry_srch_key srch_key;
-	size_t pos;
-
-	srch_key.path = path;
-	srch_key.stage = 0;
-
-	git_vector_sort(&index->entries);
-	git_vector_bsearch2(
-		&pos, &index->entries, index->entries_search, &srch_key);
-
-	return pos;
-}
-
 int git_index_conflict_add(git_index *index,
 	const git_index_entry *ancestor_entry,
 	const git_index_entry *our_entry,
@@ -957,21 +1245,22 @@ int git_index_conflict_add(git_index *index,
 
 	assert (index);
 
-	if ((ancestor_entry != NULL && (entries[0] = index_entry_dup(ancestor_entry)) == NULL) ||
-		(our_entry != NULL && (entries[1] = index_entry_dup(our_entry)) == NULL) ||
-		(their_entry != NULL && (entries[2] = index_entry_dup(their_entry)) == NULL))
-		return -1;
+	if ((ret = index_entry_dup(&entries[0], ancestor_entry)) < 0 ||
+		(ret = index_entry_dup(&entries[1], our_entry)) < 0 ||
+		(ret = index_entry_dup(&entries[2], their_entry)) < 0)
+		goto on_error;
 
 	for (i = 0; i < 3; i++) {
 		if (entries[i] == NULL)
 			continue;
 
 		/* Make sure stage is correct */
-		entries[i]->flags = (entries[i]->flags & ~GIT_IDXENTRY_STAGEMASK) |
-			((i+1) << GIT_IDXENTRY_STAGESHIFT);
+		GIT_IDXENTRY_STAGE_SET(entries[i], i + 1);
 
-		if ((ret = index_insert(index, entries[i], 1)) < 0)
+		if ((ret = index_insert(index, &entries[i], 1)) < 0)
 			goto on_error;
+
+		entries[i] = NULL; /* don't free if later entry fails */
 	}
 
 	return 0;
@@ -1061,23 +1350,24 @@ int git_index_conflict_get(
 	return 0;
 }
 
-int git_index_conflict_remove(git_index *index, const char *path)
+static int index_conflict_remove(git_index *index, const char *path)
 {
-	size_t pos, posmax;
+	size_t pos = 0;
 	git_index_entry *conflict_entry;
 	int error = 0;
 
-	assert(index && path);
-
-	if (git_index_find(&pos, index, path) < 0)
+	if (path != NULL && git_index_find(&pos, index, path) < 0)
 		return GIT_ENOTFOUND;
 
-	posmax = git_index_entrycount(index);
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to lock index");
+		return -1;
+	}
 
-	while (pos < posmax) {
-		conflict_entry = git_vector_get(&index->entries, pos);
+	while ((conflict_entry = git_vector_get(&index->entries, pos)) != NULL) {
 
-		if (index->entries_cmp_path(conflict_entry->path, path) != 0)
+		if (path != NULL &&
+			index->entries_cmp_path(conflict_entry->path, path) != 0)
 			break;
 
 		if (GIT_IDXENTRY_STAGE(conflict_entry) == 0) {
@@ -1085,32 +1375,25 @@ int git_index_conflict_remove(git_index *index, const char *path)
 			continue;
 		}
 
-		if ((error = git_vector_remove(&index->entries, pos)) < 0)
-			return error;
-
-		index_entry_free(conflict_entry);
-		posmax--;
+		if ((error = index_remove_entry(index, pos)) < 0)
+			break;
 	}
 
-	return 0;
+	git_mutex_unlock(&index->lock);
+
+	return error;
 }
 
-static int index_conflicts_match(const git_vector *v, size_t idx)
+int git_index_conflict_remove(git_index *index, const char *path)
 {
-	git_index_entry *entry = git_vector_get(v, idx);
-
-	if (GIT_IDXENTRY_STAGE(entry) > 0) {
-		index_entry_free(entry);
-		return 1;
-	}
-
-	return 0;
+	assert(index && path);
+	return index_conflict_remove(index, path);
 }
 
-void git_index_conflict_cleanup(git_index *index)
+int git_index_conflict_cleanup(git_index *index)
 {
 	assert(index);
-	git_vector_remove_matching(&index->entries, index_conflicts_match);
+	return index_conflict_remove(index, NULL);
 }
 
 int git_index_has_conflicts(const git_index *index)
@@ -1205,32 +1488,36 @@ const git_index_name_entry *git_index_name_get_byindex(
 	return git_vector_get(&index->names, n);
 }
 
+static void index_name_entry_free(git_index_name_entry *ne)
+{
+	if (!ne)
+		return;
+	git__free(ne->ancestor);
+	git__free(ne->ours);
+	git__free(ne->theirs);
+	git__free(ne);
+}
+
 int git_index_name_add(git_index *index,
 	const char *ancestor, const char *ours, const char *theirs)
 {
 	git_index_name_entry *conflict_name;
 
-	assert ((ancestor && ours) || (ancestor && theirs) || (ours && theirs));
+	assert((ancestor && ours) || (ancestor && theirs) || (ours && theirs));
 
 	conflict_name = git__calloc(1, sizeof(git_index_name_entry));
 	GITERR_CHECK_ALLOC(conflict_name);
 
-	if (ancestor) {
-		conflict_name->ancestor = git__strdup(ancestor);
-		GITERR_CHECK_ALLOC(conflict_name->ancestor);
-	}
-
-	if (ours) {
-		conflict_name->ours = git__strdup(ours);
-		GITERR_CHECK_ALLOC(conflict_name->ours);
-	}
-
-	if (theirs) {
-		conflict_name->theirs = git__strdup(theirs);
-		GITERR_CHECK_ALLOC(conflict_name->theirs);
+	if ((ancestor && !(conflict_name->ancestor = git__strdup(ancestor))) ||
+		(ours     && !(conflict_name->ours     = git__strdup(ours))) ||
+		(theirs   && !(conflict_name->theirs   = git__strdup(theirs))) ||
+		git_vector_insert(&index->names, conflict_name) < 0)
+	{
+		index_name_entry_free(conflict_name);
+		return -1;
 	}
 
-	return git_vector_insert(&index->names, conflict_name);
+	return 0;
 }
 
 void git_index_name_clear(git_index *index)
@@ -1240,18 +1527,8 @@ void git_index_name_clear(git_index *index)
 
 	assert(index);
 
-	git_vector_foreach(&index->names, i, conflict_name) {
-		if (conflict_name->ancestor)
-			git__free(conflict_name->ancestor);
-
-		if (conflict_name->ours)
-			git__free(conflict_name->ours);
-
-		if (conflict_name->theirs)
-			git__free(conflict_name->theirs);
-
-		git__free(conflict_name);
-	}
+	git_vector_foreach(&index->names, i, conflict_name)
+		index_name_entry_free(conflict_name);
 
 	git_vector_clear(&index->names);
 }
@@ -1279,7 +1556,6 @@ static int index_reuc_insert(
 		return git_vector_insert(&index->reuc, reuc);
 
 	/* exists, replace it */
-	git__free((*existing)->path);
 	git__free(*existing);
 	*existing = reuc;
 
@@ -1296,15 +1572,13 @@ int git_index_reuc_add(git_index *index, const char *path,
 
 	assert(index && path);
 
-	if ((error = index_entry_reuc_init(&reuc, path, ancestor_mode, ancestor_oid, our_mode, our_oid, their_mode, their_oid)) < 0 ||
+	if ((error = index_entry_reuc_init(&reuc, path, ancestor_mode,
+			ancestor_oid, our_mode, our_oid, their_mode, their_oid)) < 0 ||
 		(error = index_reuc_insert(index, reuc, 1)) < 0)
-	{
 		index_entry_reuc_free(reuc);
-		return error;
-	}
 
 	return error;
-} 
+}
 
 int git_index_reuc_find(size_t *at_pos, git_index *index, const char *path)
 {
@@ -1389,13 +1663,9 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 		if (size <= len)
 			return index_error_invalid("reading reuc entries");
 
-		lost = git__calloc(1, sizeof(git_index_reuc_entry));
+		lost = reuc_entry_alloc(buffer);
 		GITERR_CHECK_ALLOC(lost);
 
-		/* read NUL-terminated pathname for entry */
-		lost->path = git__strdup(buffer);
-		GITERR_CHECK_ALLOC(lost->path);
-
 		size -= len;
 		buffer += len;
 
@@ -1442,7 +1712,7 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 	}
 
 	/* entries are guaranteed to be sorted on-disk */
-	index->reuc.sorted = 1;
+	git_vector_set_sorted(&index->reuc, true);
 
 	return 0;
 }
@@ -1488,46 +1758,46 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size
 #undef read_conflict_name
 
 	/* entries are guaranteed to be sorted on-disk */
-	index->names.sorted = 1;
+	git_vector_set_sorted(&index->names, true);
 
 	return 0;
 }
 
-static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size)
+static size_t read_entry(
+	git_index_entry **out, const void *buffer, size_t buffer_size)
 {
 	size_t path_length, entry_size;
 	uint16_t flags_raw;
 	const char *path_ptr;
 	const struct entry_short *source = buffer;
+	git_index_entry entry = {{0}};
 
 	if (INDEX_FOOTER_SIZE + minimal_entry_size > buffer_size)
 		return 0;
 
-	memset(dest, 0x0, sizeof(git_index_entry));
-
-	dest->ctime.seconds = (git_time_t)ntohl(source->ctime.seconds);
-	dest->ctime.nanoseconds = ntohl(source->ctime.nanoseconds);
-	dest->mtime.seconds = (git_time_t)ntohl(source->mtime.seconds);
-	dest->mtime.nanoseconds = ntohl(source->mtime.nanoseconds);
-	dest->dev = ntohl(source->dev);
-	dest->ino = ntohl(source->ino);
-	dest->mode = ntohl(source->mode);
-	dest->uid = ntohl(source->uid);
-	dest->gid = ntohl(source->gid);
-	dest->file_size = ntohl(source->file_size);
-	git_oid_cpy(&dest->oid, &source->oid);
-	dest->flags = ntohs(source->flags);
-
-	if (dest->flags & GIT_IDXENTRY_EXTENDED) {
+	entry.ctime.seconds = (git_time_t)ntohl(source->ctime.seconds);
+	entry.ctime.nanoseconds = ntohl(source->ctime.nanoseconds);
+	entry.mtime.seconds = (git_time_t)ntohl(source->mtime.seconds);
+	entry.mtime.nanoseconds = ntohl(source->mtime.nanoseconds);
+	entry.dev = ntohl(source->dev);
+	entry.ino = ntohl(source->ino);
+	entry.mode = ntohl(source->mode);
+	entry.uid = ntohl(source->uid);
+	entry.gid = ntohl(source->gid);
+	entry.file_size = ntohl(source->file_size);
+	git_oid_cpy(&entry.id, &source->oid);
+	entry.flags = ntohs(source->flags);
+
+	if (entry.flags & GIT_IDXENTRY_EXTENDED) {
 		const struct entry_long *source_l = (const struct entry_long *)source;
 		path_ptr = source_l->path;
 
 		flags_raw = ntohs(source_l->flags_extended);
-		memcpy(&dest->flags_extended, &flags_raw, 2);
+		memcpy(&entry.flags_extended, &flags_raw, 2);
 	} else
 		path_ptr = source->path;
 
-	path_length = dest->flags & GIT_IDXENTRY_NAMEMASK;
+	path_length = entry.flags & GIT_IDXENTRY_NAMEMASK;
 
 	/* if this is a very long string, we must find its
 	 * real length without overflowing */
@@ -1541,7 +1811,7 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 		path_length = path_end - path_ptr;
 	}
 
-	if (dest->flags & GIT_IDXENTRY_EXTENDED)
+	if (entry.flags & GIT_IDXENTRY_EXTENDED)
 		entry_size = long_entry_size(path_length);
 	else
 		entry_size = short_entry_size(path_length);
@@ -1549,8 +1819,10 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 	if (INDEX_FOOTER_SIZE + entry_size > buffer_size)
 		return 0;
 
-	dest->path = git__strdup(path_ptr);
-	assert(dest->path);
+	entry.path = (char *)path_ptr;
+
+	if (index_entry_dup(out, &entry) < 0)
+		return 0;
 
 	return entry_size;
 }
@@ -1616,13 +1888,15 @@ static size_t read_extension(git_index *index, const char *buffer, size_t buffer
 
 static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 {
+	int error = 0;
 	unsigned int i;
 	struct index_header header = { 0 };
 	git_oid checksum_calculated, checksum_expected;
 
 #define seek_forward(_increase) { \
-	if (_increase >= buffer_size) \
-		return index_error_invalid("ran out of data while parsing"); \
+	if (_increase >= buffer_size) { \
+		error = index_error_invalid("ran out of data while parsing"); \
+		goto done; } \
 	buffer += _increase; \
 	buffer_size -= _increase;\
 }
@@ -1635,35 +1909,41 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 	git_hash_buf(&checksum_calculated, buffer, buffer_size - INDEX_FOOTER_SIZE);
 
 	/* Parse header */
-	if (read_header(&header, buffer) < 0)
-		return -1;
+	if ((error = read_header(&header, buffer)) < 0)
+		return error;
 
 	seek_forward(INDEX_HEADER_SIZE);
 
-	git_vector_clear(&index->entries);
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire index lock");
+		return -1;
+	}
+
+	assert(!index->entries.length);
 
 	/* Parse all the entries */
 	for (i = 0; i < header.entry_count && buffer_size > INDEX_FOOTER_SIZE; ++i) {
-		size_t entry_size;
 		git_index_entry *entry;
-
-		entry = git__malloc(sizeof(git_index_entry));
-		GITERR_CHECK_ALLOC(entry);
-
-		entry_size = read_entry(entry, buffer, buffer_size);
+		size_t entry_size = read_entry(&entry, buffer, buffer_size);
 
 		/* 0 bytes read means an object corruption */
-		if (entry_size == 0)
-			return index_error_invalid("invalid entry");
+		if (entry_size == 0) {
+			error = index_error_invalid("invalid entry");
+			goto done;
+		}
 
-		if (git_vector_insert(&index->entries, entry) < 0)
-			return -1;
+		if ((error = git_vector_insert(&index->entries, entry)) < 0) {
+			index_entry_free(entry);
+			goto done;
+		}
 
 		seek_forward(entry_size);
 	}
 
-	if (i != header.entry_count)
-		return index_error_invalid("header entries changed while parsing");
+	if (i != header.entry_count) {
+		error = index_error_invalid("header entries changed while parsing");
+		goto done;
+	}
 
 	/* There's still space for some extensions! */
 	while (buffer_size > INDEX_FOOTER_SIZE) {
@@ -1672,28 +1952,40 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 		extension_size = read_extension(index, buffer, buffer_size);
 
 		/* see if we have read any bytes from the extension */
-		if (extension_size == 0)
-			return index_error_invalid("extension is truncated");
+		if (extension_size == 0) {
+			error = index_error_invalid("extension is truncated");
+			goto done;
+		}
 
 		seek_forward(extension_size);
 	}
 
-	if (buffer_size != INDEX_FOOTER_SIZE)
-		return index_error_invalid("buffer size does not match index footer size");
+	if (buffer_size != INDEX_FOOTER_SIZE) {
+		error = index_error_invalid(
+			"buffer size does not match index footer size");
+		goto done;
+	}
 
 	/* 160-bit SHA-1 over the content of the index file before this checksum. */
 	git_oid_fromraw(&checksum_expected, (const unsigned char *)buffer);
 
-	if (git_oid__cmp(&checksum_calculated, &checksum_expected) != 0)
-		return index_error_invalid("calculated checksum does not match expected");
+	if (git_oid__cmp(&checksum_calculated, &checksum_expected) != 0) {
+		error = index_error_invalid(
+			"calculated checksum does not match expected");
+		goto done;
+	}
 
 #undef seek_forward
 
-	/* Entries are stored case-sensitively on disk. */
-	index->entries.sorted = !index->ignore_case;
-	git_vector_sort(&index->entries);
+	/* Entries are stored case-sensitively on disk, so re-sort now if
+	 * in-memory index is supposed to be case-insensitive
+	 */
+	git_vector_set_sorted(&index->entries, !index->ignore_case);
+	error = index_sort_if_needed(index, false);
 
-	return 0;
+done:
+	git_mutex_unlock(&index->lock);
+	return error;
 }
 
 static bool is_index_extended(git_index *index)
@@ -1721,7 +2013,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 	size_t path_len, disk_size;
 	char *path;
 
-	path_len = strlen(entry->path);
+	path_len = ((struct entry_internal *)entry)->pathlen;
 
 	if (entry->flags & GIT_IDXENTRY_EXTENDED)
 		disk_size = long_entry_size(path_len);
@@ -1756,7 +2048,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 	ondisk->gid = htonl(entry->gid);
 	ondisk->file_size = htonl((uint32_t)entry->file_size);
 
-	git_oid_cpy(&ondisk->oid, &entry->oid);
+	git_oid_cpy(&ondisk->oid, &entry->id);
 
 	ondisk->flags = htons(entry->flags);
 
@@ -1778,22 +2070,30 @@ static int write_entries(git_index *index, git_filebuf *file)
 {
 	int error = 0;
 	size_t i;
-	git_vector case_sorted;
+	git_vector case_sorted, *entries;
 	git_index_entry *entry;
-	git_vector *out = &index->entries;
+
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
+	}
 
 	/* If index->entries is sorted case-insensitively, then we need
 	 * to re-sort it case-sensitively before writing */
 	if (index->ignore_case) {
-		git_vector_dup(&case_sorted, &index->entries, index_cmp);
+		git_vector_dup(&case_sorted, &index->entries, git_index_entry_cmp);
 		git_vector_sort(&case_sorted);
-		out = &case_sorted;
+		entries = &case_sorted;
+	} else {
+		entries = &index->entries;
 	}
 
-	git_vector_foreach(out, i, entry)
+	git_vector_foreach(entries, i, entry)
 		if ((error = write_disk_entry(file, entry)) < 0)
 			break;
 
+	git_mutex_unlock(&index->lock);
+
 	if (index->ignore_case)
 		git_vector_free(&case_sorted);
 
@@ -1923,13 +2223,15 @@ static int write_index(git_index *index, git_filebuf *file)
 	git_oid hash_final;
 	struct index_header header;
 	bool is_extended;
+	uint32_t index_version_number;
 
 	assert(index && file);
 
 	is_extended = is_index_extended(index);
+	index_version_number = is_extended ? INDEX_VERSION_NUMBER_EXT : INDEX_VERSION_NUMBER;
 
 	header.signature = htonl(INDEX_HEADER_SIG);
-	header.version = htonl(is_extended ? INDEX_VERSION_NUMBER_EXT : INDEX_VERSION_NUMBER);
+	header.version = htonl(index_version_number);
 	header.entry_count = htonl((uint32_t)index->entries.length);
 
 	if (git_filebuf_write(file, &header, sizeof(struct index_header)) < 0)
@@ -1963,7 +2265,7 @@ int git_index_entry_stage(const git_index_entry *entry)
 typedef struct read_tree_data {
 	git_vector *old_entries;
 	git_vector *new_entries;
-	git_vector_cmp entries_search;
+	git_vector_cmp entry_cmp;
 } read_tree_data;
 
 static int read_tree_cb(
@@ -1972,6 +2274,7 @@ static int read_tree_cb(
 	read_tree_data *data = payload;
 	git_index_entry *entry = NULL, *old_entry;
 	git_buf path = GIT_BUF_INIT;
+	size_t pos;
 
 	if (git_tree_entry__is_tree(tentry))
 		return 0;
@@ -1979,29 +2282,22 @@ static int read_tree_cb(
 	if (git_buf_joinpath(&path, root, tentry->filename) < 0)
 		return -1;
 
-	entry = git__calloc(1, sizeof(git_index_entry));
+	entry = index_entry_alloc(path.ptr);
 	GITERR_CHECK_ALLOC(entry);
 
 	entry->mode = tentry->attr;
-	entry->oid = tentry->oid;
+	entry->id = tentry->oid;
 
 	/* look for corresponding old entry and copy data to new entry */
-	if (data->old_entries) {
-		size_t pos;
-		struct entry_srch_key skey;
-
-		skey.path = path.ptr;
-		skey.stage = 0;
-
-		if (!git_vector_bsearch2(
-				&pos, data->old_entries, data->entries_search, &skey) &&
-			(old_entry = git_vector_get(data->old_entries, pos)) != NULL &&
-			entry->mode == old_entry->mode &&
-			git_oid_equal(&entry->oid, &old_entry->oid))
-		{
-			memcpy(entry, old_entry, sizeof(*entry));
-			entry->flags_extended = 0;
-		}
+	if (data->old_entries != NULL &&
+		!index_find_in_entries(
+			&pos, data->old_entries, data->entry_cmp, path.ptr, 0, 0) &&
+		(old_entry = git_vector_get(data->old_entries, pos)) != NULL &&
+		entry->mode == old_entry->mode &&
+		git_oid_equal(&entry->id, &old_entry->id))
+	{
+		index_entry_cpy(entry, old_entry);
+		entry->flags_extended = 0;
 	}
 
 	if (path.size < GIT_IDXENTRY_NAMEMASK)
@@ -2009,7 +2305,6 @@ static int read_tree_cb(
 	else
 		entry->flags = GIT_IDXENTRY_NAMEMASK;
 
-	entry->path = git_buf_detach(&path);
 	git_buf_free(&path);
 
 	if (git_vector_insert(data->new_entries, entry) < 0) {
@@ -2030,17 +2325,27 @@ int git_index_read_tree(git_index *index, const git_tree *tree)
 
 	data.old_entries = &index->entries;
 	data.new_entries = &entries;
-	data.entries_search = index->entries_search;
+	data.entry_cmp   = index->entries_search;
 
-	git_vector_sort(&index->entries);
+	if (index_sort_if_needed(index, true) < 0)
+		return -1;
 
 	error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data);
 
-	git_vector_sort(&entries);
+	if (!error) {
+		git_vector_sort(&entries);
 
-	git_index_clear(index);
+		if ((error = git_index_clear(index)) < 0)
+			/* well, this isn't good */;
+		else if (git_mutex_lock(&index->lock) < 0) {
+			giterr_set(GITERR_OS, "Unable to acquire index lock");
+			error = -1;
+		} else {
+			git_vector_swap(&entries, &index->entries);
+			git_mutex_unlock(&index->lock);
+		}
+	}
 
-	git_vector_swap(&entries, &index->entries);
 	git_vector_free(&entries);
 
 	return error;
@@ -2108,7 +2413,7 @@ int git_index_add_all(
 		/* skip ignored items that are not already in the index */
 		if ((flags & GIT_INDEX_ADD_FORCE) == 0 &&
 			git_iterator_current_is_ignored(wditer) &&
-			git_index__find(&existing, index, wd->path, 0) < 0)
+			index_find(&existing, index, wd->path, 0, 0, true) < 0)
 			continue;
 
 		/* issue notification callback if requested */
@@ -2116,8 +2421,7 @@ int git_index_add_all(
 			if (error > 0) /* return > 0 means skip this one */
 				continue;
 			if (error < 0) { /* return < 0 means abort */
-				giterr_clear();
-				error = GIT_EUSER;
+				giterr_set_after_callback(error);
 				break;
 			}
 		}
@@ -2131,17 +2435,14 @@ int git_index_add_all(
 			break;
 
 		/* make the new entry to insert */
-		if ((entry = index_entry_dup(wd)) == NULL) {
-			error = -1;
+		if ((error = index_entry_dup(&entry, wd)) < 0)
 			break;
-		}
-		entry->oid = blobid;
+
+		entry->id = blobid;
 
 		/* add working directory item to index */
-		if ((error = index_insert(index, entry, 1)) < 0) {
-			index_entry_free(entry);
+		if ((error = index_insert(index, &entry, 1)) < 0)
 			break;
-		}
 
 		git_tree_cache_invalidate_path(index->tree, wd->path);
 
@@ -2204,11 +2505,8 @@ static int index_apply_to_all(
 				error = 0;
 				continue;
 			}
-			if (error < 0) { /* return < 0 means abort */
-				giterr_clear();
-				error = GIT_EUSER;
+			if (error < 0)   /* return < 0 means abort */
 				break;
-			}
 		}
 
 		/* index manipulation may alter entry, so don't depend on it */
@@ -2253,8 +2551,13 @@ int git_index_remove_all(
 	git_index_matched_path_cb cb,
 	void *payload)
 {
-	return index_apply_to_all(
+	int error = index_apply_to_all(
 		index, INDEX_ACTION_REMOVE, pathspec, cb, payload);
+
+	if (error) /* make sure error is set if callback stopped iteration */
+		giterr_set_after_callback(error);
+
+	return error;
 }
 
 int git_index_update_all(
@@ -2263,6 +2566,56 @@ int git_index_update_all(
 	git_index_matched_path_cb cb,
 	void *payload)
 {
-	return index_apply_to_all(
+	int error = index_apply_to_all(
 		index, INDEX_ACTION_UPDATE, pathspec, cb, payload);
+
+	if (error) /* make sure error is set if callback stopped iteration */
+		giterr_set_after_callback(error);
+
+	return error;
+}
+
+int git_index_snapshot_new(git_vector *snap, git_index *index)
+{
+	int error;
+
+	GIT_REFCOUNT_INC(index);
+
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
+	}
+
+	git_atomic_inc(&index->readers);
+	git_vector_sort(&index->entries);
+
+	error = git_vector_dup(snap, &index->entries, index->entries._cmp);
+
+	git_mutex_unlock(&index->lock);
+
+	if (error < 0)
+		git_index_free(index);
+
+	return error;
+}
+
+void git_index_snapshot_release(git_vector *snap, git_index *index)
+{
+	git_vector_free(snap);
+
+	git_atomic_dec(&index->readers);
+
+	if (!git_mutex_lock(&index->lock)) {
+		index_free_deleted(index); /* try to free pending deleted items */
+		git_mutex_unlock(&index->lock);
+	}
+
+	git_index_free(index);
+}
+
+int git_index_snapshot_find(
+	size_t *out, git_vector *entries, git_vector_cmp entry_srch,
+	const char *path, size_t path_len, int stage)
+{
+	return index_find_in_entries(out, entries, entry_srch, path, path_len, stage);
 }
diff --git a/src/index.h b/src/index.h
index 4c448fabf..50a0b4b6c 100644
--- a/src/index.h
+++ b/src/index.h
@@ -21,12 +21,15 @@ struct git_index {
 	git_refcount rc;
 
 	char *index_file_path;
-
 	git_futils_filestamp stamp;
+
 	git_vector entries;
 
-	unsigned int on_disk:1;
+	git_mutex  lock;    /* lock held while entries is being changed */
+	git_vector deleted; /* deleted entries if readers > 0 */
+	git_atomic readers; /* number of active iterators */
 
+	unsigned int on_disk:1;
 	unsigned int ignore_case:1;
 	unsigned int distrust_filemode:1;
 	unsigned int no_symlinks:1;
@@ -50,14 +53,44 @@ struct git_index_conflict_iterator {
 extern void git_index_entry__init_from_stat(
 	git_index_entry *entry, struct stat *st, bool trust_mode);
 
-extern size_t git_index__prefix_position(git_index *index, const char *path);
+/* Index entry comparison functions for array sorting */
+extern int git_index_entry_cmp(const void *a, const void *b);
+extern int git_index_entry_icmp(const void *a, const void *b);
 
-extern int git_index_entry__cmp(const void *a, const void *b);
-extern int git_index_entry__cmp_icase(const void *a, const void *b);
+/* Index entry search functions for search using a search spec */
+extern int git_index_entry_srch(const void *a, const void *b);
+extern int git_index_entry_isrch(const void *a, const void *b);
 
-extern int git_index__find(
-	size_t *at_pos, git_index *index, const char *path, int stage);
+/* Search index for `path`, returning GIT_ENOTFOUND if it does not exist
+ * (but not setting an error message).
+ *
+ * `at_pos` is set to the position where it is or would be inserted.
+ * Pass `path_len` as strlen of path or 0 to call strlen internally.
+ */
+extern int git_index__find_pos(
+	size_t *at_pos, git_index *index, const char *path, size_t path_len, int stage);
 
 extern void git_index__set_ignore_case(git_index *index, bool ignore_case);
 
+extern unsigned int git_index__create_mode(unsigned int mode);
+
+GIT_INLINE(const git_futils_filestamp *) git_index__filestamp(git_index *index)
+{
+   return &index->stamp;
+}
+
+extern int git_index__changed_relative_to(git_index *index, const git_futils_filestamp *fs);
+
+/* Copy the current entries vector *and* increment the index refcount.
+ * Call `git_index__release_snapshot` when done.
+ */
+extern int git_index_snapshot_new(git_vector *snap, git_index *index);
+extern void git_index_snapshot_release(git_vector *snap, git_index *index);
+
+/* Allow searching in a snapshot; entries must already be sorted! */
+extern int git_index_snapshot_find(
+	size_t *at_pos, git_vector *snap, git_vector_cmp entry_srch,
+	const char *path, size_t path_len, int stage);
+
+
 #endif
diff --git a/src/indexer.c b/src/indexer.c
index df1ce7cfb..25c3d0537 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -5,8 +5,6 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
-#include <zlib.h>
-
 #include "git2/indexer.h"
 #include "git2/object.h"
 
@@ -18,7 +16,7 @@
 #include "filebuf.h"
 #include "oid.h"
 #include "oidmap.h"
-#include "compress.h"
+#include "zstream.h"
 
 #define UINT31_MAX (0x7FFFFFFF)
 
@@ -36,7 +34,6 @@ struct git_indexer {
 		have_delta :1;
 	struct git_pack_header hdr;
 	struct git_pack_file *pack;
-	git_filebuf pack_file;
 	unsigned int mode;
 	git_off_t off;
 	git_off_t entry_start;
@@ -47,7 +44,7 @@ struct git_indexer {
 	unsigned int fanout[256];
 	git_hash_ctx hash_ctx;
 	git_oid hash;
-	git_transfer_progress_callback progress_cb;
+	git_transfer_progress_cb progress_cb;
 	void *progress_payload;
 	char objbuf[8*1024];
 
@@ -69,33 +66,18 @@ const git_oid *git_indexer_hash(const git_indexer *idx)
 	return &idx->hash;
 }
 
-static int open_pack(struct git_pack_file **out, const char *filename)
-{
-	struct git_pack_file *pack;
-
-	if (git_packfile_alloc(&pack, filename) < 0)
-		return -1;
-
-	if ((pack->mwf.fd = p_open(pack->pack_name, O_RDONLY)) < 0) {
-		giterr_set(GITERR_OS, "Failed to open packfile.");
-		git_packfile_free(pack);
-		return -1;
-	}
-
-	*out = pack;
-	return 0;
-}
-
 static int parse_header(struct git_pack_header *hdr, struct git_pack_file *pack)
 {
 	int error;
+	git_map map;
 
-	/* Verify we recognize this pack file format. */
-	if ((error = p_read(pack->mwf.fd, hdr, sizeof(*hdr))) < 0) {
-		giterr_set(GITERR_OS, "Failed to read in pack header");
+	if ((error = p_mmap(&map, sizeof(*hdr), GIT_PROT_READ, GIT_MAP_SHARED, pack->mwf.fd, 0)) < 0)
 		return error;
-	}
 
+	memcpy(hdr, map.data, sizeof(*hdr));
+	p_munmap(&map);
+
+	/* Verify we recognize this pack file format. */
 	if (hdr->hdr_signature != ntohl(PACK_SIGNATURE)) {
 		giterr_set(GITERR_INDEXER, "Wrong pack signature");
 		return -1;
@@ -122,13 +104,13 @@ int git_indexer_new(
 		const char *prefix,
 		unsigned int mode,
 		git_odb *odb,
-		git_transfer_progress_callback progress_cb,
+		git_transfer_progress_cb progress_cb,
 		void *progress_payload)
 {
 	git_indexer *idx;
-	git_buf path = GIT_BUF_INIT;
+	git_buf path = GIT_BUF_INIT, tmp_path = GIT_BUF_INIT;
 	static const char suff[] = "/pack";
-	int error;
+	int error, fd = -1;
 
 	idx = git__calloc(1, sizeof(git_indexer));
 	GITERR_CHECK_ALLOC(idx);
@@ -142,19 +124,30 @@ int git_indexer_new(
 	if (error < 0)
 		goto cleanup;
 
-	error = git_filebuf_open(&idx->pack_file, path.ptr,
-		GIT_FILEBUF_TEMPORARY | GIT_FILEBUF_DO_NOT_BUFFER,
-		idx->mode);
+	fd = git_futils_mktmp(&tmp_path, git_buf_cstr(&path), idx->mode);
 	git_buf_free(&path);
+	if (fd < 0)
+		goto cleanup;
+
+	error = git_packfile_alloc(&idx->pack, git_buf_cstr(&tmp_path));
+	git_buf_free(&tmp_path);
+
 	if (error < 0)
 		goto cleanup;
 
+	idx->pack->mwf.fd = fd;
+	if ((error = git_mwindow_file_register(&idx->pack->mwf)) < 0)
+		goto cleanup;
+
 	*out = idx;
 	return 0;
 
 cleanup:
+	if (fd != -1)
+		p_close(fd);
+
 	git_buf_free(&path);
-	git_filebuf_cleanup(&idx->pack_file);
+	git_buf_free(&tmp_path);
 	git__free(idx);
 	return -1;
 }
@@ -355,7 +348,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, git_off_t entry_star
 	git_oid oid;
 	size_t entry_size;
 	struct entry *entry;
-	struct git_pack_entry *pentry;
+	struct git_pack_entry *pentry = NULL;
 
 	entry = git__calloc(1, sizeof(*entry));
 	GITERR_CHECK_ALLOC(entry);
@@ -379,6 +372,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, git_off_t entry_star
 	return save_entry(idx, entry, pentry, entry_start);
 
 on_error:
+	git__free(pentry);
 	git__free(entry);
 	git__free(obj->data);
 	return -1;
@@ -386,8 +380,11 @@ on_error:
 
 static int do_progress_callback(git_indexer *idx, git_transfer_progress *stats)
 {
-	if (!idx->progress_cb) return 0;
-	return idx->progress_cb(stats, idx->progress_payload);
+	if (idx->progress_cb)
+		return giterr_set_after_callback_function(
+			idx->progress_cb(stats, idx->progress_payload),
+			"indexer progress");
+	return 0;
 }
 
 /* Hash everything but the last 20B of input */
@@ -427,6 +424,42 @@ static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
 	idx->inbuf_len += size - to_expell;
 }
 
+static int write_at(git_indexer *idx, const void *data, git_off_t offset, size_t size)
+{
+	git_file fd = idx->pack->mwf.fd;
+	long page_size = git__page_size();
+	git_off_t page_start, page_offset;
+	unsigned char *map_data;
+	git_map map;
+	int error;
+
+	/* the offset needs to be at the beginning of the a page boundary */
+	page_start = (offset / page_size) * page_size;
+	page_offset = offset - page_start;
+
+	if ((error = p_mmap(&map, page_offset + size, GIT_PROT_WRITE, GIT_MAP_SHARED, fd, page_start)) < 0)
+		return error;
+
+	map_data = (unsigned char *)map.data;
+	memcpy(map_data + page_offset, data, size);
+	p_munmap(&map);
+
+	return 0;
+}
+
+static int append_to_pack(git_indexer *idx, const void *data, size_t size)
+{
+	git_off_t current_size = idx->pack->mwf.size;
+
+	/* add the extra space we need at the end */
+	if (p_ftruncate(idx->pack->mwf.fd, current_size + size) < 0) {
+		giterr_system_set(errno);
+		return -1;
+	}
+
+	return write_at(idx, data, idx->pack->mwf.size, size);
+}
+
 int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats)
 {
 	int error = -1;
@@ -438,22 +471,13 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 
 	processed = stats->indexed_objects;
 
-	if (git_filebuf_write(&idx->pack_file, data, size) < 0)
-		return -1;
+	if ((error = append_to_pack(idx, data, size)) < 0)
+		return error;
 
 	hash_partially(idx, data, (int)size);
 
 	/* Make sure we set the new size of the pack */
-	if (idx->opened_pack) {
-		idx->pack->mwf.size += size;
-	} else {
-		if (open_pack(&idx->pack, idx->pack_file.path_lock) < 0)
-			return -1;
-		idx->opened_pack = 1;
-		mwf = &idx->pack->mwf;
-		if (git_mwindow_file_register(&idx->pack->mwf) < 0)
-			return -1;
-	}
+	idx->pack->mwf.size += size;
 
 	if (!idx->parsed_header) {
 		unsigned int total_objects;
@@ -461,8 +485,8 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 		if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header))
 			return 0;
 
-		if (parse_header(&idx->hdr, idx->pack) < 0)
-			return -1;
+		if ((error = parse_header(&idx->hdr, idx->pack)) < 0)
+			return error;
 
 		idx->parsed_header = 1;
 		idx->nr_objects = ntohl(hdr->hdr_entries);
@@ -491,13 +515,16 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 		stats->indexed_deltas = 0;
 		processed = stats->indexed_objects = 0;
 		stats->total_objects = total_objects;
-		do_progress_callback(idx, stats);
+
+		if ((error = do_progress_callback(idx, stats)) != 0)
+			return error;
 	}
 
 	/* Now that we have data in the pack, let's try to parse it */
 
 	/* As the file grows any windows we try to use will be out of date */
 	git_mwindow_free_all(mwf);
+
 	while (processed < idx->nr_objects) {
 		git_packfile_stream *stream = &idx->stream;
 		git_off_t entry_start = idx->off;
@@ -515,7 +542,7 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 				return 0;
 			}
 			if (error < 0)
-				return -1;
+				goto on_error;
 
 			git_mwindow_close(&w);
 			idx->entry_start = entry_start;
@@ -528,7 +555,7 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 					return 0;
 				}
 				if (error < 0)
-					return -1;
+					goto on_error;
 
 				idx->have_delta = 1;
 			} else {
@@ -537,9 +564,10 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 			}
 
 			idx->have_stream = 1;
-			if (git_packfile_stream_open(stream, idx->pack, idx->off) < 0)
-				goto on_error;
 
+			error = git_packfile_stream_open(stream, idx->pack, idx->off);
+			if (error < 0)
+				goto on_error;
 		}
 
 		if (idx->have_delta) {
@@ -573,11 +601,8 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
 		}
 		stats->received_objects++;
 
-		if (do_progress_callback(idx, stats) != 0) {
-			giterr_clear();
-			error = GIT_EUSER;
+		if ((error = do_progress_callback(idx, stats)) != 0)
 			goto on_error;
-		}
 	}
 
 	return 0;
@@ -613,24 +638,17 @@ static int index_path(git_buf *path, git_indexer *idx, const char *suffix)
  * Rewind the packfile by the trailer, as we might need to fix the
  * packfile by injecting objects at the tail and must overwrite it.
  */
-static git_off_t seek_back_trailer(git_indexer *idx)
+static void seek_back_trailer(git_indexer *idx)
 {
-	git_off_t off;
-
-	if ((off = p_lseek(idx->pack_file.fd, -GIT_OID_RAWSZ, SEEK_CUR)) < 0)
-		return -1;
-
 	idx->pack->mwf.size -= GIT_OID_RAWSZ;
 	git_mwindow_free_all(&idx->pack->mwf);
-
-	return off;
 }
 
 static int inject_object(git_indexer *idx, git_oid *id)
 {
 	git_odb_object *obj;
 	struct entry *entry;
-	struct git_pack_entry *pentry;
+	struct git_pack_entry *pentry = NULL;
 	git_oid foo = {{0}};
 	unsigned char hdr[64];
 	git_buf buf = GIT_BUF_INIT;
@@ -639,10 +657,8 @@ static int inject_object(git_indexer *idx, git_oid *id)
 	size_t len, hdr_len;
 	int error;
 
-	entry = git__calloc(1, sizeof(*entry));
-	GITERR_CHECK_ALLOC(entry);
-
-	entry_start = seek_back_trailer(idx);
+	seek_back_trailer(idx);
+	entry_start = idx->pack->mwf.size;
 
 	if (git_odb_read(&obj, idx->odb, id) < 0)
 		return -1;
@@ -650,25 +666,33 @@ static int inject_object(git_indexer *idx, git_oid *id)
 	data = git_odb_object_data(obj);
 	len = git_odb_object_size(obj);
 
+	entry = git__calloc(1, sizeof(*entry));
+	GITERR_CHECK_ALLOC(entry);
+
 	entry->crc = crc32(0L, Z_NULL, 0);
 
 	/* Write out the object header */
 	hdr_len = git_packfile__object_header(hdr, len, git_odb_object_type(obj));
-	git_filebuf_write(&idx->pack_file, hdr, hdr_len);
+	if ((error = append_to_pack(idx, hdr, hdr_len)) < 0)
+		goto cleanup;
+
 	idx->pack->mwf.size += hdr_len;
-	entry->crc = crc32(entry->crc, hdr, hdr_len);
+	entry->crc = crc32(entry->crc, hdr, (uInt)hdr_len);
 
-	if ((error = git__compress(&buf, data, len)) < 0)
+	if ((error = git_zstream_deflatebuf(&buf, data, len)) < 0)
 		goto cleanup;
 
 	/* And then the compressed object */
-	git_filebuf_write(&idx->pack_file, buf.ptr, buf.size);
+	if ((error = append_to_pack(idx, buf.ptr, buf.size)) < 0)
+		goto cleanup;
+
 	idx->pack->mwf.size += buf.size;
 	entry->crc = htonl(crc32(entry->crc, (unsigned char *)buf.ptr, (uInt)buf.size));
 	git_buf_free(&buf);
 
 	/* Write a fake trailer so the pack functions play ball */
-	if ((error = git_filebuf_write(&idx->pack_file, &foo, GIT_OID_RAWSZ)) < 0)
+
+	if ((error = append_to_pack(idx, &foo, GIT_OID_RAWSZ)) < 0)
 		goto cleanup;
 
 	idx->pack->mwf.size += GIT_OID_RAWSZ;
@@ -680,10 +704,14 @@ static int inject_object(git_indexer *idx, git_oid *id)
 	git_oid_cpy(&entry->oid, id);
 	idx->off = entry_start + hdr_len + len;
 
-	if ((error = save_entry(idx, entry, pentry, entry_start)) < 0)
-		git__free(pentry);
+	error = save_entry(idx, entry, pentry, entry_start);
 
 cleanup:
+	if (error) {
+		git__free(entry);
+		git__free(pentry);
+	}
+
 	git_odb_object_free(obj);
 	return error;
 }
@@ -696,7 +724,7 @@ static int fix_thin_pack(git_indexer *idx, git_transfer_progress *stats)
 	size_t size;
 	git_otype type;
 	git_mwindow *w = NULL;
-	git_off_t curpos;
+	git_off_t curpos = 0;
 	unsigned char *base_info;
 	unsigned int left = 0;
 	git_oid base;
@@ -710,6 +738,9 @@ static int fix_thin_pack(git_indexer *idx, git_transfer_progress *stats)
 
 	/* Loop until we find the first REF delta */
 	git_vector_foreach(&idx->deltas, i, delta) {
+		if (!delta)
+			continue;
+
 		curpos = delta->delta_off;
 		error = git_packfile_unpack_header(&size, &type, &idx->pack->mwf, &w, &curpos);
 		git_mwindow_close(&w);
@@ -749,13 +780,18 @@ static int resolve_deltas(git_indexer *idx, git_transfer_progress *stats)
 {
 	unsigned int i;
 	struct delta_info *delta;
-	int progressed = 0;
+	int progressed = 0, non_null = 0, progress_cb_result;
 
 	while (idx->deltas.length > 0) {
 		progressed = 0;
+		non_null = 0;
 		git_vector_foreach(&idx->deltas, i, delta) {
 			git_rawobj obj;
 
+			if (!delta)
+				continue;
+
+			non_null = 1;
 			idx->off = delta->delta_off;
 			if (git_packfile_unpack(&obj, idx->pack, &idx->off) < 0)
 				continue;
@@ -767,18 +803,18 @@ static int resolve_deltas(git_indexer *idx, git_transfer_progress *stats)
 			stats->indexed_objects++;
 			stats->indexed_deltas++;
 			progressed = 1;
-			do_progress_callback(idx, stats);
-
-			/*
-			 * Remove this delta from the list and
-			 * decrease i so we don't skip over the next
-			 * delta.
-			 */
-			git_vector_remove(&idx->deltas, i);
+			if ((progress_cb_result = do_progress_callback(idx, stats)) < 0)
+				return progress_cb_result;
+
+			/* remove from the list */
+			git_vector_set(NULL, &idx->deltas, i, NULL);
 			git__free(delta);
-			i--;
 		}
 
+		/* if none were actually set, we're done */
+		if (!non_null)
+			break;
+
 		if (!progressed && (fix_thin_pack(idx, stats) < 0)) {
 			giterr_set(GITERR_INDEXER, "missing delta bases");
 			return -1;
@@ -802,19 +838,12 @@ static int update_header_and_rehash(git_indexer *idx, git_transfer_progress *sta
 	ctx = &idx->trailer;
 
 	git_hash_ctx_init(ctx);
-	git_mwindow_free_all(mwf);
+
 
 	/* Update the header to include the numer of local objects we injected */
 	idx->hdr.hdr_entries = htonl(stats->total_objects + stats->local_objects);
-	if (p_lseek(idx->pack_file.fd, 0, SEEK_SET) < 0) {
-		giterr_set(GITERR_OS, "failed to seek to the beginning of the pack");
-		return -1;
-	}
-
-	if (p_write(idx->pack_file.fd, &idx->hdr, sizeof(struct git_pack_header)) < 0) {
-		giterr_set(GITERR_OS, "failed to update the pack header");
+	if (write_at(idx, &idx->hdr, 0, sizeof(struct git_pack_header)) < 0)
 		return -1;
-	}
 
 	/*
 	 * We now use the same technique as before to determine the
@@ -822,6 +851,7 @@ static int update_header_and_rehash(git_indexer *idx, git_transfer_progress *sta
 	 * hash_partially() keep the existing trailer out of the
 	 * calculation.
 	 */
+	git_mwindow_free_all(mwf);
 	idx->inbuf_len = 0;
 	while (hashed < mwf->size) {
 		ptr = git_mwindow_open(mwf, &w, hashed, chunk, &left);
@@ -841,6 +871,7 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 {
 	git_mwindow *w = NULL;
 	unsigned int i, long_offsets = 0, left;
+	int error;
 	struct git_pack_idx_header hdr;
 	git_buf filename = GIT_BUF_INIT;
 	struct entry *entry;
@@ -854,7 +885,7 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 
 	/* Test for this before resolve_deltas(), as it plays with idx->off */
 	if (idx->off < idx->pack->mwf.size - 20) {
-		giterr_set(GITERR_INDEXER, "unexpected data at the end of the pack");
+		giterr_set(GITERR_INDEXER, "Unexpected data at the end of the pack");
 		return -1;
 	}
 
@@ -877,8 +908,8 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 	/* Freeze the number of deltas */
 	stats->total_deltas = stats->total_objects - stats->indexed_objects;
 
-	if (resolve_deltas(idx, stats) < 0)
-		return -1;
+	if ((error = resolve_deltas(idx, stats)) < 0)
+		return error;
 
 	if (stats->indexed_objects != stats->total_objects) {
 		giterr_set(GITERR_INDEXER, "early EOF");
@@ -890,13 +921,7 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 			return -1;
 
 		git_hash_final(&trailer_hash, &idx->trailer);
-		if (p_lseek(idx->pack_file.fd, -GIT_OID_RAWSZ, SEEK_END) < 0)
-			return -1;
-
-		if (p_write(idx->pack_file.fd, &trailer_hash, GIT_OID_RAWSZ) < 0) {
-			giterr_set(GITERR_OS, "failed to update pack trailer");
-			return -1;
-		}
+		write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
 	}
 
 	git_vector_sort(&idx->objects);
@@ -979,14 +1004,18 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 
 	git_mwindow_free_all(&idx->pack->mwf);
 	/* We need to close the descriptor here so Windows doesn't choke on commit_at */
-	p_close(idx->pack->mwf.fd);
+	if (p_close(idx->pack->mwf.fd) < 0) {
+		giterr_set(GITERR_OS, "failed to close packfile");
+		goto on_error;
+	}
+
 	idx->pack->mwf.fd = -1;
 
 	if (index_path(&filename, idx, ".pack") < 0)
 		goto on_error;
+
 	/* And don't forget to rename the packfile to its new place. */
-	if (git_filebuf_commit_at(&idx->pack_file, filename.ptr) < 0)
-		return -1;
+	p_rename(idx->pack->pack_name, git_buf_cstr(&filename));
 
 	git_buf_free(&filename);
 	return 0;
@@ -1001,31 +1030,20 @@ on_error:
 
 void git_indexer_free(git_indexer *idx)
 {
-	khiter_t k;
-	unsigned int i;
-	struct entry *e;
-	struct delta_info *delta;
-
 	if (idx == NULL)
 		return;
 
-	git_vector_foreach(&idx->objects, i, e)
-		git__free(e);
-	git_vector_free(&idx->objects);
+	git_vector_free_deep(&idx->objects);
 
-	if (idx->pack) {
-		for (k = kh_begin(idx->pack->idx_cache); k != kh_end(idx->pack->idx_cache); k++) {
-			if (kh_exist(idx->pack->idx_cache, k))
-				git__free(kh_value(idx->pack->idx_cache, k));
-		}
+	if (idx->pack && idx->pack->idx_cache) {
+		struct git_pack_entry *pentry;
+		kh_foreach_value(
+			idx->pack->idx_cache, pentry, { git__free(pentry); });
 
 		git_oidmap_free(idx->pack->idx_cache);
 	}
 
-	git_vector_foreach(&idx->deltas, i, delta)
-		git__free(delta);
-	git_vector_free(&idx->deltas);
+	git_vector_free_deep(&idx->deltas);
 	git_packfile_free(idx->pack);
-	git_filebuf_cleanup(&idx->pack_file);
 	git__free(idx);
 }
diff --git a/src/iterator.c b/src/iterator.c
index 8646399ab..c664f17cd 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -10,7 +10,7 @@
 #include "index.h"
 #include "ignore.h"
 #include "buffer.h"
-#include "git2/submodule.h"
+#include "submodule.h"
 #include <ctype.h>
 
 #define ITERATOR_SET_CB(P,NAME_LC) do { \
@@ -447,7 +447,7 @@ static int tree_iterator__update_entry(tree_iterator *ti)
     te = tf->entries[tf->current]->te;
 
 	ti->entry.mode = te->attr;
-	git_oid_cpy(&ti->entry.oid, &te->oid);
+	git_oid_cpy(&ti->entry.id, &te->oid);
 
 	ti->entry.path = tree_iterator__current_filename(ti, te);
 	GITERR_CHECK_ALLOC(ti->entry.path);
@@ -644,6 +644,8 @@ typedef struct {
 	git_iterator base;
 	git_iterator_callbacks cb;
 	git_index *index;
+	git_vector entries;
+	git_vector_cmp entry_srch;
 	size_t current;
 	/* when not in autoexpand mode, use these to represent "tree" state */
 	git_buf partial;
@@ -654,10 +656,10 @@ typedef struct {
 
 static const git_index_entry *index_iterator__index_entry(index_iterator *ii)
 {
-	const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current);
+	const git_index_entry *ie = git_vector_get(&ii->entries, ii->current);
 
 	if (ie != NULL && iterator__past_end(ii, ie->path)) {
-		ii->current = git_index_entrycount(ii->index);
+		ii->current = git_vector_length(&ii->entries);
 		ie = NULL;
 	}
 
@@ -726,7 +728,7 @@ static int index_iterator__current(
 	const git_index_entry **entry, git_iterator *self)
 {
 	index_iterator *ii = (index_iterator *)self;
-	const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current);
+	const git_index_entry *ie = git_vector_get(&ii->entries, ii->current);
 
 	if (ie != NULL && index_iterator__at_tree(ii)) {
 		ii->tree_entry.path = ii->partial.ptr;
@@ -744,14 +746,14 @@ static int index_iterator__current(
 static int index_iterator__at_end(git_iterator *self)
 {
 	index_iterator *ii = (index_iterator *)self;
-	return (ii->current >= git_index_entrycount(ii->index));
+	return (ii->current >= git_vector_length(&ii->entries));
 }
 
 static int index_iterator__advance(
 	const git_index_entry **entry, git_iterator *self)
 {
 	index_iterator *ii = (index_iterator *)self;
-	size_t entrycount = git_index_entrycount(ii->index);
+	size_t entrycount = git_vector_length(&ii->entries);
 	const git_index_entry *ie;
 
 	if (!iterator__has_been_accessed(ii))
@@ -766,7 +768,7 @@ static int index_iterator__advance(
 			while (ii->current < entrycount) {
 				ii->current++;
 
-				if (!(ie = git_index_get_byindex(ii->index, ii->current)) ||
+				if (!(ie = git_vector_get(&ii->entries, ii->current)) ||
 					ii->base.prefixcomp(ie->path, ii->partial.ptr) != 0)
 					break;
 			}
@@ -789,7 +791,7 @@ static int index_iterator__advance_into(
 	const git_index_entry **entry, git_iterator *self)
 {
 	index_iterator *ii = (index_iterator *)self;
-	const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current);
+	const git_index_entry *ie = git_vector_get(&ii->entries, ii->current);
 
 	if (ie != NULL && index_iterator__at_tree(ii)) {
 		if (ii->restore_terminator)
@@ -815,8 +817,11 @@ static int index_iterator__reset(
 	if (iterator__reset_range(self, start, end) < 0)
 		return -1;
 
-	ii->current = ii->base.start ?
-		git_index__prefix_position(ii->index, ii->base.start) : 0;
+	ii->current = 0;
+
+	if (ii->base.start)
+		git_index_snapshot_find(
+			&ii->current, &ii->entries, ii->entry_srch, ii->base.start, 0, 0);
 
 	if ((ie = index_iterator__skip_conflicts(ii)) == NULL)
 		return 0;
@@ -841,9 +846,8 @@ static int index_iterator__reset(
 static void index_iterator__free(git_iterator *self)
 {
 	index_iterator *ii = (index_iterator *)self;
-	git_index_free(ii->index);
+	git_index_snapshot_release(&ii->entries, ii->index);
 	ii->index = NULL;
-
 	git_buf_free(&ii->partial);
 }
 
@@ -854,18 +858,29 @@ int git_iterator_for_index(
 	const char *start,
 	const char *end)
 {
+	int error = 0;
 	index_iterator *ii = git__calloc(1, sizeof(index_iterator));
 	GITERR_CHECK_ALLOC(ii);
 
+	if ((error = git_index_snapshot_new(&ii->entries, index)) < 0) {
+		git__free(ii);
+		return error;
+	}
+	ii->index = index;
+
 	ITERATOR_BASE_INIT(ii, index, INDEX, git_index_owner(index));
 
-	if (index->ignore_case) {
-		ii->base.flags |= GIT_ITERATOR_IGNORE_CASE;
-		ii->base.prefixcomp = git__prefixcmp_icase;
+	if ((error = iterator__update_ignore_case((git_iterator *)ii, flags)) < 0) {
+		git_iterator_free((git_iterator *)ii);
+		return error;
 	}
 
-	ii->index = index;
-	GIT_REFCOUNT_INC(index);
+	ii->entry_srch = iterator__ignore_case(ii) ?
+		git_index_entry_isrch : git_index_entry_srch;
+
+	git_vector_set_cmp(&ii->entries, iterator__ignore_case(ii) ?
+		git_index_entry_icmp : git_index_entry_cmp);
+	git_vector_sort(&ii->entries);
 
 	git_buf_init(&ii->partial, 0);
 	ii->tree_entry.mode = GIT_FILEMODE_TREE;
@@ -873,7 +888,6 @@ int git_iterator_for_index(
 	index_iterator__reset((git_iterator *)ii, NULL, NULL);
 
 	*iter = (git_iterator *)ii;
-
 	return 0;
 }
 
@@ -883,6 +897,7 @@ struct fs_iterator_frame {
 	fs_iterator_frame *next;
 	git_vector entries;
 	size_t index;
+	int is_ignored;
 };
 
 typedef struct fs_iterator fs_iterator;
@@ -920,12 +935,7 @@ static fs_iterator_frame *fs_iterator__alloc_frame(fs_iterator *fi)
 
 static void fs_iterator__free_frame(fs_iterator_frame *ff)
 {
-	size_t i;
-	git_path_with_stat *path;
-
-	git_vector_foreach(&ff->entries, i, path)
-		git__free(path);
-	git_vector_free(&ff->entries);
+	git_vector_free_deep(&ff->entries);
 	git__free(ff);
 }
 
@@ -991,9 +1001,8 @@ static int fs_iterator__expand_dir(fs_iterator *fi)
 		fi->base.start, fi->base.end, &ff->entries);
 
 	if (error < 0) {
-		git_error last_error = {0};
-
-		giterr_detach(&last_error);
+		git_error_state last_error = { 0 };
+		giterr_capture(&last_error, error);
 
 		/* these callbacks may clear the error message */
 		fs_iterator__free_frame(ff);
@@ -1001,18 +1010,14 @@ static int fs_iterator__expand_dir(fs_iterator *fi)
 		/* next time return value we skipped to */
 		fi->base.flags &= ~GIT_ITERATOR_FIRST_ACCESS;
 
-		if (last_error.message) {
-			giterr_set_str(last_error.klass, last_error.message);
-			free(last_error.message);
-		}
-
-		return error;
+		return giterr_restore(&last_error);
 	}
 
 	if (ff->entries.length == 0) {
 		fs_iterator__free_frame(ff);
 		return GIT_ENOTFOUND;
 	}
+	fi->base.stat_calls += ff->entries.length;
 
 	fs_iterator__seek_frame_start(fi, ff);
 
@@ -1286,14 +1291,51 @@ GIT_INLINE(bool) workdir_path_is_dotgit(const git_buf *path)
 
 static int workdir_iterator__enter_dir(fs_iterator *fi)
 {
-	/* only push new ignores if this is not top level directory */
-	if (fi->stack->next != NULL) {
-		workdir_iterator *wi = (workdir_iterator *)fi;
+	workdir_iterator *wi = (workdir_iterator *)fi;
+	fs_iterator_frame *ff = fi->stack;
+	size_t pos;
+	git_path_with_stat *entry;
+	bool found_submodules = false;
+
+	/* check if this directory is ignored */
+	if (git_ignore__lookup(
+			&ff->is_ignored, &wi->ignores, fi->path.ptr + fi->root_len) < 0) {
+		giterr_clear();
+		ff->is_ignored = GIT_IGNORE_NOTFOUND;
+	}
+
+	/* if this is not the top level directory... */
+	if (ff->next != NULL) {
 		ssize_t slash_pos = git_buf_rfind_next(&fi->path, '/');
 
+		/* inherit ignored from parent if no rule specified */
+		if (ff->is_ignored <= GIT_IGNORE_NOTFOUND)
+			ff->is_ignored = ff->next->is_ignored;
+
+		/* push new ignores for files in this directory */
 		(void)git_ignore__push_dir(&wi->ignores, &fi->path.ptr[slash_pos + 1]);
 	}
 
+	/* convert submodules to GITLINK and remove trailing slashes */
+	git_vector_foreach(&ff->entries, pos, entry) {
+		if (!S_ISDIR(entry->st.st_mode) || !strcmp(GIT_DIR, entry->path))
+			continue;
+
+		if (git_submodule__is_submodule(fi->base.repo, entry->path)) {
+			entry->st.st_mode = GIT_FILEMODE_COMMIT;
+			entry->path_len--;
+			entry->path[entry->path_len] = '\0';
+			found_submodules = true;
+		}
+	}
+
+	/* if we renamed submodules, re-sort and re-seek to start */
+	if (found_submodules) {
+		git_vector_set_sorted(&ff->entries, 0);
+		git_vector_sort(&ff->entries);
+		fs_iterator__seek_frame_start(fi, ff);
+	}
+
 	return 0;
 }
 
@@ -1306,7 +1348,6 @@ static int workdir_iterator__leave_dir(fs_iterator *fi)
 
 static int workdir_iterator__update_entry(fs_iterator *fi)
 {
-	int error = 0;
 	workdir_iterator *wi = (workdir_iterator *)fi;
 
 	/* skip over .git entries */
@@ -1314,21 +1355,7 @@ static int workdir_iterator__update_entry(fs_iterator *fi)
 		return GIT_ENOTFOUND;
 
 	/* reset is_ignored since we haven't checked yet */
-	wi->is_ignored = -1;
-
-	/* check if apparent tree entries are actually submodules */
-	if (fi->entry.mode != GIT_FILEMODE_TREE)
-		return 0;
-
-	error = git_submodule_lookup(NULL, fi->base.repo, fi->entry.path);
-	if (error < 0)
-		giterr_clear();
-
-	/* mark submodule (or any dir with .git) as GITLINK and remove slash */
-	if (!error || error == GIT_EEXISTS) {
-		fi->entry.mode = S_IFGITLINK;
-		fi->entry.path[strlen(fi->entry.path) - 1] = '\0';
-	}
+	wi->is_ignored = GIT_IGNORE_UNCHECKED;
 
 	return 0;
 }
@@ -1473,6 +1500,19 @@ int git_iterator_current_parent_tree(
 	return 0;
 }
 
+static void workdir_iterator_update_is_ignored(workdir_iterator *wi)
+{
+	if (git_ignore__lookup(
+			&wi->is_ignored, &wi->ignores, wi->fi.entry.path) < 0) {
+		giterr_clear();
+		wi->is_ignored = GIT_IGNORE_NOTFOUND;
+	}
+
+	/* use ignore from containing frame stack */
+	if (wi->is_ignored <= GIT_IGNORE_NOTFOUND)
+		wi->is_ignored = wi->fi.stack->is_ignored;
+}
+
 bool git_iterator_current_is_ignored(git_iterator *iter)
 {
 	workdir_iterator *wi = (workdir_iterator *)iter;
@@ -1480,14 +1520,22 @@ bool git_iterator_current_is_ignored(git_iterator *iter)
 	if (iter->type != GIT_ITERATOR_TYPE_WORKDIR)
 		return false;
 
-	if (wi->is_ignored != -1)
-		return (bool)(wi->is_ignored != 0);
+	if (wi->is_ignored != GIT_IGNORE_UNCHECKED)
+		return (bool)(wi->is_ignored == GIT_IGNORE_TRUE);
 
-	if (git_ignore__lookup(
-			&wi->ignores, wi->fi.entry.path, &wi->is_ignored) < 0)
-		wi->is_ignored = true;
+	workdir_iterator_update_is_ignored(wi);
+
+	return (bool)(wi->is_ignored == GIT_IGNORE_TRUE);
+}
+
+bool git_iterator_current_tree_is_ignored(git_iterator *iter)
+{
+	workdir_iterator *wi = (workdir_iterator *)iter;
 
-	return (bool)wi->is_ignored;
+	if (iter->type != GIT_ITERATOR_TYPE_WORKDIR)
+		return false;
+
+	return (bool)(wi->fi.stack->is_ignored == GIT_IGNORE_TRUE);
 }
 
 int git_iterator_cmp(git_iterator *iter, const char *path_prefix)
@@ -1516,3 +1564,73 @@ int git_iterator_current_workdir_path(git_buf **path, git_iterator *iter)
 
 	return 0;
 }
+
+int git_iterator_advance_over_with_status(
+	const git_index_entry **entryptr,
+	git_iterator_status_t *status,
+	git_iterator *iter)
+{
+	int error = 0;
+	workdir_iterator *wi = (workdir_iterator *)iter;
+	char *base = NULL;
+	const git_index_entry *entry;
+
+	*status = GIT_ITERATOR_STATUS_NORMAL;
+
+	if (iter->type != GIT_ITERATOR_TYPE_WORKDIR)
+		return git_iterator_advance(entryptr, iter);
+	if ((error = git_iterator_current(&entry, iter)) < 0)
+		return error;
+
+	if (!S_ISDIR(entry->mode)) {
+		workdir_iterator_update_is_ignored(wi);
+		if (wi->is_ignored == GIT_IGNORE_TRUE)
+			*status = GIT_ITERATOR_STATUS_IGNORED;
+		return git_iterator_advance(entryptr, iter);
+	}
+
+	*status = GIT_ITERATOR_STATUS_EMPTY;
+
+	base = git__strdup(entry->path);
+	GITERR_CHECK_ALLOC(base);
+
+	/* scan inside directory looking for a non-ignored item */
+	while (entry && !iter->prefixcomp(entry->path, base)) {
+		workdir_iterator_update_is_ignored(wi);
+
+		/* if we found an explicitly ignored item, then update from
+		 * EMPTY to IGNORED
+		 */
+		if (wi->is_ignored == GIT_IGNORE_TRUE)
+			*status = GIT_ITERATOR_STATUS_IGNORED;
+		else if (S_ISDIR(entry->mode)) {
+			error = git_iterator_advance_into(&entry, iter);
+
+			if (!error)
+				continue;
+			else if (error == GIT_ENOTFOUND) {
+				error = 0;
+				wi->is_ignored = GIT_IGNORE_TRUE; /* mark empty dirs ignored */
+			} else
+				break; /* real error, stop here */
+		} else {
+			/* we found a non-ignored item, treat parent as untracked */
+			*status = GIT_ITERATOR_STATUS_NORMAL;
+			break;
+		}
+
+		if ((error = git_iterator_advance(&entry, iter)) < 0)
+			break;
+	}
+
+	/* wrap up scan back to base directory */
+	while (entry && !iter->prefixcomp(entry->path, base))
+		if ((error = git_iterator_advance(&entry, iter)) < 0)
+			break;
+
+	*entryptr = entry;
+	git__free(base);
+
+	return error;
+}
+
diff --git a/src/iterator.h b/src/iterator.h
index 751e139d0..d88ad5191 100644
--- a/src/iterator.h
+++ b/src/iterator.h
@@ -52,6 +52,7 @@ struct git_iterator {
 	char *start;
 	char *end;
 	int (*prefixcomp)(const char *str, const char *prefix);
+	size_t stat_calls;
 	unsigned int flags;
 };
 
@@ -244,6 +245,8 @@ extern int git_iterator_current_parent_tree(
 
 extern bool git_iterator_current_is_ignored(git_iterator *iter);
 
+extern bool git_iterator_current_tree_is_ignored(git_iterator *iter);
+
 extern int git_iterator_cmp(
 	git_iterator *iter, const char *path_prefix);
 
@@ -258,4 +261,23 @@ extern int git_iterator_current_workdir_path(
 /* Return index pointer if index iterator, else NULL */
 extern git_index *git_iterator_get_index(git_iterator *iter);
 
+typedef enum {
+	GIT_ITERATOR_STATUS_NORMAL = 0,
+	GIT_ITERATOR_STATUS_IGNORED = 1,
+	GIT_ITERATOR_STATUS_EMPTY = 2
+} git_iterator_status_t;
+
+/* Advance over a directory and check if it contains no files or just
+ * ignored files.
+ *
+ * In a tree or the index, all directories will contain files, but in the
+ * working directory it is possible to have an empty directory tree or a
+ * tree that only contains ignored files.  Many Git operations treat these
+ * cases specially.  This advances over a directory (presumably an
+ * untracked directory) but checks during the scan if there are any files
+ * and any non-ignored files.
+ */
+extern int git_iterator_advance_over_with_status(
+	const git_index_entry **entry, git_iterator_status_t *status, git_iterator *iter);
+
 #endif
diff --git a/src/map.h b/src/map.h
index da3d1e19a..722eb7a30 100644
--- a/src/map.h
+++ b/src/map.h
@@ -42,5 +42,6 @@ typedef struct { /* memory mapped buffer	*/
 
 extern int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset);
 extern int p_munmap(git_map *map);
+extern long git__page_size(void);
 
 #endif /* INCLUDE_map_h__ */
diff --git a/src/merge.c b/src/merge.c
index 115867971..a279d31d4 100644
--- a/src/merge.c
+++ b/src/merge.c
@@ -26,6 +26,7 @@
 #include "oid.h"
 #include "index.h"
 #include "filebuf.h"
+#include "config.h"
 
 #include "git2/types.h"
 #include "git2/repository.h"
@@ -41,6 +42,7 @@
 #include "git2/sys/index.h"
 
 #define GIT_MERGE_INDEX_ENTRY_EXISTS(X)	((X).mode != 0)
+#define GIT_MERGE_INDEX_ENTRY_ISFILE(X) S_ISREG((X).mode)
 
 typedef enum {
 	TREE_IDX_ANCESTOR = 0,
@@ -112,6 +114,31 @@ cleanup:
 	return error;
 }
 
+int git_merge_base_octopus(git_oid *out, git_repository *repo, size_t length, const git_oid input_array[])
+{
+	git_oid result;
+	unsigned int i;
+	int error = -1;
+
+	assert(out && repo && input_array);
+
+	if (length < 2) {
+		giterr_set(GITERR_INVALID, "At least two commits are required to find an ancestor. Provided 'length' was %u.", length);
+		return -1;
+	}
+
+	result = input_array[0];
+	for (i = 1; i < length; i++) {
+		error = git_merge_base(&result, repo, &result, &input_array[i]);
+		if (error < 0)
+			return error;
+	}
+
+	*out = result;
+
+	return 0;
+}
+
 int git_merge_base(git_oid *out, git_repository *repo, const git_oid *one, const git_oid *two)
 {
 	git_revwalk *walk;
@@ -159,10 +186,10 @@ on_error:
 
 static int interesting(git_pqueue *list)
 {
-	unsigned int i;
-	/* element 0 isn't used - we need to start at 1 */
-	for (i = 1; i < list->size; i++) {
-		git_commit_list_node *commit = list->d[i];
+	size_t i;
+
+	for (i = 0; i < git_pqueue_size(list); i++) {
+		git_commit_list_node *commit = git_pqueue_get(list, i);
 		if ((commit->flags & STALE) == 0)
 			return 1;
 	}
@@ -178,13 +205,19 @@ int git_merge__bases_many(git_commit_list **out, git_revwalk *walk, git_commit_l
 	git_commit_list *result = NULL, *tmp = NULL;
 	git_pqueue list;
 
+	/* If there's only the one commit, there can be no merge bases */
+	if (twos->length == 0) {
+		*out = NULL;
+		return 0;
+	}
+
 	/* if the commit is repeated, we have a our merge base already */
 	git_vector_foreach(twos, i, two) {
 		if (one == two)
 			return git_commit_list_insert(one, out) ? 0 : -1;
 	}
 
-	if (git_pqueue_init(&list, twos->length * 2, git_commit_list_time_cmp) < 0)
+	if (git_pqueue_init(&list, 0, twos->length * 2, git_commit_list_time_cmp) < 0)
 		return -1;
 
 	if (git_commit_list_parse(walk, one) < 0)
@@ -203,10 +236,11 @@ int git_merge__bases_many(git_commit_list **out, git_revwalk *walk, git_commit_l
 
 	/* as long as there are non-STALE commits */
 	while (interesting(&list)) {
-		git_commit_list_node *commit;
+		git_commit_list_node *commit = git_pqueue_pop(&list);
 		int flags;
 
-		commit = git_pqueue_pop(&list);
+		if (commit == NULL)
+			break;
 
 		flags = commit->flags & (PARENT1 | PARENT2 | STALE);
 		if (flags == (PARENT1 | PARENT2)) {
@@ -253,7 +287,8 @@ int git_merge__bases_many(git_commit_list **out, git_revwalk *walk, git_commit_l
 	return 0;
 }
 
-int git_repository_mergehead_foreach(git_repository *repo,
+int git_repository_mergehead_foreach(
+	git_repository *repo,
 	git_repository_mergehead_foreach_cb cb,
 	void *payload)
 {
@@ -285,8 +320,8 @@ int git_repository_mergehead_foreach(git_repository *repo,
 		if ((error = git_oid_fromstr(&oid, line)) < 0)
 			goto cleanup;
 
-		if (cb(&oid, payload) != 0) {
-			error = GIT_EUSER;
+		if ((error = cb(&oid, payload)) != 0) {
+			giterr_set_after_callback(error);
 			goto cleanup;
 		}
 
@@ -314,7 +349,7 @@ GIT_INLINE(int) index_entry_cmp(const git_index_entry *a, const git_index_entry
 		return (b->path == NULL) ? 0 : 1;
 
 	if ((value = a->mode - b->mode) == 0 &&
-		(value = git_oid__cmp(&a->oid, &b->oid)) == 0)
+		(value = git_oid__cmp(&a->id, &b->id)) == 0)
 		value = strcmp(a->path, b->path);
 
 	return value;
@@ -445,7 +480,6 @@ static int merge_conflict_resolve_one_removed(
 	return error;
 }
 
-
 static int merge_conflict_resolve_one_renamed(
 	int *resolved,
 	git_merge_diff_list *diff_list,
@@ -476,12 +510,12 @@ static int merge_conflict_resolve_one_renamed(
 		conflict->type == GIT_MERGE_DIFF_RENAMED_ADDED)
 		return 0;
 
-	ours_changed = (git_oid__cmp(&conflict->ancestor_entry.oid, &conflict->our_entry.oid) != 0);
-	theirs_changed = (git_oid__cmp(&conflict->ancestor_entry.oid, &conflict->their_entry.oid) != 0);
+	ours_changed = (git_oid__cmp(&conflict->ancestor_entry.id, &conflict->our_entry.id) != 0);
+	theirs_changed = (git_oid__cmp(&conflict->ancestor_entry.id, &conflict->their_entry.id) != 0);
 
 	/* if both are modified (and not to a common target) require a merge */
 	if (ours_changed && theirs_changed &&
-		git_oid__cmp(&conflict->our_entry.oid, &conflict->their_entry.oid) != 0)
+		git_oid__cmp(&conflict->our_entry.id, &conflict->their_entry.id) != 0)
 		return 0;
 
 	if ((merged = git_pool_malloc(&diff_list->pool, sizeof(git_index_entry))) == NULL)
@@ -509,12 +543,11 @@ static int merge_conflict_resolve_automerge(
 	int *resolved,
 	git_merge_diff_list *diff_list,
 	const git_merge_diff *conflict,
-	unsigned int automerge_flags)
+	unsigned int merge_file_favor)
 {
-	git_merge_file_input ancestor = GIT_MERGE_FILE_INPUT_INIT,
-		ours = GIT_MERGE_FILE_INPUT_INIT,
-		theirs = GIT_MERGE_FILE_INPUT_INIT;
-	git_merge_file_result result = GIT_MERGE_FILE_RESULT_INIT;
+	const git_index_entry *ancestor = NULL, *ours = NULL, *theirs = NULL;
+	git_merge_file_options opts = GIT_MERGE_FILE_OPTIONS_INIT;
+	git_merge_file_result result = {0};
 	git_index_entry *index_entry;
 	git_odb *odb = NULL;
 	git_oid automerge_oid;
@@ -524,13 +557,20 @@ static int merge_conflict_resolve_automerge(
 
 	*resolved = 0;
 
-	if (automerge_flags == GIT_MERGE_AUTOMERGE_NONE)
+	if (!GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->our_entry) ||
+		!GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->their_entry))
 		return 0;
 
 	/* Reject D/F conflicts */
 	if (conflict->type == GIT_MERGE_DIFF_DIRECTORY_FILE)
 		return 0;
 
+	/* Reject submodules. */
+	if (S_ISGITLINK(conflict->ancestor_entry.mode) ||
+		S_ISGITLINK(conflict->our_entry.mode) ||
+		S_ISGITLINK(conflict->their_entry.mode))
+		return 0;
+
 	/* Reject link/file conflicts. */
 	if ((S_ISLNK(conflict->ancestor_entry.mode) ^ S_ISLNK(conflict->our_entry.mode)) ||
 		(S_ISLNK(conflict->ancestor_entry.mode) ^ S_ISLNK(conflict->their_entry.mode)))
@@ -546,13 +586,23 @@ static int merge_conflict_resolve_automerge(
 		strcmp(conflict->ancestor_entry.path, conflict->their_entry.path) != 0)
 		return 0;
 
+	/* Reject binary conflicts */
+	if (conflict->binary)
+		return 0;
+
+	ancestor = GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->ancestor_entry) ?
+		&conflict->ancestor_entry : NULL;
+	ours = GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->our_entry) ?
+		&conflict->our_entry : NULL;
+	theirs = GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->their_entry) ?
+		&conflict->their_entry : NULL;
+
+	opts.favor = merge_file_favor;
+
 	if ((error = git_repository_odb(&odb, diff_list->repo)) < 0 ||
-		(error = git_merge_file_input_from_index_entry(&ancestor, diff_list->repo, &conflict->ancestor_entry)) < 0 ||
-		(error = git_merge_file_input_from_index_entry(&ours, diff_list->repo, &conflict->our_entry)) < 0 ||
-		(error = git_merge_file_input_from_index_entry(&theirs, diff_list->repo, &conflict->their_entry)) < 0 ||
-		(error = git_merge_files(&result, &ancestor, &ours, &theirs, automerge_flags)) < 0 ||
+		(error = git_merge_file_from_index(&result, diff_list->repo, ancestor, ours, theirs, &opts)) < 0 ||
 		!result.automergeable ||
-		(error = git_odb_write(&automerge_oid, odb, result.data, result.len, GIT_OBJ_BLOB)) < 0)
+		(error = git_odb_write(&automerge_oid, odb, result.ptr, result.len, GIT_OBJ_BLOB)) < 0)
 		goto done;
 
 	if ((index_entry = git_pool_malloc(&diff_list->pool, sizeof(git_index_entry))) == NULL)
@@ -563,7 +613,7 @@ static int merge_conflict_resolve_automerge(
 
 	index_entry->file_size = result.len;
 	index_entry->mode = result.mode;
-	git_oid_cpy(&index_entry->oid, &automerge_oid);
+	git_oid_cpy(&index_entry->id, &automerge_oid);
 
 	git_vector_insert(&diff_list->staged, index_entry);
 	git_vector_insert(&diff_list->resolved, (git_merge_diff *)conflict);
@@ -571,9 +621,6 @@ static int merge_conflict_resolve_automerge(
 	*resolved = 1;
 
 done:
-	git_merge_file_input_free(&ancestor);
-	git_merge_file_input_free(&ours);
-	git_merge_file_input_free(&theirs);
 	git_merge_file_result_free(&result);
 	git_odb_free(odb);
 
@@ -584,7 +631,7 @@ static int merge_conflict_resolve(
 	int *out,
 	git_merge_diff_list *diff_list,
 	const git_merge_diff *conflict,
-	unsigned int automerge_flags)
+	unsigned int merge_file_favor)
 {
 	int resolved = 0;
 	int error = 0;
@@ -594,16 +641,14 @@ static int merge_conflict_resolve(
 	if ((error = merge_conflict_resolve_trivial(&resolved, diff_list, conflict)) < 0)
 		goto done;
 
-	if (automerge_flags != GIT_MERGE_AUTOMERGE_NONE) {
-		if (!resolved && (error = merge_conflict_resolve_one_removed(&resolved, diff_list, conflict)) < 0)
-			goto done;
+	if (!resolved && (error = merge_conflict_resolve_one_removed(&resolved, diff_list, conflict)) < 0)
+		goto done;
 
-		if (!resolved && (error = merge_conflict_resolve_one_renamed(&resolved, diff_list, conflict)) < 0)
-			goto done;
+	if (!resolved && (error = merge_conflict_resolve_one_renamed(&resolved, diff_list, conflict)) < 0)
+		goto done;
 
-		if (!resolved && (error = merge_conflict_resolve_automerge(&resolved, diff_list, conflict, automerge_flags)) < 0)
-			goto done;
-	}
+	if (!resolved && (error = merge_conflict_resolve_automerge(&resolved, diff_list, conflict, merge_file_favor)) < 0)
+		goto done;
 
 	*out = resolved;
 
@@ -625,7 +670,7 @@ static int index_entry_similarity_exact(
 	git_index_entry *b,
 	size_t b_idx,
 	void **cache,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	GIT_UNUSED(repo);
 	GIT_UNUSED(a_idx);
@@ -633,7 +678,7 @@ static int index_entry_similarity_exact(
 	GIT_UNUSED(cache);
 	GIT_UNUSED(opts);
 
-	if (git_oid__cmp(&a->oid, &b->oid) == 0)
+	if (git_oid__cmp(&a->id, &b->id) == 0)
 		return 100;
 
 	return 0;
@@ -643,7 +688,7 @@ static int index_entry_similarity_calc(
 	void **out,
 	git_repository *repo,
 	git_index_entry *entry,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	git_blob *blob;
 	git_diff_file diff_file = {{{0}}};
@@ -652,10 +697,10 @@ static int index_entry_similarity_calc(
 
 	*out = NULL;
 
-	if ((error = git_blob_lookup(&blob, repo, &entry->oid)) < 0)
+	if ((error = git_blob_lookup(&blob, repo, &entry->id)) < 0)
 		return error;
 
-	git_oid_cpy(&diff_file.oid, &entry->oid);
+	git_oid_cpy(&diff_file.id, &entry->id);
 	diff_file.path = entry->path;
 	diff_file.size = entry->file_size;
 	diff_file.mode = entry->mode;
@@ -683,7 +728,7 @@ static int index_entry_similarity_inexact(
 	git_index_entry *b,
 	size_t b_idx,
 	void **cache,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	int score = 0;
 	int error = 0;
@@ -720,9 +765,9 @@ static int merge_diff_mark_similarity(
 	git_merge_diff_list *diff_list,
 	struct merge_diff_similarity *similarity_ours,
 	struct merge_diff_similarity *similarity_theirs,
-	int (*similarity_fn)(git_repository *, git_index_entry *, size_t, git_index_entry *, size_t, void **, const git_merge_tree_opts *),
+	int (*similarity_fn)(git_repository *, git_index_entry *, size_t, git_index_entry *, size_t, void **, const git_merge_options *),
 	void **cache,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	size_t i, j;
 	git_merge_diff *conflict_src, *conflict_tgt;
@@ -823,7 +868,7 @@ static void merge_diff_mark_rename_conflict(
 	bool theirs_renamed,
 	size_t theirs_source_idx,
 	git_merge_diff *target,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	git_merge_diff *ours_source = NULL, *theirs_source = NULL;
 
@@ -893,7 +938,7 @@ static void merge_diff_list_coalesce_renames(
 	git_merge_diff_list *diff_list,
 	struct merge_diff_similarity *similarity_ours,
 	struct merge_diff_similarity *similarity_theirs,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	size_t i;
 	bool ours_renamed = 0, theirs_renamed = 0;
@@ -950,10 +995,12 @@ static void merge_diff_list_coalesce_renames(
 	}
 }
 
-static int merge_diff_empty(const git_vector *conflicts, size_t idx)
+static int merge_diff_empty(const git_vector *conflicts, size_t idx, void *p)
 {
 	git_merge_diff *conflict = conflicts->contents[idx];
 
+	GIT_UNUSED(p);
+
 	return (!GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->ancestor_entry) &&
 		!GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->our_entry) &&
 		!GIT_MERGE_INDEX_ENTRY_EXISTS(conflict->their_entry));
@@ -983,7 +1030,7 @@ static void merge_diff_list_count_candidates(
 int git_merge_diff_list__find_renames(
 	git_repository *repo,
 	git_merge_diff_list *diff_list,
-	const git_merge_tree_opts *opts)
+	const git_merge_options *opts)
 {
 	struct merge_diff_similarity *similarity_ours, *similarity_theirs;
 	void **cache = NULL;
@@ -1034,7 +1081,7 @@ int git_merge_diff_list__find_renames(
 	merge_diff_list_coalesce_renames(diff_list, similarity_ours, similarity_theirs, opts);
 
 	/* And remove any entries that were merged and are now empty. */
-	git_vector_remove_matching(&diff_list->conflicts, merge_diff_empty);
+	git_vector_remove_matching(&diff_list->conflicts, merge_diff_empty, NULL);
 
 done:
 	if (cache != NULL) {
@@ -1145,7 +1192,45 @@ GIT_INLINE(int) merge_diff_detect_type(
 	return 0;
 }
 
-GIT_INLINE(int) index_entry_dup(
+GIT_INLINE(int) merge_diff_detect_binary(
+	git_repository *repo,
+	git_merge_diff *conflict)
+{
+	git_blob *ancestor_blob = NULL, *our_blob = NULL, *their_blob = NULL;
+	int error = 0;
+
+	if (GIT_MERGE_INDEX_ENTRY_ISFILE(conflict->ancestor_entry)) {
+		if ((error = git_blob_lookup(&ancestor_blob, repo, &conflict->ancestor_entry.id)) < 0)
+			goto done;
+
+		conflict->binary = git_blob_is_binary(ancestor_blob);
+	}
+
+	if (!conflict->binary &&
+		GIT_MERGE_INDEX_ENTRY_ISFILE(conflict->our_entry)) {
+		if ((error = git_blob_lookup(&our_blob, repo, &conflict->our_entry.id)) < 0)
+			goto done;
+
+		conflict->binary = git_blob_is_binary(our_blob);
+	}
+
+	if (!conflict->binary &&
+		GIT_MERGE_INDEX_ENTRY_ISFILE(conflict->their_entry)) {
+		if ((error = git_blob_lookup(&their_blob, repo, &conflict->their_entry.id)) < 0)
+			goto done;
+
+		conflict->binary = git_blob_is_binary(their_blob);
+	}
+
+done:
+	git_blob_free(ancestor_blob);
+	git_blob_free(our_blob);
+	git_blob_free(their_blob);
+
+	return error;
+}
+
+GIT_INLINE(int) index_entry_dup_pool(
 	git_index_entry *out,
 	git_pool *pool,
 	const git_index_entry *src)
@@ -1174,7 +1259,7 @@ GIT_INLINE(int) merge_delta_type_from_index_entries(
 		return GIT_DELTA_TYPECHANGE;
 	else if(S_ISLNK(ancestor->mode) ^ S_ISLNK(other->mode))
 		return GIT_DELTA_TYPECHANGE;
-	else if (git_oid__cmp(&ancestor->oid, &other->oid) ||
+	else if (git_oid__cmp(&ancestor->id, &other->id) ||
 			 ancestor->mode != other->mode)
 		return GIT_DELTA_MODIFIED;
 
@@ -1191,9 +1276,9 @@ static git_merge_diff *merge_diff_from_index_entries(
 	if ((conflict = git_pool_malloc(pool, sizeof(git_merge_diff))) == NULL)
 		return NULL;
 
-	if (index_entry_dup(&conflict->ancestor_entry, pool, entries[TREE_IDX_ANCESTOR]) < 0 ||
-		index_entry_dup(&conflict->our_entry, pool, entries[TREE_IDX_OURS]) < 0 ||
-		index_entry_dup(&conflict->their_entry, pool, entries[TREE_IDX_THEIRS]) < 0)
+	if (index_entry_dup_pool(&conflict->ancestor_entry, pool, entries[TREE_IDX_ANCESTOR]) < 0 ||
+		index_entry_dup_pool(&conflict->our_entry, pool, entries[TREE_IDX_OURS]) < 0 ||
+		index_entry_dup_pool(&conflict->their_entry, pool, entries[TREE_IDX_THEIRS]) < 0)
 		return NULL;
 
 	conflict->our_status = merge_delta_type_from_index_entries(
@@ -1206,7 +1291,7 @@ static git_merge_diff *merge_diff_from_index_entries(
 
 /* Merge trees */
 
-static int merge_index_insert_conflict(
+static int merge_diff_list_insert_conflict(
 	git_merge_diff_list *diff_list,
 	struct merge_diff_df_data *merge_df_data,
 	const git_index_entry *tree_items[3])
@@ -1216,13 +1301,14 @@ static int merge_index_insert_conflict(
 	if ((conflict = merge_diff_from_index_entries(diff_list, tree_items)) == NULL ||
 		merge_diff_detect_type(conflict) < 0 ||
 		merge_diff_detect_df_conflict(merge_df_data, conflict) < 0 ||
+		merge_diff_detect_binary(diff_list->repo, conflict) < 0 ||
 		git_vector_insert(&diff_list->conflicts, conflict) < 0)
 		return -1;
 
 	return 0;
 }
 
-static int merge_index_insert_unmodified(
+static int merge_diff_list_insert_unmodified(
 	git_merge_diff_list *diff_list,
 	const git_index_entry *tree_items[3])
 {
@@ -1232,7 +1318,7 @@ static int merge_index_insert_unmodified(
 	entry = git_pool_malloc(&diff_list->pool, sizeof(git_index_entry));
 	GITERR_CHECK_ALLOC(entry);
 
-	if ((error = index_entry_dup(entry, &diff_list->pool, tree_items[0])) >= 0)
+	if ((error = index_entry_dup_pool(entry, &diff_list->pool, tree_items[0])) >= 0)
 		error = git_vector_insert(&diff_list->staged, entry);
 
 	return error;
@@ -1246,13 +1332,13 @@ int git_merge_diff_list__find_differences(
 {
 	git_iterator *iterators[3] = {0};
 	const git_index_entry *items[3] = {0}, *best_cur_item, *cur_items[3];
-	git_vector_cmp entry_compare = git_index_entry__cmp;
+	git_vector_cmp entry_compare = git_index_entry_cmp;
 	struct merge_diff_df_data df_data = {0};
 	int cur_item_modified;
 	size_t i, j;
 	int error = 0;
 
-	assert(diff_list && our_tree && their_tree);
+	assert(diff_list && (our_tree || their_tree));
 
 	if ((error = git_iterator_for_tree(&iterators[TREE_IDX_ANCESTOR], (git_tree *)ancestor_tree, GIT_ITERATOR_DONT_IGNORE_CASE, NULL, NULL)) < 0 ||
 		(error = git_iterator_for_tree(&iterators[TREE_IDX_OURS], (git_tree *)our_tree, GIT_ITERATOR_DONT_IGNORE_CASE, NULL, NULL)) < 0 ||
@@ -1262,6 +1348,7 @@ int git_merge_diff_list__find_differences(
 	/* Set up the iterators */
 	for (i = 0; i < 3; i++) {
 		error = git_iterator_current(&items[i], iterators[i]);
+
 		if (error < 0 && error != GIT_ITEROVER)
 			goto done;
 	}
@@ -1313,9 +1400,9 @@ int git_merge_diff_list__find_differences(
 			break;
 
 		if (cur_item_modified)
-			error = merge_index_insert_conflict(diff_list, &df_data, cur_items);
+			error = merge_diff_list_insert_conflict(diff_list, &df_data, cur_items);
 		else
-			error = merge_index_insert_unmodified(diff_list, cur_items);
+			error = merge_diff_list_insert_unmodified(diff_list, cur_items);
 		if (error < 0)
 			goto done;
 
@@ -1325,6 +1412,7 @@ int git_merge_diff_list__find_differences(
 				continue;
 
 			error = git_iterator_advance(&items[i], iterators[i]);
+
 			if (error < 0 && error != GIT_ITEROVER)
 				goto done;
 		}
@@ -1370,10 +1458,10 @@ void git_merge_diff_list__free(git_merge_diff_list *diff_list)
 	git__free(diff_list);
 }
 
-static int merge_tree_normalize_opts(
+static int merge_normalize_opts(
 	git_repository *repo,
-	git_merge_tree_opts *opts,
-	const git_merge_tree_opts *given)
+	git_merge_options *opts,
+	const git_merge_options *given)
 {
 	git_config *cfg = NULL;
 	int error = 0;
@@ -1384,9 +1472,9 @@ static int merge_tree_normalize_opts(
 		return error;
 
 	if (given != NULL)
-		memcpy(opts, given, sizeof(git_merge_tree_opts));
+		memcpy(opts, given, sizeof(git_merge_options));
 	else {
-		git_merge_tree_opts init = GIT_MERGE_TREE_OPTS_INIT;
+		git_merge_options init = GIT_MERGE_OPTIONS_INIT;
 		memcpy(opts, &init, sizeof(init));
 
 		opts->flags = GIT_MERGE_TREE_FIND_RENAMES;
@@ -1394,19 +1482,13 @@ static int merge_tree_normalize_opts(
 	}
 
 	if (!opts->target_limit) {
-		int32_t limit = 0;
-
-		opts->target_limit = GIT_MERGE_TREE_TARGET_LIMIT;
+		int limit = git_config__get_int_force(cfg, "merge.renamelimit", 0);
 
-		if (git_config_get_int32(&limit, cfg, "merge.renameLimit") < 0) {
-			giterr_clear();
-
-			if (git_config_get_int32(&limit, cfg, "diff.renameLimit") < 0)
-				giterr_clear();
-		}
+		if (!limit)
+			limit = git_config__get_int_force(cfg, "diff.renamelimit", 0);
 
-		if (limit > 0)
-			opts->target_limit = limit;
+		opts->target_limit = (limit <= 0) ?
+			GIT_MERGE_TREE_TARGET_LIMIT : (unsigned int)limit;
 	}
 
 	/* assign the internal metric with whitespace flag as payload */
@@ -1452,7 +1534,7 @@ static int merge_index_insert_reuc(
 	}
 
 	mode[idx] = entry->mode;
-	oid[idx] = &entry->oid;
+	oid[idx] = &entry->id;
 
 	return git_index_reuc_add(index, entry->path,
 		mode[0], oid[0], mode[1], oid[1], mode[2], oid[2]);
@@ -1560,20 +1642,22 @@ int git_merge_trees(
 	const git_tree *ancestor_tree,
 	const git_tree *our_tree,
 	const git_tree *their_tree,
-	const git_merge_tree_opts *given_opts)
+	const git_merge_options *given_opts)
 {
 	git_merge_diff_list *diff_list;
-	git_merge_tree_opts opts;
+	git_merge_options opts;
 	git_merge_diff *conflict;
 	git_vector changes;
 	size_t i;
 	int error = 0;
 
-	assert(out && repo && our_tree && their_tree);
+	assert(out && repo && (our_tree || their_tree));
 
 	*out = NULL;
 
-	if ((error = merge_tree_normalize_opts(repo, &opts, given_opts)) < 0)
+	GITERR_CHECK_VERSION(given_opts, GIT_MERGE_OPTIONS_VERSION, "git_merge_options");
+
+	if ((error = merge_normalize_opts(repo, &opts, given_opts)) < 0)
 		return error;
 
 	diff_list = git_merge_diff_list__alloc(repo);
@@ -1589,7 +1673,7 @@ int git_merge_trees(
 	git_vector_foreach(&changes, i, conflict) {
 		int resolved = 0;
 
-		if ((error = merge_conflict_resolve(&resolved, diff_list, conflict, opts.automerge_flags)) < 0)
+		if ((error = merge_conflict_resolve(&resolved, diff_list, conflict, opts.file_favor)) < 0)
 			goto done;
 
 		if (!resolved)
@@ -1607,6 +1691,40 @@ done:
 	return error;
 }
 
+int git_merge_commits(
+	git_index **out,
+	git_repository *repo,
+	const git_commit *our_commit,
+	const git_commit *their_commit,
+	const git_merge_options *opts)
+{
+	git_oid ancestor_oid;
+	git_commit *ancestor_commit = NULL;
+	git_tree *our_tree = NULL, *their_tree = NULL, *ancestor_tree = NULL;
+	int error = 0;
+
+	if ((error = git_merge_base(&ancestor_oid, repo, git_commit_id(our_commit), git_commit_id(their_commit))) < 0 &&
+		error == GIT_ENOTFOUND)
+		giterr_clear();
+	else if (error < 0 ||
+		(error = git_commit_lookup(&ancestor_commit, repo, &ancestor_oid)) < 0 ||
+		(error = git_commit_tree(&ancestor_tree, ancestor_commit)) < 0)
+		goto done;
+
+	if ((error = git_commit_tree(&our_tree, our_commit)) < 0 ||
+		(error = git_commit_tree(&their_tree, their_commit)) < 0 ||
+		(error = git_merge_trees(out, repo, ancestor_tree, our_tree, their_tree, opts)) < 0)
+		goto done;
+
+done:
+	git_commit_free(ancestor_commit);
+	git_tree_free(our_tree);
+	git_tree_free(their_tree);
+	git_tree_free(ancestor_tree);
+
+	return error;
+}
+
 /* Merge setup / cleanup */
 
 static int write_orig_head(
@@ -1664,31 +1782,20 @@ cleanup:
 	return error;
 }
 
-static int write_merge_mode(git_repository *repo, unsigned int flags)
+static int write_merge_mode(git_repository *repo)
 {
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf file_path = GIT_BUF_INIT;
 	int error = 0;
 
-	/* For future expansion */
-	GIT_UNUSED(flags);
-
 	assert(repo);
 
 	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_MODE_FILE)) < 0 ||
 		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_MERGE_FILE_MODE)) < 0)
 		goto cleanup;
 
-	/*
-	 * no-ff is the only thing allowed here at present.  One would
-	 * presume they would be space-delimited when there are more, but
-	 * this needs to be revisited.
-	 */
-
-	if (flags & GIT_MERGE_NO_FASTFORWARD) {
-		if ((error = git_filebuf_write(&file, "no-ff", 5)) < 0)
-			goto cleanup;
-	}
+	if ((error = git_filebuf_write(&file, "no-ff", 5)) < 0)
+		goto cleanup;
 
 	error = git_filebuf_commit(&file);
 
@@ -2004,6 +2111,25 @@ cleanup:
 	return error;
 }
 
+int git_merge__setup(
+	git_repository *repo,
+	const git_merge_head *our_head,
+	const git_merge_head *heads[],
+	size_t heads_len)
+{
+	int error = 0;
+
+	assert (repo && our_head && heads);
+
+	if ((error = write_orig_head(repo, our_head)) == 0 &&
+		(error = write_merge_head(repo, heads, heads_len)) == 0 &&
+		(error = write_merge_mode(repo)) == 0) {
+		error = write_merge_msg(repo, heads, heads_len);
+	}
+
+	return error;
+}
+
 /* Merge branches */
 
 static int merge_ancestor_head(
@@ -2030,44 +2156,13 @@ static int merge_ancestor_head(
 	if ((error = git_merge_base_many(&ancestor_oid, repo, their_heads_len + 1, oids)) < 0)
 		goto on_error;
 
-	error = git_merge_head_from_oid(ancestor_head, repo, &ancestor_oid);
+	error = git_merge_head_from_id(ancestor_head, repo, &ancestor_oid);
 
 on_error:
 	git__free(oids);
 	return error;
 }
 
-GIT_INLINE(bool) merge_check_uptodate(
-	git_merge_result *result,
-	const git_merge_head *ancestor_head,
-	const git_merge_head *their_head)
-{
-	if (git_oid_cmp(&ancestor_head->oid, &their_head->oid) == 0) {
-		result->is_uptodate = 1;
-		return true;
-	}
-
-	return false;
-}
-
-GIT_INLINE(bool) merge_check_fastforward(
-	git_merge_result *result,
-	const git_merge_head *ancestor_head,
-	const git_merge_head *our_head,
-	const git_merge_head *their_head,
-	unsigned int flags)
-{
-	if ((flags & GIT_MERGE_NO_FASTFORWARD) == 0 &&
-		git_oid_cmp(&ancestor_head->oid, &our_head->oid) == 0) {
-		result->is_fastforward = 1;
-		git_oid_cpy(&result->fastforward_oid, &their_head->oid);
-
-		return true;
-	}
-
-	return false;
-}
-
 const char *merge_their_label(const char *branchname)
 {
 	const char *slash;
@@ -2081,39 +2176,51 @@ const char *merge_their_label(const char *branchname)
 	return slash+1;
 }
 
-static int merge_normalize_opts(
+static int merge_normalize_checkout_opts(
 	git_repository *repo,
-	git_merge_opts *opts,
-	const git_merge_opts *given,
+	git_checkout_options *checkout_opts,
+	const git_checkout_options *given_checkout_opts,
+	const git_merge_head *ancestor_head,
+	const git_merge_head *our_head,
 	size_t their_heads_len,
 	const git_merge_head **their_heads)
 {
 	int error = 0;
-	unsigned int default_checkout_strategy = GIT_CHECKOUT_SAFE_CREATE |
-		GIT_CHECKOUT_ALLOW_CONFLICTS;
 
 	GIT_UNUSED(repo);
 
-	if (given != NULL)
-		memcpy(opts, given, sizeof(git_merge_opts));
+	if (given_checkout_opts != NULL)
+		memcpy(checkout_opts, given_checkout_opts, sizeof(git_checkout_options));
 	else {
-		git_merge_opts default_opts = GIT_MERGE_OPTS_INIT;
-		memcpy(opts, &default_opts, sizeof(git_merge_opts));
+		git_checkout_options default_checkout_opts = GIT_CHECKOUT_OPTIONS_INIT;
+		default_checkout_opts.checkout_strategy =  GIT_CHECKOUT_SAFE |
+			GIT_CHECKOUT_ALLOW_CONFLICTS;
+
+		memcpy(checkout_opts, &default_checkout_opts, sizeof(git_checkout_options));
 	}
 
-	if (!opts->checkout_opts.checkout_strategy)
-		opts->checkout_opts.checkout_strategy = default_checkout_strategy;
+	/* TODO: for multiple ancestors in merge-recursive, this is "merged common ancestors" */
+	if (!checkout_opts->ancestor_label) {
+		if (ancestor_head && ancestor_head->commit)
+			checkout_opts->ancestor_label = git_commit_summary(ancestor_head->commit);
+		else
+			checkout_opts->ancestor_label = "ancestor";
+	}
 
-	if (!opts->checkout_opts.our_label)
-		opts->checkout_opts.our_label = "HEAD";
+	if (!checkout_opts->our_label) {
+		if (our_head && our_head->ref_name)
+			checkout_opts->our_label = our_head->ref_name;
+		else
+			checkout_opts->our_label = "ours";
+	}
 
-	if (!opts->checkout_opts.their_label) {
+	if (!checkout_opts->their_label) {
 		if (their_heads_len == 1 && their_heads[0]->ref_name)
-			opts->checkout_opts.their_label = merge_their_label(their_heads[0]->ref_name);
+			checkout_opts->their_label = merge_their_label(their_heads[0]->ref_name);
 		else if (their_heads_len == 1)
-			opts->checkout_opts.their_label = their_heads[0]->oid_str;
+			checkout_opts->their_label = their_heads[0]->oid_str;
 		else
-			opts->checkout_opts.their_label = "theirs";
+			checkout_opts->their_label = "theirs";
 	}
 
 	return error;
@@ -2235,7 +2342,7 @@ done:
 
 static int merge_check_workdir(size_t *conflicts, git_repository *repo, git_index *index_new, git_vector *merged_paths)
 {
-	git_tree *head_tree = NULL;
+	git_index *index_repo = NULL;
 	git_diff *wd_diff_list = NULL;
 	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
 	int error = 0;
@@ -2246,9 +2353,6 @@ static int merge_check_workdir(size_t *conflicts, git_repository *repo, git_inde
 
 	opts.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
 
-	if ((error = git_repository_head_tree(&head_tree, repo)) < 0)
-		goto done;
-
 	/* Workdir changes may exist iff they do not conflict with changes that
 	 * will be applied by the merge (including conflicts).  Ensure that there
 	 * are no changes in the workdir to these paths.
@@ -2256,22 +2360,22 @@ static int merge_check_workdir(size_t *conflicts, git_repository *repo, git_inde
 	opts.pathspec.count = merged_paths->length;
 	opts.pathspec.strings = (char **)merged_paths->contents;
 
-	if ((error = git_diff_tree_to_workdir(&wd_diff_list, repo, head_tree, &opts)) < 0)
+	if ((error = git_diff_index_to_workdir(&wd_diff_list, repo, index_repo, &opts)) < 0)
 		goto done;
 
 	*conflicts = wd_diff_list->deltas.length;
 
 done:
-	git_tree_free(head_tree);
+	git_index_free(index_repo);
 	git_diff_free(wd_diff_list);
 
 	return error;
 }
 
-static int merge_indexes(git_repository *repo, git_index *index_new)
+int git_merge__indexes(git_repository *repo, git_index *index_new)
 {
-	git_index *index_repo;
-	unsigned int index_repo_caps;
+	git_index *index_repo = NULL;
+	int index_repo_caps = 0;
 	git_vector paths = GIT_VECTOR_INIT;
 	size_t index_conflicts = 0, wd_conflicts = 0, conflicts, i;
 	char *path;
@@ -2303,12 +2407,21 @@ static int merge_indexes(git_repository *repo, git_index *index_new)
 		goto done;
 	}
 
-	/* Update the new index */
+	/* Remove removed items from the index */
 	git_vector_foreach(&paths, i, path) {
-		if ((e = git_index_get_bypath(index_new, path, 0)) != NULL)
-			error = git_index_add(index_repo, e);
-		else
-			error = git_index_remove(index_repo, path, 0);
+		if (git_index_get_bypath(index_new, path, 0) == NULL) {
+			if ((error = git_index_remove(index_repo, path, 0)) < 0 &&
+				error != GIT_ENOTFOUND)
+				goto done;
+		}
+	}
+
+	/* Add updated items to the index */
+	git_vector_foreach(&paths, i, path) {
+		if ((e = git_index_get_bypath(index_new, path, 0)) != NULL) {
+			if ((error = git_index_add(index_repo, e)) < 0)
+				goto done;
+		}
 	}
 
 	/* Add conflicts */
@@ -2351,78 +2464,222 @@ done:
         git_index_set_caps(index_repo, index_repo_caps);
 
 	git_index_free(index_repo);
+	git_vector_free_deep(&paths);
 
-	git_vector_foreach(&paths, i, path)
-		git__free(path);
+	return error;
+}
+
+int git_merge__append_conflicts_to_merge_msg(
+	git_repository *repo,
+	git_index *index)
+{
+	git_filebuf file = GIT_FILEBUF_INIT;
+	git_buf file_path = GIT_BUF_INIT;
+	const char *last = NULL;
+	size_t i;
+	int error;
+
+	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_MSG_FILE)) < 0 ||
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_APPEND, GIT_MERGE_FILE_MODE)) < 0)
+		goto cleanup;
+
+	if (git_index_has_conflicts(index))
+		git_filebuf_printf(&file, "\nConflicts:\n");
+
+	for (i = 0; i < git_index_entrycount(index); i++) {
+		const git_index_entry *e = git_index_get_byindex(index, i);
+
+		if (git_index_entry_stage(e) == 0)
+			continue;
+
+		if (last == NULL || strcmp(e->path, last) != 0)
+			git_filebuf_printf(&file, "\t%s\n", e->path);
+
+		last = e->path;
+	}
+
+	error = git_filebuf_commit(&file);
 
-	git_vector_free(&paths);
+cleanup:
+	if (error < 0)
+		git_filebuf_cleanup(&file);
+
+	git_buf_free(&file_path);
+
+	return error;
+}
+
+
+static int merge_state_cleanup(git_repository *repo)
+{
+	const char *state_files[] = {
+		GIT_MERGE_HEAD_FILE,
+		GIT_MERGE_MODE_FILE,
+		GIT_MERGE_MSG_FILE,
+	};
+
+	return git_repository__cleanup_files(repo, state_files, ARRAY_SIZE(state_files));
+}
+
+static int merge_heads(
+	git_merge_head **ancestor_head_out,
+	git_merge_head **our_head_out,
+	git_repository *repo,
+	const git_merge_head **their_heads,
+	size_t their_heads_len)
+{
+	git_merge_head *ancestor_head = NULL, *our_head = NULL;
+	git_reference *our_ref = NULL;
+	int error = 0;
+
+	*ancestor_head_out = NULL;
+	*our_head_out = NULL;
+
+	if ((error = git_repository__ensure_not_bare(repo, "merge")) < 0)
+		goto done;
+
+	if ((error = git_reference_lookup(&our_ref, repo, GIT_HEAD_FILE)) < 0 ||
+		(error = git_merge_head_from_ref(&our_head, repo, our_ref)) < 0)
+		goto done;
+
+	if ((error = merge_ancestor_head(&ancestor_head, repo, our_head, their_heads, their_heads_len)) < 0) {
+		if (error != GIT_ENOTFOUND)
+			goto done;
+
+		giterr_clear();
+		error = 0;
+	}
+
+	*ancestor_head_out = ancestor_head;
+	*our_head_out = our_head;
+
+done:
+	if (error < 0) {
+		git_merge_head_free(ancestor_head);
+		git_merge_head_free(our_head);
+	}
+
+	git_reference_free(our_ref);
 
 	return error;
 }
 
+static int merge_preference(git_merge_preference_t *out, git_repository *repo)
+{
+	git_config *config;
+	const char *value;
+	int bool_value, error = 0;
+
+	*out = GIT_MERGE_PREFERENCE_NONE;
+
+	if ((error = git_repository_config_snapshot(&config, repo)) < 0)
+		goto done;
+
+	if ((error = git_config_get_string(&value, config, "merge.ff")) < 0) {
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			error = 0;
+		}
+
+		goto done;
+	}
+
+	if (git_config_parse_bool(&bool_value, value) == 0) {
+		if (!bool_value)
+			*out |= GIT_MERGE_PREFERENCE_NO_FASTFORWARD;
+	} else {
+		if (strcasecmp(value, "only") == 0)
+			*out |= GIT_MERGE_PREFERENCE_FASTFORWARD_ONLY;
+	}
+
+done:
+	git_config_free(config);
+	return error;
+}
+
+int git_merge_analysis(
+	git_merge_analysis_t *analysis_out,
+	git_merge_preference_t *preference_out,
+	git_repository *repo,
+	const git_merge_head **their_heads,
+	size_t their_heads_len)
+{
+	git_merge_head *ancestor_head = NULL, *our_head = NULL;
+	int error = 0;
+
+	assert(analysis_out && preference_out && repo && their_heads);
+
+	if (their_heads_len != 1) {
+		giterr_set(GITERR_MERGE, "Can only merge a single branch");
+		error = -1;
+		goto done;
+	}
+
+	*analysis_out = GIT_MERGE_ANALYSIS_NONE;
+
+	if ((error = merge_preference(preference_out, repo)) < 0)
+		goto done;
+
+	if (git_repository_head_unborn(repo)) {
+		*analysis_out |= GIT_MERGE_ANALYSIS_FASTFORWARD | GIT_MERGE_ANALYSIS_UNBORN;
+		goto done;
+	}
+
+	if ((error = merge_heads(&ancestor_head, &our_head, repo, their_heads, their_heads_len)) < 0)
+		goto done;
+
+	/* We're up-to-date if we're trying to merge our own common ancestor. */
+	if (ancestor_head && git_oid_equal(&ancestor_head->oid, &their_heads[0]->oid))
+		*analysis_out |= GIT_MERGE_ANALYSIS_UP_TO_DATE;
+
+	/* We're fastforwardable if we're our own common ancestor. */
+	else if (ancestor_head && git_oid_equal(&ancestor_head->oid, &our_head->oid))
+		*analysis_out |= GIT_MERGE_ANALYSIS_FASTFORWARD | GIT_MERGE_ANALYSIS_NORMAL;
+
+	/* Otherwise, just a normal merge is possible. */
+	else
+		*analysis_out |= GIT_MERGE_ANALYSIS_NORMAL;
+
+done:
+	git_merge_head_free(ancestor_head);
+	git_merge_head_free(our_head);
+	return error;
+}
+
 int git_merge(
-	git_merge_result **out,
 	git_repository *repo,
 	const git_merge_head **their_heads,
 	size_t their_heads_len,
-	const git_merge_opts *given_opts)
+	const git_merge_options *merge_opts,
+	const git_checkout_options *given_checkout_opts)
 {
-	git_merge_result *result;
-	git_merge_opts opts;
 	git_reference *our_ref = NULL;
+	git_checkout_options checkout_opts;
 	git_merge_head *ancestor_head = NULL, *our_head = NULL;
 	git_tree *ancestor_tree = NULL, *our_tree = NULL, **their_trees = NULL;
 	git_index *index_new = NULL, *index_repo = NULL;
 	size_t i;
 	int error = 0;
 
-	assert(out && repo && their_heads);
-
-	*out = NULL;
+	assert(repo && their_heads);
 
 	if (their_heads_len != 1) {
 		giterr_set(GITERR_MERGE, "Can only merge a single branch");
 		return -1;
 	}
 
-	result = git__calloc(1, sizeof(git_merge_result));
-	GITERR_CHECK_ALLOC(result);
-
 	their_trees = git__calloc(their_heads_len, sizeof(git_tree *));
 	GITERR_CHECK_ALLOC(their_trees);
 
-	if ((error = merge_normalize_opts(repo, &opts, given_opts, their_heads_len, their_heads)) < 0)
-		goto on_error;
-
-	if ((error = git_repository__ensure_not_bare(repo, "merge")) < 0)
-		goto on_error;
-
-	if ((error = git_reference_lookup(&our_ref, repo, GIT_HEAD_FILE)) < 0 ||
-		(error = git_merge_head_from_ref(&our_head, repo, our_ref)) < 0)
+	if ((error = merge_heads(&ancestor_head, &our_head, repo, their_heads, their_heads_len)) < 0)
 		goto on_error;
 
-	if ((error = merge_ancestor_head(&ancestor_head, repo, our_head, their_heads, their_heads_len)) < 0 &&
-		error != GIT_ENOTFOUND)
+	if ((error = merge_normalize_checkout_opts(repo, &checkout_opts, given_checkout_opts,
+		ancestor_head, our_head, their_heads_len, their_heads)) < 0)
 		goto on_error;
 
-	if (their_heads_len == 1 &&
-		ancestor_head != NULL &&
-		(merge_check_uptodate(result, ancestor_head, their_heads[0]) ||
-		merge_check_fastforward(result, ancestor_head, our_head, their_heads[0], opts.merge_flags))) {
-		*out = result;
-		goto done;
-	}
-
-	/* If FASTFORWARD_ONLY is specified, fail. */
-	if ((opts.merge_flags & GIT_MERGE_FASTFORWARD_ONLY) ==
-		GIT_MERGE_FASTFORWARD_ONLY) {
-		giterr_set(GITERR_MERGE, "Not a fast-forward.");
-		error = GIT_ENONFASTFORWARD;
-		goto on_error;
-	}
-
 	/* Write the merge files to the repository. */
-	if ((error = git_merge__setup(repo, our_head, their_heads, their_heads_len, opts.merge_flags)) < 0)
+	if ((error = git_merge__setup(repo, our_head, their_heads, their_heads_len)) < 0)
 		goto on_error;
 
 	if (ancestor_head != NULL &&
@@ -2439,24 +2696,20 @@ int git_merge(
 
 	/* TODO: recursive, octopus, etc... */
 
-	if ((error = git_merge_trees(&index_new, repo, ancestor_tree, our_tree, their_trees[0], &opts.merge_tree_opts)) < 0 ||
-		(error = merge_indexes(repo, index_new)) < 0 ||
+	if ((error = git_merge_trees(&index_new, repo, ancestor_tree, our_tree, their_trees[0], merge_opts)) < 0 ||
+		(error = git_merge__indexes(repo, index_new)) < 0 ||
 		(error = git_repository_index(&index_repo, repo)) < 0 ||
-		(error = git_checkout_index(repo, index_repo, &opts.checkout_opts)) < 0)
+		(error = git_merge__append_conflicts_to_merge_msg(repo, index_repo)) < 0 ||
+		(error = git_checkout_index(repo, index_repo, &checkout_opts)) < 0)
 		goto on_error;
 
-	result->index = index_new;
-
-	*out = result;
 	goto done;
 
 on_error:
-	git_repository_merge_cleanup(repo);
-
-	git_index_free(index_new);
-	git__free(result);
+	merge_state_cleanup(repo);
 
 done:
+	git_index_free(index_new);
 	git_index_free(index_repo);
 
 	git_tree_free(ancestor_tree);
@@ -2475,94 +2728,6 @@ done:
 	return error;
 }
 
-int git_merge__setup(
-	git_repository *repo,
-	const git_merge_head *our_head,
-	const git_merge_head *heads[],
-	size_t heads_len,
-	unsigned int flags)
-{
-	int error = 0;
-
-	assert (repo && our_head && heads);
-
-	if ((error = write_orig_head(repo, our_head)) == 0 &&
-		(error = write_merge_head(repo, heads, heads_len)) == 0 &&
-		(error = write_merge_mode(repo, flags)) == 0) {
-		error = write_merge_msg(repo, heads, heads_len);
-	}
-
-	return error;
-}
-
-int git_repository_merge_cleanup(git_repository *repo)
-{
-	int error = 0;
-	git_buf merge_head_path = GIT_BUF_INIT,
-		merge_mode_path = GIT_BUF_INIT,
-		merge_msg_path = GIT_BUF_INIT;
-
-	assert(repo);
-
-	if (git_buf_joinpath(&merge_head_path, repo->path_repository, GIT_MERGE_HEAD_FILE) < 0 ||
-		git_buf_joinpath(&merge_mode_path, repo->path_repository, GIT_MERGE_MODE_FILE) < 0 ||
-		git_buf_joinpath(&merge_msg_path, repo->path_repository, GIT_MERGE_MSG_FILE) < 0)
-		return -1;
-
-	if (git_path_isfile(merge_head_path.ptr)) {
-		if ((error = p_unlink(merge_head_path.ptr)) < 0)
-			goto cleanup;
-	}
-
-	if (git_path_isfile(merge_mode_path.ptr))
-		(void)p_unlink(merge_mode_path.ptr);
-
-	if (git_path_isfile(merge_msg_path.ptr))
-		(void)p_unlink(merge_msg_path.ptr);
-
-cleanup:
-	git_buf_free(&merge_msg_path);
-	git_buf_free(&merge_mode_path);
-	git_buf_free(&merge_head_path);
-
-	return error;
-}
-
-/* Merge result data */
-
-int git_merge_result_is_uptodate(git_merge_result *merge_result)
-{
-	assert(merge_result);
-
-	return merge_result->is_uptodate;
-}
-
-int git_merge_result_is_fastforward(git_merge_result *merge_result)
-{
-	assert(merge_result);
-
-	return merge_result->is_fastforward;
-}
-
-int git_merge_result_fastforward_oid(git_oid *out, git_merge_result *merge_result)
-{
-	assert(out && merge_result);
-
-	git_oid_cpy(out, &merge_result->fastforward_oid);
-	return 0;
-}
-
-void git_merge_result_free(git_merge_result *merge_result)
-{
-	if (merge_result == NULL)
-		return;
-
-	git_index_free(merge_result->index);
-	merge_result->index = NULL;
-
-	git__free(merge_result);
-}
-
 /* Merge heads are the input to merge */
 
 static int merge_head_init(
@@ -2609,7 +2774,7 @@ static int merge_head_init(
 int git_merge_head_from_ref(
 	git_merge_head **out,
 	git_repository *repo,
-	git_reference *ref)
+	const git_reference *ref)
 {
 	git_reference *resolved;
 	int error = 0;
@@ -2628,7 +2793,7 @@ int git_merge_head_from_ref(
 	return error;
 }
 
-int git_merge_head_from_oid(
+int git_merge_head_from_id(
 	git_merge_head **out,
 	git_repository *repo,
 	const git_oid *oid)
@@ -2650,6 +2815,14 @@ int git_merge_head_from_fetchhead(
 	return merge_head_init(out, repo, branch_name, remote_url, oid);
 }
 
+const git_oid *git_merge_head_id(
+	const git_merge_head *head)
+{
+	assert(head);
+
+	return &head->oid;
+}
+
 void git_merge_head_free(git_merge_head *head)
 {
 	if (head == NULL)
@@ -2666,3 +2839,25 @@ void git_merge_head_free(git_merge_head *head)
 
 	git__free(head);
 }
+
+int git_merge_init_options(git_merge_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_merge_options, GIT_MERGE_OPTIONS_INIT);
+	return 0;
+}
+
+int git_merge_file_init_input(git_merge_file_input *input, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		input, version, git_merge_file_input, GIT_MERGE_FILE_INPUT_INIT);
+	return 0;
+}
+
+int git_merge_file_init_options(
+	git_merge_file_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_merge_file_options, GIT_MERGE_FILE_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/merge.h b/src/merge.h
index d7d1c67b7..00f6197bf 100644
--- a/src/merge.h
+++ b/src/merge.h
@@ -106,6 +106,8 @@ typedef struct {
 
 	git_index_entry their_entry;
 	git_delta_t their_status;
+
+	int binary:1;
 } git_merge_diff;
 
 /** Internal structure for merge inputs */
@@ -118,16 +120,6 @@ struct git_merge_head {
 	git_commit *commit;
 };
 
-/** Internal structure for merge results */
-struct git_merge_result {
-	bool is_uptodate;
-
-	bool is_fastforward;
-	git_oid fastforward_oid;
-
-	git_index *index;
-};
-
 int git_merge__bases_many(
 	git_commit_list **out,
 	git_revwalk *walk,
@@ -145,7 +137,7 @@ int git_merge_diff_list__find_differences(git_merge_diff_list *merge_diff_list,
 	const git_tree *ours_tree,
 	const git_tree *theirs_tree);
 
-int git_merge_diff_list__find_renames(git_repository *repo, git_merge_diff_list *merge_diff_list, const git_merge_tree_opts *opts);
+int git_merge_diff_list__find_renames(git_repository *repo, git_merge_diff_list *merge_diff_list, const git_merge_options *opts);
 
 void git_merge_diff_list__free(git_merge_diff_list *diff_list);
 
@@ -154,8 +146,11 @@ void git_merge_diff_list__free(git_merge_diff_list *diff_list);
 int git_merge__setup(
 	git_repository *repo,
 	const git_merge_head *our_head,
-	const git_merge_head *their_heads[],
-	size_t their_heads_len,
-	unsigned int flags);
+	const git_merge_head *heads[],
+	size_t heads_len);
+
+int git_merge__indexes(git_repository *repo, git_index *index_new);
+
+int git_merge__append_conflicts_to_merge_msg(git_repository *repo, git_index *index);
 
 #endif
diff --git a/src/merge_file.c b/src/merge_file.c
index 48fc46e57..ff0364432 100644
--- a/src/merge_file.c
+++ b/src/merge_file.c
@@ -8,6 +8,9 @@
 #include "common.h"
 #include "repository.h"
 #include "merge_file.h"
+#include "posix.h"
+#include "fileops.h"
+#include "index.h"
 
 #include "git2/repository.h"
 #include "git2/object.h"
@@ -22,17 +25,17 @@ GIT_INLINE(const char *) merge_file_best_path(
 	const git_merge_file_input *ours,
 	const git_merge_file_input *theirs)
 {
-	if (!GIT_MERGE_FILE_SIDE_EXISTS(ancestor)) {
-		if (strcmp(ours->path, theirs->path) == 0)
+	if (!ancestor) {
+		if (ours && theirs && strcmp(ours->path, theirs->path) == 0)
 			return ours->path;
 
 		return NULL;
 	}
 
-	if (strcmp(ancestor->path, ours->path) == 0)
-		return theirs->path;
-	else if(strcmp(ancestor->path, theirs->path) == 0)
-		return ours->path;
+	if (ours && strcmp(ancestor->path, ours->path) == 0)
+		return theirs ? theirs->path : NULL;
+	else if(theirs && strcmp(ancestor->path, theirs->path) == 0)
+		return ours ? ours->path : NULL;
 
 	return NULL;
 }
@@ -47,128 +50,230 @@ GIT_INLINE(int) merge_file_best_mode(
 	 * assume executable.  Otherwise, if any mode changed from the ancestor,
 	 * use that one.
 	 */
-	if (!GIT_MERGE_FILE_SIDE_EXISTS(ancestor)) {
-		if (ours->mode == GIT_FILEMODE_BLOB_EXECUTABLE ||
-			theirs->mode == GIT_FILEMODE_BLOB_EXECUTABLE)
+	if (!ancestor) {
+		if ((ours && ours->mode == GIT_FILEMODE_BLOB_EXECUTABLE) ||
+			(theirs && theirs->mode == GIT_FILEMODE_BLOB_EXECUTABLE))
 			return GIT_FILEMODE_BLOB_EXECUTABLE;
 
 		return GIT_FILEMODE_BLOB;
-	}
+	} else if (ours && theirs) {
+		if (ancestor->mode == ours->mode)
+			return theirs->mode;
 
-	if (ancestor->mode == ours->mode)
-		return theirs->mode;
-	else if(ancestor->mode == theirs->mode)
 		return ours->mode;
+	}
 
 	return 0;
 }
 
-int git_merge_file_input_from_index_entry(
-	git_merge_file_input *input,
-	git_repository *repo,
+int git_merge_file__input_from_index(
+	git_merge_file_input *input_out,
+	git_odb_object **odb_object_out,
+	git_odb *odb,
 	const git_index_entry *entry)
 {
-	git_odb *odb = NULL;
 	int error = 0;
 
-	assert(input && repo && entry);
-
-	if (entry->mode == 0)
-		return 0;
+	assert(input_out && odb_object_out && odb && entry);
 
-	if ((error = git_repository_odb(&odb, repo)) < 0 ||
-		(error = git_odb_read(&input->odb_object, odb, &entry->oid)) < 0)
+	if ((error = git_odb_read(odb_object_out, odb, &entry->id)) < 0)
 		goto done;
 
-	input->mode = entry->mode;
-	input->path = git__strdup(entry->path);
-	input->mmfile.size = git_odb_object_size(input->odb_object);
-	input->mmfile.ptr = (char *)git_odb_object_data(input->odb_object);
-
-	if (input->label == NULL)
-		input->label = entry->path;
+	input_out->path = entry->path;
+	input_out->mode = entry->mode;
+	input_out->ptr = (char *)git_odb_object_data(*odb_object_out);
+	input_out->size = git_odb_object_size(*odb_object_out);
 
 done:
-	git_odb_free(odb);
-
 	return error;
 }
 
-int git_merge_file_input_from_diff_file(
-	git_merge_file_input *input,
-	git_repository *repo,
-	const git_diff_file *file)
+static void merge_file_normalize_opts(
+	git_merge_file_options *out,
+	const git_merge_file_options *given_opts)
 {
-	git_odb *odb = NULL;
+	if (given_opts)
+		memcpy(out, given_opts, sizeof(git_merge_file_options));
+	else {
+		git_merge_file_options default_opts = GIT_MERGE_FILE_OPTIONS_INIT;
+		memcpy(out, &default_opts, sizeof(git_merge_file_options));
+	}
+}
+
+static int git_merge_file__from_inputs(
+	git_merge_file_result *out,
+	const git_merge_file_input *ancestor,
+	const git_merge_file_input *ours,
+	const git_merge_file_input *theirs,
+	const git_merge_file_options *given_opts)
+{
+	xmparam_t xmparam;
+	mmfile_t ancestor_mmfile = {0}, our_mmfile = {0}, their_mmfile = {0};
+	mmbuffer_t mmbuffer;
+	git_merge_file_options options = GIT_MERGE_FILE_OPTIONS_INIT;
+	const char *path;
+	int xdl_result;
 	int error = 0;
 
-	assert(input && repo && file);
+	memset(out, 0x0, sizeof(git_merge_file_result));
 
-	if (file->mode == 0)
-		return 0;
+	merge_file_normalize_opts(&options, given_opts);
 
-	if ((error = git_repository_odb(&odb, repo)) < 0 ||
-		(error = git_odb_read(&input->odb_object, odb, &file->oid)) < 0)
+	memset(&xmparam, 0x0, sizeof(xmparam_t));
+
+	if (ancestor) {
+		xmparam.ancestor = (options.ancestor_label) ?
+			options.ancestor_label : ancestor->path;
+		ancestor_mmfile.ptr = (char *)ancestor->ptr;
+		ancestor_mmfile.size = ancestor->size;
+	}
+
+	xmparam.file1 = (options.our_label) ?
+		options.our_label : ours->path;
+	our_mmfile.ptr = (char *)ours->ptr;
+	our_mmfile.size = ours->size;
+
+	xmparam.file2 = (options.their_label) ?
+		options.their_label : theirs->path;
+	their_mmfile.ptr = (char *)theirs->ptr;
+	their_mmfile.size = theirs->size;
+
+	if (options.favor == GIT_MERGE_FILE_FAVOR_OURS)
+		xmparam.favor = XDL_MERGE_FAVOR_OURS;
+	else if (options.favor == GIT_MERGE_FILE_FAVOR_THEIRS)
+		xmparam.favor = XDL_MERGE_FAVOR_THEIRS;
+	else if (options.favor == GIT_MERGE_FILE_FAVOR_UNION)
+		xmparam.favor = XDL_MERGE_FAVOR_UNION;
+
+	xmparam.level = (options.flags & GIT_MERGE_FILE_SIMPLIFY_ALNUM) ?
+		XDL_MERGE_ZEALOUS_ALNUM : XDL_MERGE_ZEALOUS;
+
+	if (options.flags & GIT_MERGE_FILE_STYLE_DIFF3)
+		xmparam.style = XDL_MERGE_DIFF3;
+
+	if ((xdl_result = xdl_merge(&ancestor_mmfile, &our_mmfile,
+		&their_mmfile, &xmparam, &mmbuffer)) < 0) {
+		giterr_set(GITERR_MERGE, "Failed to merge files.");
+		error = -1;
 		goto done;
+	}
 
-	input->mode = file->mode;
-	input->path = git__strdup(file->path);
-	input->mmfile.size = git_odb_object_size(input->odb_object);
-	input->mmfile.ptr = (char *)git_odb_object_data(input->odb_object);
+	if ((path = merge_file_best_path(ancestor, ours, theirs)) != NULL &&
+		(out->path = strdup(path)) == NULL) {
+		error = -1;
+		goto done;
+	}
 
-	if (input->label == NULL)
-		input->label = file->path;
+	out->automergeable = (xdl_result == 0);
+	out->ptr = (const char *)mmbuffer.ptr;
+	out->len = mmbuffer.size;
+	out->mode = merge_file_best_mode(ancestor, ours, theirs);
 
 done:
-	git_odb_free(odb);
+	if (error < 0)
+		git_merge_file_result_free(out);
 
 	return error;
 }
 
-int git_merge_files(
+static git_merge_file_input *git_merge_file__normalize_inputs(
+	git_merge_file_input *out,
+	const git_merge_file_input *given)
+{
+	memcpy(out, given, sizeof(git_merge_file_input));
+
+	if (!out->path)
+		out->path = "file.txt";
+
+	if (!out->mode)
+		out->mode = 0100644;
+
+	return out;
+}
+
+int git_merge_file(
 	git_merge_file_result *out,
-	git_merge_file_input *ancestor,
-	git_merge_file_input *ours,
-	git_merge_file_input *theirs,
-	git_merge_automerge_flags flags)
+	const git_merge_file_input *ancestor,
+	const git_merge_file_input *ours,
+	const git_merge_file_input *theirs,
+	const git_merge_file_options *options)
 {
-	xmparam_t xmparam;
-	mmbuffer_t mmbuffer;
-	int xdl_result;
-	int error = 0;
+	git_merge_file_input inputs[3] = { {0} };
 
-	assert(out && ancestor && ours && theirs);
+	assert(out && ours && theirs);
 
 	memset(out, 0x0, sizeof(git_merge_file_result));
 
-	if (!GIT_MERGE_FILE_SIDE_EXISTS(ours) || !GIT_MERGE_FILE_SIDE_EXISTS(theirs))
-		return 0;
+	if (ancestor)
+		ancestor = git_merge_file__normalize_inputs(&inputs[0], ancestor);
 
-	memset(&xmparam, 0x0, sizeof(xmparam_t));
-	xmparam.ancestor = ancestor->label;
-	xmparam.file1 = ours->label;
-	xmparam.file2 = theirs->label;
+	ours = git_merge_file__normalize_inputs(&inputs[1], ours);
+	theirs = git_merge_file__normalize_inputs(&inputs[2], theirs);
 
-	out->path = merge_file_best_path(ancestor, ours, theirs);
-	out->mode = merge_file_best_mode(ancestor, ours, theirs);
+	return git_merge_file__from_inputs(out, ancestor, ours, theirs, options);
+}
 
-	if (flags == GIT_MERGE_AUTOMERGE_FAVOR_OURS)
-		xmparam.favor = XDL_MERGE_FAVOR_OURS;
+int git_merge_file_from_index(
+	git_merge_file_result *out,
+	git_repository *repo,
+	const git_index_entry *ancestor,
+	const git_index_entry *ours,
+	const git_index_entry *theirs,
+	const git_merge_file_options *options)
+{
+	git_merge_file_input inputs[3] = { {0} },
+		*ancestor_input = NULL, *our_input = NULL, *their_input = NULL;
+	git_odb *odb = NULL;
+	git_odb_object *odb_object[3] = { 0 };
+	int error = 0;
 
-	if (flags == GIT_MERGE_AUTOMERGE_FAVOR_THEIRS)
-		xmparam.favor = XDL_MERGE_FAVOR_THEIRS;
+	assert(out && repo && ours && theirs);
 
-	if ((xdl_result = xdl_merge(&ancestor->mmfile, &ours->mmfile,
-		&theirs->mmfile, &xmparam, &mmbuffer)) < 0) {
-		giterr_set(GITERR_MERGE, "Failed to merge files.");
-		error = -1;
+	memset(out, 0x0, sizeof(git_merge_file_result));
+
+	if ((error = git_repository_odb(&odb, repo)) < 0)
 		goto done;
+
+	if (ancestor) {
+		if ((error = git_merge_file__input_from_index(
+			&inputs[0], &odb_object[0], odb, ancestor)) < 0)
+			goto done;
+
+		ancestor_input = &inputs[0];
 	}
 
-	out->automergeable = (xdl_result == 0);
-	out->data = (unsigned char *)mmbuffer.ptr;
-	out->len = mmbuffer.size;
+	if ((error = git_merge_file__input_from_index(
+		&inputs[1], &odb_object[1], odb, ours)) < 0)
+		goto done;
+
+	our_input = &inputs[1];
+
+	if ((error = git_merge_file__input_from_index(
+		&inputs[2], &odb_object[2], odb, theirs)) < 0)
+		goto done;
+
+	their_input = &inputs[2];
+
+	if ((error = git_merge_file__from_inputs(out,
+		ancestor_input, our_input, their_input, options)) < 0)
+		goto done;
 
 done:
+	git_odb_object_free(odb_object[0]);
+	git_odb_object_free(odb_object[1]);
+	git_odb_object_free(odb_object[2]);
+	git_odb_free(odb);
+
 	return error;
 }
+
+void git_merge_file_result_free(git_merge_file_result *result)
+{
+	if (result == NULL)
+		return;
+
+	git__free((char *)result->path);
+
+	/* xdiff uses malloc() not git_malloc, so we use free(), not git_free() */
+	free((char *)result->ptr);
+}
diff --git a/src/merge_file.h b/src/merge_file.h
index 0af2f0a57..263391ee3 100644
--- a/src/merge_file.h
+++ b/src/merge_file.h
@@ -11,61 +11,4 @@
 
 #include "git2/merge.h"
 
-typedef struct {
-	const char *label;
-	char *path;
-	unsigned int mode;
-	mmfile_t mmfile;
-
-	git_odb_object *odb_object;
-} git_merge_file_input;
-
-#define GIT_MERGE_FILE_INPUT_INIT	{0}
-
-typedef struct {
-	bool automergeable;
-
-	const char *path;
-	int mode;
-
-	unsigned char *data;
-	size_t len;
-} git_merge_file_result;
-
-#define GIT_MERGE_FILE_RESULT_INIT	{0}
-
-int git_merge_file_input_from_index_entry(
-	git_merge_file_input *input,
-	git_repository *repo,
-	const git_index_entry *entry);
-
-int git_merge_file_input_from_diff_file(
-	git_merge_file_input *input,
-	git_repository *repo,
-	const git_diff_file *file);
-
-int git_merge_files(
-	git_merge_file_result *out,
-	git_merge_file_input *ancestor,
-	git_merge_file_input *ours,
-	git_merge_file_input *theirs,
-	git_merge_automerge_flags flags);
-
-GIT_INLINE(void) git_merge_file_input_free(git_merge_file_input *input)
-{
-	assert(input);
-	git__free(input->path);
-	git_odb_object_free(input->odb_object);
-}
-
-GIT_INLINE(void) git_merge_file_result_free(git_merge_file_result *filediff)
-{
-	if (filediff == NULL)
-		return;
-
-	/* xdiff uses malloc() not git_malloc, so we use free(), not git_free() */
-	if (filediff->data != NULL)
-		free(filediff->data);
-}
-
 #endif
diff --git a/src/message.c b/src/message.c
index 0eff426f2..6c5a2379f 100644
--- a/src/message.c
+++ b/src/message.c
@@ -21,7 +21,7 @@ static size_t line_length_without_trailing_spaces(const char *line, size_t len)
 
 /* Greatly inspired from git.git "stripspace" */
 /* see https://github.com/git/git/blob/497215d8811ac7b8955693ceaad0899ecd894ed2/builtin/stripspace.c#L4-67 */
-int git_message__prettify(git_buf *message_out, const char *message, int strip_comments)
+int git_message_prettify(git_buf *message_out, const char *message, int strip_comments, char comment_char)
 {
 	const size_t message_len = strlen(message);
 
@@ -29,6 +29,8 @@ int git_message__prettify(git_buf *message_out, const char *message, int strip_c
 	size_t i, line_length, rtrimmed_line_length;
 	char *next_newline;
 
+	git_buf_sanitize(message_out);
+
 	for (i = 0; i < strlen(message); i += line_length) {
 		next_newline = memchr(message + i, '\n', message_len - i);
 
@@ -38,7 +40,7 @@ int git_message__prettify(git_buf *message_out, const char *message, int strip_c
 			line_length = message_len - i;
 		}
 
-		if (strip_comments && line_length && message[i] == '#')
+		if (strip_comments && line_length && message[i] == comment_char)
 			continue;
 
 		rtrimmed_line_length = line_length_without_trailing_spaces(message + i, line_length);
@@ -58,29 +60,3 @@ int git_message__prettify(git_buf *message_out, const char *message, int strip_c
 
 	return git_buf_oom(message_out) ? -1 : 0;
 }
-
-int git_message_prettify(char *message_out, size_t buffer_size, const char *message, int strip_comments)
-{
-	git_buf buf = GIT_BUF_INIT;
-	ssize_t out_size = -1;
-
-	if (message_out && buffer_size)
-		*message_out = '\0';
-
-	if (git_message__prettify(&buf, message, strip_comments) < 0)
-		goto done;
-
-	if (message_out && buf.size + 1 > buffer_size) { /* +1 for NUL byte */
-		giterr_set(GITERR_INVALID, "Buffer too short to hold the cleaned message");
-		goto done;
-	}
-
-	if (message_out)
-		git_buf_copy_cstr(message_out, buffer_size, &buf);
-
-	out_size = buf.size + 1;
-
-done:
-	git_buf_free(&buf);
-	return (int)out_size;
-}
diff --git a/src/netops.c b/src/netops.c
index ad27d84cf..965e4775d 100644
--- a/src/netops.c
+++ b/src/netops.c
@@ -33,6 +33,7 @@
 #include "posix.h"
 #include "buffer.h"
 #include "http_parser.h"
+#include "global.h"
 
 #ifdef GIT_WIN32
 static void net_set_error(const char *str)
@@ -157,7 +158,7 @@ void gitno_buffer_setup_callback(
 void gitno_buffer_setup(gitno_socket *socket, gitno_buffer *buf, char *data, size_t len)
 {
 #ifdef GIT_SSL
-	if (socket->ssl.ctx) {
+	if (socket->ssl.ssl) {
 		gitno_buffer_setup_callback(socket, buf, data, len, gitno__recv_ssl, NULL);
 		return;
 	}
@@ -202,12 +203,13 @@ static int gitno_ssl_teardown(gitno_ssl *ssl)
 		ret = 0;
 
 	SSL_free(ssl->ssl);
-	SSL_CTX_free(ssl->ctx);
 	return ret;
 }
 
+#endif
+
 /* Match host names according to RFC 2818 rules */
-static int match_host(const char *pattern, const char *host)
+int gitno__match_host(const char *pattern, const char *host)
 {
 	for (;;) {
 		char c = tolower(*pattern++);
@@ -230,9 +232,9 @@ static int match_host(const char *pattern, const char *host)
 			while(*host) {
 				char h = tolower(*host);
 				if (c == h)
-					return match_host(pattern, host++);
+					return gitno__match_host(pattern, host++);
 				if (h == '.')
-					return match_host(pattern, host);
+					return gitno__match_host(pattern, host);
 				host++;
 			}
 			return -1;
@@ -250,12 +252,14 @@ static int check_host_name(const char *name, const char *host)
 	if (!strcasecmp(name, host))
 		return 0;
 
-	if (match_host(name, host) < 0)
+	if (gitno__match_host(name, host) < 0)
 		return -1;
 
 	return 0;
 }
 
+#ifdef GIT_SSL
+
 static int verify_server_cert(gitno_ssl *ssl, const char *host)
 {
 	X509 *cert;
@@ -287,6 +291,10 @@ static int verify_server_cert(gitno_ssl *ssl, const char *host)
 
 
 	cert = SSL_get_peer_certificate(ssl->ssl);
+	if (!cert) {
+		giterr_set(GITERR_SSL, "the server did not provide a certificate");
+		return -1;
+	}
 
 	/* Check the alternative names */
 	alts = X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
@@ -321,7 +329,7 @@ static int verify_server_cert(gitno_ssl *ssl, const char *host)
 	GENERAL_NAMES_free(alts);
 
 	if (matched == 0)
-		goto cert_fail;
+		goto cert_fail_name;
 
 	if (matched == 1)
 		return 0;
@@ -358,11 +366,11 @@ static int verify_server_cert(gitno_ssl *ssl, const char *host)
 		int size = ASN1_STRING_to_UTF8(&peer_cn, str);
 		GITERR_CHECK_ALLOC(peer_cn);
 		if (memchr(peer_cn, '\0', size))
-			goto cert_fail;
+			goto cert_fail_name;
 	}
 
 	if (check_host_name((char *)peer_cn, host) < 0)
-		goto cert_fail;
+		goto cert_fail_name;
 
 	OPENSSL_free(peer_cn);
 
@@ -372,9 +380,9 @@ on_error:
 	OPENSSL_free(peer_cn);
 	return ssl_set_error(ssl, 0);
 
-cert_fail:
+cert_fail_name:
 	OPENSSL_free(peer_cn);
-	giterr_set(GITERR_SSL, "Certificate host name check failed");
+	giterr_set(GITERR_SSL, "hostname does not match certificate");
 	return -1;
 }
 
@@ -382,18 +390,12 @@ static int ssl_setup(gitno_socket *socket, const char *host, int flags)
 {
 	int ret;
 
-	SSL_library_init();
-	SSL_load_error_strings();
-	socket->ssl.ctx = SSL_CTX_new(SSLv23_method());
-	if (socket->ssl.ctx == NULL)
-		return ssl_set_error(&socket->ssl, 0);
-
-	SSL_CTX_set_mode(socket->ssl.ctx, SSL_MODE_AUTO_RETRY);
-	SSL_CTX_set_verify(socket->ssl.ctx, SSL_VERIFY_NONE, NULL);
-	if (!SSL_CTX_set_default_verify_paths(socket->ssl.ctx))
-		return ssl_set_error(&socket->ssl, 0);
+	if (git__ssl_ctx == NULL) {
+		giterr_set(GITERR_NET, "OpenSSL initialization failed");
+		return -1;
+	}
 
-	socket->ssl.ssl = SSL_new(socket->ssl.ctx);
+	socket->ssl.ssl = SSL_new(git__ssl_ctx);
 	if (socket->ssl.ssl == NULL)
 		return ssl_set_error(&socket->ssl, 0);
 
@@ -530,7 +532,7 @@ int gitno_send(gitno_socket *socket, const char *msg, size_t len, int flags)
 	size_t off = 0;
 
 #ifdef GIT_SSL
-	if (socket->ssl.ctx)
+	if (socket->ssl.ssl)
 		return gitno_send_ssl(&socket->ssl, msg, len, flags);
 #endif
 
@@ -551,7 +553,7 @@ int gitno_send(gitno_socket *socket, const char *msg, size_t len, int flags)
 int gitno_close(gitno_socket *s)
 {
 #ifdef GIT_SSL
-	if (s->ssl.ctx &&
+	if (s->ssl.ssl &&
 		gitno_ssl_teardown(&s->ssl) < 0)
 		return -1;
 #endif
diff --git a/src/netops.h b/src/netops.h
index 666d66b12..dfb4ab7b4 100644
--- a/src/netops.h
+++ b/src/netops.h
@@ -16,7 +16,6 @@
 
 struct gitno_ssl {
 #ifdef GIT_SSL
-	SSL_CTX *ctx;
 	SSL *ssl;
 #else
 	size_t dummy;
@@ -54,6 +53,19 @@ enum {
 	GITNO_CONNECT_SSL_NO_CHECK_CERT = 2,
 };
 
+/**
+ * Check if the name in a cert matches the wanted hostname
+ *
+ * Check if a pattern from a certificate matches the hostname we
+ * wanted to connect to according to RFC2818 rules (which specifies
+ * HTTP over TLS). Mainly, an asterisk matches anything, but is
+ * limited to a single url component.
+ *
+ * Note that this does not set an error message. It expects the user
+ * to provide the message for the user.
+ */
+int gitno__match_host(const char *pattern, const char *host);
+
 void gitno_buffer_setup(gitno_socket *t, gitno_buffer *buf, char *data, size_t len);
 void gitno_buffer_setup_callback(gitno_socket *t, gitno_buffer *buf, char *data, size_t len, int (*recv)(gitno_buffer *buf), void *cb_data);
 int gitno_recv(gitno_buffer *buf);
diff --git a/src/notes.c b/src/notes.c
index beace1b50..ffe5d345a 100644
--- a/src/notes.c
+++ b/src/notes.c
@@ -313,7 +313,7 @@ static int note_new(git_note **out, git_oid *note_oid, git_blob *blob)
 	note = (git_note *)git__malloc(sizeof(git_note));
 	GITERR_CHECK_ALLOC(note);
 
-	git_oid_cpy(&note->oid, note_oid);
+	git_oid_cpy(&note->id, note_oid);
 	note->message = git__strdup((char *)git_blob_rawcontent(blob));
 	GITERR_CHECK_ALLOC(note->message);
 
@@ -378,20 +378,11 @@ cleanup:
 
 static int note_get_default_ref(const char **out, git_repository *repo)
 {
-	int ret;
 	git_config *cfg;
+	int ret = git_repository_config__weakptr(&cfg, repo);
 
-	*out = NULL;
-
-	if (git_repository_config__weakptr(&cfg, repo) < 0)
-		return -1;
-
-	ret = git_config_get_string(out, cfg, "core.notesRef");
-	if (ret == GIT_ENOTFOUND) {
-		giterr_clear();
-		*out = GIT_NOTES_DEFAULT_REF;
-		return 0;
-	}
+	*out = (ret != 0) ? NULL : git_config__get_string_force(
+		cfg, "core.notesref", GIT_NOTES_DEFAULT_REF);
 
 	return ret;
 }
@@ -517,10 +508,10 @@ const char * git_note_message(const git_note *note)
 	return note->message;
 }
 
-const git_oid * git_note_oid(const git_note *note)
+const git_oid * git_note_id(const git_note *note)
 {
 	assert(note);
-	return &note->oid;
+	return &note->id;
 }
 
 void git_note_free(git_note *note)
@@ -592,8 +583,8 @@ int git_note_foreach(
 		return error;
 
 	while (!(error = git_note_next(&note_id, &annotated_id, iter))) {
-		if (note_cb(&note_id, &annotated_id, payload)) {
-			error = GIT_EUSER;
+		if ((error = note_cb(&note_id, &annotated_id, payload)) != 0) {
+			giterr_set_after_callback(error);
 			break;
 		}
 	}
@@ -649,7 +640,7 @@ int git_note_next(
 	if ((error = git_iterator_current(&item, it)) < 0)
 		return error;
 
-	git_oid_cpy(note_id, &item->oid);
+	git_oid_cpy(note_id, &item->id);
 
 	if (!(error = process_entry_path(item->path, annotated_id)))
 		git_iterator_advance(NULL, it);
diff --git a/src/notes.h b/src/notes.h
index 39e18b621..e9cfa00fa 100644
--- a/src/notes.h
+++ b/src/notes.h
@@ -21,7 +21,7 @@
 	"Notes removed by 'git_note_remove' from libgit2"
 
 struct git_note {
-	git_oid oid;
+	git_oid id;
 
 	char *message;
 };
diff --git a/src/object.c b/src/object.c
index 3fc984b45..93068b85f 100644
--- a/src/object.c
+++ b/src/object.c
@@ -4,8 +4,6 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
-#include <stdarg.h>
-
 #include "git2/object.h"
 
 #include "common.h"
@@ -377,7 +375,7 @@ int git_object_lookup_bypath(
 
 	assert(out && treeish && path);
 
-	if ((error = git_object_peel((git_object**)&tree, treeish, GIT_OBJ_TREE) < 0) ||
+	if ((error = git_object_peel((git_object**)&tree, treeish, GIT_OBJ_TREE)) < 0 ||
 		 (error = git_tree_entry_bypath(&entry, tree, path)) < 0)
 	{
 		goto cleanup;
@@ -399,3 +397,46 @@ cleanup:
 	git_tree_free(tree);
 	return error;
 }
+
+int git_object_short_id(git_buf *out, const git_object *obj)
+{
+	git_repository *repo;
+	int len = GIT_ABBREV_DEFAULT, error;
+	git_oid id = {{0}};
+	git_odb *odb;
+
+	assert(out && obj);
+
+	git_buf_sanitize(out);
+	repo = git_object_owner(obj);
+
+	if ((error = git_repository__cvar(&len, repo, GIT_CVAR_ABBREV)) < 0)
+		return error;
+
+	if ((error = git_repository_odb(&odb, repo)) < 0)
+		return error;
+
+	while (len < GIT_OID_HEXSZ) {
+		/* set up short oid */
+		memcpy(&id.id, &obj->cached.oid.id, (len + 1) / 2);
+		if (len & 1)
+			id.id[len / 2] &= 0xf0;
+
+		error = git_odb_exists_prefix(NULL, odb, &id, len);
+		if (error != GIT_EAMBIGUOUS)
+			break;
+
+		giterr_clear();
+		len++;
+	}
+
+	if (!error && !(error = git_buf_grow(out, len + 1))) {
+		git_oid_tostr(out->ptr, len + 1, &id);
+		out->size = len;
+	}
+
+	git_odb_free(odb);
+
+	return error;
+}
+
diff --git a/src/odb.c b/src/odb.c
index b208b279e..a4fc02686 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -445,7 +445,7 @@ int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
 {
 	backend_internal *internal;
 
-	assert(odb && odb);
+	assert(out && odb);
 	internal = git_vector_get(&odb->backends, pos);
 
 	if (internal && internal->backend) {
@@ -635,6 +635,66 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 	return (int)found;
 }
 
+int git_odb_exists_prefix(
+	git_oid *out, git_odb *db, const git_oid *short_id, size_t len)
+{
+	int error = GIT_ENOTFOUND, num_found = 0;
+	size_t i;
+	git_oid key = {{0}}, last_found = {{0}}, found;
+
+	assert(db && short_id);
+
+	if (len < GIT_OID_MINPREFIXLEN)
+		return git_odb__error_ambiguous("prefix length too short");
+	if (len > GIT_OID_HEXSZ)
+		len = GIT_OID_HEXSZ;
+
+	if (len == GIT_OID_HEXSZ) {
+		if (git_odb_exists(db, short_id)) {
+			if (out)
+				git_oid_cpy(out, short_id);
+			return 0;
+		} else {
+			return git_odb__error_notfound("no match for id prefix", short_id);
+		}
+	}
+
+	/* just copy valid part of short_id */
+	memcpy(&key.id, short_id->id, (len + 1) / 2);
+	if (len & 1)
+		key.id[len / 2] &= 0xF0;
+
+	for (i = 0; i < db->backends.length; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		if (!b->exists_prefix)
+			continue;
+
+		error = b->exists_prefix(&found, b, &key, len);
+		if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
+			continue;
+		if (error)
+			return error;
+
+		/* make sure found item doesn't introduce ambiguity */
+		if (num_found) {
+			if (git_oid__cmp(&last_found, &found))
+				return git_odb__error_ambiguous("multiple matches for prefix");
+		} else {
+			git_oid_cpy(&last_found, &found);
+			num_found++;
+		}
+	}
+
+	if (!num_found)
+		return git_odb__error_notfound("no match for id prefix", &key);
+	if (out)
+		git_oid_cpy(out, &last_found);
+
+	return 0;
+}
+
 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
 {
 	int error;
@@ -723,6 +783,7 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 		return error;
 	}
 
+	giterr_clear();
 	if ((object = odb_object__alloc(id, &raw)) == NULL)
 		return -1;
 
@@ -735,7 +796,7 @@ int git_odb_read_prefix(
 {
 	size_t i;
 	int error = GIT_ENOTFOUND;
-	git_oid found_full_oid = {{0}};
+	git_oid key = {{0}}, found_full_oid = {{0}};
 	git_rawobj raw;
 	void *data = NULL;
 	bool found = false;
@@ -745,7 +806,6 @@ int git_odb_read_prefix(
 
 	if (len < GIT_OID_MINPREFIXLEN)
 		return git_odb__error_ambiguous("prefix length too short");
-
 	if (len > GIT_OID_HEXSZ)
 		len = GIT_OID_HEXSZ;
 
@@ -755,13 +815,18 @@ int git_odb_read_prefix(
 			return 0;
 	}
 
+	/* just copy valid part of short_id */
+	memcpy(&key.id, short_id->id, (len + 1) / 2);
+	if (len & 1)
+		key.id[len / 2] &= 0xF0;
+
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
 		if (b->read_prefix != NULL) {
 			git_oid full_oid;
-			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
+			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, &key, len);
 			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
 				continue;
 
@@ -782,7 +847,7 @@ int git_odb_read_prefix(
 	}
 
 	if (!found)
-		return git_odb__error_notfound("no match for prefix", short_id);
+		return git_odb__error_notfound("no match for prefix", &key);
 
 	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL)
 		return -1;
@@ -862,7 +927,7 @@ int git_odb_open_wstream(
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
-	git_hash_ctx *ctx;
+	git_hash_ctx *ctx = NULL;
 
 	assert(stream && db);
 
@@ -883,22 +948,28 @@ int git_odb_open_wstream(
 		}
 	}
 
-	if (error == GIT_PASSTHROUGH)
-		error = 0;
-	if (error < 0 && !writes)
-		error = git_odb__error_unsupported_in_backend("write object");
+	if (error < 0) {
+		if (error == GIT_PASSTHROUGH)
+			error = 0;
+		else if (!writes)
+			error = git_odb__error_unsupported_in_backend("write object");
+
+		goto done;
+	}
 
 	ctx = git__malloc(sizeof(git_hash_ctx));
 	GITERR_CHECK_ALLOC(ctx);
 
+	if ((error = git_hash_ctx_init(ctx)) < 0)
+		goto done;
 
-	git_hash_ctx_init(ctx);
 	hash_header(ctx, size, type);
 	(*stream)->hash_ctx = ctx;
 
 	(*stream)->declared_size = size;
 	(*stream)->received_bytes = 0;
 
+done:
 	return error;
 }
 
@@ -949,6 +1020,9 @@ int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
 
 void git_odb_stream_free(git_odb_stream *stream)
 {
+	if (stream == NULL)
+		return;
+
 	git__free(stream->hash_ctx);
 	stream->free(stream);
 }
@@ -978,7 +1052,7 @@ int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oi
 	return error;
 }
 
-int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_callback progress_cb, void *progress_payload)
+int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_cb progress_cb, void *progress_payload)
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
@@ -1050,3 +1124,9 @@ int git_odb__error_ambiguous(const char *message)
 	return GIT_EAMBIGUOUS;
 }
 
+int git_odb_init_backend(git_odb_backend *backend, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		backend, version, git_odb_backend, GIT_ODB_BACKEND_INIT);
+	return 0;
+}
diff --git a/src/odb_loose.c b/src/odb_loose.c
index ced272b33..b2e8bed4d 100644
--- a/src/odb_loose.c
+++ b/src/odb_loose.c
@@ -519,11 +519,11 @@ static int locate_object_short_oid(
 	loose_locate_object_state state;
 	int error;
 
-	/* prealloc memory for OBJ_DIR/xx/ */
-	if (git_buf_grow(object_location, dir_len + 5) < 0)
+	/* prealloc memory for OBJ_DIR/xx/xx..38x..xx */
+	if (git_buf_grow(object_location, dir_len + 3 + GIT_OID_HEXSZ) < 0)
 		return -1;
 
-	git_buf_sets(object_location, objects_dir);
+	git_buf_set(object_location, objects_dir, dir_len);
 	git_path_to_dir(object_location);
 
 	/* save adjusted position at end of dir so it can be restored later */
@@ -533,8 +533,9 @@ static int locate_object_short_oid(
 	git_oid_fmt((char *)state.short_oid, short_oid);
 
 	/* Explore OBJ_DIR/xx/ where xx is the beginning of hex formatted short oid */
-	if (git_buf_printf(object_location, "%.2s/", state.short_oid) < 0)
+	if (git_buf_put(object_location, (char *)state.short_oid, 3) < 0)
 		return -1;
+	object_location->ptr[object_location->size - 1] = '/';
 
 	/* Check that directory exists */
 	if (git_path_isdir(object_location->ptr) == false)
@@ -547,8 +548,7 @@ static int locate_object_short_oid(
 	/* Explore directory to find a unique object matching short_oid */
 	error = git_path_direach(
 		object_location, 0, fn_locate_object_short_oid, &state);
-
-	if (error && error != GIT_EUSER)
+	if (error < 0 && error != GIT_EAMBIGUOUS)
 		return error;
 
 	if (!state.found)
@@ -647,12 +647,9 @@ static int loose_backend__read_prefix(
 {
 	int error = 0;
 
-	assert(len <= GIT_OID_HEXSZ);
-
-	if (len < GIT_OID_MINPREFIXLEN)
-		error = git_odb__error_ambiguous("prefix length too short");
+	assert(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_HEXSZ);
 
-	else if (len == GIT_OID_HEXSZ) {
+	if (len == GIT_OID_HEXSZ) {
 		/* We can fall back to regular read method */
 		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
 		if (!error)
@@ -692,11 +689,26 @@ static int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
 	return !error;
 }
 
+static int loose_backend__exists_prefix(
+	git_oid *out, git_odb_backend *backend, const git_oid *short_id, size_t len)
+{
+	git_buf object_path = GIT_BUF_INIT;
+	int error;
+
+	assert(backend && out && short_id && len >= GIT_OID_MINPREFIXLEN);
+
+	error = locate_object_short_oid(
+		&object_path, out, (loose_backend *)backend, short_id, len);
+
+	git_buf_free(&object_path);
+
+	return error;
+}
+
 struct foreach_state {
 	size_t dir_len;
 	git_odb_foreach_cb cb;
 	void *data;
-	int cb_error;
 };
 
 GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
@@ -735,18 +747,18 @@ static int foreach_object_dir_cb(void *_state, git_buf *path)
 	if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
 		return 0;
 
-	if (state->cb(&oid, state->data)) {
-		state->cb_error = GIT_EUSER;
-		return -1;
-	}
-
-	return 0;
+	return giterr_set_after_callback_function(
+		state->cb(&oid, state->data), "git_odb_foreach");
 }
 
 static int foreach_cb(void *_state, git_buf *path)
 {
 	struct foreach_state *state = (struct foreach_state *) _state;
 
+	/* non-dir is some stray file, ignore it */
+	if (!git_path_isdir(git_buf_cstr(path)))
+		return 0;
+
 	return git_path_direach(path, 0, foreach_object_dir_cb, state);
 }
 
@@ -764,6 +776,8 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
 
 	git_buf_sets(&buf, objects_dir);
 	git_path_to_dir(&buf);
+	if (git_buf_oom(&buf))
+		return -1;
 
 	memset(&state, 0, sizeof(state));
 	state.cb = cb;
@@ -774,7 +788,7 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
 
 	git_buf_free(&buf);
 
-	return state.cb_error ? state.cb_error : error;
+	return error;
 }
 
 static int loose_backend__stream_fwrite(git_odb_stream *_stream, const git_oid *oid)
@@ -943,6 +957,7 @@ int git_odb_backend_loose(
 	backend->parent.read_header = &loose_backend__read_header;
 	backend->parent.writestream = &loose_backend__stream;
 	backend->parent.exists = &loose_backend__exists;
+	backend->parent.exists_prefix = &loose_backend__exists_prefix;
 	backend->parent.foreach = &loose_backend__foreach;
 	backend->parent.free = &loose_backend__free;
 
diff --git a/src/odb_mempack.c b/src/odb_mempack.c
new file mode 100644
index 000000000..d9b3a1824
--- /dev/null
+++ b/src/odb_mempack.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "git2/object.h"
+#include "git2/sys/odb_backend.h"
+#include "fileops.h"
+#include "hash.h"
+#include "odb.h"
+#include "array.h"
+#include "oidmap.h"
+
+#include "git2/odb_backend.h"
+#include "git2/types.h"
+#include "git2/pack.h"
+
+GIT__USE_OIDMAP;
+
+struct memobject {
+	git_oid oid;
+	size_t len;
+	git_otype type;
+	char data[];
+};
+
+struct memory_packer_db {
+	git_odb_backend parent;
+	git_oidmap *objects;
+	git_array_t(struct memobject *) commits;
+};
+
+static int impl__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)_backend;
+	struct memobject *obj = NULL; 
+	khiter_t pos;
+	int rval;
+
+	pos = kh_put(oid, db->objects, oid, &rval);
+	if (rval < 0)
+		return -1;
+
+	if (rval == 0)
+		return 0;
+
+	obj = git__malloc(sizeof(struct memobject) + len);
+	GITERR_CHECK_ALLOC(obj);
+
+	memcpy(obj->data, data, len);
+	git_oid_cpy(&obj->oid, oid);
+	obj->len = len;
+	obj->type = type;
+
+	kh_key(db->objects, pos) = &obj->oid;
+	kh_val(db->objects, pos) = obj;
+
+	if (type == GIT_OBJ_COMMIT) {
+		struct memobject **store = git_array_alloc(db->commits);
+		GITERR_CHECK_ALLOC(store);
+		*store = obj;
+	}
+
+	return 0;
+}
+
+static int impl__exists(git_odb_backend *backend, const git_oid *oid)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)backend;
+	khiter_t pos;
+
+	pos = kh_get(oid, db->objects, oid);
+	if (pos != kh_end(db->objects))
+		return 1;
+
+	return 0;
+}
+
+static int impl__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)backend;
+	struct memobject *obj = NULL;
+	khiter_t pos;
+
+	pos = kh_get(oid, db->objects, oid);
+	if (pos == kh_end(db->objects))
+		return GIT_ENOTFOUND;
+
+	obj = kh_val(db->objects, pos);
+
+	*len_p = obj->len;
+	*type_p = obj->type;
+	*buffer_p = git__malloc(obj->len);
+	GITERR_CHECK_ALLOC(*buffer_p);
+
+	memcpy(*buffer_p, obj->data, obj->len);
+	return 0;
+}
+
+static int impl__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)backend;
+	struct memobject *obj = NULL;
+	khiter_t pos;
+
+	pos = kh_get(oid, db->objects, oid);
+	if (pos == kh_end(db->objects))
+		return GIT_ENOTFOUND;
+
+	obj = kh_val(db->objects, pos);
+
+	*len_p = obj->len;
+	*type_p = obj->type;
+	return 0;
+}
+
+int git_mempack_dump(git_buf *pack, git_repository *repo, git_odb_backend *_backend)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)_backend;
+	git_packbuilder *packbuilder;
+	uint32_t i;
+	int err = -1;
+
+	if (git_packbuilder_new(&packbuilder, repo) < 0)
+		return -1;
+
+	for (i = 0; i < db->commits.size; ++i) {
+		struct memobject *commit = db->commits.ptr[i];
+
+		err = git_packbuilder_insert_commit(packbuilder, &commit->oid);
+		if (err < 0)
+			goto cleanup;
+	}
+
+	err = git_packbuilder_write_buf(pack, packbuilder);
+
+cleanup:
+	git_packbuilder_free(packbuilder);
+	return err;
+}
+
+void git_mempack_reset(git_odb_backend *_backend)
+{
+	struct memory_packer_db *db = (struct memory_packer_db *)_backend;
+	struct memobject *object = NULL;
+
+	kh_foreach_value(db->objects, object, {
+		git__free(object);
+	});
+
+	git_array_clear(db->commits);
+}
+
+static void impl__free(git_odb_backend *_backend)
+{
+	git_mempack_reset(_backend);
+	git__free(_backend);
+}
+
+int git_mempack_new(git_odb_backend **out)
+{
+	struct memory_packer_db *db;
+
+	assert(out);
+
+	db = git__calloc(1, sizeof(struct memory_packer_db));
+	GITERR_CHECK_ALLOC(db);
+
+	db->objects = git_oidmap_alloc();
+
+	db->parent.read = &impl__read;
+	db->parent.write = &impl__write;
+	db->parent.read_header = &impl__read_header;
+	db->parent.exists = &impl__exists;
+	db->parent.free = &impl__free;
+
+	*out = (git_odb_backend *)db;
+	return 0;
+}
diff --git a/src/odb_pack.c b/src/odb_pack.c
index fd2ca0fd8..3750da37f 100644
--- a/src/odb_pack.c
+++ b/src/odb_pack.c
@@ -190,31 +190,39 @@ static int packfile_sort__cb(const void *a_, const void *b_)
 }
 
 
-
-static int packfile_load__cb(void *_data, git_buf *path)
+static int packfile_load__cb(void *data, git_buf *path)
 {
-	struct pack_backend *backend = (struct pack_backend *)_data;
+	struct pack_backend *backend = data;
 	struct git_pack_file *pack;
+	const char *path_str = git_buf_cstr(path);
+	size_t i, cmp_len = git_buf_len(path);
 	int error;
-	size_t i;
 
-	if (git__suffixcmp(path->ptr, ".idx") != 0)
+	if (cmp_len <= strlen(".idx") || git__suffixcmp(path_str, ".idx") != 0)
 		return 0; /* not an index */
 
+	cmp_len -= strlen(".idx");
+
 	for (i = 0; i < backend->packs.length; ++i) {
 		struct git_pack_file *p = git_vector_get(&backend->packs, i);
-		if (memcmp(p->pack_name, git_buf_cstr(path), git_buf_len(path) - strlen(".idx")) == 0)
+
+		if (memcmp(p->pack_name, path_str, cmp_len) == 0)
 			return 0;
 	}
 
 	error = git_packfile_alloc(&pack, path->ptr);
-	if (error == GIT_ENOTFOUND)
-		/* ignore missing .pack file as git does */
+
+	/* ignore missing .pack file as git does */
+	if (error == GIT_ENOTFOUND) {
+		giterr_clear();
 		return 0;
-	else if (error < 0)
-		return error;
+	}
+
+	if (!error)
+		error = git_vector_insert(&backend->packs, pack);
+
+	return error;
 
-	return git_vector_insert(&backend->packs, pack);
 }
 
 static int pack_entry_find_inner(
@@ -314,13 +322,12 @@ static int pack_entry_find_prefix(
  * Implement the git_odb_backend API calls
  *
  ***********************************************************/
-static int pack_backend__refresh(git_odb_backend *_backend)
+static int pack_backend__refresh(git_odb_backend *backend_)
 {
-	struct pack_backend *backend = (struct pack_backend *)_backend;
-
 	int error;
 	struct stat st;
 	git_buf path = GIT_BUF_INIT;
+	struct pack_backend *backend = (struct pack_backend *)backend_;
 
 	if (backend->pack_folder == NULL)
 		return 0;
@@ -334,12 +341,9 @@ static int pack_backend__refresh(git_odb_backend *_backend)
 	error = git_path_direach(&path, 0, packfile_load__cb, backend);
 
 	git_buf_free(&path);
-
-	if (error < 0)
-		return -1;
-
 	git_vector_sort(&backend->packs);
-	return 0;
+
+	return error;
 }
 
 static int pack_backend__read_header_internal(
@@ -489,6 +493,23 @@ static int pack_backend__exists(git_odb_backend *backend, const git_oid *oid)
 	return pack_entry_find(&e, (struct pack_backend *)backend, oid) == 0;
 }
 
+static int pack_backend__exists_prefix(
+	git_oid *out, git_odb_backend *backend, const git_oid *short_id, size_t len)
+{
+	int error;
+	struct pack_backend *pb = (struct pack_backend *)backend;
+	struct git_pack_entry e = {0};
+
+	error = pack_entry_find_prefix(&e, pb, short_id, len);
+
+	if (error == GIT_ENOTFOUND && !(error = pack_backend__refresh(backend)))
+		error = pack_entry_find_prefix(&e, pb, short_id, len);
+
+	git_oid_cpy(out, &e.sha1);
+
+	return error;
+}
+
 static int pack_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb cb, void *data)
 {
 	int error;
@@ -542,7 +563,7 @@ static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
 static int pack_backend__writepack(struct git_odb_writepack **out,
 	git_odb_backend *_backend,
         git_odb *odb,
-	git_transfer_progress_callback progress_cb,
+	git_transfer_progress_cb progress_cb,
 	void *progress_payload)
 {
 	struct pack_backend *backend;
@@ -608,6 +629,7 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
 	backend->parent.read_prefix = &pack_backend__read_prefix;
 	backend->parent.read_header = &pack_backend__read_header;
 	backend->parent.exists = &pack_backend__exists;
+	backend->parent.exists_prefix = &pack_backend__exists_prefix;
 	backend->parent.refresh = &pack_backend__refresh;
 	backend->parent.foreach = &pack_backend__foreach;
 	backend->parent.writepack = &pack_backend__writepack;
diff --git a/src/oid.c b/src/oid.c
index d56b6af24..b640cadd1 100644
--- a/src/oid.c
+++ b/src/oid.c
@@ -24,30 +24,24 @@ int git_oid_fromstrn(git_oid *out, const char *str, size_t length)
 	size_t p;
 	int v;
 
-	if (length > GIT_OID_HEXSZ)
-		return oid_error_invalid("too long");
+	assert(out && str);
 
-	for (p = 0; p < length - 1; p += 2) {
-		v = (git__fromhex(str[p + 0]) << 4)
-				| git__fromhex(str[p + 1]);
+	if (!length)
+		return oid_error_invalid("too short");
 
-		if (v < 0)
-			return oid_error_invalid("contains invalid characters");
+	if (length > GIT_OID_HEXSZ)
+		return oid_error_invalid("too long");
 
-		out->id[p / 2] = (unsigned char)v;
-	}
+	memset(out->id, 0, GIT_OID_RAWSZ);
 
-	if (length % 2) {
-		v = (git__fromhex(str[p + 0]) << 4);
+	for (p = 0; p < length; p++) {
+		v = git__fromhex(str[p]);
 		if (v < 0)
 			return oid_error_invalid("contains invalid characters");
 
-		out->id[p / 2] = (unsigned char)v;
-		p += 2;
+		out->id[p / 2] |= (unsigned char)(v << (p % 2 ? 0 : 4));
 	}
 
-	memset(out->id + p / 2, 0, (GIT_OID_HEXSZ - p) / 2);
-
 	return 0;
 }
 
@@ -179,6 +173,11 @@ int git_oid_cmp(const git_oid *a, const git_oid *b)
 	return git_oid__cmp(a, b);
 }
 
+int git_oid_equal(const git_oid *a, const git_oid *b)
+{
+	return (git_oid__cmp(a, b) == 0);
+}
+
 int git_oid_ncmp(const git_oid *oid_a, const git_oid *oid_b, size_t len)
 {
 	const unsigned char *a = oid_a->id;
@@ -314,6 +313,9 @@ git_oid_shorten *git_oid_shorten_new(size_t min_length)
 
 void git_oid_shorten_free(git_oid_shorten *os)
 {
+	if (os == NULL)
+		return;
+
 	git__free(os->nodes);
 	git__free(os);
 }
diff --git a/src/pack-objects.c b/src/pack-objects.c
index 2d62507f2..0040a826b 100644
--- a/src/pack-objects.c
+++ b/src/pack-objects.c
@@ -7,7 +7,7 @@
 
 #include "pack-objects.h"
 
-#include "compress.h"
+#include "zstream.h"
 #include "delta.h"
 #include "iterator.h"
 #include "netops.h"
@@ -61,6 +61,9 @@ struct pack_write_context {
 /* The minimal interval between progress updates (in seconds). */
 #define MIN_PROGRESS_UPDATE_INTERVAL 0.5
 
+/* Size of the buffer to feed to zlib */
+#define COMPRESS_BUFLEN (1024 * 1024)
+
 static unsigned name_hash(const char *name)
 {
 	unsigned c, hash = 0;
@@ -87,8 +90,8 @@ static int packbuilder_config(git_packbuilder *pb)
 	int ret;
 	int64_t val;
 
-	if (git_repository_config__weakptr(&config, pb->repo) < 0)
-		return -1;
+	if ((ret = git_repository_config_snapshot(&config, pb->repo)) < 0)
+		return ret;
 
 #define config_get(KEY,DST,DFLT) do { \
 	ret = git_config_get_int64(&val, config, KEY); \
@@ -106,6 +109,8 @@ static int packbuilder_config(git_packbuilder *pb)
 
 #undef config_get
 
+	git_config_free(config);
+
 	return 0;
 }
 
@@ -127,6 +132,7 @@ int git_packbuilder_new(git_packbuilder **out, git_repository *repo)
 	pb->nr_threads = 1; /* do not spawn any thread by default */
 
 	if (git_hash_ctx_init(&pb->ctx) < 0 ||
+		git_zstream_init(&pb->zstream) < 0 ||
 		git_repository_odb(&pb->odb, repo) < 0 ||
 		packbuilder_config(pb) < 0)
 		goto on_error;
@@ -205,14 +211,18 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
 	po = pb->object_list + pb->nr_objects;
 	memset(po, 0x0, sizeof(*po));
 
-	if (git_odb_read_header(&po->size, &po->type, pb->odb, oid) < 0)
-		return -1;
+	if ((ret = git_odb_read_header(&po->size, &po->type, pb->odb, oid)) < 0)
+		return ret;
 
 	pb->nr_objects++;
 	git_oid_cpy(&po->id, oid);
 	po->hash = name_hash(name);
 
 	pos = kh_put(oid, pb->object_ix, &po->id, &ret);
+	if (ret < 0) {
+		giterr_set_oom();
+		return ret;
+	}
 	assert(ret != 0);
 	kh_value(pb->object_ix, pos) = po;
 
@@ -220,12 +230,17 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
 
 	if (pb->progress_cb) {
 		double current_time = git__timer();
-		if ((current_time - pb->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
+		double elapsed = current_time - pb->last_progress_report_time;
+
+		if (elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
 			pb->last_progress_report_time = current_time;
-			if (pb->progress_cb(GIT_PACKBUILDER_ADDING_OBJECTS, pb->nr_objects, 0, pb->progress_cb_payload)) {
-				giterr_clear();
-				return GIT_EUSER;
-			}
+
+			ret = pb->progress_cb(
+				GIT_PACKBUILDER_ADDING_OBJECTS,
+				pb->nr_objects, 0, pb->progress_cb_payload);
+
+			if (ret)
+				return giterr_set_after_callback(ret);
 		}
 	}
 
@@ -266,76 +281,96 @@ on_error:
 	return -1;
 }
 
-static int write_object(git_buf *buf, git_packbuilder *pb, git_pobject *po)
+static int write_object(
+	git_packbuilder *pb,
+	git_pobject *po,
+	int (*write_cb)(void *buf, size_t size, void *cb_data),
+	void *cb_data)
 {
 	git_odb_object *obj = NULL;
-	git_buf zbuf = GIT_BUF_INIT;
 	git_otype type;
-	unsigned char hdr[10];
-	size_t hdr_len;
-	unsigned long size;
-	void *data;
+	unsigned char hdr[10], *zbuf = NULL;
+	void *data = NULL;
+	size_t hdr_len, zbuf_len = COMPRESS_BUFLEN, data_len;
+	int error;
 
+	/*
+	 * If we have a delta base, let's use the delta to save space.
+	 * Otherwise load the whole object. 'data' ends up pointing to
+	 * whatever data we want to put into the packfile.
+	 */
 	if (po->delta) {
 		if (po->delta_data)
 			data = po->delta_data;
-		else if (get_delta(&data, pb->odb, po) < 0)
-				goto on_error;
-		size = po->delta_size;
+		else if ((error = get_delta(&data, pb->odb, po)) < 0)
+				goto done;
+
+		data_len = po->delta_size;
 		type = GIT_OBJ_REF_DELTA;
 	} else {
-		if (git_odb_read(&obj, pb->odb, &po->id))
-			goto on_error;
+		if ((error = git_odb_read(&obj, pb->odb, &po->id)) < 0)
+			goto done;
 
 		data = (void *)git_odb_object_data(obj);
-		size = (unsigned long)git_odb_object_size(obj);
+		data_len = git_odb_object_size(obj);
 		type = git_odb_object_type(obj);
 	}
 
 	/* Write header */
-	hdr_len = git_packfile__object_header(hdr, size, type);
+	hdr_len = git_packfile__object_header(hdr, data_len, type);
 
-	if (git_buf_put(buf, (char *)hdr, hdr_len) < 0)
-		goto on_error;
-
-	if (git_hash_update(&pb->ctx, hdr, hdr_len) < 0)
-		goto on_error;
+	if ((error = write_cb(hdr, hdr_len, cb_data)) < 0 ||
+		(error = git_hash_update(&pb->ctx, hdr, hdr_len)) < 0)
+		goto done;
 
 	if (type == GIT_OBJ_REF_DELTA) {
-		if (git_buf_put(buf, (char *)po->delta->id.id, GIT_OID_RAWSZ) < 0 ||
-			git_hash_update(&pb->ctx, po->delta->id.id, GIT_OID_RAWSZ) < 0)
-			goto on_error;
+		if ((error = write_cb(po->delta->id.id, GIT_OID_RAWSZ, cb_data)) < 0 ||
+			(error = git_hash_update(&pb->ctx, po->delta->id.id, GIT_OID_RAWSZ)) < 0)
+			goto done;
 	}
 
 	/* Write data */
-	if (po->z_delta_size)
-		size = po->z_delta_size;
-	else if (git__compress(&zbuf, data, size) < 0)
-		goto on_error;
-	else {
-		if (po->delta)
-			git__free(data);
-		data = zbuf.ptr;
-		size = (unsigned long)zbuf.size;
-	}
+	if (po->z_delta_size) {
+		data_len = po->z_delta_size;
 
-	if (git_buf_put(buf, data, size) < 0 ||
-		git_hash_update(&pb->ctx, data, size) < 0)
-		goto on_error;
+		if ((error = write_cb(data, data_len, cb_data)) < 0 ||
+			(error = git_hash_update(&pb->ctx, data, data_len)) < 0)
+			goto done;
+	} else {
+		zbuf = git__malloc(zbuf_len);
+		GITERR_CHECK_ALLOC(zbuf);
 
-	if (po->delta_data)
-		git__free(po->delta_data);
+		git_zstream_reset(&pb->zstream);
+		git_zstream_set_input(&pb->zstream, data, data_len);
 
-	git_odb_object_free(obj);
-	git_buf_free(&zbuf);
+		while (!git_zstream_done(&pb->zstream)) {
+			if ((error = git_zstream_get_output(zbuf, &zbuf_len, &pb->zstream)) < 0 ||
+				(error = write_cb(zbuf, zbuf_len, cb_data)) < 0 ||
+				(error = git_hash_update(&pb->ctx, zbuf, zbuf_len)) < 0)
+				goto done;
+
+			zbuf_len = COMPRESS_BUFLEN; /* reuse buffer */
+		}
+	}
+
+	/*
+	 * If po->delta is true, data is a delta and it is our
+	 * responsibility to free it (otherwise it's a git_object's
+	 * data). We set po->delta_data to NULL in case we got the
+	 * data from there instead of get_delta(). If we didn't,
+	 * there's no harm.
+	 */
+	if (po->delta) {
+		git__free(data);
+		po->delta_data = NULL;
+	}
 
 	pb->nr_written++;
-	return 0;
 
-on_error:
+done:
+	git__free(zbuf);
 	git_odb_object_free(obj);
-	git_buf_free(&zbuf);
-	return -1;
+	return error;
 }
 
 enum write_one_status {
@@ -345,9 +380,15 @@ enum write_one_status {
 	WRITE_ONE_RECURSIVE = 2 /* already scheduled to be written */
 };
 
-static int write_one(git_buf *buf, git_packbuilder *pb, git_pobject *po,
-		     enum write_one_status *status)
+static int write_one(
+	enum write_one_status *status,
+	git_packbuilder *pb,
+	git_pobject *po,
+	int (*write_cb)(void *buf, size_t size, void *cb_data),
+	void *cb_data)
 {
+	int error;
+
 	if (po->recursing) {
 		*status = WRITE_ONE_RECURSIVE;
 		return 0;
@@ -358,21 +399,20 @@ static int write_one(git_buf *buf, git_packbuilder *pb, git_pobject *po,
 
 	if (po->delta) {
 		po->recursing = 1;
-		if (write_one(buf, pb, po->delta, status) < 0)
-			return -1;
-		switch (*status) {
-		case WRITE_ONE_RECURSIVE:
-			/* we cannot depend on this one */
+
+		if ((error = write_one(status, pb, po->delta, write_cb, cb_data)) < 0)
+			return error;
+
+		/* we cannot depend on this one */
+		if (*status == WRITE_ONE_RECURSIVE)
 			po->delta = NULL;
-			break;
-		default:
-			break;
-		}
 	}
 
+	*status = WRITE_ONE_WRITTEN;
 	po->written = 1;
 	po->recursing = 0;
-	return write_object(buf, pb, po);
+
+	return write_object(pb, po, write_cb, cb_data);
 }
 
 GIT_INLINE(void) add_to_write_order(git_pobject **wo, unsigned int *endp,
@@ -552,12 +592,11 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 }
 
 static int write_pack(git_packbuilder *pb,
-		      int (*cb)(void *buf, size_t size, void *data),
-		      void *data)
+	int (*write_cb)(void *buf, size_t size, void *cb_data),
+	void *cb_data)
 {
 	git_pobject **write_order;
 	git_pobject *po;
-	git_buf buf = GIT_BUF_INIT;
 	enum write_one_status status;
 	struct git_pack_header ph;
 	git_oid entry_oid;
@@ -575,10 +614,8 @@ static int write_pack(git_packbuilder *pb,
 	ph.hdr_version = htonl(PACK_VERSION);
 	ph.hdr_entries = htonl(pb->nr_objects);
 
-	if ((error = cb(&ph, sizeof(ph), data)) < 0)
-		goto done;
-
-	if ((error = git_hash_update(&pb->ctx, &ph, sizeof(ph))) < 0)
+	if ((error = write_cb(&ph, sizeof(ph), cb_data)) < 0 ||
+		(error = git_hash_update(&pb->ctx, &ph, sizeof(ph))) < 0)
 		goto done;
 
 	pb->nr_remaining = pb->nr_objects;
@@ -586,25 +623,30 @@ static int write_pack(git_packbuilder *pb,
 		pb->nr_written = 0;
 		for ( ; i < pb->nr_objects; ++i) {
 			po = write_order[i];
-			if ((error = write_one(&buf, pb, po, &status)) < 0)
-				goto done;
-			if ((error = cb(buf.ptr, buf.size, data)) < 0)
+
+			if ((error = write_one(&status, pb, po, write_cb, cb_data)) < 0)
 				goto done;
-			git_buf_clear(&buf);
 		}
 
 		pb->nr_remaining -= pb->nr_written;
 	} while (pb->nr_remaining && i < pb->nr_objects);
 
-
 	if ((error = git_hash_final(&entry_oid, &pb->ctx)) < 0)
 		goto done;
 
-	error = cb(entry_oid.id, GIT_OID_RAWSZ, data);
+	error = write_cb(entry_oid.id, GIT_OID_RAWSZ, cb_data);
 
 done:
+	/* if callback cancelled writing, we must still free delta_data */
+	for ( ; i < pb->nr_objects; ++i) {
+		po = write_order[i];
+		if (po->delta_data) {
+			git__free(po->delta_data);
+			po->delta_data = NULL;
+		}
+	}
+
 	git__free(write_order);
-	git_buf_free(&buf);
 	return error;
 }
 
@@ -911,7 +953,7 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 		 * between writes at that moment.
 		 */
 		if (po->delta_data) {
-			if (git__compress(&zbuf, po->delta_data, po->delta_size) < 0)
+			if (git_zstream_deflatebuf(&zbuf, po->delta_data, po->delta_size) < 0)
 				goto on_error;
 
 			git__free(po->delta_data);
@@ -1167,7 +1209,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 		git_mutex_unlock(&target->mutex);
 
 		if (!sub_size) {
-			git_thread_join(target->thread, NULL);
+			git_thread_join(&target->thread, NULL);
 			git_cond_free(&target->cond);
 			git_mutex_free(&target->mutex);
 			active_threads--;
@@ -1236,6 +1278,7 @@ int git_packbuilder_foreach(git_packbuilder *pb, int (*cb)(void *buf, size_t siz
 int git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb)
 {
 	PREPARE_PACK;
+	git_buf_sanitize(buf);
 	return write_pack(pb, &write_pack_buf, buf);
 }
 
@@ -1249,7 +1292,7 @@ int git_packbuilder_write(
 	git_packbuilder *pb,
 	const char *path,
 	unsigned int mode,
-	git_transfer_progress_callback progress_cb,
+	git_transfer_progress_cb progress_cb,
 	void *progress_cb_payload)
 {
 	git_indexer *indexer;
@@ -1284,21 +1327,22 @@ const git_oid *git_packbuilder_hash(git_packbuilder *pb)
 	return &pb->pack_oid;
 }
 
-static int cb_tree_walk(const char *root, const git_tree_entry *entry, void *payload)
+static int cb_tree_walk(
+	const char *root, const git_tree_entry *entry, void *payload)
 {
+	int error;
 	struct tree_walk_context *ctx = payload;
 
 	/* A commit inside a tree represents a submodule commit and should be skipped. */
 	if (git_tree_entry_type(entry) == GIT_OBJ_COMMIT)
 		return 0;
 
-	if (git_buf_sets(&ctx->buf, root) < 0 ||
-		git_buf_puts(&ctx->buf, git_tree_entry_name(entry)) < 0)
-		return -1;
+	if (!(error = git_buf_sets(&ctx->buf, root)) &&
+		!(error = git_buf_puts(&ctx->buf, git_tree_entry_name(entry))))
+		error = git_packbuilder_insert(
+			ctx->pb, git_tree_entry_id(entry), git_buf_cstr(&ctx->buf));
 
-	return git_packbuilder_insert(ctx->pb,
-		git_tree_entry_id(entry),
-		git_buf_cstr(&ctx->buf));
+	return error;
 }
 
 int git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *oid)
@@ -1318,22 +1362,17 @@ int git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *oid)
 
 int git_packbuilder_insert_tree(git_packbuilder *pb, const git_oid *oid)
 {
-	git_tree *tree;
+	int error;
+	git_tree *tree = NULL;
 	struct tree_walk_context context = { pb, GIT_BUF_INIT };
 
-	if (git_tree_lookup(&tree, pb->repo, oid) < 0 ||
-	    git_packbuilder_insert(pb, oid, NULL) < 0)
-		return -1;
-
-	if (git_tree_walk(tree, GIT_TREEWALK_PRE, cb_tree_walk, &context) < 0) {
-		git_tree_free(tree);
-		git_buf_free(&context.buf);
-		return -1;
-	}
+	if (!(error = git_tree_lookup(&tree, pb->repo, oid)) &&
+	    !(error = git_packbuilder_insert(pb, oid, NULL)))
+		error = git_tree_walk(tree, GIT_TREEWALK_PRE, cb_tree_walk, &context);
 
 	git_tree_free(tree);
 	git_buf_free(&context.buf);
-	return 0;
+	return error;
 }
 
 uint32_t git_packbuilder_object_count(git_packbuilder *pb)
@@ -1380,6 +1419,7 @@ void git_packbuilder_free(git_packbuilder *pb)
 		git__free(pb->object_list);
 
 	git_hash_ctx_cleanup(&pb->ctx);
+	git_zstream_free(&pb->zstream);
 
 	git__free(pb);
 }
diff --git a/src/pack-objects.h b/src/pack-objects.h
index 0c94a5a7a..4647df75a 100644
--- a/src/pack-objects.h
+++ b/src/pack-objects.h
@@ -14,6 +14,7 @@
 #include "hash.h"
 #include "oidmap.h"
 #include "netops.h"
+#include "zstream.h"
 
 #include "git2/oid.h"
 #include "git2/pack.h"
@@ -54,6 +55,7 @@ struct git_packbuilder {
 	git_odb *odb; /* associated object database */
 
 	git_hash_ctx ctx;
+	git_zstream zstream;
 
 	uint32_t nr_objects,
 		 nr_alloc,
diff --git a/src/pack.c b/src/pack.c
index 644b2d465..ace7abb58 100644
--- a/src/pack.c
+++ b/src/pack.c
@@ -83,16 +83,12 @@ static void cache_free(git_pack_cache *cache)
 		}
 
 		git_offmap_free(cache->entries);
-		git_mutex_free(&cache->lock);
+		cache->entries = NULL;
 	}
-
-	memset(cache, 0, sizeof(*cache));
 }
 
 static int cache_init(git_pack_cache *cache)
 {
-	memset(cache, 0, sizeof(*cache));
-
 	cache->entries = git_offmap_alloc();
 	GITERR_CHECK_ALLOC(cache->entries);
 
@@ -514,72 +510,102 @@ int git_packfile_resolve_header(
 	return error;
 }
 
-static int packfile_unpack_delta(
-		git_rawobj *obj,
-		struct git_pack_file *p,
-		git_mwindow **w_curs,
-		git_off_t *curpos,
-		size_t delta_size,
-		git_otype delta_type,
-		git_off_t obj_offset)
+#define SMALL_STACK_SIZE 64
+
+/**
+ * Generate the chain of dependencies which we need to get to the
+ * object at `off`. `chain` is used a stack, popping gives the right
+ * order to apply deltas on. If an object is found in the pack's base
+ * cache, we stop calculating there.
+ */
+static int pack_dependency_chain(git_dependency_chain *chain_out,
+				 git_pack_cache_entry **cached_out, git_off_t *cached_off,
+				 struct pack_chain_elem *small_stack, size_t *stack_sz,
+				 struct git_pack_file *p, git_off_t obj_offset)
 {
-	git_off_t base_offset, base_key;
-	git_rawobj base, delta;
-	git_pack_cache_entry *cached = NULL;
-	int error, found_base = 0;
+	git_dependency_chain chain = GIT_ARRAY_INIT;
+	git_mwindow *w_curs = NULL;
+	git_off_t curpos = obj_offset, base_offset;
+	int error = 0, use_heap = 0;
+	size_t size, elem_pos;
+	git_otype type;
 
-	base_offset = get_delta_base(p, w_curs, curpos, delta_type, obj_offset);
-	git_mwindow_close(w_curs);
-	if (base_offset == 0)
-		return packfile_error("delta offset is zero");
-	if (base_offset < 0) /* must actually be an error code */
-		return (int)base_offset;
+	elem_pos = 0;
+	while (true) {
+		struct pack_chain_elem *elem;
+		git_pack_cache_entry *cached = NULL;
 
-	if (!p->bases.entries && (cache_init(&p->bases) < 0))
-		return -1;
+		/* if we have a base cached, we can stop here instead */
+		if ((cached = cache_get(&p->bases, obj_offset)) != NULL) {
+			*cached_out = cached;
+			*cached_off = obj_offset;
+			break;
+		}
 
-	base_key = base_offset; /* git_packfile_unpack modifies base_offset */
-	if ((cached = cache_get(&p->bases, base_offset)) != NULL) {
-		memcpy(&base, &cached->raw, sizeof(git_rawobj));
-		found_base = 1;
-	}
+		/* if we run out of space on the small stack, use the array */
+		if (elem_pos == SMALL_STACK_SIZE) {
+			git_array_init_to_size(chain, elem_pos);
+			GITERR_CHECK_ARRAY(chain);
+			memcpy(chain.ptr, small_stack, elem_pos * sizeof(struct pack_chain_elem));
+			chain.size = elem_pos;
+			use_heap = 1;
+		}
 
-	if (!cached) { /* have to inflate it */
-		error = git_packfile_unpack(&base, p, &base_offset);
+		curpos = obj_offset;
+		if (!use_heap) {
+			elem = &small_stack[elem_pos];
+		} else {
+			elem = git_array_alloc(chain);
+			if (!elem) {
+				error = -1;
+				goto on_error;
+			}
+		}
+
+		elem->base_key = obj_offset;
+
+		error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
+		git_mwindow_close(&w_curs);
 
-		/*
-		 * TODO: git.git tries to load the base from other packfiles
-		 * or loose objects.
-		 *
-		 * We'll need to do this in order to support thin packs.
-		 */
 		if (error < 0)
-			return error;
-	}
+			goto on_error;
 
-	error = packfile_unpack_compressed(&delta, p, w_curs, curpos, delta_size, delta_type);
-	git_mwindow_close(w_curs);
+		elem->offset = curpos;
+		elem->size = size;
+		elem->type = type;
+		elem->base_key = obj_offset;
 
-	if (error < 0) {
-		if (!found_base)
-			git__free(base.data);
-		return error;
-	}
+		if (type != GIT_OBJ_OFS_DELTA && type != GIT_OBJ_REF_DELTA)
+			break;
 
-	obj->type = base.type;
-	error = git__delta_apply(obj, base.data, base.len, delta.data, delta.len);
-	if (error < 0)
-		goto on_error;
+		base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
+		git_mwindow_close(&w_curs);
 
-	if (found_base)
-		git_atomic_dec(&cached->refcount);
-	else if (cache_add(&p->bases, &base, base_key) < 0)
-		git__free(base.data);
+		if (base_offset == 0) {
+			error = packfile_error("delta offset is zero");
+			goto on_error;
+		}
+		if (base_offset < 0) { /* must actually be an error code */
+			error = (int)base_offset;
+			goto on_error;
+		}
 
-on_error:
-	git__free(delta.data);
+		/* we need to pass the pos *after* the delta-base bit */
+		elem->offset = curpos;
+
+		/* go through the loop again, but with the new object */
+		obj_offset = base_offset;
+		elem_pos++;
+	}
+
+	
+	*stack_sz = elem_pos + 1;
+	*chain_out = chain;
+	return error;
 
-	return error; /* error set by git__delta_apply */
+on_error:
+	git_array_clear(chain);
+	return error;
 }
 
 int git_packfile_unpack(
@@ -589,48 +615,138 @@ int git_packfile_unpack(
 {
 	git_mwindow *w_curs = NULL;
 	git_off_t curpos = *obj_offset;
-	int error;
-
-	size_t size = 0;
-	git_otype type;
+	int error, free_base = 0;
+	git_dependency_chain chain = GIT_ARRAY_INIT;
+	struct pack_chain_elem *elem = NULL, *stack;
+	git_pack_cache_entry *cached = NULL;
+	struct pack_chain_elem small_stack[SMALL_STACK_SIZE];
+	size_t stack_size, elem_pos;
+	git_otype base_type;
 
 	/*
 	 * TODO: optionally check the CRC on the packfile
 	 */
 
+	error = pack_dependency_chain(&chain, &cached, obj_offset, small_stack, &stack_size, p, *obj_offset);
+	if (error < 0)
+		return error;
+
 	obj->data = NULL;
 	obj->len = 0;
 	obj->type = GIT_OBJ_BAD;
 
-	error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
-	git_mwindow_close(&w_curs);
+	/* let's point to the right stack */
+	stack = chain.ptr ? chain.ptr : small_stack;
 
-	if (error < 0)
-		return error;
+	elem_pos = stack_size;
+	if (cached) {
+		memcpy(obj, &cached->raw, sizeof(git_rawobj));
+		base_type = obj->type;
+		elem_pos--;	/* stack_size includes the base, which isn't actually there */
+	} else {
+		elem = &stack[--elem_pos];
+		base_type = elem->type;
+	}
 
-	switch (type) {
-	case GIT_OBJ_OFS_DELTA:
-	case GIT_OBJ_REF_DELTA:
-		error = packfile_unpack_delta(
-				obj, p, &w_curs, &curpos,
-				size, type, *obj_offset);
-		break;
+	if (error < 0)
+		goto cleanup;
 
+	switch (base_type) {
 	case GIT_OBJ_COMMIT:
 	case GIT_OBJ_TREE:
 	case GIT_OBJ_BLOB:
 	case GIT_OBJ_TAG:
-		error = packfile_unpack_compressed(
-				obj, p, &w_curs, &curpos,
-				size, type);
+		if (!cached) {
+			curpos = elem->offset;
+			error = packfile_unpack_compressed(obj, p, &w_curs, &curpos, elem->size, elem->type);
+			git_mwindow_close(&w_curs);
+			base_type = elem->type;
+		}
+		if (error < 0)
+			goto cleanup;
 		break;
-
+	case GIT_OBJ_OFS_DELTA:
+	case GIT_OBJ_REF_DELTA:
+		error = packfile_error("dependency chain ends in a delta");
+		goto cleanup;
 	default:
-		error = packfile_error("invalid packfile type in header");;
-		break;
+		error = packfile_error("invalid packfile type in header");
+		goto cleanup;
+	}
+
+	/*
+	 * Finding the object we want a cached base element is
+	 * problematic, as we need to make sure we don't accidentally
+	 * give the caller the cached object, which it would then feel
+	 * free to free, so we need to copy the data.
+	 */
+	if (cached && stack_size == 1) {
+		void *data = obj->data;
+		obj->data = git__malloc(obj->len + 1);
+		GITERR_CHECK_ALLOC(obj->data);
+		memcpy(obj->data, data, obj->len + 1);
+		git_atomic_dec(&cached->refcount);
+		goto cleanup;
+	}
+
+	/* we now apply each consecutive delta until we run out */
+	while (elem_pos > 0 && !error) {
+		git_rawobj base, delta;
+
+		/*
+		 * We can now try to add the base to the cache, as
+		 * long as it's not already the cached one.
+		 */
+		if (!cached)
+			free_base = !!cache_add(&p->bases, obj, elem->base_key);
+
+		elem = &stack[elem_pos - 1];
+		curpos = elem->offset;
+		error = packfile_unpack_compressed(&delta, p, &w_curs, &curpos, elem->size, elem->type);
+		git_mwindow_close(&w_curs);
+
+		if (error < 0)
+			break;
+
+		/* the current object becomes the new base, on which we apply the delta */
+		base = *obj;
+		obj->data = NULL;
+		obj->len = 0;
+		obj->type = GIT_OBJ_BAD;
+
+		error = git__delta_apply(obj, base.data, base.len, delta.data, delta.len);
+		obj->type = base_type;
+		/*
+		 * We usually don't want to free the base at this
+		 * point, as we put it into the cache in the previous
+		 * iteration. free_base lets us know that we got the
+		 * base object directly from the packfile, so we can free it.
+		 */
+		git__free(delta.data);
+		if (free_base) {
+			free_base = 0;
+			git__free(base.data);
+		}
+
+		if (cached) {
+			git_atomic_dec(&cached->refcount);
+			cached = NULL;
+		}
+
+		if (error < 0)
+			break;
+
+		elem_pos--;
 	}
 
-	*obj_offset = curpos;
+cleanup:
+	if (error < 0)
+		git__free(obj->data);
+
+	if (elem)
+		*obj_offset = elem->offset;
+
+	git_array_clear(chain);
 	return error;
 }
 
@@ -660,7 +776,7 @@ int git_packfile_stream_open(git_packfile_stream *obj, struct git_pack_file *p,
 	st = inflateInit(&obj->zstream);
 	if (st != Z_OK) {
 		git__free(obj);
-		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		giterr_set(GITERR_ZLIB, "failed to init packfile stream");
 		return -1;
 	}
 
@@ -691,7 +807,7 @@ ssize_t git_packfile_stream_read(git_packfile_stream *obj, void *buffer, size_t
 	written = len - obj->zstream.avail_out;
 
 	if (st != Z_OK && st != Z_STREAM_END) {
-		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		giterr_set(GITERR_ZLIB, "error reading from the zlib stream");
 		return -1;
 	}
 
@@ -736,7 +852,7 @@ int packfile_unpack_compressed(
 	st = inflateInit(&stream);
 	if (st != Z_OK) {
 		git__free(buffer);
-		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		giterr_set(GITERR_ZLIB, "failed to init zlib stream on unpack");
 
 		return -1;
 	}
@@ -763,7 +879,7 @@ int packfile_unpack_compressed(
 
 	if ((st != Z_STREAM_END) || stream.total_out != size) {
 		git__free(buffer);
-		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		giterr_set(GITERR_ZLIB, "error inflating zlib stream");
 		return -1;
 	}
 
@@ -862,6 +978,7 @@ void git_packfile_free(struct git_pack_file *p)
 	git__free(p->bad_object_sha1);
 
 	git_mutex_free(&p->lock);
+	git_mutex_free(&p->bases.lock);
 	git__free(p);
 }
 
@@ -997,10 +1114,10 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
 		return -1;
 	}
 
-	/* see if we can parse the sha1 oid in the packfile name */
-	if (path_len < 40 ||
-		git_oid_fromstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < 0)
-		memset(&p->sha1, 0x0, GIT_OID_RAWSZ);
+	if (cache_init(&p->bases) < 0) {
+		git__free(p);
+		return -1;
+	}
 
 	*pack_out = p;
 
@@ -1042,10 +1159,9 @@ int git_pack_foreach_entry(
 {
 	const unsigned char *index = p->index_map.data, *current;
 	uint32_t i;
+	int error = 0;
 
 	if (index == NULL) {
-		int error;
-
 		if ((error = pack_index_open(p)) < 0)
 			return error;
 
@@ -1062,7 +1178,6 @@ int git_pack_foreach_entry(
 
 	if (p->oids == NULL) {
 		git_vector offsets, oids;
-		int error;
 
 		if ((error = git_vector_init(&oids, p->num_objects, NULL)))
 			return error;
@@ -1084,15 +1199,16 @@ int git_pack_foreach_entry(
 			git_vector_foreach(&offsets, i, current)
 				git_vector_insert(&oids, (void*)&current[4]);
 		}
+
 		git_vector_free(&offsets);
-		p->oids = (git_oid **)oids.contents;
+		p->oids = (git_oid **)git_vector_detach(NULL, NULL, &oids);
 	}
 
 	for (i = 0; i < p->num_objects; i++)
-		if (cb(p->oids[i], data))
-			return GIT_EUSER;
+		if ((error = cb(p->oids[i], data)) != 0)
+			return giterr_set_after_callback(error);
 
-	return 0;
+	return error;
 }
 
 static int pack_entry_find_offset(
diff --git a/src/pack.h b/src/pack.h
index 28146ab30..610e70c18 100644
--- a/src/pack.h
+++ b/src/pack.h
@@ -17,6 +17,7 @@
 #include "mwindow.h"
 #include "odb.h"
 #include "oidmap.h"
+#include "array.h"
 
 #define GIT_PACK_FILE_MODE 0444
 
@@ -60,6 +61,15 @@ typedef struct git_pack_cache_entry {
 	git_rawobj raw;
 } git_pack_cache_entry;
 
+struct pack_chain_elem {
+	git_off_t base_key;
+	git_off_t offset;
+	size_t size;
+	git_otype type;
+};
+
+typedef git_array_t(struct pack_chain_elem) git_dependency_chain;
+
 #include "offmap.h"
 
 GIT__USE_OFFMAP;
@@ -88,7 +98,6 @@ struct git_pack_file {
 	int index_version;
 	git_time_t mtime;
 	unsigned pack_local:1, pack_keep:1, has_cache:1;
-	git_oid sha1;
 	git_oidmap *idx_cache;
 	git_oid **oids;
 
diff --git a/src/path.c b/src/path.c
index 750dd3ef7..5beab97ed 100644
--- a/src/path.c
+++ b/src/path.c
@@ -9,10 +9,10 @@
 #include "posix.h"
 #ifdef GIT_WIN32
 #include "win32/posix.h"
+#include "win32/w32_util.h"
 #else
 #include <dirent.h>
 #endif
-#include <stdarg.h>
 #include <stdio.h>
 #include <ctype.h>
 
@@ -436,8 +436,12 @@ int git_path_walk_up(
 	while (scan >= stop) {
 		error = cb(data, &iter);
 		iter.ptr[scan] = oldc;
-		if (error < 0)
+
+		if (error) {
+			giterr_set_after_callback(error);
 			break;
+		}
+
 		scan = git_buf_rfind_next(&iter, '/');
 		if (scan >= 0) {
 			scan++;
@@ -483,33 +487,33 @@ bool git_path_isfile(const char *path)
 
 bool git_path_is_empty_dir(const char *path)
 {
-	HANDLE hFind = INVALID_HANDLE_VALUE;
-	git_win32_path wbuf;
-	int wbufsz;
-	WIN32_FIND_DATAW ffd;
-	bool retval = true;
-
-	if (!git_path_isdir(path))
-		return false;
-
-	wbufsz = git_win32_path_from_c(wbuf, path);
-	if (!wbufsz || wbufsz + 2 > GIT_WIN_PATH_UTF16)
-		return false;
-	memcpy(&wbuf[wbufsz - 1], L"\\*", 3 * sizeof(wchar_t));
-
-	hFind = FindFirstFileW(wbuf, &ffd);
-	if (INVALID_HANDLE_VALUE == hFind)
-		return false;
-
-	do {
-		if (!git_path_is_dot_or_dotdotW(ffd.cFileName)) {
-			retval = false;
-			break;
+	git_win32_path filter_w;
+	bool empty = false;
+
+	if (git_win32__findfirstfile_filter(filter_w, path)) {
+		WIN32_FIND_DATAW findData;
+		HANDLE hFind = FindFirstFileW(filter_w, &findData);
+
+		/* If the find handle was created successfully, then it's a directory */
+		if (hFind != INVALID_HANDLE_VALUE) {
+			empty = true;
+
+			do {
+				/* Allow the enumeration to return . and .. and still be considered
+				 * empty. In the special case of drive roots (i.e. C:\) where . and
+				 * .. do not occur, we can still consider the path to be an empty
+				 * directory if there's nothing there. */
+				if (!git_path_is_dot_or_dotdotW(findData.cFileName)) {
+					empty = false;
+					break;
+				}
+			} while (FindNextFileW(hFind, &findData));
+
+			FindClose(hFind);
 		}
-	} while (FindNextFileW(hFind, &ffd) != 0);
+	}
 
-	FindClose(hFind);
-	return retval;
+	return empty;
 }
 
 #else
@@ -528,7 +532,9 @@ bool git_path_is_empty_dir(const char *path)
 	if (!git_path_isdir(path))
 		return false;
 
-	if (!(error = git_buf_sets(&dir, path)))
+	if ((error = git_buf_sets(&dir, path)) != 0)
+		giterr_clear();
+	else
 		error = git_path_direach(&dir, 0, path_found_entry, NULL);
 
 	git_buf_free(&dir);
@@ -619,7 +625,7 @@ int git_path_find_dir(git_buf *dir, const char *path, const char *base)
 
 	/* call dirname if this is not a directory */
 	if (!error) /* && git_path_isdir(dir->ptr) == false) */
-		error = git_path_dirname_r(dir, dir->ptr);
+		error = (git_path_dirname_r(dir, dir->ptr) < 0) ? -1 : 0;
 
 	if (!error)
 		error = git_path_to_dir(dir);
@@ -777,8 +783,10 @@ int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen)
 		!git_path_has_non_ascii(*in, *inlen))
 		return 0;
 
+	git_buf_clear(&ic->buf);
+
 	while (1) {
-		if (git_buf_grow(&ic->buf, wantlen) < 0)
+		if (git_buf_grow(&ic->buf, wantlen + 1) < 0)
 			return -1;
 
 		nfc    = ic->buf.ptr   + ic->buf.size;
@@ -791,8 +799,11 @@ int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen)
 		if (rv != (size_t)-1)
 			break;
 
+		/* if we cannot convert the data (probably because iconv thinks
+		 * it is not valid UTF-8 source data), then use original data
+		 */
 		if (errno != E2BIG)
-			goto fail;
+			return 0;
 
 		/* make space for 2x the remaining data to be converted
 		 * (with per retry overhead to avoid infinite loops)
@@ -815,6 +826,64 @@ fail:
 	return -1;
 }
 
+static const char *nfc_file = "\xC3\x85\x73\x74\x72\xC3\xB6\x6D.XXXXXX";
+static const char *nfd_file = "\x41\xCC\x8A\x73\x74\x72\x6F\xCC\x88\x6D.XXXXXX";
+
+/* Check if the platform is decomposing unicode data for us.  We will
+ * emulate core Git and prefer to use precomposed unicode data internally
+ * on these platforms, composing the decomposed unicode on the fly.
+ *
+ * This mainly happens on the Mac where HDFS stores filenames as
+ * decomposed unicode.  Even on VFAT and SAMBA file systems, the Mac will
+ * return decomposed unicode from readdir() even when the actual
+ * filesystem is storing precomposed unicode.
+ */
+bool git_path_does_fs_decompose_unicode(const char *root)
+{
+	git_buf path = GIT_BUF_INIT;
+	int fd;
+	bool found_decomposed = false;
+	char tmp[6];
+
+	/* Create a file using a precomposed path and then try to find it
+	 * using the decomposed name.  If the lookup fails, then we will mark
+	 * that we should precompose unicode for this repository.
+	 */
+	if (git_buf_joinpath(&path, root, nfc_file) < 0 ||
+		(fd = p_mkstemp(path.ptr)) < 0)
+		goto done;
+	p_close(fd);
+
+	/* record trailing digits generated by mkstemp */
+	memcpy(tmp, path.ptr + path.size - sizeof(tmp), sizeof(tmp));
+
+	/* try to look up as NFD path */
+	if (git_buf_joinpath(&path, root, nfd_file) < 0)
+		goto done;
+	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
+
+	found_decomposed = git_path_exists(path.ptr);
+
+	/* remove temporary file (using original precomposed path) */
+	if (git_buf_joinpath(&path, root, nfc_file) < 0)
+		goto done;
+	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
+
+	(void)p_unlink(path.ptr);
+
+done:
+	git_buf_free(&path);
+	return found_decomposed;
+}
+
+#else
+
+bool git_path_does_fs_decompose_unicode(const char *root)
+{
+	GIT_UNUSED(root);
+	return false;
+}
+
 #endif
 
 #if defined(__sun) || defined(__GNU__)
@@ -848,6 +917,9 @@ int git_path_direach(
 
 	if ((dir = opendir(path->ptr)) == NULL) {
 		giterr_set(GITERR_OS, "Failed to open directory '%s'", path->ptr);
+		if (errno == ENOENT)
+			return GIT_ENOTFOUND;
+
 		return -1;
 	}
 
@@ -867,7 +939,7 @@ int git_path_direach(
 		if ((error = git_path_iconv(&ic, &de_path, &de_len)) < 0)
 			break;
 #endif
-				
+
 		if ((error = git_buf_put(path, de_path, de_len)) < 0)
 			break;
 
@@ -875,8 +947,8 @@ int git_path_direach(
 
 		git_buf_truncate(path, wd_len); /* restore path */
 
-		if (error) {
-			error = GIT_EUSER;
+		if (error != 0) {
+			giterr_set_after_callback(error);
 			break;
 		}
 	}
@@ -1043,15 +1115,8 @@ int git_path_dirload_with_stat(
 		}
 
 		if (S_ISDIR(ps->st.st_mode)) {
-			if ((error = git_buf_joinpath(&full, full.ptr, ".git")) < 0)
-				break;
-
-			if (p_access(full.ptr, F_OK) == 0) {
-				ps->st.st_mode = GIT_FILEMODE_COMMIT;
-			} else {
-				ps->path[ps->path_len++] = '/';
-				ps->path[ps->path_len] = '\0';
-			}
+			ps->path[ps->path_len++] = '/';
+			ps->path[ps->path_len] = '\0';
 		}
 	}
 
@@ -1062,3 +1127,21 @@ int git_path_dirload_with_stat(
 
 	return error;
 }
+
+int git_path_from_url_or_path(git_buf *local_path_out, const char *url_or_path)
+{
+	int error;
+
+	/* If url_or_path begins with file:// treat it as a URL */
+	if (!git__prefixcmp(url_or_path, "file://")) {
+		if ((error = git_path_fromurl(local_path_out, url_or_path)) < 0) {
+			return error;
+		}
+	} else { /* We assume url_or_path is already a path */
+		if ((error = git_buf_sets(local_path_out, url_or_path)) < 0) {
+			return error;
+		}
+	}
+
+	return 0;
+}
diff --git a/src/path.h b/src/path.h
index 3daafd265..3e6efe3de 100644
--- a/src/path.h
+++ b/src/path.h
@@ -119,6 +119,14 @@ GIT_INLINE(void) git_path_mkposix(char *path)
 #	define git_path_mkposix(p) /* blank */
 #endif
 
+/**
+ * Check if string is a relative path (i.e. starts with "./" or "../")
+ */
+GIT_INLINE(int) git_path_is_relative(const char *p)
+{
+	return (p[0] == '.' && (p[1] == '/' || (p[1] == '.' && p[2] == '/')));
+}
+
 extern int git__percent_decode(git_buf *decoded_out, const char *input);
 
 /**
@@ -255,9 +263,10 @@ enum {
  * @param flags Combination of GIT_PATH_DIR flags.
  * @param callback Callback for each entry. Passed the `payload` and each
  *		successive path inside the directory as a full path.  This may
- *		safely append text to the pathbuf if needed.
+ *		safely append text to the pathbuf if needed.  Return non-zero to
+ *		cancel iteration (and return value will be propagated back).
  * @param payload Passed to callback as first argument.
- * @return 0 on success, GIT_EUSER on non-zero callback, or error code
+ * @return 0 on success or error code from OS error or from callback
  */
 extern int git_path_direach(
 	git_buf *pathbuf,
@@ -288,7 +297,7 @@ extern int git_path_cmp(
  *		original input path.
  * @param callback Function to invoke on each path.  Passed the `payload`
  *		and the buffer containing the current path.  The path should not
- *		be modified in any way.
+ *		be modified in any way. Return non-zero to stop iteration.
  * @param state Passed to fn as the first ath.
  */
 extern int git_path_walk_up(
@@ -427,4 +436,9 @@ extern int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen);
 
 #endif /* GIT_USE_ICONV */
 
+extern bool git_path_does_fs_decompose_unicode(const char *root);
+
+/* Used for paths to repositories on the filesystem */
+extern int git_path_from_url_or_path(git_buf *local_path_out, const char *url_or_path);
+
 #endif
diff --git a/src/pathspec.c b/src/pathspec.c
index 1e7e65e90..a01d74f07 100644
--- a/src/pathspec.c
+++ b/src/pathspec.c
@@ -83,14 +83,17 @@ int git_pathspec__vinit(
 		if (!match)
 			return -1;
 
-		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG;
+		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE |
+			GIT_ATTR_FNMATCH_ALLOWNEG | GIT_ATTR_FNMATCH_NOLEADINGDIR;
 
 		ret = git_attr_fnmatch__parse(match, strpool, NULL, &pattern);
 		if (ret == GIT_ENOTFOUND) {
 			git__free(match);
 			continue;
-		} else if (ret < 0)
+		} else if (ret < 0) {
+			git__free(match);
 			return ret;
+		}
 
 		if (git_vector_insert(vspec, match) < 0)
 			return -1;
@@ -102,15 +105,7 @@ int git_pathspec__vinit(
 /* free data from the pathspec vector */
 void git_pathspec__vfree(git_vector *vspec)
 {
-	git_attr_fnmatch *match;
-	unsigned int i;
-
-	git_vector_foreach(vspec, i, match) {
-		git__free(match);
-		vspec->contents[i] = NULL;
-	}
-
-	git_vector_free(vspec);
+	git_vector_free_deep(vspec);
 }
 
 struct pathspec_match_context {
@@ -451,7 +446,7 @@ static int pathspec_match_from_iterator(
 		/* check if path is ignored and untracked */
 		if (index != NULL &&
 			git_iterator_current_is_ignored(iter) &&
-			git_index__find(NULL, index, entry->path, GIT_INDEX_STAGE_ANY) < 0)
+			git_index__find_pos(NULL, index, entry->path, 0, GIT_INDEX_STAGE_ANY) < 0)
 			continue;
 
 		/* mark the matched pattern as used */
diff --git a/src/pool.c b/src/pool.c
index d484769e9..146f118b4 100644
--- a/src/pool.c
+++ b/src/pool.c
@@ -190,19 +190,18 @@ void *git_pool_malloc(git_pool *pool, uint32_t items)
 
 char *git_pool_strndup(git_pool *pool, const char *str, size_t n)
 {
-	void *ptr = NULL;
+	char *ptr = NULL;
 
 	assert(pool && str && pool->item_size == sizeof(char));
 
-	if (n + 1 == 0) {
-		giterr_set_oom();
+	if ((uint32_t)(n + 1) < n)
 		return NULL;
-	}
 
 	if ((ptr = git_pool_malloc(pool, (uint32_t)(n + 1))) != NULL) {
 		memcpy(ptr, str, n);
-		*(((char *)ptr) + n) = '\0';
+		ptr[n] = '\0';
 	}
+
 	pool->has_string_alloc = 1;
 
 	return ptr;
diff --git a/src/posix.c b/src/posix.c
index b75109b83..7aeb0e6c1 100644
--- a/src/posix.c
+++ b/src/posix.c
@@ -99,7 +99,7 @@ const char *p_gai_strerror(int ret)
 
 #endif /* NO_ADDRINFO */
 
-int p_open(const char *path, int flags, ...)
+int p_open(const char *path, volatile int flags, ...)
 {
 	mode_t mode = 0;
 
@@ -189,7 +189,7 @@ int p_write(git_file fd, const void *buf, size_t cnt)
 		r = write(fd, b, cnt);
 #endif
 		if (r < 0) {
-			if (errno == EINTR || errno == EAGAIN)
+			if (errno == EINTR || GIT_ISBLOCKED(errno))
 				continue;
 			return -1;
 		}
@@ -203,4 +203,47 @@ int p_write(git_file fd, const void *buf, size_t cnt)
 	return 0;
 }
 
+#ifdef NO_MMAP
+
+#include "map.h"
+
+long git__page_size(void)
+{
+	/* dummy; here we don't need any alignment anyway */
+	return 4096;
+}
+
+
+int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
+{
+	GIT_MMAP_VALIDATE(out, len, prot, flags);
+
+	out->data = NULL;
+	out->len = 0;
+
+	if ((prot & GIT_PROT_WRITE) && ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED)) {
+		giterr_set(GITERR_OS, "Trying to map shared-writeable");
+		return -1;
+	}
+
+	out->data = malloc(len);
+	GITERR_CHECK_ALLOC(out->data);
 
+	if ((p_lseek(fd, offset, SEEK_SET) < 0) || ((size_t)p_read(fd, out->data, len) != len)) {
+		giterr_set(GITERR_OS, "mmap emulation failed");
+		return -1;
+	}
+
+	out->len = len;
+	return 0;
+}
+
+int p_munmap(git_map *map)
+{
+	assert(map != NULL);
+	free(map->data);
+
+	return 0;
+}
+
+#endif
diff --git a/src/posix.h b/src/posix.h
index 14c92b93d..965cd98d5 100644
--- a/src/posix.h
+++ b/src/posix.h
@@ -29,6 +29,15 @@
 #define O_CLOEXEC 0
 #endif
 
+/* Determine whether an errno value indicates that a read or write failed
+ * because the descriptor is blocked.
+ */
+#if defined(EWOULDBLOCK)
+#define GIT_ISBLOCKED(e) ((e) == EAGAIN || (e) == EWOULDBLOCK)
+#else
+#define GIT_ISBLOCKED(e) ((e) == EAGAIN)
+#endif
+
 typedef int git_file;
 
 /**
@@ -64,6 +73,7 @@ extern int p_rename(const char *from, const char *to);
 #define p_rmdir(p) rmdir(p)
 #define p_chmod(p,m) chmod(p, m)
 #define p_access(p,m) access(p,m)
+#define p_ftruncate(fd, sz) ftruncate(fd, sz)
 #define p_recv(s,b,l,f) recv(s,b,l,f)
 #define p_send(s,b,l,f) send(s,b,l,f)
 typedef int GIT_SOCKET;
@@ -89,9 +99,7 @@ extern struct tm * p_gmtime_r (const time_t *timer, struct tm *result);
 #	include "unix/posix.h"
 #endif
 
-#ifndef __MINGW32__
-# define p_strnlen strnlen
-#endif
+#include "strnlen.h"
 
 #ifdef NO_READDIR_R
 #	include <dirent.h>
diff --git a/src/pqueue.c b/src/pqueue.c
index 7819ed41e..54a60ca04 100644
--- a/src/pqueue.c
+++ b/src/pqueue.c
@@ -3,161 +3,115 @@
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
- *
- * This file is based on a modified version of the priority queue found
- * in the Apache project and libpqueue library.
- * 
- * https://github.com/vy/libpqueue
- * 
- * Original file notice:
- * 
- * Copyright 2010 Volkan Yazici <volkan.yazici@gmail.com>
- * Copyright 2006-2010 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
  */
 
-#include "common.h"
 #include "pqueue.h"
+#include "util.h"
 
-#define left(i)	((i) << 1)
-#define right(i) (((i) << 1) + 1)
-#define parent(i) ((i) >> 1)
+#define PQUEUE_LCHILD_OF(I) (((I)<<1)+1)
+#define PQUEUE_RCHILD_OF(I) (((I)<<1)+2)
+#define PQUEUE_PARENT_OF(I) (((I)-1)>>1)
 
-int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri)
+int git_pqueue_init(
+	git_pqueue *pq,
+	uint32_t flags,
+	size_t init_size,
+	git_vector_cmp cmp)
 {
-	assert(q);
+	int error = git_vector_init(pq, init_size, cmp);
 
-	/* Need to allocate n+1 elements since element 0 isn't used. */
-	q->d = git__malloc((n + 1) * sizeof(void *));
-	GITERR_CHECK_ALLOC(q->d);
+	if (!error) {
+		/* mix in our flags */
+		pq->flags |= flags;
 
-	q->size = 1;
-	q->avail = q->step = (n + 1); /* see comment above about n+1 */
-	q->cmppri = cmppri;
+		/* if fixed size heap, pretend vector is exactly init_size elements */
+		if ((flags & GIT_PQUEUE_FIXED_SIZE) && init_size > 0)
+			pq->_alloc_size = init_size;
+	}
 
-	return 0;
+	return error;
 }
 
-
-void git_pqueue_free(git_pqueue *q)
+static void pqueue_up(git_pqueue *pq, size_t el)
 {
-	git__free(q->d);
-	q->d = NULL;
-}
+	size_t parent_el = PQUEUE_PARENT_OF(el);
+	void *kid = git_vector_get(pq, el);
 
-void git_pqueue_clear(git_pqueue *q)
-{
-	q->size = 1;
-}
+	while (el > 0) {
+		void *parent = pq->contents[parent_el];
 
-size_t git_pqueue_size(git_pqueue *q)
-{
-	/* queue element 0 exists but doesn't count since it isn't used. */
-	return (q->size - 1);
-}
+		if (pq->_cmp(parent, kid) <= 0)
+			break;
 
+		pq->contents[el] = parent;
 
-static void bubble_up(git_pqueue *q, size_t i)
-{
-	size_t parent_node;
-	void *moving_node = q->d[i];
-
-	for (parent_node = parent(i);
-			((i > 1) && q->cmppri(q->d[parent_node], moving_node));
-			i = parent_node, parent_node = parent(i)) {
-		q->d[i] = q->d[parent_node];
+		el = parent_el;
+		parent_el = PQUEUE_PARENT_OF(el);
 	}
 
-	q->d[i] = moving_node;
+	pq->contents[el] = kid;
 }
 
-
-static size_t maxchild(git_pqueue *q, size_t i)
+static void pqueue_down(git_pqueue *pq, size_t el)
 {
-	size_t child_node = left(i);
+	void *parent = git_vector_get(pq, el), *kid, *rkid;
 
-	if (child_node >= q->size)
-		return 0;
+	while (1) {
+		size_t kid_el = PQUEUE_LCHILD_OF(el);
 
-	if ((child_node + 1) < q->size &&
-		q->cmppri(q->d[child_node], q->d[child_node + 1]))
-		child_node++; /* use right child instead of left */
+		if ((kid = git_vector_get(pq, kid_el)) == NULL)
+			break;
 
-	return child_node;
-}
+		if ((rkid = git_vector_get(pq, kid_el + 1)) != NULL &&
+			pq->_cmp(kid, rkid) > 0) {
+			kid    = rkid;
+			kid_el += 1;
+		}
 
+		if (pq->_cmp(parent, kid) <= 0)
+			break;
 
-static void percolate_down(git_pqueue *q, size_t i)
-{
-	size_t child_node;
-	void *moving_node = q->d[i];
-
-	while ((child_node = maxchild(q, i)) != 0 &&
-			q->cmppri(moving_node, q->d[child_node])) {
-		q->d[i] = q->d[child_node];
-		i = child_node;
+		pq->contents[el] = kid;
+		el = kid_el;
 	}
 
-	q->d[i] = moving_node;
+	pq->contents[el] = parent;
 }
 
-
-int git_pqueue_insert(git_pqueue *q, void *d)
+int git_pqueue_insert(git_pqueue *pq, void *item)
 {
-	void *tmp;
-	size_t i;
-	size_t newsize;
-
-	if (!q) return 1;
-
-	/* allocate more memory if necessary */
-	if (q->size >= q->avail) {
-		newsize = q->size + q->step;
-		tmp = git__realloc(q->d, sizeof(void *) * newsize);
-		GITERR_CHECK_ALLOC(tmp);
-
-		q->d = tmp;
-		q->avail = newsize;
+	int error = 0;
+
+	/* if heap is full, pop the top element if new one should replace it */
+	if ((pq->flags & GIT_PQUEUE_FIXED_SIZE) != 0 &&
+		pq->length >= pq->_alloc_size)
+	{
+		/* skip this item if below min item in heap */
+		if (pq->_cmp(item, git_vector_get(pq, 0)) <= 0)
+			return 0;
+		/* otherwise remove the min item before inserting new */
+		(void)git_pqueue_pop(pq);
 	}
 
-	/* insert item */
-	i = q->size++;
-	q->d[i] = d;
-	bubble_up(q, i);
+	if (!(error = git_vector_insert(pq, item)))
+		pqueue_up(pq, pq->length - 1);
 
-	return 0;
+	return error;
 }
 
-
-void *git_pqueue_pop(git_pqueue *q)
+void *git_pqueue_pop(git_pqueue *pq)
 {
-	void *head;
-
-	if (!q || q->size == 1)
-		return NULL;
-
-	head = q->d[1];
-	q->d[1] = q->d[--q->size];
-	percolate_down(q, 1);
-
-	return head;
-}
-
+	void *rval = git_pqueue_get(pq, 0);
+
+	if (git_pqueue_size(pq) > 1) {
+		/* move last item to top of heap, shrink, and push item down */
+		pq->contents[0] = git_vector_last(pq);
+		git_vector_pop(pq);
+		pqueue_down(pq, 0);
+	} else {
+		/* all we need to do is shrink the heap in this case */
+		git_vector_pop(pq);
+	}
 
-void *git_pqueue_peek(git_pqueue *q)
-{
-	if (!q || q->size == 1)
-		return NULL;
-	return q->d[1];
+	return rval;
 }
diff --git a/src/pqueue.h b/src/pqueue.h
index 9061f8279..da7b74edf 100644
--- a/src/pqueue.h
+++ b/src/pqueue.h
@@ -3,99 +3,54 @@
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
- * 
- * This file is based on a modified version of the priority queue found
- * in the Apache project and libpqueue library.
- * 
- * https://github.com/vy/libpqueue
- * 
- * Original file notice:
- * 
- * Copyright 2010 Volkan Yazici <volkan.yazici@gmail.com>
- * Copyright 2006-2010 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
  */
-
 #ifndef INCLUDE_pqueue_h__
 #define INCLUDE_pqueue_h__
 
-/** callback functions to get/set/compare the priority of an element */
-typedef int (*git_pqueue_cmp)(void *a, void *b);
+#include "vector.h"
 
-/** the priority queue handle */
-typedef struct {
-	size_t size, avail, step;
-	git_pqueue_cmp cmppri;
-	void **d;
-} git_pqueue;
+typedef git_vector git_pqueue;
 
+enum {
+	/* flag meaning: don't grow heap, keep highest values only */
+	GIT_PQUEUE_FIXED_SIZE = (GIT_VECTOR_FLAG_MAX << 1),
+};
 
 /**
- * initialize the queue
+ * Initialize priority queue
  *
- * @param n the initial estimate of the number of queue items for which memory
- *			should be preallocated
- * @param cmppri the callback function to compare two nodes of the queue
- *
- * @return the handle or NULL for insufficent memory
- */
-int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri);
-
-
-/**
- * free all memory used by the queue
- * @param q the queue
- */
-void git_pqueue_free(git_pqueue *q);
-
-/**
- * clear all the elements in the queue
- * @param q the queue
- */
-void git_pqueue_clear(git_pqueue *q);
+ * @param pq The priority queue struct to initialize
+ * @param flags Flags (see above) to control queue behavior
+ * @param init_size The initial queue size
+ * @param cmp The entry priority comparison function
+ * @return 0 on success, <0 on error
+ */
+extern int git_pqueue_init(
+	git_pqueue *pq,
+	uint32_t flags,
+	size_t init_size,
+	git_vector_cmp cmp);
+
+#define git_pqueue_free  git_vector_free
+#define git_pqueue_clear git_vector_clear
+#define git_pqueue_size  git_vector_length
+#define git_pqueue_get   git_vector_get
 
 /**
- * return the size of the queue.
- * @param q the queue
- */
-size_t git_pqueue_size(git_pqueue *q);
-
-
-/**
- * insert an item into the queue.
- * @param q the queue
- * @param d the item
- * @return 0 on success
- */
-int git_pqueue_insert(git_pqueue *q, void *d);
-
-
-/**
- * pop the highest-ranking item from the queue.
- * @param q the queue
- * @return NULL on error, otherwise the entry
+ * Insert a new item into the queue
+ *
+ * @param pq The priority queue
+ * @param item Pointer to the item data
+ * @return 0 on success, <0 on failure
  */
-void *git_pqueue_pop(git_pqueue *q);
-
+extern int git_pqueue_insert(git_pqueue *pq, void *item);
 
 /**
- * access highest-ranking item without removing it.
- * @param q the queue
- * @return NULL on error, otherwise the entry
+ * Remove the top item in the priority queue
+ *
+ * @param pq The priority queue
+ * @return item from heap on success, NULL if queue is empty
  */
-void *git_pqueue_peek(git_pqueue *q);
-
-#endif /* PQUEUE_H */
-/** @} */
+extern void *git_pqueue_pop(git_pqueue *pq);
 
+#endif
diff --git a/src/push.c b/src/push.c
index 3c9d5bb35..be5ec1c0e 100644
--- a/src/push.c
+++ b/src/push.c
@@ -194,7 +194,10 @@ int git_push_add_refspec(git_push *push, const char *refspec)
 	return 0;
 }
 
-int git_push_update_tips(git_push *push)
+int git_push_update_tips(
+		git_push *push,
+		const git_signature *signature,
+		const char *reflog_message)
 {
 	git_buf remote_ref_name = GIT_BUF_INIT;
 	size_t i, j;
@@ -205,16 +208,14 @@ int git_push_update_tips(git_push *push)
 	int error = 0;
 
 	git_vector_foreach(&push->status, i, status) {
-		/* If this ref update was successful (ok, not ng), it will have an empty message */
-		if (status->msg)
-			continue;
+		int fire_callback = 1;
 
 		/* Find the corresponding remote ref */
 		fetch_spec = git_remote__matching_refspec(push->remote, status->ref);
 		if (!fetch_spec)
 			continue;
 
-		if ((error = git_refspec_transform_r(&remote_ref_name, fetch_spec, status->ref)) < 0)
+		if ((error = git_refspec_transform(&remote_ref_name, fetch_spec, status->ref)) < 0)
 			goto on_error;
 
 		/* Find matching  push ref spec */
@@ -227,22 +228,38 @@ int git_push_update_tips(git_push *push)
 		if (j == push->specs.length)
 			continue;
 
-		/* Update the remote ref */
-		if (git_oid_iszero(&push_spec->loid)) {
-			error = git_reference_lookup(&remote_ref, push->remote->repo, git_buf_cstr(&remote_ref_name));
+		/* If this ref update was successful (ok, not ng), it will have an empty message */
+		if (status->msg == NULL) {
+			/* Update the remote ref */
+			if (git_oid_iszero(&push_spec->loid)) {
+				error = git_reference_lookup(&remote_ref, push->remote->repo, git_buf_cstr(&remote_ref_name));
 
-			if (!error) {
-				if ((error = git_reference_delete(remote_ref)) < 0) {
+				if (error >= 0) {
+					error = git_reference_delete(remote_ref);
 					git_reference_free(remote_ref);
-					goto on_error;
 				}
-				git_reference_free(remote_ref);
-			} else if (error == GIT_ENOTFOUND)
-				giterr_clear();
-			else
+			} else {
+				error = git_reference_create(NULL, push->remote->repo,
+							git_buf_cstr(&remote_ref_name), &push_spec->loid, 1, signature,
+							reflog_message ? reflog_message : "update by push");
+			}
+		}
+
+		if (error < 0) {
+			if (error != GIT_ENOTFOUND)
 				goto on_error;
-		} else if ((error = git_reference_create(NULL, push->remote->repo, git_buf_cstr(&remote_ref_name), &push_spec->loid, 1)) < 0)
-			goto on_error;
+
+			giterr_clear();
+			fire_callback = 0;
+		}
+
+		if (fire_callback && push->remote->callbacks.update_tips) {
+			error = push->remote->callbacks.update_tips(git_buf_cstr(&remote_ref_name),
+						&push_spec->roid, &push_spec->loid, push->remote->callbacks.payload);
+
+			if (error < 0)
+				goto on_error;
+		}
 	}
 
 	error = 0;
@@ -541,10 +558,7 @@ static int queue_objects(git_push *push)
 	error = 0;
 
 on_error:
-	git_vector_foreach(&commits, i, oid)
-		git__free(oid);
-
-	git_vector_free(&commits);
+	git_vector_free_deep(&commits);
 	return error;
 }
 
@@ -649,7 +663,7 @@ int git_push_finish(git_push *push)
 	return 0;
 }
 
-int git_push_unpack_ok(git_push *push)
+int git_push_unpack_ok(const git_push *push)
 {
 	return push->unpack_ok;
 }
@@ -662,8 +676,9 @@ int git_push_status_foreach(git_push *push,
 	unsigned int i;
 
 	git_vector_foreach(&push->status, i, status) {
-		if (cb(status->ref, status->msg, data) < 0)
-			return GIT_EUSER;
+		int error = cb(status->ref, status->msg, data);
+		if (error)
+			return giterr_set_after_callback(error);
 	}
 
 	return 0;
@@ -674,9 +689,7 @@ void git_push_status_free(push_status *status)
 	if (status == NULL)
 		return;
 
-	if (status->msg)
-		git__free(status->msg);
-
+	git__free(status->msg);
 	git__free(status->ref);
 	git__free(status);
 }
@@ -702,3 +715,10 @@ void git_push_free(git_push *push)
 
 	git__free(push);
 }
+
+int git_push_init_options(git_push_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_push_options, GIT_PUSH_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/refdb.c b/src/refdb.c
index adb58806e..69bf74734 100644
--- a/src/refdb.c
+++ b/src/refdb.c
@@ -167,14 +167,14 @@ void git_refdb_iterator_free(git_reference_iterator *iter)
 	iter->free(iter);
 }
 
-int git_refdb_write(git_refdb *db, git_reference *ref, int force)
+int git_refdb_write(git_refdb *db, git_reference *ref, int force, const git_signature *who, const char *message, const git_oid *old_id, const char *old_target)
 {
 	assert(db && db->backend);
 
 	GIT_REFCOUNT_INC(db);
 	ref->db = db;
 
-	return db->backend->write(db->backend, ref, force);
+	return db->backend->write(db->backend, ref, force, who, message, old_id, old_target);
 }
 
 int git_refdb_rename(
@@ -182,12 +182,14 @@ int git_refdb_rename(
 	git_refdb *db,
 	const char *old_name,
 	const char *new_name,
-	int force)
+	int force,
+	const git_signature *who,
+	const char *message)
 {
 	int error;
 
 	assert(db && db->backend);
-	error = db->backend->rename(out, db->backend, old_name, new_name, force);
+	error = db->backend->rename(out, db->backend, old_name, new_name, force, who, message);
 	if (error < 0)
 		return error;
 
@@ -199,10 +201,10 @@ int git_refdb_rename(
 	return 0;
 }
 
-int git_refdb_delete(struct git_refdb *db, const char *ref_name)
+int git_refdb_delete(struct git_refdb *db, const char *ref_name, const git_oid *old_id, const char *old_target)
 {
 	assert(db && db->backend);
-	return db->backend->del(db->backend, ref_name);
+	return db->backend->del(db->backend, ref_name, old_id, old_target);
 }
 
 int git_refdb_reflog_read(git_reflog **out, git_refdb *db,  const char *name)
@@ -219,3 +221,24 @@ int git_refdb_reflog_read(git_reflog **out, git_refdb *db,  const char *name)
 
 	return 0;
 }
+
+int git_refdb_has_log(git_refdb *db, const char *refname)
+{
+	assert(db && refname);
+
+	return db->backend->has_log(db->backend, refname);
+}
+
+int git_refdb_ensure_log(git_refdb *db, const char *refname)
+{
+	assert(db && refname);
+
+	return db->backend->ensure_log(db->backend, refname);
+}
+
+int git_refdb_init_backend(git_refdb_backend *backend, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		backend, version, git_refdb_backend, GIT_REFDB_BACKEND_INIT);
+	return 0;
+}
diff --git a/src/refdb.h b/src/refdb.h
index 0ee60d911..cbad86faf 100644
--- a/src/refdb.h
+++ b/src/refdb.h
@@ -33,18 +33,24 @@ int git_refdb_rename(
 	git_refdb *db,
 	const char *old_name,
 	const char *new_name,
-	int force);
+	int force,
+	const git_signature *who,
+	const char *message);
 
 int git_refdb_iterator(git_reference_iterator **out, git_refdb *db, const char *glob);
 int git_refdb_iterator_next(git_reference **out, git_reference_iterator *iter);
 int git_refdb_iterator_next_name(const char **out, git_reference_iterator *iter);
 void git_refdb_iterator_free(git_reference_iterator *iter);
 
-int git_refdb_write(git_refdb *refdb, git_reference *ref, int force);
-int git_refdb_delete(git_refdb *refdb, const char *ref_name);
+int git_refdb_write(git_refdb *refdb, git_reference *ref, int force, const git_signature *who, const char *message, const git_oid *old_id, const char *old_target);
+int git_refdb_delete(git_refdb *refdb, const char *ref_name, const git_oid *old_id, const char *old_target);
 
 int git_refdb_reflog_read(git_reflog **out, git_refdb *db,  const char *name);
 int git_refdb_reflog_write(git_reflog *reflog);
 
+int git_refdb_has_log(git_refdb *db, const char *refname);
+int git_refdb_ensure_log(git_refdb *refdb, const char *refname);
+
+
 
 #endif
diff --git a/src/refdb_fs.c b/src/refdb_fs.c
index 62d5c1047..0e36ca8ac 100644
--- a/src/refdb_fs.c
+++ b/src/refdb_fs.c
@@ -21,6 +21,7 @@
 #include <git2/tag.h>
 #include <git2/object.h>
 #include <git2/refdb.h>
+#include <git2/branch.h>
 #include <git2/sys/refdb_backend.h>
 #include <git2/sys/refs.h>
 #include <git2/sys/reflog.h>
@@ -264,17 +265,25 @@ done:
 	return error;
 }
 
-static int _dirent_loose_load(void *data, git_buf *full_path)
+static int _dirent_loose_load(void *payload, git_buf *full_path)
 {
-	refdb_fs_backend *backend = (refdb_fs_backend *)data;
+	refdb_fs_backend *backend = payload;
 	const char *file_path;
 
 	if (git__suffixcmp(full_path->ptr, ".lock") == 0)
 		return 0;
 
-	if (git_path_isdir(full_path->ptr))
-		return git_path_direach(
+	if (git_path_isdir(full_path->ptr)) {
+		int error = git_path_direach(
 			full_path, backend->direach_flags, _dirent_loose_load, backend);
+		/* Race with the filesystem, ignore it */
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			return 0;
+		}
+
+		return error;
+	}
 
 	file_path = full_path->ptr + strlen(backend->path);
 
@@ -305,7 +314,7 @@ static int packed_loadloose(refdb_fs_backend *backend)
 
 	git_buf_free(&refs_path);
 
-	return (error == GIT_EUSER) ? -1 : error;
+	return error;
 }
 
 static int refdb_fs_backend__exists(
@@ -449,6 +458,7 @@ typedef struct {
 	git_pool pool;
 	git_vector loose;
 
+	git_sortedcache *cache;
 	size_t loose_pos;
 	size_t packed_pos;
 } refdb_fs_iter;
@@ -459,6 +469,7 @@ static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter)
 
 	git_vector_free(&iter->loose);
 	git_pool_clear(&iter->pool);
+	git_sortedcache_free(iter->cache);
 	git__free(iter);
 }
 
@@ -530,10 +541,14 @@ static int refdb_fs_backend__iterator_next(
 		giterr_clear();
 	}
 
-	git_sortedcache_rlock(backend->refcache);
+	if (!iter->cache) {
+		if ((error = git_sortedcache_copy(&iter->cache, backend->refcache, 1, NULL, NULL)) < 0)
+			return error;
+	}
 
-	while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
-		ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
+	error = GIT_ITEROVER;
+	while (iter->packed_pos < git_sortedcache_entrycount(iter->cache)) {
+		ref = git_sortedcache_entry(iter->cache, iter->packed_pos++);
 		if (!ref) /* stop now if another thread deleted refs and we past end */
 			break;
 
@@ -547,7 +562,6 @@ static int refdb_fs_backend__iterator_next(
 		break;
 	}
 
-	git_sortedcache_runlock(backend->refcache);
 	return error;
 }
 
@@ -570,10 +584,14 @@ static int refdb_fs_backend__iterator_next_name(
 		giterr_clear();
 	}
 
-	git_sortedcache_rlock(backend->refcache);
+	if (!iter->cache) {
+		if ((error = git_sortedcache_copy(&iter->cache, backend->refcache, 1, NULL, NULL)) < 0)
+			return error;
+	}
 
-	while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
-		ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
+	error = GIT_ITEROVER;
+	while (iter->packed_pos < git_sortedcache_entrycount(iter->cache)) {
+		ref = git_sortedcache_entry(iter->cache, iter->packed_pos++);
 		if (!ref) /* stop now if another thread deleted refs and we past end */
 			break;
 
@@ -587,7 +605,6 @@ static int refdb_fs_backend__iterator_next_name(
 		break;
 	}
 
-	git_sortedcache_runlock(backend->refcache);
 	return error;
 }
 
@@ -688,39 +705,44 @@ static int reference_path_available(
 	return 0;
 }
 
-static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
+static int loose_lock(git_filebuf *file, refdb_fs_backend *backend, const char *name)
 {
-	git_filebuf file = GIT_FILEBUF_INIT;
+        int error;
 	git_buf ref_path = GIT_BUF_INIT;
 
+	assert(file && backend && name);
+
 	/* Remove a possibly existing empty directory hierarchy
 	 * which name would collide with the reference name
 	 */
-	if (git_futils_rmdir_r(ref->name, backend->path, GIT_RMDIR_SKIP_NONEMPTY) < 0)
+	if (git_futils_rmdir_r(name, backend->path, GIT_RMDIR_SKIP_NONEMPTY) < 0)
 		return -1;
 
-	if (git_buf_joinpath(&ref_path, backend->path, ref->name) < 0)
+	if (git_buf_joinpath(&ref_path, backend->path, name) < 0)
 		return -1;
 
-	if (git_filebuf_open(&file, ref_path.ptr, GIT_FILEBUF_FORCE, GIT_REFS_FILE_MODE) < 0) {
-		git_buf_free(&ref_path);
-		return -1;
-	}
+	error = git_filebuf_open(file, ref_path.ptr, GIT_FILEBUF_FORCE, GIT_REFS_FILE_MODE);
 
 	git_buf_free(&ref_path);
+        return error;
+}
+
+static int loose_commit(git_filebuf *file, const git_reference *ref)
+{
+	assert(file && ref);
 
 	if (ref->type == GIT_REF_OID) {
 		char oid[GIT_OID_HEXSZ + 1];
 		git_oid_nfmt(oid, sizeof(oid), &ref->target.oid);
 
-		git_filebuf_printf(&file, "%s\n", oid);
+		git_filebuf_printf(file, "%s\n", oid);
 	} else if (ref->type == GIT_REF_SYMBOLIC) {
-		git_filebuf_printf(&file, GIT_SYMREF "%s\n", ref->target.symbolic);
+		git_filebuf_printf(file, GIT_SYMREF "%s\n", ref->target.symbolic);
 	} else {
 		assert(0); /* don't let this happen */
 	}
 
-	return git_filebuf_commit(&file);
+	return git_filebuf_commit(file);
 }
 
 /*
@@ -907,13 +929,155 @@ fail:
 	return -1;
 }
 
+static int reflog_append(refdb_fs_backend *backend, const git_reference *ref, const git_oid *old, const git_oid *new, const git_signature *author, const char *message);
+static int has_reflog(git_repository *repo, const char *name);
+
+/* We only write if it's under heads/, remotes/ or notes/ or if it already has a log */
+static int should_write_reflog(int *write, git_repository *repo, const char *name)
+{
+	int error, logall;
+
+	error = git_repository__cvar(&logall, repo, GIT_CVAR_LOGALLREFUPDATES);
+	if (error < 0)
+		return error;
+
+	/* Defaults to the opposite of the repo being bare */
+	if (logall == GIT_LOGALLREFUPDATES_UNSET)
+		logall = !git_repository_is_bare(repo);
+
+	if (!logall) {
+		*write = 0;
+	} else if (has_reflog(repo, name)) {
+		*write = 1;
+	} else if (!git__prefixcmp(name, GIT_REFS_HEADS_DIR) ||
+		   !git__strcmp(name, GIT_HEAD_FILE) ||
+		   !git__prefixcmp(name, GIT_REFS_REMOTES_DIR) ||
+		   !git__prefixcmp(name, GIT_REFS_NOTES_DIR)) {
+		*write = 1;
+	} else {
+		*write = 0;
+	}
+
+	return 0;
+}
+
+static int cmp_old_ref(int *cmp, git_refdb_backend *backend, const char *name,
+	const git_oid *old_id, const char *old_target)
+{
+	int error = 0;
+	git_reference *old_ref = NULL;
+
+	*cmp = 0;
+	/* It "matches" if there is no old value to compare against */
+	if (!old_id && !old_target)
+		return 0;
+
+	if ((error = refdb_fs_backend__lookup(&old_ref, backend, name)) < 0)
+		goto out;
+
+	/* If the types don't match, there's no way the values do */
+	if (old_id && old_ref->type != GIT_REF_OID) {
+		*cmp = -1;
+		goto out;
+	}
+	if (old_target && old_ref->type != GIT_REF_SYMBOLIC) {
+		*cmp = 1;
+		goto out;
+	}
+
+	if (old_id && old_ref->type == GIT_REF_OID)
+		*cmp = git_oid_cmp(old_id, &old_ref->target.oid);
+
+	if (old_target && old_ref->type == GIT_REF_SYMBOLIC)
+		*cmp = git__strcmp(old_target, old_ref->target.symbolic);
+
+out:
+	git_reference_free(old_ref);
+
+	return error;
+}
+
+/*
+ * The git.git comment regarding this, for your viewing pleasure:
+ *
+ * Special hack: If a branch is updated directly and HEAD
+ * points to it (may happen on the remote side of a push
+ * for example) then logically the HEAD reflog should be
+ * updated too.
+ * A generic solution implies reverse symref information,
+ * but finding all symrefs pointing to the given branch
+ * would be rather costly for this rare event (the direct
+ * update of a branch) to be worth it.  So let's cheat and
+ * check with HEAD only which should cover 99% of all usage
+ * scenarios (even 100% of the default ones).
+ */
+static int maybe_append_head(refdb_fs_backend *backend, const git_reference *ref, const git_signature *who, const char *message)
+{
+	int error;
+	git_oid old_id = {{0}};
+	git_reference *tmp = NULL, *head = NULL, *peeled = NULL;
+	const char *name;
+
+	if (ref->type == GIT_REF_SYMBOLIC)
+		return 0;
+
+	/* if we can't resolve, we use {0}*40 as old id */
+	git_reference_name_to_id(&old_id, backend->repo, ref->name);
+
+	if ((error = git_reference_lookup(&head, backend->repo, GIT_HEAD_FILE)) < 0)
+		return error;
+
+	if (git_reference_type(head) == GIT_REF_OID)
+		goto cleanup;
+
+	if ((error = git_reference_lookup(&tmp, backend->repo, GIT_HEAD_FILE)) < 0)
+		goto cleanup;
+
+	/* Go down the symref chain until we find the branch */
+	while (git_reference_type(tmp) == GIT_REF_SYMBOLIC) {
+		error = git_reference_lookup(&peeled, backend->repo, git_reference_symbolic_target(tmp));
+		if (error < 0)
+			break;
+
+		git_reference_free(tmp);
+		tmp = peeled;
+	}
+
+	if (error == GIT_ENOTFOUND) {
+		error = 0;
+		name = git_reference_symbolic_target(tmp);
+	} else if (error < 0) {
+		goto cleanup;
+	} else {
+		name = git_reference_name(tmp);
+	}
+
+	if (strcmp(name, ref->name))
+		goto cleanup;
+
+	error = reflog_append(backend, head, &old_id, git_reference_target(ref), who, message);
+
+cleanup:
+	git_reference_free(tmp);
+	git_reference_free(head);
+	return error;
+}
+
+
 static int refdb_fs_backend__write(
 	git_refdb_backend *_backend,
 	const git_reference *ref,
-	int force)
+	int force,
+	const git_signature *who,
+	const char *message,
+	const git_oid *old_id,
+	const char *old_target)
 {
 	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
-	int error;
+	git_filebuf file = GIT_FILEBUF_INIT;
+	int error = 0, cmp = 0, should_write;
+	const char *new_target = NULL;
+	const git_oid *new_id = NULL;
 
 	assert(backend);
 
@@ -921,21 +1085,78 @@ static int refdb_fs_backend__write(
 	if (error < 0)
 		return error;
 
-	return loose_write(backend, ref);
+	/* We need to perform the reflog append and old value check under the ref's lock */
+	if ((error = loose_lock(&file, backend, ref->name)) < 0)
+		return error;
+
+	if ((error = cmp_old_ref(&cmp, _backend, ref->name, old_id, old_target)) < 0)
+		goto on_error;
+
+	if (cmp) {
+		giterr_set(GITERR_REFERENCE, "old reference value does not match");
+		error = GIT_EMODIFIED;
+		goto on_error;
+	}
+
+	if (ref->type == GIT_REF_SYMBOLIC)
+		new_target = ref->target.symbolic;
+	else
+		new_id = &ref->target.oid;
+
+	error = cmp_old_ref(&cmp, _backend, ref->name, new_id, new_target);
+	if (error < 0 && error != GIT_ENOTFOUND)
+		goto on_error;
+
+	/* Don't update if we have the same value */
+	if (!error && !cmp) {
+		error = 0;
+		goto on_error; /* not really error */
+	}
+
+	if ((error = should_write_reflog(&should_write, backend->repo, ref->name)) < 0)
+		goto on_error;
+
+	if (should_write) {
+		if ((error = reflog_append(backend, ref, NULL, NULL, who, message)) < 0)
+			goto on_error;
+		if ((error = maybe_append_head(backend, ref, who, message)) < 0)
+			goto on_error;
+	}
+
+	return loose_commit(&file, ref);
+
+on_error:
+        git_filebuf_cleanup(&file);
+        return error;
 }
 
 static int refdb_fs_backend__delete(
 	git_refdb_backend *_backend,
-	const char *ref_name)
+	const char *ref_name,
+	const git_oid *old_id, const char *old_target)
 {
 	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_buf loose_path = GIT_BUF_INIT;
 	size_t pack_pos;
-	int error = 0;
+	git_filebuf file = GIT_FILEBUF_INIT;
+	int error = 0, cmp = 0;
 	bool loose_deleted = 0;
 
 	assert(backend && ref_name);
 
+	if ((error = loose_lock(&file, backend, ref_name)) < 0)
+		return error;
+
+	error = cmp_old_ref(&cmp, _backend, ref_name, old_id, old_target);
+	if (error < 0)
+		goto cleanup;
+
+	if (cmp) {
+		giterr_set(GITERR_REFERENCE, "old reference value does not match");
+		error = GIT_EMODIFIED;
+		goto cleanup;
+	}
+
 	/* If a loose reference exists, remove it from the filesystem */
 	if (git_buf_joinpath(&loose_path, backend->path, ref_name) < 0)
 		return -1;
@@ -948,14 +1169,14 @@ static int refdb_fs_backend__delete(
 	git_buf_free(&loose_path);
 
 	if (error != 0)
-		return error;
+		goto cleanup;
 
-	if (packed_reload(backend) < 0)
-		return -1;
+	if ((error = packed_reload(backend)) < 0)
+		goto cleanup;
 
 	/* If a packed reference exists, remove it from the packfile and repack */
-	if (git_sortedcache_wlock(backend->refcache) < 0)
-		return -1;
+	if ((error = git_sortedcache_wlock(backend->refcache)) < 0)
+		goto cleanup;
 
 	if (!(error = git_sortedcache_lookup_index(
 			&pack_pos, backend->refcache, ref_name)))
@@ -963,21 +1184,33 @@ static int refdb_fs_backend__delete(
 
 	git_sortedcache_wunlock(backend->refcache);
 
-	if (error == GIT_ENOTFOUND)
-		return loose_deleted ? 0 : ref_error_notfound(ref_name);
+	if (error == GIT_ENOTFOUND) {
+		error = loose_deleted ? 0 : ref_error_notfound(ref_name);
+		goto cleanup;
+	}
+
+	error = packed_write(backend);
 
-	return packed_write(backend);
+cleanup:
+	git_filebuf_cleanup(&file);
+
+	return error;
 }
 
+static int refdb_reflog_fs__rename(git_refdb_backend *_backend, const char *old_name, const char *new_name);
+
 static int refdb_fs_backend__rename(
 	git_reference **out,
 	git_refdb_backend *_backend,
 	const char *old_name,
 	const char *new_name,
-	int force)
+	int force,
+	const git_signature *who,
+	const char *message)
 {
 	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_reference *old, *new;
+	git_filebuf file = GIT_FILEBUF_INIT;
 	int error;
 
 	assert(backend);
@@ -987,7 +1220,7 @@ static int refdb_fs_backend__rename(
 		(error = refdb_fs_backend__lookup(&old, _backend, old_name)) < 0)
 		return error;
 
-	if ((error = refdb_fs_backend__delete(_backend, old_name)) < 0) {
+	if ((error = refdb_fs_backend__delete(_backend, old_name, NULL, NULL)) < 0) {
 		git_reference_free(old);
 		return error;
 	}
@@ -998,7 +1231,28 @@ static int refdb_fs_backend__rename(
 		return -1;
 	}
 
-	if ((error = loose_write(backend, new)) < 0 || out == NULL) {
+	if ((error = loose_lock(&file, backend, new->name)) < 0) {
+		git_reference_free(new);
+		return error;
+	}
+
+	/* Try to rename the refog; it's ok if the old doesn't exist */
+	error = refdb_reflog_fs__rename(_backend, old_name, new_name);
+	if (((error == 0) || (error == GIT_ENOTFOUND)) &&
+	    ((error = reflog_append(backend, new, NULL, NULL, who, message)) < 0)) {
+		git_reference_free(new);
+		git_filebuf_cleanup(&file);
+		return error;
+	}
+
+	if (error < 0) {
+		git_reference_free(new);
+		git_filebuf_cleanup(&file);
+		return error;
+	}
+
+
+	if ((error = loose_commit(&file, new)) < 0 || out == NULL) {
 		git_reference_free(new);
 		return error;
 	}
@@ -1173,7 +1427,7 @@ static int create_new_reflog_file(const char *filepath)
 		return error;
 
 	if ((fd = p_open(filepath,
-			O_WRONLY | O_CREAT | O_TRUNC,
+			O_WRONLY | O_CREAT,
 			GIT_REFLOG_FILE_MODE)) < 0)
 		return -1;
 
@@ -1182,7 +1436,54 @@ static int create_new_reflog_file(const char *filepath)
 
 GIT_INLINE(int) retrieve_reflog_path(git_buf *path, git_repository *repo, const char *name)
 {
-	return git_buf_join_n(path, '/', 3, repo->path_repository, GIT_REFLOG_DIR, name);
+	return git_buf_join3(path, '/', repo->path_repository, GIT_REFLOG_DIR, name);
+}
+
+static int refdb_reflog_fs__ensure_log(git_refdb_backend *_backend, const char *name)
+{
+	refdb_fs_backend *backend;
+	git_repository *repo;
+	git_buf path = GIT_BUF_INIT;
+	int error;
+
+	assert(_backend && name);
+
+	backend = (refdb_fs_backend *) _backend;
+	repo = backend->repo;
+
+	if ((error = retrieve_reflog_path(&path, repo, name)) < 0)
+		return error;
+
+	error = create_new_reflog_file(git_buf_cstr(&path));
+	git_buf_free(&path);
+
+	return error;
+}
+
+static int has_reflog(git_repository *repo, const char *name)
+{
+	int ret = 0;
+	git_buf path = GIT_BUF_INIT;
+
+	if (retrieve_reflog_path(&path, repo, name) < 0)
+		goto cleanup;
+
+	ret = git_path_isfile(git_buf_cstr(&path));
+
+cleanup:
+	git_buf_free(&path);
+	return ret;
+}
+
+static int refdb_reflog_fs__has_log(git_refdb_backend *_backend, const char *name)
+{
+	refdb_fs_backend *backend;
+
+	assert(_backend && name);
+
+	backend = (refdb_fs_backend *) _backend;
+
+	return has_reflog(backend->repo, name);
 }
 
 static int refdb_reflog_fs__read(git_reflog **out, git_refdb_backend *_backend, const char *name)
@@ -1264,34 +1565,48 @@ static int serialize_reflog_entry(
 	return git_buf_oom(buf);
 }
 
+static int lock_reflog(git_filebuf *file, refdb_fs_backend *backend, const char *refname)
+{
+	git_repository *repo;
+	git_buf log_path = GIT_BUF_INIT;
+	int error;
+
+	repo = backend->repo;
+
+	if (retrieve_reflog_path(&log_path, repo, refname) < 0)
+		return -1;
+
+	if (!git_path_isfile(git_buf_cstr(&log_path))) {
+		giterr_set(GITERR_INVALID,
+			"Log file for reference '%s' doesn't exist.", refname);
+		error = -1;
+		goto cleanup;
+	}
+
+	error = git_filebuf_open(file, git_buf_cstr(&log_path), 0, GIT_REFLOG_FILE_MODE);
+
+cleanup:
+	git_buf_free(&log_path);
+
+	return error;
+}
+
 static int refdb_reflog_fs__write(git_refdb_backend *_backend, git_reflog *reflog)
 {
 	int error = -1;
 	unsigned int i;
 	git_reflog_entry *entry;
-	git_repository *repo;
 	refdb_fs_backend *backend;
-	git_buf log_path = GIT_BUF_INIT;
 	git_buf log = GIT_BUF_INIT;
 	git_filebuf fbuf = GIT_FILEBUF_INIT;
 
 	assert(_backend && reflog);
 
 	backend = (refdb_fs_backend *) _backend;
-	repo = backend->repo;
 
-	if (retrieve_reflog_path(&log_path, repo, reflog->ref_name) < 0)
+	if ((error = lock_reflog(&fbuf, backend, reflog->ref_name)) < 0)
 		return -1;
 
-	if (!git_path_isfile(git_buf_cstr(&log_path))) {
-		giterr_set(GITERR_INVALID,
-			"Log file for reference '%s' doesn't exist.", reflog->ref_name);
-		goto cleanup;
-	}
-
-	if ((error = git_filebuf_open(&fbuf, git_buf_cstr(&log_path), 0, GIT_REFLOG_FILE_MODE)) < 0)
-		goto cleanup;
-
 	git_vector_foreach(&reflog->entries, i, entry) {
 		if (serialize_reflog_entry(&log, &(entry->oid_old), &(entry->oid_cur), entry->committer, entry->msg) < 0)
 			goto cleanup;
@@ -1308,7 +1623,70 @@ cleanup:
 
 success:
 	git_buf_free(&log);
-	git_buf_free(&log_path);
+
+	return error;
+}
+
+/* Append to the reflog, must be called under reference lock */
+static int reflog_append(refdb_fs_backend *backend, const git_reference *ref, const git_oid *old, const git_oid *new, const git_signature *who, const char *message)
+{
+	int error, is_symbolic;
+	git_oid old_id = {{0}}, new_id = {{0}};
+	git_buf buf = GIT_BUF_INIT, path = GIT_BUF_INIT;
+	git_repository *repo = backend->repo;
+
+	is_symbolic = ref->type == GIT_REF_SYMBOLIC;
+
+	/* "normal" symbolic updates do not write */
+	if (is_symbolic &&
+	    strcmp(ref->name, GIT_HEAD_FILE) &&
+	    !(old && new))
+		return 0;
+
+	/* From here on is_symoblic also means that it's HEAD */
+
+	if (old) {
+		git_oid_cpy(&old_id, old);
+	} else {
+		error = git_reference_name_to_id(&old_id, repo, ref->name);
+		if (error < 0 && error != GIT_ENOTFOUND)
+			return error;
+	}
+
+	if (new) {
+		git_oid_cpy(&new_id, new);
+	} else {
+		if (!is_symbolic) {
+			git_oid_cpy(&new_id, git_reference_target(ref));
+		} else {
+			error = git_reference_name_to_id(&new_id, repo, git_reference_symbolic_target(ref));
+			if (error < 0 && error != GIT_ENOTFOUND)
+				return error;
+			/* detaching HEAD does not create an entry */
+			if (error == GIT_ENOTFOUND)
+				return 0;
+
+			giterr_clear();
+		}
+	}
+
+	if ((error = serialize_reflog_entry(&buf, &old_id, &new_id, who, message)) < 0)
+		goto cleanup;
+
+	if ((error = retrieve_reflog_path(&path, repo, ref->name)) < 0)
+		goto cleanup;
+
+	if (((error = git_futils_mkpath2file(git_buf_cstr(&path), 0777)) < 0) &&
+	    (error != GIT_EEXISTS)) {
+		goto cleanup;
+	}
+
+	error = git_futils_writebuffer(&buf, git_buf_cstr(&path), O_WRONLY|O_CREAT|O_APPEND, GIT_REFLOG_FILE_MODE);
+
+cleanup:
+	git_buf_free(&buf);
+	git_buf_free(&path);
+
 	return error;
 }
 
@@ -1340,6 +1718,11 @@ static int refdb_reflog_fs__rename(git_refdb_backend *_backend, const char *old_
 	if (git_buf_joinpath(&new_path, git_buf_cstr(&temp_path), git_buf_cstr(&normalized)) < 0)
 		return -1;
 
+	if (!git_path_exists(git_buf_cstr(&old_path))) {
+		error = GIT_ENOTFOUND;
+		goto cleanup;
+	}
+
 	/*
 	 * Move the reflog to a temporary place. This two-phase renaming is required
 	 * in order to cope with funny renaming use cases when one tries to move a reference
@@ -1454,6 +1837,8 @@ int git_refdb_backend_fs(
 	backend->parent.del = &refdb_fs_backend__delete;
 	backend->parent.rename = &refdb_fs_backend__rename;
 	backend->parent.compress = &refdb_fs_backend__compress;
+	backend->parent.has_log = &refdb_reflog_fs__has_log;
+	backend->parent.ensure_log = &refdb_reflog_fs__ensure_log;
 	backend->parent.free = &refdb_fs_backend__free;
 	backend->parent.reflog_read = &refdb_reflog_fs__read;
 	backend->parent.reflog_write = &refdb_reflog_fs__write;
diff --git a/src/reflog.c b/src/reflog.c
index cebb87d86..8e41621ea 100644
--- a/src/reflog.c
+++ b/src/reflog.c
@@ -82,7 +82,7 @@ int git_reflog_append(git_reflog *reflog, const git_oid *new_oid, const git_sign
 	entry = git__calloc(1, sizeof(git_reflog_entry));
 	GITERR_CHECK_ALLOC(entry);
 
-	if ((entry->committer = git_signature_dup(committer)) == NULL)
+	if ((git_signature_dup(&entry->committer, committer)) < 0)
 		goto cleanup;
 
 	if (msg != NULL) {
@@ -230,22 +230,3 @@ int git_reflog_drop(git_reflog *reflog, size_t idx, int rewrite_previous_entry)
 
 	return 0;
 }
-
-int git_reflog_append_to(git_repository *repo, const char *name, const git_oid *id,
-			 const git_signature *committer, const char *msg)
-{
-	int error;
-	git_reflog *reflog;
-
-	if ((error = git_reflog_read(&reflog, repo, name)) < 0)
-		return error;
-
-	if ((error = git_reflog_append(reflog, id, committer, msg)) < 0)
-		goto cleanup;
-
-	error = git_reflog_write(reflog);
-
-cleanup:
-	git_reflog_free(reflog);
-	return error;
-}
diff --git a/src/refs.c b/src/refs.c
index 472a79890..1603876da 100644
--- a/src/refs.c
+++ b/src/refs.c
@@ -21,6 +21,7 @@
 #include <git2/refs.h>
 #include <git2/refdb.h>
 #include <git2/sys/refs.h>
+#include <git2/signature.h>
 
 GIT__USE_STRMAP;
 
@@ -115,7 +116,26 @@ void git_reference_free(git_reference *reference)
 
 int git_reference_delete(git_reference *ref)
 {
-	return git_refdb_delete(ref->db, ref->name);
+	const git_oid *old_id = NULL;
+	const char *old_target = NULL;
+
+	if (ref->type == GIT_REF_OID)
+		old_id = &ref->target.oid;
+	else
+		old_target = ref->target.symbolic;
+
+	return git_refdb_delete(ref->db, ref->name, old_id, old_target);
+}
+
+int git_reference_remove(git_repository *repo, const char *name)
+{
+	git_refdb *db;
+	int error;
+
+	if ((error = git_repository_refdb__weakptr(&db, repo)) < 0)
+		return error;
+
+	return git_refdb_delete(db, name, NULL, NULL);
 }
 
 int git_reference_lookup(git_reference **ref_out,
@@ -139,8 +159,7 @@ int git_reference_name_to_id(
 }
 
 static int reference_normalize_for_repo(
-	char *out,
-	size_t out_size,
+	git_refname_t out,
 	git_repository *repo,
 	const char *name)
 {
@@ -151,7 +170,7 @@ static int reference_normalize_for_repo(
 		precompose)
 		flags |= GIT_REF_FORMAT__PRECOMPOSE_UNICODE;
 
-	return git_reference_normalize_name(out, out_size, name, flags);
+	return git_reference_normalize_name(out, GIT_REFNAME_MAX, name, flags);
 }
 
 int git_reference_lookup_resolved(
@@ -160,7 +179,7 @@ int git_reference_lookup_resolved(
 	const char *name,
 	int max_nesting)
 {
-	char scan_name[GIT_REFNAME_MAX];
+	git_refname_t scan_name;
 	git_ref_t scan_type;
 	int error = 0, nesting;
 	git_reference *ref = NULL;
@@ -177,8 +196,7 @@ int git_reference_lookup_resolved(
 
 	scan_type = GIT_REF_SYMBOLIC;
 
-	if ((error = reference_normalize_for_repo(
-			scan_name, sizeof(scan_name), repo, name)) < 0)
+	if ((error = reference_normalize_for_repo(scan_name, repo, name)) < 0)
 		return error;
 
 	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
@@ -328,17 +346,24 @@ static int reference__create(
 	const char *name,
 	const git_oid *oid,
 	const char *symbolic,
-	int force)
+	int force,
+	const git_signature *signature,
+	const char *log_message,
+	const git_oid *old_id,
+	const char *old_target)
 {
-	char normalized[GIT_REFNAME_MAX];
+	git_refname_t normalized;
 	git_refdb *refdb;
 	git_reference *ref = NULL;
 	int error = 0;
 
+	assert(repo && name);
+	assert(symbolic || signature);
+
 	if (ref_out)
 		*ref_out = NULL;
 
-	error = git_reference__normalize_name_lax(normalized, sizeof(normalized), name);
+	error = reference_normalize_for_repo(normalized, repo, name);
 	if (error < 0)
 		return error;
 
@@ -347,15 +372,33 @@ static int reference__create(
 		return error;
 
 	if (oid != NULL) {
+		git_odb *odb;
+
 		assert(symbolic == NULL);
+
+		/* Sanity check the reference being created - target must exist. */
+		if ((error = git_repository_odb__weakptr(&odb, repo)) < 0)
+			return error;
+
+		if (!git_odb_exists(odb, oid)) {
+			giterr_set(GITERR_REFERENCE,
+				"Target OID for the reference doesn't exist on the repository");
+			return -1;
+		}
+
 		ref = git_reference__alloc(normalized, oid, NULL);
 	} else {
-		ref = git_reference__alloc_symbolic(normalized, symbolic);
+		git_refname_t normalized_target;
+
+		if ((error = reference_normalize_for_repo(normalized_target, repo, symbolic)) < 0)
+			return error;
+
+		ref = git_reference__alloc_symbolic(normalized, normalized_target);
 	}
 
 	GITERR_CHECK_ALLOC(ref);
 
-	if ((error = git_refdb_write(refdb, ref, force)) < 0) {
+	if ((error = git_refdb_write(refdb, ref, force, signature, log_message, old_id, old_target)) < 0) {
 		git_reference_free(ref);
 		return error;
 	}
@@ -368,29 +411,88 @@ static int reference__create(
 	return 0;
 }
 
+static int log_signature(git_signature **out, git_repository *repo)
+{
+	int error;
+	git_signature *who;
+
+	if(((error = git_signature_default(&who, repo)) < 0) &&
+	   ((error = git_signature_now(&who, "unknown", "unknown")) < 0))
+		return error;
+
+	*out = who;
+	return 0;
+}
+
+int git_reference_create_matching(
+	git_reference **ref_out,
+	git_repository *repo,
+	const char *name,
+	const git_oid *id,
+	int force,
+	const git_oid *old_id,
+	const git_signature *signature,
+	const char *log_message)
+
+{
+	int error;
+	git_signature *who = NULL;
+	
+	assert(id);
+
+	if (!signature) {
+		if ((error = log_signature(&who, repo)) < 0)
+			return error;
+		else
+			signature = who;
+	}
+
+	error = reference__create(
+		ref_out, repo, name, id, NULL, force, signature, log_message, old_id, NULL);
+
+	git_signature_free(who);
+	return error;
+}
+
 int git_reference_create(
 	git_reference **ref_out,
 	git_repository *repo,
 	const char *name,
-	const git_oid *oid,
-	int force)
+	const git_oid *id,
+	int force,
+	const git_signature *signature,
+	const char *log_message)
 {
-	git_odb *odb;
-	int error = 0;
+        return git_reference_create_matching(ref_out, repo, name, id, force, NULL, signature, log_message);
+}
 
-	assert(repo && name && oid);
+int git_reference_symbolic_create_matching(
+	git_reference **ref_out,
+	git_repository *repo,
+	const char *name,
+	const char *target,
+	int force,
+	const char *old_target,
+	const git_signature *signature,
+	const char *log_message)
+{
+	int error;
+	git_signature *who = NULL;
 
-	/* Sanity check the reference being created - target must exist. */
-	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0)
-		return error;
+	assert(target);
 
-	if (!git_odb_exists(odb, oid)) {
-		giterr_set(GITERR_REFERENCE,
-			"Target OID for the reference doesn't exist on the repository");
-		return -1;
+	if (!signature) {
+		if ((error = log_signature(&who, repo)) < 0)
+			return error;
+		else
+			signature = who;
 	}
 
-	return reference__create(ref_out, repo, name, oid, NULL, force);
+	error = reference__create(
+		ref_out, repo, name, NULL, target, force, signature, log_message, NULL, old_target);
+
+	git_signature_free(who);
+	return error;
 }
 
 int git_reference_symbolic_create(
@@ -398,95 +500,127 @@ int git_reference_symbolic_create(
 	git_repository *repo,
 	const char *name,
 	const char *target,
-	int force)
+	int force,
+	const git_signature *signature,
+	const char *log_message)
 {
-	char normalized[GIT_REFNAME_MAX];
-	int error = 0;
-
-	assert(repo && name && target);
+	return git_reference_symbolic_create_matching(ref_out, repo, name, target, force, NULL, signature, log_message);
+}
 
-	if ((error = git_reference__normalize_name_lax(
-		normalized, sizeof(normalized), target)) < 0)
-		return error;
+static int ensure_is_an_updatable_direct_reference(git_reference *ref)
+{
+	if (ref->type == GIT_REF_OID)
+		return 0;
 
-	return reference__create(ref_out, repo, name, NULL, normalized, force);
+	giterr_set(GITERR_REFERENCE, "Cannot set OID on symbolic reference");
+	return -1;
 }
 
 int git_reference_set_target(
 	git_reference **out,
 	git_reference *ref,
-	const git_oid *id)
+	const git_oid *id,
+	const git_signature *signature,
+	const char *log_message)
 {
+	int error;
+	git_repository *repo;
+
 	assert(out && ref && id);
 
-	if (ref->type != GIT_REF_OID) {
-		giterr_set(GITERR_REFERENCE, "Cannot set OID on symbolic reference");
-		return -1;
-	}
+	repo = ref->db->repo;
+
+	if ((error = ensure_is_an_updatable_direct_reference(ref)) < 0)
+		return error;
 
-	return git_reference_create(out, ref->db->repo, ref->name, id, 1);
+	return git_reference_create_matching(out, repo, ref->name, id, 1, &ref->target.oid, signature, log_message);
+}
+
+static int ensure_is_an_updatable_symbolic_reference(git_reference *ref)
+{
+	if (ref->type == GIT_REF_SYMBOLIC)
+		return 0;
+
+	giterr_set(GITERR_REFERENCE, "Cannot set symbolic target on a direct reference");
+	return -1;
 }
 
 int git_reference_symbolic_set_target(
 	git_reference **out,
 	git_reference *ref,
-	const char *target)
+	const char *target,
+	const git_signature *signature,
+	const char *log_message)
 {
+	int error;
+
 	assert(out && ref && target);
 
-	if (ref->type != GIT_REF_SYMBOLIC) {
-		giterr_set(GITERR_REFERENCE,
-			"Cannot set symbolic target on a direct reference");
-		return -1;
-	}
+	if ((error = ensure_is_an_updatable_symbolic_reference(ref)) < 0)
+		return error;
 
-	return git_reference_symbolic_create(out, ref->db->repo, ref->name, target, 1);
+	return git_reference_symbolic_create_matching(
+		out, ref->db->repo, ref->name, target, 1, ref->target.symbolic, signature, log_message);
 }
 
-int git_reference_rename(
-	git_reference **out,
-	git_reference *ref,
-	const char *new_name,
-	int force)
+static int reference__rename(git_reference **out, git_reference *ref, const char *new_name, int force,
+				 const git_signature *signature, const char *message)
 {
-	unsigned int normalization_flags;
-	char normalized[GIT_REFNAME_MAX];
+	git_refname_t normalized;
 	bool should_head_be_updated = false;
 	int error = 0;
-	int reference_has_log;
 
-	normalization_flags = ref->type == GIT_REF_SYMBOLIC ?
-		GIT_REF_FORMAT_ALLOW_ONELEVEL : GIT_REF_FORMAT_NORMAL;
+	assert(ref && new_name && signature);
 
-	if ((error = git_reference_normalize_name(
-			normalized, sizeof(normalized), new_name, normalization_flags)) < 0)
+	if ((error = reference_normalize_for_repo(
+			normalized, git_reference_owner(ref), new_name)) < 0)
 		return error;
 
+
 	/* Check if we have to update HEAD. */
 	if ((error = git_branch_is_head(ref)) < 0)
 		return error;
 
 	should_head_be_updated = (error > 0);
 
-	if ((error = git_refdb_rename(out, ref->db, ref->name, new_name, force)) < 0)
+	if ((error = git_refdb_rename(out, ref->db, ref->name, normalized, force, signature, message)) < 0)
 		return error;
 
-	/* Update HEAD it was poiting to the reference being renamed. */
+	/* Update HEAD it was pointing to the reference being renamed */
 	if (should_head_be_updated &&
-		(error = git_repository_set_head(ref->db->repo, new_name)) < 0) {
+		(error = git_repository_set_head(ref->db->repo, normalized, signature, message)) < 0) {
 		giterr_set(GITERR_REFERENCE, "Failed to update HEAD after renaming reference");
 		return error;
 	}
 
-	/* Rename the reflog file, if it exists. */
-	reference_has_log = git_reference_has_log(ref);
-	if (reference_has_log < 0)
-		return reference_has_log;
+	return 0;
+}
+
+
+int git_reference_rename(
+	git_reference **out,
+	git_reference *ref,
+	const char *new_name,
+	int force,
+	const git_signature *signature,
+	const char *log_message)
+{
+	git_signature *who = (git_signature*)signature;
+	int error;
 
-	if (reference_has_log && (error = git_reflog_rename(git_reference_owner(ref), git_reference_name(ref), new_name)) < 0)
+	/* Should we return an error if there is no default? */
+	if (!who &&
+	    ((error = git_signature_default(&who, ref->db->repo)) < 0) &&
+	    ((error = git_signature_now(&who, "unknown", "unknown")) < 0)) {
 		return error;
+	}
 
-	return 0;
+	error = reference__rename(out, ref, new_name, force, who, log_message);
+
+	if (!signature)
+		git_signature_free(who);
+
+	return error;
 }
 
 int git_reference_resolve(git_reference **ref_out, const git_reference *ref)
@@ -513,20 +647,19 @@ int git_reference_foreach(
 	git_reference *ref;
 	int error;
 
-	if (git_reference_iterator_new(&iter, repo) < 0)
-		return -1;
+	if ((error = git_reference_iterator_new(&iter, repo)) < 0)
+		return error;
 
-	while ((error = git_reference_next(&ref, iter)) == 0) {
-		if (callback(ref, payload)) {
-			error = GIT_EUSER;
-			goto out;
+	while (!(error = git_reference_next(&ref, iter))) {
+		if ((error = callback(ref, payload)) != 0) {
+			giterr_set_after_callback(error);
+			break;
 		}
 	}
 
 	if (error == GIT_ITEROVER)
 		error = 0;
 
-out:
 	git_reference_iterator_free(iter);
 	return error;
 }
@@ -540,20 +673,19 @@ int git_reference_foreach_name(
 	const char *refname;
 	int error;
 
-	if (git_reference_iterator_new(&iter, repo) < 0)
-		return -1;
+	if ((error = git_reference_iterator_new(&iter, repo)) < 0)
+		return error;
 
-	while ((error = git_reference_next_name(&refname, iter)) == 0) {
-		if (callback(refname, payload)) {
-			error = GIT_EUSER;
-			goto out;
+	while (!(error = git_reference_next_name(&refname, iter))) {
+		if ((error = callback(refname, payload)) != 0) {
+			giterr_set_after_callback(error);
+			break;
 		}
 	}
 
 	if (error == GIT_ITEROVER)
 		error = 0;
 
-out:
 	git_reference_iterator_free(iter);
 	return error;
 }
@@ -568,20 +700,19 @@ int git_reference_foreach_glob(
 	const char *refname;
 	int error;
 
-	if (git_reference_iterator_glob_new(&iter, repo, glob) < 0)
-		return -1;
+	if ((error = git_reference_iterator_glob_new(&iter, repo, glob)) < 0)
+		return error;
 
-	while ((error = git_reference_next_name(&refname, iter)) == 0) {
-		if (callback(refname, payload)) {
-			error = GIT_EUSER;
-			goto out;
+	while (!(error = git_reference_next_name(&refname, iter))) {
+		if ((error = callback(refname, payload)) != 0) {
+			giterr_set_after_callback(error);
+			break;
 		}
 	}
 
 	if (error == GIT_ITEROVER)
 		error = 0;
 
-out:
 	git_reference_iterator_free(iter);
 	return error;
 }
@@ -619,12 +750,17 @@ int git_reference_next_name(const char **out, git_reference_iterator *iter)
 
 void git_reference_iterator_free(git_reference_iterator *iter)
 {
+	if (iter == NULL)
+		return;
+
 	git_refdb_iterator_free(iter);
 }
 
 static int cb__reflist_add(const char *ref, void *data)
 {
-	return git_vector_insert((git_vector *)data, git__strdup(ref));
+	char *name = git__strdup(ref);
+	GITERR_CHECK_ALLOC(name);
+	return git_vector_insert((git_vector *)data, name);
 }
 
 int git_reference_list(
@@ -647,8 +783,8 @@ int git_reference_list(
 		return -1;
 	}
 
-	array->strings = (char **)ref_list.contents;
-	array->count = ref_list.length;
+	array->strings = (char **)git_vector_detach(&array->count, NULL, &ref_list);
+
 	return 0;
 }
 
@@ -875,20 +1011,11 @@ cleanup:
 	return error;
 }
 
-int git_reference__normalize_name_lax(
-	char *buffer_out,
-	size_t out_size,
-	const char *name)
-{
-	return git_reference_normalize_name(
-		buffer_out,
-		out_size,
-		name,
-		GIT_REF_FORMAT_ALLOW_ONELEVEL);
-}
 #define GIT_REF_TYPEMASK (GIT_REF_OID | GIT_REF_SYMBOLIC)
 
-int git_reference_cmp(git_reference *ref1, git_reference *ref2)
+int git_reference_cmp(
+	const git_reference *ref1,
+	const git_reference *ref2)
 {
 	git_ref_t type1, type2;
 	assert(ref1 && ref2);
@@ -910,7 +1037,9 @@ static int reference__update_terminal(
 	git_repository *repo,
 	const char *ref_name,
 	const git_oid *oid,
-	int nesting)
+	int nesting,
+	const git_signature *signature,
+	const char *log_message)
 {
 	git_reference *ref;
 	int error = 0;
@@ -925,7 +1054,7 @@ static int reference__update_terminal(
 	/* If we haven't found the reference at all, create a new reference. */
 	if (error == GIT_ENOTFOUND) {
 		giterr_clear();
-		return git_reference_create(NULL, repo, ref_name, oid, 0);
+		return git_reference_create(NULL, repo, ref_name, oid, 0, signature, log_message);
 	}
 
 	if (error < 0)
@@ -934,11 +1063,13 @@ static int reference__update_terminal(
 	/* If the ref is a symbolic reference, follow its target. */
 	if (git_reference_type(ref) == GIT_REF_SYMBOLIC) {
 		error = reference__update_terminal(repo, git_reference_symbolic_target(ref), oid,
-			nesting+1);
+			nesting+1, signature, log_message);
 		git_reference_free(ref);
 	} else {
+		/* If we're not moving the target, don't recreate the ref */
+		if (0 != git_oid_cmp(git_reference_target(ref), oid))
+			error = git_reference_create(NULL, repo, ref_name, oid, 1, signature, log_message);
 		git_reference_free(ref);
-		error = git_reference_create(NULL, repo, ref_name, oid, 1);
 	}
 
 	return error;
@@ -952,27 +1083,37 @@ static int reference__update_terminal(
 int git_reference__update_terminal(
 	git_repository *repo,
 	const char *ref_name,
-	const git_oid *oid)
+	const git_oid *oid,
+	const git_signature *signature,
+	const char *log_message)
 {
-	return reference__update_terminal(repo, ref_name, oid, 0);
+	return reference__update_terminal(repo, ref_name, oid, 0, signature, log_message);
 }
 
-int git_reference_has_log(
-	git_reference *ref)
+int git_reference_has_log(git_repository *repo, const char *refname)
 {
-	git_buf path = GIT_BUF_INIT;
-	int result;
+	int error;
+	git_refdb *refdb;
 
-	assert(ref);
+	assert(repo && refname);
 
-	if (git_buf_join_n(&path, '/', 3, ref->db->repo->path_repository,
-		GIT_REFLOG_DIR, ref->name) < 0)
-		return -1;
+	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
+		return error;
 
-	result = git_path_isfile(git_buf_cstr(&path));
-	git_buf_free(&path);
+	return git_refdb_has_log(refdb, refname);
+}
 
-	return result;
+int git_reference_ensure_log(git_repository *repo, const char *refname)
+{
+	int error;
+	git_refdb *refdb;
+
+	assert(repo && refname);
+
+	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
+		return error;
+
+	return git_refdb_ensure_log(refdb, refname);
 }
 
 int git_reference__is_branch(const char *ref_name)
@@ -980,7 +1121,7 @@ int git_reference__is_branch(const char *ref_name)
 	return git__prefixcmp(ref_name, GIT_REFS_HEADS_DIR) == 0;
 }
 
-int git_reference_is_branch(git_reference *ref)
+int git_reference_is_branch(const git_reference *ref)
 {
 	assert(ref);
 	return git_reference__is_branch(ref->name);
@@ -991,7 +1132,7 @@ int git_reference__is_remote(const char *ref_name)
 	return git__prefixcmp(ref_name, GIT_REFS_REMOTES_DIR) == 0;
 }
 
-int git_reference_is_remote(git_reference *ref)
+int git_reference_is_remote(const git_reference *ref)
 {
 	assert(ref);
 	return git_reference__is_remote(ref->name);
@@ -1002,12 +1143,23 @@ int git_reference__is_tag(const char *ref_name)
 	return git__prefixcmp(ref_name, GIT_REFS_TAGS_DIR) == 0;
 }
 
-int git_reference_is_tag(git_reference *ref)
+int git_reference_is_tag(const git_reference *ref)
 {
 	assert(ref);
 	return git_reference__is_tag(ref->name);
 }
 
+int git_reference__is_note(const char *ref_name)
+{
+	return git__prefixcmp(ref_name, GIT_REFS_NOTES_DIR) == 0;
+}
+
+int git_reference_is_note(const git_reference *ref)
+{
+	assert(ref);
+	return git_reference__is_note(ref->name);
+}
+
 static int peel_error(int error, git_reference *ref, const char* msg)
 {
 	giterr_set(
@@ -1076,7 +1228,7 @@ int git_reference_is_valid_name(const char *refname)
 	return git_reference__is_valid_name(refname, GIT_REF_FORMAT_ALLOW_ONELEVEL);
 }
 
-const char *git_reference_shorthand(git_reference *ref)
+const char *git_reference_shorthand(const git_reference *ref)
 {
 	const char *name = ref->name;
 
diff --git a/src/refs.h b/src/refs.h
index 80c7703fc..f75a4bf7e 100644
--- a/src/refs.h
+++ b/src/refs.h
@@ -19,6 +19,7 @@
 #define GIT_REFS_HEADS_DIR GIT_REFS_DIR "heads/"
 #define GIT_REFS_TAGS_DIR GIT_REFS_DIR "tags/"
 #define GIT_REFS_REMOTES_DIR GIT_REFS_DIR "remotes/"
+#define GIT_REFS_NOTES_DIR GIT_REFS_DIR "notes/"
 #define GIT_REFS_DIR_MODE 0777
 #define GIT_REFS_FILE_MODE 0666
 
@@ -50,6 +51,8 @@
 
 #define GIT_REFNAME_MAX 1024
 
+typedef char git_refname_t[GIT_REFNAME_MAX];
+
 struct git_reference {
 	git_refdb *db;
 	git_ref_t type;
@@ -65,9 +68,8 @@ struct git_reference {
 
 git_reference *git_reference__set_name(git_reference *ref, const char *name);
 
-int git_reference__normalize_name_lax(char *buffer_out, size_t out_size, const char *name);
 int git_reference__normalize_name(git_buf *buf, const char *name, unsigned int flags);
-int git_reference__update_terminal(git_repository *repo, const char *ref_name, const git_oid *oid);
+int git_reference__update_terminal(git_repository *repo, const char *ref_name, const git_oid *oid, const git_signature *signature, const char *log_message);
 int git_reference__is_valid_name(const char *refname, unsigned int flags);
 int git_reference__is_branch(const char *ref_name);
 int git_reference__is_remote(const char *ref_name);
diff --git a/src/refspec.c b/src/refspec.c
index a97340071..fa60aa7aa 100644
--- a/src/refspec.c
+++ b/src/refspec.c
@@ -178,54 +178,6 @@ int git_refspec_dst_matches(const git_refspec *refspec, const char *refname)
 	return (p_fnmatch(refspec->dst, refname, 0) == 0);
 }
 
-static int refspec_transform_internal(char *out, size_t outlen, const char *from, const char *to, const char *name)
-{
-	size_t baselen, namelen;
-
-	baselen = strlen(to);
-	if (outlen <= baselen) {
-		giterr_set(GITERR_INVALID, "Reference name too long");
-		return GIT_EBUFS;
-	}
-
-	/*
-	 * No '*' at the end means that it's mapped to one specific local
-	 * branch, so no actual transformation is needed.
-	 */
-	if (to[baselen - 1] != '*') {
-		memcpy(out, to, baselen + 1); /* include '\0' */
-		return 0;
-	}
-
-	/* There's a '*' at the end, so remove its length */
-	baselen--;
-
-	/* skip the prefix, -1 is for the '*' */
-	name += strlen(from) - 1;
-
-	namelen = strlen(name);
-
-	if (outlen <= baselen + namelen) {
-		giterr_set(GITERR_INVALID, "Reference name too long");
-		return GIT_EBUFS;
-	}
-
-	memcpy(out, to, baselen);
-	memcpy(out + baselen, name, namelen + 1);
-
-	return 0;
-}
-
-int git_refspec_transform(char *out, size_t outlen, const git_refspec *spec, const char *name)
-{
-	return refspec_transform_internal(out, outlen, spec->src, spec->dst, name);
-}
-
-int git_refspec_rtransform(char *out, size_t outlen, const git_refspec *spec, const char *name)
-{
-	return refspec_transform_internal(out, outlen, spec->dst, spec->src, name);
-}
-
 static int refspec_transform(
 	git_buf *out, const char *from, const char *to, const char *name)
 {
@@ -233,6 +185,8 @@ static int refspec_transform(
 	size_t from_len = from ? strlen(from) : 0;
 	size_t name_len = name ? strlen(name) : 0;
 
+	git_buf_sanitize(out);
+
 	if (git_buf_set(out, to, to_len) < 0)
 		return -1;
 
@@ -253,12 +207,12 @@ static int refspec_transform(
 	return git_buf_put(out, name + from_len, name_len - from_len);
 }
 
-int git_refspec_transform_r(git_buf *out, const git_refspec *spec, const char *name)
+int git_refspec_transform(git_buf *out, const git_refspec *spec, const char *name)
 {
 	return refspec_transform(out, spec->src, spec->dst, name);
 }
 
-int git_refspec_transform_l(git_buf *out, const git_refspec *spec, const char *name)
+int git_refspec_rtransform(git_buf *out, const git_refspec *spec, const char *name)
 {
 	return refspec_transform(out, spec->dst, spec->src, name);
 }
diff --git a/src/refspec.h b/src/refspec.h
index 51b7bfee9..9a87c97a5 100644
--- a/src/refspec.h
+++ b/src/refspec.h
@@ -23,7 +23,6 @@ struct git_refspec {
 
 #define GIT_REFSPEC_TAGS "refs/tags/*:refs/tags/*"
 
-int git_refspec_parse(struct git_refspec *refspec, const char *str);
 int git_refspec__parse(
 	struct git_refspec *refspec,
 	const char *str,
@@ -31,28 +30,6 @@ int git_refspec__parse(
 
 void git_refspec__free(git_refspec *refspec);
 
-/**
- * Transform a reference to its target following the refspec's rules,
- * and writes the results into a git_buf.
- *
- * @param out where to store the target name
- * @param spec the refspec
- * @param name the name of the reference to transform
- * @return 0 or error if buffer allocation fails
- */
-int git_refspec_transform_r(git_buf *out, const git_refspec *spec, const char *name);
-
-/**
- * Transform a reference from its target following the refspec's rules,
- * and writes the results into a git_buf.
- *
- * @param out where to store the source name
- * @param spec the refspec
- * @param name the name of the reference to transform
- * @return 0 or error if buffer allocation fails
- */
-int git_refspec_transform_l(git_buf *out, const git_refspec *spec, const char *name);
-
 int git_refspec__serialize(git_buf *out, const git_refspec *refspec);
 
 /**
diff --git a/src/remote.c b/src/remote.c
index 3d890a5f1..47b61b1b1 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -45,7 +45,7 @@ static int add_refspec(git_remote *remote, const char *string, bool is_fetch)
 
 static int download_tags_value(git_remote *remote, git_config *cfg)
 {
-	const char *val;
+	const git_config_entry *ce;
 	git_buf buf = GIT_BUF_INIT;
 	int error;
 
@@ -53,16 +53,14 @@ static int download_tags_value(git_remote *remote, git_config *cfg)
 	if (git_buf_printf(&buf, "remote.%s.tagopt", remote->name) < 0)
 		return -1;
 
-	error = git_config_get_string(&val, cfg, git_buf_cstr(&buf));
+	error = git_config__lookup_entry(&ce, cfg, git_buf_cstr(&buf), false);
 	git_buf_free(&buf);
-	if (!error && !strcmp(val, "--no-tags"))
-		remote->download_tags = GIT_REMOTE_DOWNLOAD_TAGS_NONE;
-	else if (!error && !strcmp(val, "--tags"))
-		remote->download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL;
 
-	if (error == GIT_ENOTFOUND) {
-		giterr_clear();
-		error = 0;
+	if (!error && ce && ce->value) {
+		if (!strcmp(ce->value, "--no-tags"))
+			remote->download_tags = GIT_REMOTE_DOWNLOAD_TAGS_NONE;
+		else if (!strcmp(ce->value, "--tags"))
+			remote->download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL;
 	}
 
 	return error;
@@ -75,7 +73,7 @@ static int ensure_remote_name_is_valid(const char *name)
 	if (!git_remote_is_valid_name(name)) {
 		giterr_set(
 			GITERR_CONFIG,
-			"'%s' is not a valid remote name.", name);
+			"'%s' is not a valid remote name.", name ? name : "(null)");
 		error = GIT_EINVALIDSPEC;
 	}
 
@@ -104,12 +102,7 @@ static int get_check_cert(int *out, git_repository *repo)
 	if ((error = git_repository_config__weakptr(&cfg, repo)) < 0)
 		return error;
 
-	if ((error = git_config_get_bool(out, cfg, "http.sslVerify")) == 0)
-		return 0;
-	else if (error != GIT_ENOTFOUND)
-		return error;
-
-	giterr_clear();
+	*out = git_config__get_bool_force(cfg, "http.sslverify", 1);
 	return 0;
 }
 
@@ -243,7 +236,7 @@ on_error:
 	return -1;
 }
 
-int git_remote_create_inmemory(git_remote **out, git_repository *repo, const char *fetch, const char *url)
+int git_remote_create_anonymous(git_remote **out, git_repository *repo, const char *url, const char *fetch)
 {
 	int error;
 	git_remote *remote;
@@ -255,6 +248,62 @@ int git_remote_create_inmemory(git_remote **out, git_repository *repo, const cha
 	return 0;
 }
 
+int git_remote_dup(git_remote **dest, git_remote *source)
+{
+	int error = 0;
+	git_strarray refspecs = { 0 };
+	git_remote *remote = git__calloc(1, sizeof(git_remote));
+	GITERR_CHECK_ALLOC(remote);
+
+	if (source->name != NULL) {
+		remote->name = git__strdup(source->name);
+		GITERR_CHECK_ALLOC(remote->name);
+	}
+
+	if (source->url != NULL) {
+		remote->url = git__strdup(source->url);
+		GITERR_CHECK_ALLOC(remote->url);		
+	}
+
+	if (source->pushurl != NULL) {
+		remote->pushurl = git__strdup(source->pushurl);
+		GITERR_CHECK_ALLOC(remote->pushurl);		
+	}
+
+	remote->repo = source->repo;
+	remote->download_tags = source->download_tags;
+	remote->check_cert = source->check_cert;
+	remote->update_fetchhead = source->update_fetchhead;
+
+	if (git_vector_init(&remote->refs, 32, NULL) < 0 ||
+	    git_vector_init(&remote->refspecs, 2, NULL) < 0 ||
+	    git_vector_init(&remote->active_refspecs, 2, NULL) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	if ((error = git_remote_get_fetch_refspecs(&refspecs, source)) < 0 ||
+	    (error = git_remote_set_fetch_refspecs(remote, &refspecs)) < 0)
+		goto cleanup;
+
+	git_strarray_free(&refspecs);
+
+	if ((error = git_remote_get_push_refspecs(&refspecs, source)) < 0 ||
+	    (error = git_remote_set_push_refspecs(remote, &refspecs)) < 0)
+		goto cleanup;
+
+	*dest = remote;
+
+cleanup:
+
+	git_strarray_free(&refspecs);
+
+	if (error < 0)
+		git__free(remote);
+
+	return error;
+}
+
 struct refspec_cb_data {
 	git_remote *remote;
 	int fetch;
@@ -262,8 +311,7 @@ struct refspec_cb_data {
 
 static int refspec_cb(const git_config_entry *entry, void *payload)
 {
-	const struct refspec_cb_data *data = (struct refspec_cb_data *)payload;
-
+	struct refspec_cb_data *data = (struct refspec_cb_data *)payload;
 	return add_refspec(data->remote, entry->value, data->fetch);
 }
 
@@ -290,9 +338,6 @@ static int get_optional_config(
 		error = 0;
 	}
 
-	if (error < 0)
-		error = -1;
-
 	return error;
 }
 
@@ -303,7 +348,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	const char *val;
 	int error = 0;
 	git_config *config;
-	struct refspec_cb_data data;
+	struct refspec_cb_data data = { NULL };
 	bool optional_setting_found = false, found;
 
 	assert(out && repo && name);
@@ -311,8 +356,8 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	if ((error = ensure_remote_name_is_valid(name)) < 0)
 		return error;
 
-	if (git_repository_config__weakptr(&config, repo) < 0)
-		return -1;
+	if ((error = git_repository_config_snapshot(&config, repo)) < 0)
+		return error;
 
 	remote = git__malloc(sizeof(git_remote));
 	GITERR_CHECK_ALLOC(remote);
@@ -325,17 +370,15 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	if ((error = get_check_cert(&remote->check_cert, repo)) < 0)
 		goto cleanup;
 
-	if ((git_vector_init(&remote->refs, 32, NULL) < 0) ||
-	    (git_vector_init(&remote->refspecs, 2, NULL) < 0) ||
-	    (git_vector_init(&remote->active_refspecs, 2, NULL) < 0)) {
+	if (git_vector_init(&remote->refs, 32, NULL) < 0 ||
+	    git_vector_init(&remote->refspecs, 2, NULL) < 0 ||
+	    git_vector_init(&remote->active_refspecs, 2, NULL) < 0) {
 		error = -1;
 		goto cleanup;
 	}
 
-	if (git_buf_printf(&buf, "remote.%s.url", name) < 0) {
-		error = -1;
+	if ((error = git_buf_printf(&buf, "remote.%s.url", name)) < 0)
 		goto cleanup;
-	}
 
 	if ((error = get_optional_config(&found, config, &buf, NULL, (void *)&val)) < 0)
 		goto cleanup;
@@ -360,6 +403,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 
 	if (!optional_setting_found) {
 		error = GIT_ENOTFOUND;
+		giterr_set(GITERR_CONFIG, "Remote '%s' does not exist.", name);
 		goto cleanup;
 	}
 
@@ -370,6 +414,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 
 	data.remote = remote;
 	data.fetch = true;
+
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.fetch", name);
 
@@ -393,6 +438,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	*out = remote;
 
 cleanup:
+	git_config_free(config);
 	git_buf_free(&buf);
 
 	if (error < 0)
@@ -451,57 +497,46 @@ cleanup:
 int git_remote_save(const git_remote *remote)
 {
 	int error;
-	git_config *config;
+	git_config *cfg;
 	const char *tagopt = NULL;
 	git_buf buf = GIT_BUF_INIT;
+	const git_config_entry *existing;
 
 	assert(remote);
 
 	if (!remote->name) {
-		giterr_set(GITERR_INVALID, "Can't save an in-memory remote.");
+		giterr_set(GITERR_INVALID, "Can't save an anonymous remote.");
 		return GIT_EINVALIDSPEC;
 	}
 
 	if ((error = ensure_remote_name_is_valid(remote->name)) < 0)
 		return error;
 
-	if (git_repository_config__weakptr(&config, remote->repo) < 0)
-		return -1;
+	if ((error = git_repository_config__weakptr(&cfg, remote->repo)) < 0)
+		return error;
 
-	if (git_buf_printf(&buf, "remote.%s.url", remote->name) < 0)
-		return -1;
+	if ((error = git_buf_printf(&buf, "remote.%s.url", remote->name)) < 0)
+		return error;
 
-	if (git_config_set_string(config, git_buf_cstr(&buf), remote->url) < 0) {
-		git_buf_free(&buf);
-		return -1;
-	}
+	/* after this point, buffer is allocated so end with cleanup */
+
+	if ((error = git_config_set_string(
+			cfg, git_buf_cstr(&buf), remote->url)) < 0)
+		goto cleanup;
 
 	git_buf_clear(&buf);
-	if (git_buf_printf(&buf, "remote.%s.pushurl", remote->name) < 0)
-		return -1;
+	if ((error = git_buf_printf(&buf, "remote.%s.pushurl", remote->name)) < 0)
+		goto cleanup;
 
-	if (remote->pushurl) {
-		if (git_config_set_string(config, git_buf_cstr(&buf), remote->pushurl) < 0) {
-			git_buf_free(&buf);
-			return -1;
-		}
-	} else {
-		int error = git_config_delete_entry(config, git_buf_cstr(&buf));
-		if (error == GIT_ENOTFOUND) {
-			error = 0;
-			giterr_clear();
-		}
-		if (error < 0) {
-			git_buf_free(&buf);
-			return -1;
-		}
-	}
+	if ((error = git_config__update_entry(
+			cfg, git_buf_cstr(&buf), remote->pushurl, true, false)) < 0)
+		goto cleanup;
 
-	if (update_config_refspec(remote, config, GIT_DIRECTION_FETCH) < 0)
-		goto on_error;
+	if ((error = update_config_refspec(remote, cfg, GIT_DIRECTION_FETCH)) < 0)
+		goto cleanup;
 
-	if (update_config_refspec(remote, config, GIT_DIRECTION_PUSH) < 0)
-		goto on_error;
+	if ((error = update_config_refspec(remote, cfg, GIT_DIRECTION_PUSH)) < 0)
+		goto cleanup;
 
 	/*
 	 * What action to take depends on the old and new values. This
@@ -517,31 +552,26 @@ int git_remote_save(const git_remote *remote)
 	 */
 
 	git_buf_clear(&buf);
-	if (git_buf_printf(&buf, "remote.%s.tagopt", remote->name) < 0)
-		goto on_error;
-
-	error = git_config_get_string(&tagopt, config, git_buf_cstr(&buf));
-	if (error < 0 && error != GIT_ENOTFOUND)
-		goto on_error;
+	if ((error = git_buf_printf(&buf, "remote.%s.tagopt", remote->name)) < 0)
+		goto cleanup;
 
-	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
-		if (git_config_set_string(config, git_buf_cstr(&buf), "--tags") < 0)
-			goto on_error;
-	} else if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_NONE) {
-		if (git_config_set_string(config, git_buf_cstr(&buf), "--no-tags") < 0)
-			goto on_error;
-	} else if (tagopt) {
-		if (git_config_delete_entry(config, git_buf_cstr(&buf)) < 0)
-			goto on_error;
-	}
+	if ((error = git_config__lookup_entry(
+			&existing, cfg, git_buf_cstr(&buf), false)) < 0)
+		goto cleanup;
 
-	git_buf_free(&buf);
+	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL)
+		tagopt = "--tags";
+	else if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_NONE)
+		tagopt = "--no-tags";
+	else if (existing != NULL)
+		tagopt = NULL;
 
-	return 0;
+	error = git_config__update_entry(
+		cfg, git_buf_cstr(&buf), tagopt, true, false);
 
-on_error:
+cleanup:
 	git_buf_free(&buf);
-	return -1;
+	return error;
 }
 
 const char *git_remote_name(const git_remote *remote)
@@ -635,13 +665,13 @@ int git_remote_connect(git_remote *remote, git_direction direction)
 		return error;
 
 	if (t->set_callbacks &&
-		(error = t->set_callbacks(t, remote->callbacks.progress, NULL, remote->callbacks.payload)) < 0)
+		(error = t->set_callbacks(t, remote->callbacks.sideband_progress, NULL, remote->callbacks.payload)) < 0)
 		goto on_error;
 
 	if (!remote->check_cert)
 		flags |= GIT_TRANSPORTFLAGS_NO_CHECK_CERT;
 
-	if ((error = t->connect(t, url, remote->callbacks.credentials, remote->callbacks.payload, direction, flags)) < 0)
+	if ((error = t->connect(t, url, remote->callbacks.credentials, remote->callbacks.payload, direction, flags)) != 0)
 		goto on_error;
 
 	remote->transport = t;
@@ -667,7 +697,8 @@ int git_remote_ls(const git_remote_head ***out, size_t *size, git_remote *remote
 int git_remote__get_http_proxy(git_remote *remote, bool use_ssl, char **proxy_url)
 {
 	git_config *cfg;
-	const char *val;
+	const git_config_entry *ce;
+	const char *val = NULL;
 	int error;
 
 	assert(remote);
@@ -684,44 +715,39 @@ int git_remote__get_http_proxy(git_remote *remote, bool use_ssl, char **proxy_ur
 	 * to least specific. */
 
 	/* remote.<name>.proxy config setting */
-	if (remote->name && 0 != *(remote->name)) {
+	if (remote->name && remote->name[0]) {
 		git_buf buf = GIT_BUF_INIT;
 
 		if ((error = git_buf_printf(&buf, "remote.%s.proxy", remote->name)) < 0)
 			return error;
 
-		if ((error = git_config_get_string(&val, cfg, git_buf_cstr(&buf))) == 0 &&
-			val && ('\0' != *val)) {
-			git_buf_free(&buf);
+		error = git_config__lookup_entry(&ce, cfg, git_buf_cstr(&buf), false);
+		git_buf_free(&buf);
 
-			*proxy_url = git__strdup(val);
-			GITERR_CHECK_ALLOC(*proxy_url);
-			return 0;
-		} else if (error != GIT_ENOTFOUND)
+		if (error < 0)
 			return error;
 
-		giterr_clear();
-		git_buf_free(&buf);
+		if (ce && ce->value) {
+			val = ce->value;
+			goto found;
+		}
 	}
 
 	/* http.proxy config setting */
-	if ((error = git_config_get_string(&val, cfg, "http.proxy")) == 0 &&
-		val && ('\0' != *val)) {
-		*proxy_url = git__strdup(val);
-		GITERR_CHECK_ALLOC(*proxy_url);
-		return 0;
-	} else if (error != GIT_ENOTFOUND)
+	if ((error = git_config__lookup_entry(&ce, cfg, "http.proxy", false)) < 0)
 		return error;
-
-	giterr_clear();
+	if (ce && ce->value) {
+		val = ce->value;
+		goto found;
+	}
 
 	/* HTTP_PROXY / HTTPS_PROXY environment variables */
 	val = use_ssl ? getenv("HTTPS_PROXY") : getenv("HTTP_PROXY");
 
-	if (val && ('\0' != *val)) {
+found:
+	if (val && val[0]) {
 		*proxy_url = git__strdup(val);
 		GITERR_CHECK_ALLOC(*proxy_url);
-		return 0;
 	}
 
 	return 0;
@@ -797,7 +823,7 @@ int git_remote_download(git_remote *remote)
 	git_vector_free(&refs);
 
 	if (error < 0)
-		return -1;
+		return error;
 
 	if ((error = git_fetch_negotiate(remote)) < 0)
 		return error;
@@ -805,22 +831,39 @@ int git_remote_download(git_remote *remote)
 	return git_fetch_download_pack(remote);
 }
 
-int git_remote_fetch(git_remote *remote)
+int git_remote_fetch(
+		git_remote *remote,
+		const git_signature *signature,
+		const char *reflog_message)
 {
 	int error;
+	git_buf reflog_msg_buf = GIT_BUF_INIT;
 
 	/* Connect and download everything */
-	if ((error = git_remote_connect(remote, GIT_DIRECTION_FETCH)) < 0)
+	if ((error = git_remote_connect(remote, GIT_DIRECTION_FETCH)) != 0)
 		return error;
 
-	if ((error = git_remote_download(remote)) < 0)
-		return error;
+	error = git_remote_download(remote);
 
 	/* We don't need to be connected anymore */
 	git_remote_disconnect(remote);
 
+	/* If the download failed, return the error */
+	if (error != 0)
+		return error;
+
+	/* Default reflog message */
+	if (reflog_message)
+		git_buf_sets(&reflog_msg_buf, reflog_message);
+	else {
+		git_buf_printf(&reflog_msg_buf, "fetch %s",
+				remote->name ? remote->name : remote->url);
+	}
+
 	/* Create "remote/foo" branches for all remote branches */
-	return git_remote_update_tips(remote);
+	error = git_remote_update_tips(remote, signature, git_buf_cstr(&reflog_msg_buf));
+	git_buf_free(&reflog_msg_buf);
+	return error;
 }
 
 static int remote_head_for_fetchspec_src(git_remote_head **out, git_vector *update_heads, const char *fetchspec_src)
@@ -845,19 +888,32 @@ static int remote_head_for_fetchspec_src(git_remote_head **out, git_vector *upda
 static int remote_head_for_ref(git_remote_head **out, git_refspec *spec, git_vector *update_heads, git_reference *ref)
 {
 	git_reference *resolved_ref = NULL;
-	git_reference *tracking_ref = NULL;
 	git_buf remote_name = GIT_BUF_INIT;
+	git_buf upstream_name = GIT_BUF_INIT;
+	git_repository *repo;
+	const char *ref_name;
 	int error = 0;
 
 	assert(out && spec && ref);
 
 	*out = NULL;
 
-	if ((error = git_reference_resolve(&resolved_ref, ref)) < 0 ||
-		(!git_reference_is_branch(resolved_ref)) ||
-		(error = git_branch_upstream(&tracking_ref, resolved_ref)) < 0 ||
-		(error = git_refspec_transform_l(&remote_name, spec, git_reference_name(tracking_ref))) < 0) {
-		/* Not an error if HEAD is unborn or no tracking branch */
+	repo = git_reference_owner(ref);
+
+	error = git_reference_resolve(&resolved_ref, ref);
+
+	/* If we're in an unborn branch, let's pretend nothing happened */
+	if (error == GIT_ENOTFOUND && git_reference_type(ref) == GIT_REF_SYMBOLIC) {
+		ref_name = git_reference_symbolic_target(ref);
+		error = 0;
+	} else {
+		ref_name = git_reference_name(resolved_ref);
+	}
+
+	if ((!git_reference__is_branch(ref_name)) ||
+	    (error = git_branch_upstream_name(&upstream_name, repo, ref_name)) < 0 ||
+	    (error = git_refspec_rtransform(&remote_name, spec, upstream_name.ptr)) < 0) {
+		/* Not an error if there is no upstream */
 		if (error == GIT_ENOTFOUND)
 			error = 0;
 
@@ -867,9 +923,9 @@ static int remote_head_for_ref(git_remote_head **out, git_refspec *spec, git_vec
 	error = remote_head_for_fetchspec_src(out, update_heads, git_buf_cstr(&remote_name));
 
 cleanup:
-	git_reference_free(tracking_ref);
 	git_reference_free(resolved_ref);
 	git_buf_free(&remote_name);
+	git_buf_free(&upstream_name);
 	return error;
 }
 
@@ -938,7 +994,12 @@ cleanup:
 	return error;
 }
 
-static int update_tips_for_spec(git_remote *remote, git_refspec *spec, git_vector *refs)
+static int update_tips_for_spec(
+		git_remote *remote,
+		git_refspec *spec,
+		git_vector *refs,
+		const git_signature *signature,
+		const char *log_message)
 {
 	int error = 0, autotag;
 	unsigned int i = 0;
@@ -971,7 +1032,7 @@ static int update_tips_for_spec(git_remote *remote, git_refspec *spec, git_vecto
 			continue;
 
 		if (git_refspec_src_matches(spec, head->name) && spec->dst) {
-			if (git_refspec_transform_r(&refname, spec, head->name) < 0)
+			if (git_refspec_transform(&refname, spec, head->name) < 0)
 				goto on_error;
 		} else if (remote->download_tags != GIT_REMOTE_DOWNLOAD_TAGS_NONE) {
 
@@ -1005,7 +1066,8 @@ static int update_tips_for_spec(git_remote *remote, git_refspec *spec, git_vecto
 			continue;
 
 		/* In autotag mode, don't overwrite any locally-existing tags */
-		error = git_reference_create(&ref, remote->repo, refname.ptr, &head->oid, !autotag);
+		error = git_reference_create(&ref, remote->repo, refname.ptr, &head->oid, !autotag, 
+				signature, log_message);
 		if (error < 0 && error != GIT_EEXISTS)
 			goto on_error;
 
@@ -1034,14 +1096,16 @@ on_error:
 
 }
 
-int git_remote_update_tips(git_remote *remote)
+int git_remote_update_tips(
+		git_remote *remote,
+		const git_signature *signature,
+		const char *reflog_message)
 {
 	git_refspec *spec, tagspec;
 	git_vector refs;
 	int error;
 	size_t i;
 
-
 	if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
 		return -1;
 
@@ -1050,7 +1114,7 @@ int git_remote_update_tips(git_remote *remote)
 		goto out;
 
 	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
-		error = update_tips_for_spec(remote, &tagspec, &refs);
+		error = update_tips_for_spec(remote, &tagspec, &refs, signature, reflog_message);
 		goto out;
 	}
 
@@ -1058,7 +1122,7 @@ int git_remote_update_tips(git_remote *remote)
 		if (spec->push)
 			continue;
 
-		if ((error = update_tips_for_spec(remote, spec, &refs)) < 0)
+		if ((error = update_tips_for_spec(remote, spec, &refs, signature, reflog_message)) < 0)
 			goto out;
 	}
 
@@ -1068,7 +1132,7 @@ out:
 	return error;
 }
 
-int git_remote_connected(git_remote *remote)
+int git_remote_connected(const git_remote *remote)
 {
 	assert(remote);
 
@@ -1141,40 +1205,28 @@ static int remote_list_cb(const git_config_entry *entry, void *payload)
 
 int git_remote_list(git_strarray *remotes_list, git_repository *repo)
 {
-	git_config *cfg;
-	git_vector list;
 	int error;
+	git_config *cfg;
+	git_vector list = GIT_VECTOR_INIT;
 
-	if (git_repository_config__weakptr(&cfg, repo) < 0)
-		return -1;
+	if ((error = git_repository_config__weakptr(&cfg, repo)) < 0)
+		return error;
 
-	if (git_vector_init(&list, 4, git__strcmp_cb) < 0)
-		return -1;
+	if ((error = git_vector_init(&list, 4, git__strcmp_cb)) < 0)
+		return error;
 
 	error = git_config_foreach_match(
 		cfg, "^remote\\..*\\.(push)?url$", remote_list_cb, &list);
 
 	if (error < 0) {
-		size_t i;
-		char *elem;
-
-		git_vector_foreach(&list, i, elem) {
-			git__free(elem);
-		}
-
-		git_vector_free(&list);
-
-		/* cb error is converted to GIT_EUSER by git_config_foreach */
-		if (error == GIT_EUSER)
-			error = -1;
-
+		git_vector_free_deep(&list);
 		return error;
 	}
 
 	git_vector_uniq(&list, git__free);
 
-	remotes_list->strings = (char **)list.contents;
-	remotes_list->count = list.length;
+	remotes_list->strings =
+		(char **)git_vector_detach(&remotes_list->count, NULL, &list);
 
 	return 0;
 }
@@ -1196,13 +1248,20 @@ int git_remote_set_callbacks(git_remote *remote, const git_remote_callbacks *cal
 
 	if (remote->transport && remote->transport->set_callbacks)
 		return remote->transport->set_callbacks(remote->transport,
-			remote->callbacks.progress,
+			remote->callbacks.sideband_progress,
 			NULL,
 			remote->callbacks.payload);
 
 	return 0;
 }
 
+const git_remote_callbacks *git_remote_get_callbacks(git_remote *remote)
+{
+	assert(remote);
+
+	return &remote->callbacks;
+}
+
 int git_remote_set_transport(git_remote *remote, git_transport *transport)
 {
 	assert(remote && transport);
@@ -1224,7 +1283,7 @@ const git_transfer_progress* git_remote_stats(git_remote *remote)
 	return &remote->stats;
 }
 
-git_remote_autotag_option_t git_remote_autotag(git_remote *remote)
+git_remote_autotag_option_t git_remote_autotag(const git_remote *remote)
 {
 	return remote->download_tags;
 }
@@ -1246,13 +1305,14 @@ static int rename_remote_config_section(
 	if (git_buf_printf(&old_section_name, "remote.%s", old_name) < 0)
 		goto cleanup;
 
-	if (git_buf_printf(&new_section_name, "remote.%s", new_name) < 0)
-		goto cleanup;
+	if (new_name &&
+		(git_buf_printf(&new_section_name, "remote.%s", new_name) < 0))
+			goto cleanup;
 
 	error = git_config_rename_section(
 		repo,
 		git_buf_cstr(&old_section_name),
-		git_buf_cstr(&new_section_name));
+		new_name ? git_buf_cstr(&new_section_name) : NULL);
 
 cleanup:
 	git_buf_free(&old_section_name);
@@ -1261,8 +1321,7 @@ cleanup:
 	return error;
 }
 
-struct update_data
-{
+struct update_data {
 	git_config *config;
 	const char *old_remote_name;
 	const char *new_remote_name;
@@ -1278,9 +1337,7 @@ static int update_config_entries_cb(
 		return 0;
 
 	return git_config_set_string(
-		data->config,
-		entry->name,
-		data->new_remote_name);
+		data->config, entry->name, data->new_remote_name);
 }
 
 static int update_branch_remote_config_entry(
@@ -1288,41 +1345,77 @@ static int update_branch_remote_config_entry(
 	const char *old_name,
 	const char *new_name)
 {
-	git_config *config;
-	struct update_data data;
+	int error;
+	struct update_data data = { NULL };
 
-	if (git_repository_config__weakptr(&config, repo) < 0)
-		return -1;
+	if ((error = git_repository_config__weakptr(&data.config, repo)) < 0)
+		return error;
 
-	data.config = config;
 	data.old_remote_name = old_name;
 	data.new_remote_name = new_name;
 
 	return git_config_foreach_match(
-		config,
-		"branch\\..+\\.remote",
-		update_config_entries_cb, &data);
+		data.config, "branch\\..+\\.remote", update_config_entries_cb, &data);
 }
 
 static int rename_one_remote_reference(
-	git_reference *reference,
+	git_reference *reference_in,
 	const char *old_remote_name,
 	const char *new_remote_name)
 {
-	int error = -1;
+	int error;
+	git_reference *ref = NULL, *dummy = NULL;
+	git_buf namespace = GIT_BUF_INIT, old_namespace = GIT_BUF_INIT;
 	git_buf new_name = GIT_BUF_INIT;
+	git_buf log_message = GIT_BUF_INIT;
+	size_t pfx_len;
+	const char *target;
 
-	if (git_buf_printf(
-		&new_name,
-		GIT_REFS_REMOTES_DIR "%s%s",
-		new_remote_name,
-		reference->name + strlen(GIT_REFS_REMOTES_DIR) + strlen(old_remote_name)) < 0)
-			return -1;
+	if ((error = git_buf_printf(&namespace, GIT_REFS_REMOTES_DIR "%s/", new_remote_name)) < 0)
+		return error;
 
-	error = git_reference_rename(NULL, reference, git_buf_cstr(&new_name), 0);
-	git_reference_free(reference);
+	pfx_len = strlen(GIT_REFS_REMOTES_DIR) + strlen(old_remote_name) + 1;
+	git_buf_puts(&new_name, namespace.ptr);
+	if ((error = git_buf_puts(&new_name, git_reference_name(reference_in) + pfx_len)) < 0)
+		goto cleanup;
+
+	if ((error = git_buf_printf(&log_message,
+					"renamed remote %s to %s",
+					old_remote_name, new_remote_name)) < 0)
+		goto cleanup;
+
+	if ((error = git_reference_rename(&ref, reference_in, git_buf_cstr(&new_name), 1,
+					  NULL, git_buf_cstr(&log_message))) < 0)
+		goto cleanup;
+
+	if (git_reference_type(ref) != GIT_REF_SYMBOLIC)
+		goto cleanup;
+
+	/* Handle refs like origin/HEAD -> origin/master */
+	target = git_reference_symbolic_target(ref);
+	if ((error = git_buf_printf(&old_namespace, GIT_REFS_REMOTES_DIR "%s/", old_remote_name)) < 0)
+		goto cleanup;
+
+	if (git__prefixcmp(target, old_namespace.ptr))
+		goto cleanup;
 
+	git_buf_clear(&new_name);
+	git_buf_puts(&new_name, namespace.ptr);
+	if ((error = git_buf_puts(&new_name, target + pfx_len)) < 0)
+		goto cleanup;
+
+	error = git_reference_symbolic_set_target(&dummy, ref, git_buf_cstr(&new_name),
+						  NULL, git_buf_cstr(&log_message));
+
+	git_reference_free(dummy);
+
+cleanup:
+	git_reference_free(reference_in);
+	git_reference_free(ref);
+	git_buf_free(&namespace);
+	git_buf_free(&old_namespace);
 	git_buf_free(&new_name);
+	git_buf_free(&log_message);
 	return error;
 }
 
@@ -1331,159 +1424,138 @@ static int rename_remote_references(
 	const char *old_name,
 	const char *new_name)
 {
-	int error = -1;
+	int error;
+	git_buf buf = GIT_BUF_INIT;
 	git_reference *ref;
 	git_reference_iterator *iter;
 
-	if (git_reference_iterator_new(&iter, repo) < 0)
-		return -1;
+	if ((error = git_buf_printf(&buf, GIT_REFS_REMOTES_DIR "%s/*", old_name)) < 0)
+		return error;
 
-	while ((error = git_reference_next(&ref, iter)) == 0) {
-		if (git__prefixcmp(ref->name, GIT_REFS_REMOTES_DIR)) {
-			git_reference_free(ref);
-			continue;
-		}
+	error = git_reference_iterator_glob_new(&iter, repo, git_buf_cstr(&buf));
+	git_buf_free(&buf);
 
-		if ((error = rename_one_remote_reference(ref, old_name, new_name)) < 0) {
-			git_reference_iterator_free(iter);
-			return error;
-		}
+	if (error < 0)
+		return error;
+
+	while ((error = git_reference_next(&ref, iter)) == 0) {
+		if ((error = rename_one_remote_reference(ref, old_name, new_name)) < 0)
+			break;
 	}
 
 	git_reference_iterator_free(iter);
 
-	if (error == GIT_ITEROVER)
-		return 0;
-
-	return error;
+	return (error == GIT_ITEROVER) ? 0 : error;
 }
 
-static int rename_fetch_refspecs(
-	git_remote *remote,
-	const char *new_name,
-	int (*callback)(const char *problematic_refspec, void *payload),
-	void *payload)
+static int rename_fetch_refspecs(git_vector *problems, git_remote *remote, const char *new_name)
 {
 	git_config *config;
 	git_buf base = GIT_BUF_INIT, var = GIT_BUF_INIT, val = GIT_BUF_INIT;
 	const git_refspec *spec;
 	size_t i;
-	int error = -1;
+	int error = 0;
 
-	if (git_buf_printf(&base, "+refs/heads/*:refs/remotes/%s/*", remote->name) < 0)
-		goto cleanup;
+	if ((error = git_repository_config__weakptr(&config, remote->repo)) < 0)
+		return error;
+
+	if ((error = git_vector_init(problems, 1, NULL)) < 0)
+		return error;
+
+	if ((error = git_buf_printf(
+			&base, "+refs/heads/*:refs/remotes/%s/*", remote->name)) < 0)
+		return error;
 
 	git_vector_foreach(&remote->refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
-		/* Every refspec is a problem refspec for an in-memory remote */
-		if (!remote->name) {
-			if (callback(spec->string, payload) < 0) {
-				error = GIT_EUSER;
-				goto cleanup;
-			}
+		/* Does the dst part of the refspec follow the expected format? */
+		if (strcmp(git_buf_cstr(&base), spec->string)) {
+			char *dup;
 
-			continue;
-		}
+			dup = git__strdup(spec->string);
+			GITERR_CHECK_ALLOC(dup);
 
-		/* Does the dst part of the refspec follow the extected standard format? */
-		if (strcmp(git_buf_cstr(&base), spec->string)) {
-			if (callback(spec->string, payload) < 0) {
-				error = GIT_EUSER;
-				goto cleanup;
-			}
+			if ((error = git_vector_insert(problems, dup)) < 0)
+				break;
 
 			continue;
 		}
 
 		/* If we do want to move it to the new section */
-		if (git_buf_printf(&val, "+refs/heads/*:refs/remotes/%s/*", new_name) < 0)
-			goto cleanup;
 
-		if (git_buf_printf(&var, "remote.%s.fetch", new_name) < 0)
-			goto cleanup;
+		git_buf_clear(&val);
+		git_buf_clear(&var);
 
-		if (git_repository_config__weakptr(&config, remote->repo) < 0)
-			goto cleanup;
+		if (git_buf_printf(
+				&val, "+refs/heads/*:refs/remotes/%s/*", new_name) < 0 ||
+			git_buf_printf(&var, "remote.%s.fetch", new_name) < 0)
+		{
+			error = -1;
+			break;
+		}
 
-		if (git_config_set_string(config, git_buf_cstr(&var), git_buf_cstr(&val)) < 0)
-			goto cleanup;
+		if ((error = git_config_set_string(
+				config, git_buf_cstr(&var), git_buf_cstr(&val))) < 0)
+			break;
 	}
 
-	error = 0;
-
-cleanup:
 	git_buf_free(&base);
 	git_buf_free(&var);
 	git_buf_free(&val);
+
+	if (error < 0) {
+		char *str;
+		git_vector_foreach(problems, i, str)
+			git__free(str);
+
+		git_vector_free(problems);
+	}
+
 	return error;
 }
 
-int git_remote_rename(
-	git_remote *remote,
-	const char *new_name,
-	git_remote_rename_problem_cb callback,
-	void *payload)
+int git_remote_rename(git_strarray *out, git_remote *remote, const char *new_name)
 {
 	int error;
+	git_vector problem_refspecs;
+	char *tmp, *dup;
 
-	assert(remote && new_name);
+	assert(out && remote && new_name);
 
 	if (!remote->name) {
-		giterr_set(GITERR_INVALID, "Can't rename an in-memory remote.");
+		giterr_set(GITERR_INVALID, "Can't rename an anonymous remote.");
 		return GIT_EINVALIDSPEC;
 	}
 
 	if ((error = ensure_remote_name_is_valid(new_name)) < 0)
 		return error;
 
-	if (remote->repo) {
-		if ((error = ensure_remote_doesnot_exist(remote->repo, new_name)) < 0)
-			return error;
+	if ((error = ensure_remote_doesnot_exist(remote->repo, new_name)) < 0)
+		return error;
 
-		if (!remote->name) {
-			if ((error = rename_fetch_refspecs(
-				remote,
-				new_name,
-				callback,
-				payload)) < 0)
-				return error;
+	if ((error = rename_remote_config_section(remote->repo, remote->name, new_name)) < 0)
+		return error;
 
-			remote->name = git__strdup(new_name);
+	if ((error = update_branch_remote_config_entry(remote->repo, remote->name, new_name)) < 0)
+		return error;
 
-			if (!remote->name) return 0;
-			return git_remote_save(remote);
-		}
+	if ((error = rename_remote_references(remote->repo, remote->name, new_name)) < 0)
+		return error;
 
-		if ((error = rename_remote_config_section(
-			remote->repo,
-			remote->name,
-			new_name)) < 0)
-				return error;
-
-		if ((error = update_branch_remote_config_entry(
-			remote->repo,
-			remote->name,
-			new_name)) < 0)
-				return error;
-
-		if ((error = rename_remote_references(
-			remote->repo,
-			remote->name,
-			new_name)) < 0)
-				return error;
-
-		if ((error = rename_fetch_refspecs(
-			remote,
-			new_name,
-			callback,
-			payload)) < 0)
-			return error;
-	}
+	if ((error = rename_fetch_refspecs(&problem_refspecs, remote, new_name)) < 0)
+		return error;
 
-	git__free(remote->name);
-	remote->name = git__strdup(new_name);
+	out->count = problem_refspecs.length;
+	out->strings = (char **) problem_refspecs.contents;
+
+	dup = git__strdup(new_name);
+	GITERR_CHECK_ALLOC(dup);
+
+	tmp = remote->name;
+	remote->name = dup;
+	git__free(tmp);
 
 	return 0;
 }
@@ -1626,7 +1698,7 @@ int git_remote_set_push_refspecs(git_remote *remote, git_strarray *array)
 	return set_refspecs(remote, array, true);
 }
 
-static int copy_refspecs(git_strarray *array, git_remote *remote, unsigned int push)
+static int copy_refspecs(git_strarray *array, const git_remote *remote, unsigned int push)
 {
 	size_t i;
 	git_vector refspecs;
@@ -1655,29 +1727,246 @@ static int copy_refspecs(git_strarray *array, git_remote *remote, unsigned int p
 	return 0;
 
 on_error:
-	git_vector_foreach(&refspecs, i, dup)
-		git__free(dup);
-	git_vector_free(&refspecs);
+	git_vector_free_deep(&refspecs);
 
 	return -1;
 }
 
-int git_remote_get_fetch_refspecs(git_strarray *array, git_remote *remote)
+int git_remote_get_fetch_refspecs(git_strarray *array, const git_remote *remote)
 {
 	return copy_refspecs(array, remote, false);
 }
 
-int git_remote_get_push_refspecs(git_strarray *array, git_remote *remote)
+int git_remote_get_push_refspecs(git_strarray *array, const git_remote *remote)
 {
 	return copy_refspecs(array, remote, true);
 }
 
-size_t git_remote_refspec_count(git_remote *remote)
+size_t git_remote_refspec_count(const git_remote *remote)
 {
 	return remote->refspecs.length;
 }
 
-const git_refspec *git_remote_get_refspec(git_remote *remote, size_t n)
+const git_refspec *git_remote_get_refspec(const git_remote *remote, size_t n)
 {
 	return git_vector_get(&remote->refspecs, n);
 }
+
+int git_remote_init_callbacks(git_remote_callbacks *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_remote_callbacks, GIT_REMOTE_CALLBACKS_INIT);
+	return 0;
+}
+
+/* asserts a branch.<foo>.remote format */
+static const char *name_offset(size_t *len_out, const char *name)
+{
+	size_t prefix_len;
+	const char *dot;
+
+	prefix_len = strlen("remote.");
+	dot = strchr(name + prefix_len, '.');
+
+	assert(dot);
+
+	*len_out = dot - name - prefix_len;
+	return name + prefix_len;
+}
+
+static int remove_branch_config_related_entries(
+	git_repository *repo,
+	const char *remote_name)
+{
+	int error;
+	git_config *config;
+	git_config_entry *entry;
+	git_config_iterator *iter;
+	git_buf buf = GIT_BUF_INIT;
+
+	if ((error = git_repository_config__weakptr(&config, repo)) < 0)
+		return error;
+
+	if ((error = git_config_iterator_glob_new(&iter, config, "branch\\..+\\.remote")) < 0)
+		return error;
+
+	/* find any branches with us as upstream and remove that config */
+	while ((error = git_config_next(&entry, iter)) == 0) {
+		const char *branch;
+		size_t branch_len;
+
+		if (strcmp(remote_name, entry->value))
+			continue;
+
+		branch = name_offset(&branch_len, entry->name);
+
+		git_buf_clear(&buf);
+		if (git_buf_printf(&buf, "branch.%.*s.merge", (int)branch_len, branch) < 0)
+			break;
+
+		if ((error = git_config_delete_entry(config, git_buf_cstr(&buf))) < 0)
+			break;
+
+		git_buf_clear(&buf);
+		if (git_buf_printf(&buf, "branch.%.*s.remote", (int)branch_len, branch) < 0)
+			break;
+
+		if ((error = git_config_delete_entry(config, git_buf_cstr(&buf))) < 0)
+			break;
+	}
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+	git_buf_free(&buf);
+	git_config_iterator_free(iter);
+	return error;
+}
+
+static int remove_refs(git_repository *repo, const git_refspec *spec)
+{
+	git_reference_iterator *iter = NULL;
+	git_vector refs;
+	const char *name;
+	char *dup;
+	int error;
+	size_t i;
+
+	if ((error = git_vector_init(&refs, 8, NULL)) < 0)
+		return error;
+
+	if ((error = git_reference_iterator_new(&iter, repo)) < 0)
+		goto cleanup;
+
+	while ((error = git_reference_next_name(&name, iter)) == 0) {
+		if (!git_refspec_dst_matches(spec, name))
+			continue;
+
+		dup = git__strdup(name);
+		if (!dup) {
+			error = -1;
+			goto cleanup;
+		}
+
+		if ((error = git_vector_insert(&refs, dup)) < 0)
+			goto cleanup;
+	}
+	if (error == GIT_ITEROVER)
+		error = 0;
+	if (error < 0)
+		goto cleanup;
+
+	git_vector_foreach(&refs, i, name) {
+		if ((error = git_reference_remove(repo, name)) < 0)
+			break;
+	}
+
+cleanup:
+	git_reference_iterator_free(iter);
+	git_vector_foreach(&refs, i, dup) {
+		git__free(dup);
+	}
+	git_vector_free(&refs);
+	return error;
+}
+
+static int remove_remote_tracking(git_repository *repo, const char *remote_name)
+{
+	git_remote *remote;
+	int error;
+	size_t i, count;
+
+	/* we want to use what's on the config, regardless of changes to the instance in memory */
+	if ((error = git_remote_load(&remote, repo, remote_name)) < 0)
+		return error;
+
+	count = git_remote_refspec_count(remote);
+	for (i = 0; i < count; i++) {
+		const git_refspec *refspec = git_remote_get_refspec(remote, i);
+
+		/* shouldn't ever actually happen */
+		if (refspec == NULL)
+			continue;
+
+		if ((error = remove_refs(repo, refspec)) < 0)
+			break;
+	}
+
+	git_remote_free(remote);
+	return error;
+}
+
+int git_remote_delete(git_remote *remote)
+{
+	int error;
+	git_repository *repo;
+
+	assert(remote);
+
+	if (!remote->name) {
+		giterr_set(GITERR_INVALID, "Can't delete an anonymous remote.");
+		return -1;
+	}
+
+	repo = git_remote_owner(remote);
+
+	if ((error = remove_branch_config_related_entries(repo,
+		git_remote_name(remote))) < 0)
+		return error;
+
+	if ((error = remove_remote_tracking(repo, git_remote_name(remote))) < 0)
+		return error;
+
+	if ((error = rename_remote_config_section(
+		repo, git_remote_name(remote), NULL)) < 0)
+		return error;
+
+	return 0;
+}
+
+int git_remote_default_branch(git_buf *out, git_remote *remote)
+{
+	const git_remote_head **heads;
+	const git_remote_head *guess = NULL;
+	const git_oid *head_id;
+	size_t heads_len, i;
+	int error;
+
+	if ((error = git_remote_ls(&heads, &heads_len, remote)) < 0)
+		return error;
+
+	if (heads_len == 0)
+		return GIT_ENOTFOUND;
+
+	git_buf_sanitize(out);
+	/* the first one must be HEAD so if that has the symref info, we're done */
+	if (heads[0]->symref_target)
+		return git_buf_puts(out, heads[0]->symref_target);
+
+	/*
+	 * If there's no symref information, we have to look over them
+	 * and guess. We return the first match unless the master
+	 * branch is a candidate. Then we return the master branch.
+	 */
+	head_id = &heads[0]->oid;
+
+	for (i = 1; i < heads_len; i++) {
+		if (git_oid_cmp(head_id, &heads[i]->oid))
+			continue;
+
+		if (!guess) {
+			guess = heads[i];
+			continue;
+		}
+
+		if (!git__strcmp(GIT_REFS_HEADS_MASTER_FILE, heads[i]->name)) {
+			guess = heads[i];
+			break;
+		}
+	}
+
+	if (!guess)
+		return GIT_ENOTFOUND;
+
+	return git_buf_puts(out, guess->name);
+}
diff --git a/src/repository.c b/src/repository.c
index dcc02e4fb..e8d50aed3 100644
--- a/src/repository.c
+++ b/src/repository.c
@@ -4,7 +4,6 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
-#include <stdarg.h>
 #include <ctype.h>
 
 #include "git2/object.h"
@@ -17,6 +16,7 @@
 #include "tag.h"
 #include "blob.h"
 #include "fileops.h"
+#include "sysdir.h"
 #include "filebuf.h"
 #include "index.h"
 #include "config.h"
@@ -27,6 +27,10 @@
 #include "merge.h"
 #include "diff_driver.h"
 
+#ifdef GIT_WIN32
+# include "win32/w32_util.h"
+#endif
+
 #define GIT_FILE_CONTENT_PREFIX "gitdir:"
 
 #define GIT_BRANCH_MASTER "master"
@@ -93,6 +97,7 @@ void git_repository__cleanup(git_repository *repo)
 
 	git_cache_clear(&repo->objects);
 	git_attr_cache_flush(repo);
+	git_submodule_cache_free(repo);
 
 	set_config(repo, NULL);
 	set_index(repo, NULL);
@@ -108,7 +113,6 @@ void git_repository_free(git_repository *repo)
 	git_repository__cleanup(repo);
 
 	git_cache_free(&repo->objects);
-	git_submodule_config_free(repo);
 
 	git_diff_driver_registry_free(repo->diff_drivers);
 	repo->diff_drivers = NULL;
@@ -165,13 +169,9 @@ int git_repository_new(git_repository **out)
 	return 0;
 }
 
-static int load_config_data(git_repository *repo)
+static int load_config_data(git_repository *repo, const git_config *config)
 {
 	int is_bare;
-	git_config *config;
-
-	if (git_repository_config__weakptr(&config, repo) < 0)
-		return -1;
 
 	/* Try to figure out if it's bare, default to non-bare if it's not set */
 	if (git_config_get_bool(&is_bare, config, "core.bare") < 0)
@@ -182,43 +182,37 @@ static int load_config_data(git_repository *repo)
 	return 0;
 }
 
-static int load_workdir(git_repository *repo, git_buf *parent_path)
+static int load_workdir(git_repository *repo, git_config *config, git_buf *parent_path)
 {
 	int         error;
-	git_config *config;
-	const char *worktree;
-	git_buf     worktree_buf = GIT_BUF_INIT;
+	const git_config_entry *ce;
+	git_buf     worktree = GIT_BUF_INIT;
 
 	if (repo->is_bare)
 		return 0;
 
-	if (git_repository_config__weakptr(&config, repo) < 0)
-		return -1;
+	if ((error = git_config__lookup_entry(
+			&ce, config, "core.worktree", false)) < 0)
+		return error;
 
-	error = git_config_get_string(&worktree, config, "core.worktree");
-	if (!error && worktree != NULL) {
-		error = git_path_prettify_dir(
-			&worktree_buf, worktree, repo->path_repository);
-		if (error < 0)
+	if (ce && ce->value) {
+		if ((error = git_path_prettify_dir(
+				&worktree, ce->value, repo->path_repository)) < 0)
 			return error;
-		repo->workdir = git_buf_detach(&worktree_buf);
+
+		repo->workdir = git_buf_detach(&worktree);
 	}
-	else if (error != GIT_ENOTFOUND)
-		return error;
+	else if (parent_path && git_path_isdir(parent_path->ptr))
+		repo->workdir = git_buf_detach(parent_path);
 	else {
-		giterr_clear();
+		if (git_path_dirname_r(&worktree, repo->path_repository) < 0 ||
+			git_path_to_dir(&worktree) < 0)
+			return -1;
 
-		if (parent_path && git_path_isdir(parent_path->ptr))
-			repo->workdir = git_buf_detach(parent_path);
-		else {
-			git_path_dirname_r(&worktree_buf, repo->path_repository);
-			git_path_to_dir(&worktree_buf);
-			repo->workdir = git_buf_detach(&worktree_buf);
-		}
+		repo->workdir = git_buf_detach(&worktree);
 	}
 
 	GITERR_CHECK_ALLOC(repo->workdir);
-
 	return 0;
 }
 
@@ -291,16 +285,20 @@ static int read_gitfile(git_buf *path_out, const char *file_path)
 		return -1;
 
 	git_buf_rtrim(&file);
+	/* apparently on Windows, some people use backslashes in paths */
+	git_path_mkposix(file.ptr);
 
 	if (git_buf_len(&file) <= prefix_len ||
 		memcmp(git_buf_cstr(&file), GIT_FILE_CONTENT_PREFIX, prefix_len) != 0)
 	{
-		giterr_set(GITERR_REPOSITORY, "The `.git` file at '%s' is malformed", file_path);
+		giterr_set(GITERR_REPOSITORY,
+			"The `.git` file at '%s' is malformed", file_path);
 		error = -1;
 	}
 	else if ((error = git_path_dirname_r(path_out, file_path)) >= 0) {
 		const char *gitlink = git_buf_cstr(&file) + prefix_len;
 		while (*gitlink && git__isspace(*gitlink)) gitlink++;
+
 		error = git_path_prettify_dir(
 			path_out, gitlink, git_buf_cstr(path_out));
 	}
@@ -461,16 +459,22 @@ int git_repository_open_ext(
 
 	if ((flags & GIT_REPOSITORY_OPEN_BARE) != 0)
 		repo->is_bare = 1;
-	else if ((error = load_config_data(repo)) < 0 ||
-		(error = load_workdir(repo, &parent)) < 0)
-	{
-		git_repository_free(repo);
-		return error;
+	else {
+		git_config *config = NULL;
+
+		if ((error = git_repository_config_snapshot(&config, repo)) < 0 ||
+			(error = load_config_data(repo, config)) < 0 ||
+			(error = load_workdir(repo, config, &parent)) < 0)
+			git_repository_free(repo);
+
+		git_config_free(config);
 	}
 
+	if (!error)
+		*repo_ptr = repo;
 	git_buf_free(&parent);
-	*repo_ptr = repo;
-	return 0;
+
+	return error;
 }
 
 int git_repository_open(git_repository **repo_out, const char *path)
@@ -493,34 +497,18 @@ int git_repository_wrap_odb(git_repository **repo_out, git_odb *odb)
 }
 
 int git_repository_discover(
-	char *repository_path,
-	size_t size,
+	git_buf *out,
 	const char *start_path,
 	int across_fs,
 	const char *ceiling_dirs)
 {
-	git_buf path = GIT_BUF_INIT;
 	uint32_t flags = across_fs ? GIT_REPOSITORY_OPEN_CROSS_FS : 0;
-	int error;
-
-	assert(start_path && repository_path && size > 0);
-
-	*repository_path = '\0';
 
-	if ((error = find_repo(&path, NULL, start_path, flags, ceiling_dirs)) < 0)
-		return error != GIT_ENOTFOUND ? -1 : error;
+	assert(start_path);
 
-	if (size < (size_t)(path.size + 1)) {
-		giterr_set(GITERR_REPOSITORY,
-			"The given buffer is too small to store the discovered path");
-		git_buf_free(&path);
-		return -1;
-	}
+	git_buf_sanitize(out);
 
-	/* success: we discovered a repository */
-	git_buf_copy_cstr(repository_path, size, &path);
-	git_buf_free(&path);
-	return 0;
+	return find_repo(out, NULL, start_path, flags, ceiling_dirs);
 }
 
 static int load_config(
@@ -596,9 +584,9 @@ int git_repository_config__weakptr(git_config **out, git_repository *repo)
 		git_buf system_buf = GIT_BUF_INIT;
 		git_config *config;
 
-		git_config_find_global_r(&global_buf);
-		git_config_find_xdg_r(&xdg_buf);
-		git_config_find_system_r(&system_buf);
+		git_config_find_global(&global_buf);
+		git_config_find_xdg(&xdg_buf);
+		git_config_find_system(&system_buf);
 
 		/* If there is no global file, open a backend for it anyway */
 		if (git_buf_len(&global_buf) == 0)
@@ -637,6 +625,17 @@ int git_repository_config(git_config **out, git_repository *repo)
 	return 0;
 }
 
+int git_repository_config_snapshot(git_config **out, git_repository *repo)
+{
+	int error;
+	git_config *weak;
+
+	if ((error = git_repository_config__weakptr(&weak, repo)) < 0)
+		return error;
+
+	return git_config_snapshot(out, weak);
+}
+
 void git_repository_set_config(git_repository *repo, git_config *config)
 {
 	assert(repo && config);
@@ -890,60 +889,6 @@ static bool are_symlinks_supported(const char *wd_path)
 	return symlinks_supported;
 }
 
-#ifdef GIT_USE_ICONV
-
-static const char *nfc_file = "\xC3\x85\x73\x74\x72\xC3\xB6\x6D.XXXXXX";
-static const char *nfd_file = "\x41\xCC\x8A\x73\x74\x72\x6F\xCC\x88\x6D.XXXXXX";
-
-/* Check if the platform is decomposing unicode data for us.  We will
- * emulate core Git and prefer to use precomposed unicode data internally
- * on these platforms, composing the decomposed unicode on the fly.
- *
- * This mainly happens on the Mac where HDFS stores filenames as
- * decomposed unicode.  Even on VFAT and SAMBA file systems, the Mac will
- * return decomposed unicode from readdir() even when the actual
- * filesystem is storing precomposed unicode.
- */
-static bool does_fs_decompose_unicode_paths(const char *wd_path)
-{
-	git_buf path = GIT_BUF_INIT;
-	int fd;
-	bool found_decomposed = false;
-	char tmp[6];
-
-	/* Create a file using a precomposed path and then try to find it
-	 * using the decomposed name.  If the lookup fails, then we will mark
-	 * that we should precompose unicode for this repository.
-	 */
-	if (git_buf_joinpath(&path, wd_path, nfc_file) < 0 ||
-		(fd = p_mkstemp(path.ptr)) < 0)
-		goto done;
-	p_close(fd);
-
-	/* record trailing digits generated by mkstemp */
-	memcpy(tmp, path.ptr + path.size - sizeof(tmp), sizeof(tmp));
-
-	/* try to look up as NFD path */
-	if (git_buf_joinpath(&path, wd_path, nfd_file) < 0)
-		goto done;
-	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
-
-	found_decomposed = git_path_exists(path.ptr);
-
-	/* remove temporary file (using original precomposed path) */
-	if (git_buf_joinpath(&path, wd_path, nfc_file) < 0)
-		goto done;
-	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
-
-	(void)p_unlink(path.ptr);
-
-done:
-	git_buf_free(&path);
-	return found_decomposed;
-}
-
-#endif
-
 static int create_empty_file(const char *path, mode_t mode)
 {
 	int fd;
@@ -1034,8 +979,9 @@ static int repo_init_fs_configs(
 #ifdef GIT_USE_ICONV
 	if ((error = git_config_set_bool(
 			cfg, "core.precomposeunicode",
-			does_fs_decompose_unicode_paths(work_dir))) < 0)
+			git_path_does_fs_decompose_unicode(work_dir))) < 0)
 		return error;
+	/* on non-iconv platforms, don't even set core.precomposeunicode */
 #endif
 
 	return 0;
@@ -1163,7 +1109,7 @@ static int repo_write_template(
 
 #ifdef GIT_WIN32
 	if (!error && hidden) {
-		if (p_hide_directory__w32(path.ptr) < 0)
+		if (git_win32__sethidden(path.ptr) < 0)
 			error = -1;
 	}
 #else
@@ -1244,12 +1190,13 @@ static int repo_init_structure(
 	bool external_tpl =
 		((opts->flags & GIT_REPOSITORY_INIT_EXTERNAL_TEMPLATE) != 0);
 	mode_t dmode = pick_dir_mode(opts);
+	bool chmod = opts->mode != GIT_REPOSITORY_INIT_SHARED_UMASK;
 
 	/* Hide the ".git" directory */
 #ifdef GIT_WIN32
 	if ((opts->flags & GIT_REPOSITORY_INIT__HAS_DOTGIT) != 0) {
-		if (p_hide_directory__w32(repo_dir) < 0) {
-			giterr_set(GITERR_REPOSITORY,
+		if (git_win32__sethidden(repo_dir) < 0) {
+			giterr_set(GITERR_OS,
 				"Failed to mark Git repository folder as hidden");
 			return -1;
 		}
@@ -1279,15 +1226,17 @@ static int repo_init_structure(
 		}
 
 		if (!tdir) {
-			if (!(error = git_futils_find_template_dir(&template_buf)))
+			if (!(error = git_sysdir_find_template_dir(&template_buf)))
 				tdir = template_buf.ptr;
 			default_template = true;
 		}
 
-		if (tdir)
-			error = git_futils_cp_r(tdir, repo_dir,
-				GIT_CPDIR_COPY_SYMLINKS | GIT_CPDIR_CHMOD_DIRS |
-				GIT_CPDIR_SIMPLE_TO_MODE, dmode);
+		if (tdir) {
+			uint32_t cpflags = GIT_CPDIR_COPY_SYMLINKS | GIT_CPDIR_SIMPLE_TO_MODE;
+			if (opts->mode != GIT_REPOSITORY_INIT_SHARED_UMASK)
+					cpflags |= GIT_CPDIR_CHMOD_DIRS;
+			error = git_futils_cp_r(tdir, repo_dir, cpflags, dmode);
+		}
 
 		git_buf_free(&template_buf);
 		git_config_free(cfg);
@@ -1308,9 +1257,14 @@ static int repo_init_structure(
 	 * - only create files if no external template was specified
 	 */
 	for (tpl = repo_template; !error && tpl->path; ++tpl) {
-		if (!tpl->content)
+		if (!tpl->content) {
+			uint32_t mkdir_flags = GIT_MKDIR_PATH;
+			if (chmod)
+				mkdir_flags |= GIT_MKDIR_CHMOD;
+
 			error = git_futils_mkdir(
-				tpl->path, repo_dir, dmode, GIT_MKDIR_PATH | GIT_MKDIR_CHMOD);
+				tpl->path, repo_dir, dmode, mkdir_flags);
+		}
 		else if (!external_tpl) {
 			const char *content = tpl->content;
 
@@ -1610,15 +1564,14 @@ static int at_least_one_cb(const char *refname, void *payload)
 {
 	GIT_UNUSED(refname);
 	GIT_UNUSED(payload);
-
-	return GIT_EUSER;
+	return GIT_PASSTHROUGH;
 }
 
 static int repo_contains_no_reference(git_repository *repo)
 {
 	int error = git_reference_foreach_name(repo, &at_least_one_cb, NULL);
 
-	if (error == GIT_EUSER)
+	if (error == GIT_PASSTHROUGH)
 		return 0;
 
 	if (!error)
@@ -1731,14 +1684,13 @@ cleanup:
 	return error;
 }
 
-int git_repository_message(char *buffer, size_t len, git_repository *repo)
+int git_repository_message(git_buf *out, git_repository *repo)
 {
-	git_buf buf = GIT_BUF_INIT, path = GIT_BUF_INIT;
+	git_buf path = GIT_BUF_INIT;
 	struct stat st;
 	int error;
 
-	if (buffer != NULL)
-		*buffer = '\0';
+	git_buf_sanitize(out);
 
 	if (git_buf_joinpath(&path, repo->path_repository, GIT_MERGE_MSG_FILE) < 0)
 		return -1;
@@ -1747,17 +1699,11 @@ int git_repository_message(char *buffer, size_t len, git_repository *repo)
 		if (errno == ENOENT)
 			error = GIT_ENOTFOUND;
 		giterr_set(GITERR_OS, "Could not access message file");
-	}
-	else if (buffer != NULL) {
-		error = git_futils_readbuffer(&buf, git_buf_cstr(&path));
-		git_buf_copy_cstr(buffer, len, &buf);
+	} else {
+		error = git_futils_readbuffer(out, git_buf_cstr(&path));
 	}
 
 	git_buf_free(&path);
-	git_buf_free(&buf);
-
-	if (!error)
-		error = (int)st.st_size + 1; /* add 1 for NUL byte */
 
 	return error;
 }
@@ -1807,7 +1753,8 @@ int git_repository_hashfile(
 	/* passing empty string for "as_path" indicated --no-filters */
 	if (strlen(as_path) > 0) {
 		error = git_filter_list_load(
-			&fl, repo, NULL, as_path, GIT_FILTER_TO_ODB);
+			&fl, repo, NULL, as_path,
+			GIT_FILTER_TO_ODB, GIT_FILTER_OPT_DEFAULT);
 		if (error < 0)
 			return error;
 	} else {
@@ -1852,7 +1799,9 @@ static bool looks_like_a_branch(const char *refname)
 
 int git_repository_set_head(
 	git_repository* repo,
-	const char* refname)
+	const char* refname,
+	const git_signature *signature,
+	const char *log_message)
 {
 	git_reference *ref,
 		*new_head = NULL;
@@ -1865,12 +1814,17 @@ int git_repository_set_head(
 		return error;
 
 	if (!error) {
-		if (git_reference_is_branch(ref))
-			error = git_reference_symbolic_create(&new_head, repo, GIT_HEAD_FILE, git_reference_name(ref), 1);
-		else
-			error = git_repository_set_head_detached(repo, git_reference_target(ref));
-	} else if (looks_like_a_branch(refname))
-		error = git_reference_symbolic_create(&new_head, repo, GIT_HEAD_FILE, refname, 1);
+		if (git_reference_is_branch(ref)) {
+			error = git_reference_symbolic_create(&new_head, repo, GIT_HEAD_FILE,
+					git_reference_name(ref), true, signature, log_message);
+		} else {
+			error = git_repository_set_head_detached(repo, git_reference_target(ref),
+					signature, log_message);
+		}
+	} else if (looks_like_a_branch(refname)) {
+		error = git_reference_symbolic_create(&new_head, repo, GIT_HEAD_FILE, refname,
+				true, signature, log_message);
+	}
 
 	git_reference_free(ref);
 	git_reference_free(new_head);
@@ -1879,7 +1833,9 @@ int git_repository_set_head(
 
 int git_repository_set_head_detached(
 	git_repository* repo,
-	const git_oid* commitish)
+	const git_oid* commitish,
+	const git_signature *signature,
+	const char *log_message)
 {
 	int error;
 	git_object *object,
@@ -1894,7 +1850,7 @@ int git_repository_set_head_detached(
 	if ((error = git_object_peel(&peeled, object, GIT_OBJ_COMMIT)) < 0)
 		goto cleanup;
 
-	error = git_reference_create(&new_head, repo, GIT_HEAD_FILE, git_object_id(peeled), 1);
+	error = git_reference_create(&new_head, repo, GIT_HEAD_FILE, git_object_id(peeled), true, signature, log_message);
 
 cleanup:
 	git_object_free(object);
@@ -1904,7 +1860,9 @@ cleanup:
 }
 
 int git_repository_detach_head(
-	git_repository* repo)
+	git_repository* repo,
+	const git_signature *signature,
+	const char *reflog_message)
 {
 	git_reference *old_head = NULL,
 		*new_head = NULL;
@@ -1919,7 +1877,8 @@ int git_repository_detach_head(
 	if ((error = git_object_lookup(&object, repo, git_reference_target(old_head), GIT_OBJ_COMMIT)) < 0)
 		goto cleanup;
 
-	error = git_reference_create(&new_head, repo, GIT_HEAD_FILE, git_reference_target(old_head), 1);
+	error = git_reference_create(&new_head, repo, GIT_HEAD_FILE, git_reference_target(old_head),
+			1, signature, reflog_message);
 
 cleanup:
 	git_object_free(object);
@@ -1965,6 +1924,53 @@ int git_repository_state(git_repository *repo)
 	return state;
 }
 
+int git_repository__cleanup_files(
+	git_repository *repo, const char *files[], size_t files_len)
+{
+	git_buf buf = GIT_BUF_INIT;
+	size_t i;
+	int error;
+
+	for (error = 0, i = 0; !error && i < files_len; ++i) {
+		const char *path;
+
+		if (git_buf_joinpath(&buf, repo->path_repository, files[i]) < 0)
+			return -1;
+
+		path = git_buf_cstr(&buf);
+
+		if (git_path_isfile(path)) {
+			error = p_unlink(path);
+		} else if (git_path_isdir(path)) {
+			error = git_futils_rmdir_r(path, NULL,
+				GIT_RMDIR_REMOVE_FILES | GIT_RMDIR_REMOVE_BLOCKERS);
+		}
+			
+		git_buf_clear(&buf);
+	}
+
+	git_buf_free(&buf);
+	return error;
+}
+
+static const char *state_files[] = {
+	GIT_MERGE_HEAD_FILE,
+	GIT_MERGE_MODE_FILE,
+	GIT_MERGE_MSG_FILE,
+	GIT_REVERT_HEAD_FILE,
+	GIT_CHERRY_PICK_HEAD_FILE,
+	GIT_BISECT_LOG_FILE,
+	GIT_REBASE_MERGE_DIR,
+	GIT_REBASE_APPLY_DIR,
+};
+
+int git_repository_state_cleanup(git_repository *repo)
+{
+	assert(repo);
+
+	return git_repository__cleanup_files(repo, state_files, ARRAY_SIZE(state_files));
+}
+
 int git_repository_is_shallow(git_repository *repo)
 {
 	git_buf path = GIT_BUF_INIT;
@@ -1975,9 +1981,21 @@ int git_repository_is_shallow(git_repository *repo)
 	error = git_path_lstat(path.ptr, &st);
 	git_buf_free(&path);
 
-	if (error == GIT_ENOTFOUND)
+	if (error == GIT_ENOTFOUND) {
+		giterr_clear();
 		return 0;
+	}
+
 	if (error < 0)
-		return -1;
+		return error;
 	return st.st_size == 0 ? 0 : 1;
 }
+
+int git_repository_init_init_options(
+	git_repository_init_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_repository_init_options,
+		GIT_REPOSITORY_INIT_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/repository.h b/src/repository.h
index 832df3bd2..aba16a016 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -19,7 +19,7 @@
 #include "buffer.h"
 #include "object.h"
 #include "attrcache.h"
-#include "strmap.h"
+#include "submodule.h"
 #include "diff_driver.h"
 
 #define DOT_GIT ".git"
@@ -38,6 +38,8 @@ typedef enum {
 	GIT_CVAR_TRUSTCTIME,    /* core.trustctime */
 	GIT_CVAR_ABBREV,        /* core.abbrev */
 	GIT_CVAR_PRECOMPOSE,    /* core.precomposeunicode */
+	GIT_CVAR_SAFE_CRLF,		/* core.safecrlf */
+	GIT_CVAR_LOGALLREFUPDATES, /* core.logallrefupdates */
 	GIT_CVAR_CACHE_MAX
 } git_cvar_cached;
 
@@ -89,7 +91,11 @@ typedef enum {
 	GIT_ABBREV_DEFAULT = 7,
 	/* core.precomposeunicode */
 	GIT_PRECOMPOSE_DEFAULT = GIT_CVAR_FALSE,
-
+	/* core.safecrlf */
+	GIT_SAFE_CRLF_DEFAULT = GIT_CVAR_FALSE,
+	/* core.logallrefupdates */
+	GIT_LOGALLREFUPDATES_UNSET = 2,
+	GIT_LOGALLREFUPDATES_DEFAULT = GIT_LOGALLREFUPDATES_UNSET,
 } git_cvar_value;
 
 /* internal repository init flags */
@@ -105,10 +111,10 @@ struct git_repository {
 	git_refdb *_refdb;
 	git_config *_config;
 	git_index *_index;
+	git_submodule_cache *_submodules;
 
 	git_cache objects;
-	git_attr_cache attrcache;
-	git_strmap *submodules;
+	git_attr_cache *attrcache;
 	git_diff_driver_registry *diff_drivers;
 
 	char *path_repository;
@@ -123,7 +129,7 @@ struct git_repository {
 
 GIT_INLINE(git_attr_cache *) git_repository_attr_cache(git_repository *repo)
 {
-	return &repo->attrcache;
+	return repo->attrcache;
 }
 
 int git_repository_head_tree(git_tree **tree, git_repository *repo);
@@ -149,11 +155,6 @@ int git_repository_index__weakptr(git_index **out, git_repository *repo);
 int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar);
 void git_repository__cvar_cache_clear(git_repository *repo);
 
-/*
- * Submodule cache
- */
-extern void git_submodule_config_free(git_repository *repo);
-
 GIT_INLINE(int) git_repository__ensure_not_bare(
 	git_repository *repo,
 	const char *operation_name)
@@ -169,4 +170,6 @@ GIT_INLINE(int) git_repository__ensure_not_bare(
 	return GIT_EBAREREPO;
 }
 
+int git_repository__cleanup_files(git_repository *repo, const char *files[], size_t files_len);
+
 #endif
diff --git a/src/reset.c b/src/reset.c
index a9780bfbc..248c91d3a 100644
--- a/src/reset.c
+++ b/src/reset.c
@@ -60,19 +60,24 @@ int git_reset_default(
 	for (i = 0, max_i = git_diff_num_deltas(diff); i < max_i; ++i) {
 		const git_diff_delta *delta = git_diff_get_delta(diff, i);
 
-		if ((error = git_index_conflict_remove(index, delta->old_file.path)) < 0)
-			goto cleanup;
-
 		assert(delta->status == GIT_DELTA_ADDED ||
 			delta->status == GIT_DELTA_MODIFIED ||
 			delta->status == GIT_DELTA_DELETED);
 
+		error = git_index_conflict_remove(index, delta->old_file.path);
+		if (error < 0) {
+			if (delta->status == GIT_DELTA_ADDED && error == GIT_ENOTFOUND)
+				giterr_clear();
+			else
+				goto cleanup;
+		}
+
 		if (delta->status == GIT_DELTA_DELETED) {
 			if ((error = git_index_remove(index, delta->old_file.path, 0)) < 0)
 				goto cleanup;
 		} else {
 			entry.mode = delta->new_file.mode;
-			git_oid_cpy(&entry.oid, &delta->new_file.oid);
+			git_oid_cpy(&entry.id, &delta->new_file.id);
 			entry.path = (char *)delta->new_file.path;
 
 			if ((error = git_index_add(index, &entry)) < 0)
@@ -94,13 +99,16 @@ cleanup:
 int git_reset(
 	git_repository *repo,
 	git_object *target,
-	git_reset_t reset_type)
+	git_reset_t reset_type,
+	git_signature *signature,
+	const char *log_message)
 {
 	git_object *commit = NULL;
 	git_index *index = NULL;
 	git_tree *tree = NULL;
 	int error = 0;
-	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT;
+	git_buf log_message_buf = GIT_BUF_INIT;
 
 	assert(repo && target);
 
@@ -129,9 +137,14 @@ int git_reset(
 		goto cleanup;
 	}
 
+	if (log_message)
+		git_buf_sets(&log_message_buf, log_message);
+	else
+		git_buf_sets(&log_message_buf, "reset: moving");
+
 	/* move HEAD to the new target */
 	if ((error = git_reference__update_terminal(repo, GIT_HEAD_FILE,
-		git_object_id(commit))) < 0)
+		git_object_id(commit), signature, git_buf_cstr(&log_message_buf))) < 0)
 		goto cleanup;
 
 	if (reset_type == GIT_RESET_HARD) {
@@ -149,7 +162,7 @@ int git_reset(
 			(error = git_index_write(index)) < 0)
 			goto cleanup;
 
-		if ((error = git_repository_merge_cleanup(repo)) < 0) {
+		if ((error = git_repository_state_cleanup(repo)) < 0) {
 			giterr_set(GITERR_INDEX, "%s - failed to clean up merge data", ERROR_MSG);
 			goto cleanup;
 		}
@@ -159,6 +172,7 @@ cleanup:
 	git_object_free(commit);
 	git_index_free(index);
 	git_tree_free(tree);
+	git_buf_free(&log_message_buf);
 
 	return error;
 }
diff --git a/src/revert.c b/src/revert.c
new file mode 100644
index 000000000..9c587724b
--- /dev/null
+++ b/src/revert.c
@@ -0,0 +1,228 @@
+/*
+* Copyright (C) the libgit2 contributors. All rights reserved.
+*
+* This file is part of libgit2, distributed under the GNU GPL v2 with
+* a Linking Exception. For full terms see the included COPYING file.
+*/
+
+#include "common.h"
+#include "repository.h"
+#include "filebuf.h"
+#include "merge.h"
+
+#include "git2/types.h"
+#include "git2/merge.h"
+#include "git2/revert.h"
+#include "git2/commit.h"
+#include "git2/sys/commit.h"
+
+#define GIT_REVERT_FILE_MODE		0666
+
+static int write_revert_head(
+	git_repository *repo,
+	const char *commit_oidstr)
+{
+	git_filebuf file = GIT_FILEBUF_INIT;
+	git_buf file_path = GIT_BUF_INIT;
+	int error = 0;
+
+	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_REVERT_HEAD_FILE)) >= 0 &&
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_REVERT_FILE_MODE)) >= 0 &&
+		(error = git_filebuf_printf(&file, "%s\n", commit_oidstr)) >= 0)
+		error = git_filebuf_commit(&file);
+
+	if (error < 0)
+		git_filebuf_cleanup(&file);
+
+	git_buf_free(&file_path);
+
+	return error;
+}
+
+static int write_merge_msg(
+	git_repository *repo,
+	const char *commit_oidstr,
+	const char *commit_msgline)
+{
+	git_filebuf file = GIT_FILEBUF_INIT;
+	git_buf file_path = GIT_BUF_INIT;
+	int error = 0;
+
+	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_MSG_FILE)) < 0 ||
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_REVERT_FILE_MODE)) < 0 ||
+		(error = git_filebuf_printf(&file, "Revert \"%s\"\n\nThis reverts commit %s.\n",
+		commit_msgline, commit_oidstr)) < 0)
+		goto cleanup;
+
+	error = git_filebuf_commit(&file);
+
+cleanup:
+	if (error < 0)
+		git_filebuf_cleanup(&file);
+
+	git_buf_free(&file_path);
+
+	return error;
+}
+
+static int revert_normalize_opts(
+	git_repository *repo,
+	git_revert_options *opts,
+	const git_revert_options *given,
+	const char *their_label)
+{
+	int error = 0;
+	unsigned int default_checkout_strategy = GIT_CHECKOUT_SAFE_CREATE |
+		GIT_CHECKOUT_ALLOW_CONFLICTS;
+
+	GIT_UNUSED(repo);
+
+	if (given != NULL)
+		memcpy(opts, given, sizeof(git_revert_options));
+	else {
+		git_revert_options default_opts = GIT_REVERT_OPTIONS_INIT;
+		memcpy(opts, &default_opts, sizeof(git_revert_options));
+	}
+
+	if (!opts->checkout_opts.checkout_strategy)
+		opts->checkout_opts.checkout_strategy = default_checkout_strategy;
+
+	if (!opts->checkout_opts.our_label)
+		opts->checkout_opts.our_label = "HEAD";
+
+	if (!opts->checkout_opts.their_label)
+		opts->checkout_opts.their_label = their_label;
+
+	return error;
+}
+
+static int revert_state_cleanup(git_repository *repo)
+{
+	const char *state_files[] = { GIT_REVERT_HEAD_FILE, GIT_MERGE_MSG_FILE };
+
+	return git_repository__cleanup_files(repo, state_files, ARRAY_SIZE(state_files));
+}
+
+static int revert_seterr(git_commit *commit, const char *fmt)
+{
+	char commit_oidstr[GIT_OID_HEXSZ + 1];
+
+	git_oid_fmt(commit_oidstr, git_commit_id(commit));
+	commit_oidstr[GIT_OID_HEXSZ] = '\0';
+
+	giterr_set(GITERR_REVERT, fmt, commit_oidstr);
+
+	return -1;
+}
+
+int git_revert_commit(
+	git_index **out,
+	git_repository *repo,
+	git_commit *revert_commit,
+	git_commit *our_commit,
+	unsigned int mainline,
+	const git_merge_options *merge_opts)
+{
+	git_commit *parent_commit = NULL;
+	git_tree *parent_tree = NULL, *our_tree = NULL, *revert_tree = NULL;
+	int parent = 0, error = 0;
+
+	assert(out && repo && revert_commit && our_commit);
+
+	if (git_commit_parentcount(revert_commit) > 1) {
+		if (!mainline)
+			return revert_seterr(revert_commit,
+				"Mainline branch is not specified but %s is a merge commit");
+
+		parent = mainline;
+	} else {
+		if (mainline)
+			return revert_seterr(revert_commit,
+				"Mainline branch specified but %s is not a merge commit");
+
+		parent = git_commit_parentcount(revert_commit);
+	}
+
+	if (parent &&
+		((error = git_commit_parent(&parent_commit, revert_commit, (parent - 1))) < 0 ||
+		(error = git_commit_tree(&parent_tree, parent_commit)) < 0))
+		goto done;
+
+	if ((error = git_commit_tree(&revert_tree, revert_commit)) < 0 ||
+		(error = git_commit_tree(&our_tree, our_commit)) < 0)
+		goto done;
+
+	error = git_merge_trees(out, repo, revert_tree, our_tree, parent_tree, merge_opts);
+
+done:
+	git_tree_free(parent_tree);
+	git_tree_free(our_tree);
+	git_tree_free(revert_tree);
+	git_commit_free(parent_commit);
+
+	return error;
+}
+
+int git_revert(
+	git_repository *repo,
+	git_commit *commit,
+	const git_revert_options *given_opts)
+{
+	git_revert_options opts;
+	git_reference *our_ref = NULL;
+	git_commit *our_commit = NULL;
+	char commit_oidstr[GIT_OID_HEXSZ + 1];
+	const char *commit_msg;
+	git_buf their_label = GIT_BUF_INIT;
+	git_index *index_new = NULL, *index_repo = NULL;
+	int error;
+
+	assert(repo && commit);
+
+	GITERR_CHECK_VERSION(given_opts, GIT_REVERT_OPTIONS_VERSION, "git_revert_options");
+
+	if ((error = git_repository__ensure_not_bare(repo, "revert")) < 0)
+		return error;
+
+	git_oid_fmt(commit_oidstr, git_commit_id(commit));
+	commit_oidstr[GIT_OID_HEXSZ] = '\0';
+
+	if ((commit_msg = git_commit_summary(commit)) == NULL) {
+		error = -1;
+		goto on_error;
+	}
+
+	if ((error = git_buf_printf(&their_label, "parent of %.7s... %s", commit_oidstr, commit_msg)) < 0 ||
+		(error = revert_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0 ||
+		(error = write_revert_head(repo, commit_oidstr)) < 0 ||
+		(error = write_merge_msg(repo, commit_oidstr, commit_msg)) < 0 ||
+		(error = git_repository_head(&our_ref, repo)) < 0 ||
+		(error = git_reference_peel((git_object **)&our_commit, our_ref, GIT_OBJ_COMMIT)) < 0 ||
+		(error = git_revert_commit(&index_new, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 ||
+		(error = git_merge__indexes(repo, index_new)) < 0 ||
+		(error = git_repository_index(&index_repo, repo)) < 0 ||
+		(error = git_merge__append_conflicts_to_merge_msg(repo, index_repo)) < 0 ||
+		(error = git_checkout_index(repo, index_repo, &opts.checkout_opts)) < 0)
+		goto on_error;
+
+	goto done;
+
+on_error:
+	revert_state_cleanup(repo);
+
+done:
+	git_index_free(index_new);
+	git_index_free(index_repo);
+	git_commit_free(our_commit);
+	git_reference_free(our_ref);
+	git_buf_free(&their_label);
+
+	return error;
+}
+
+int git_revert_init_options(git_revert_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_revert_options, GIT_REVERT_OPTIONS_INIT);
+	return 0;
+}
diff --git a/src/revparse.c b/src/revparse.c
index c120b466f..60872e187 100644
--- a/src/revparse.c
+++ b/src/revparse.c
@@ -168,6 +168,8 @@ static int retrieve_previously_checked_out_branch_or_revision(git_object **out,
 	for (i = 0; i < numentries; i++) {
 		entry = git_reflog_entry_byindex(reflog, i);
 		msg = git_reflog_entry_message(entry);
+		if (!msg)
+			continue;
 
 		if (regexec(&preg, msg, 2, regexmatches, 0))
 			continue;
@@ -488,8 +490,7 @@ static int handle_grep_syntax(git_object **out, git_repository *repo, const git_
 	git_revwalk_sorting(walk, GIT_SORT_TIME);
 
 	if (spec_oid == NULL) {
-		// TODO: @carlosmn: The glob should be refs/* but this makes git_revwalk_next() fails
-		if ((error = git_revwalk_push_glob(walk, GIT_REFS_HEADS_DIR "*")) < 0)
+		if ((error = git_revwalk_push_glob(walk, "refs/*")) < 0)
 			goto cleanup;
 	} else if ((error = git_revwalk_push(walk, spec_oid)) < 0)
 			goto cleanup;
diff --git a/src/revwalk.c b/src/revwalk.c
index 3dd14b419..530c9705e 100644
--- a/src/revwalk.c
+++ b/src/revwalk.c
@@ -39,26 +39,23 @@ git_commit_list_node *git_revwalk__commit_lookup(
 	return commit;
 }
 
-static int mark_uninteresting(git_commit_list_node *commit)
+static int mark_uninteresting(git_revwalk *walk, git_commit_list_node *commit)
 {
+	int error;
 	unsigned short i;
 	git_array_t(git_commit_list_node *) pending = GIT_ARRAY_INIT;
 	git_commit_list_node **tmp;
 
 	assert(commit);
 
-	git_array_alloc(pending);
+	git_array_init_to_size(pending, 2);
 	GITERR_CHECK_ARRAY(pending);
 
 	do {
 		commit->uninteresting = 1;
 
-		/* This means we've reached a merge base, so there's no need to walk any more */
-		if ((commit->flags & (RESULT | STALE)) == RESULT) {
-			tmp = git_array_pop(pending);
-			commit = tmp ? *tmp : NULL;
-			continue;
-		}
+		if ((error = git_commit_list_parse(walk, commit)) < 0)
+			return error;
 
 		for (i = 0; i < commit->out_degree; ++i)
 			if (!commit->parents[i]->uninteresting) {
@@ -70,7 +67,7 @@ static int mark_uninteresting(git_commit_list_node *commit)
 		tmp = git_array_pop(pending);
 		commit = tmp ? *tmp : NULL;
 
-	} while (git_array_size(pending) > 0);
+	} while (commit != NULL);
 
 	git_array_clear(pending);
 
@@ -81,7 +78,10 @@ static int process_commit(git_revwalk *walk, git_commit_list_node *commit, int h
 {
 	int error;
 
-	if (hide && mark_uninteresting(commit) < 0)
+	if (!hide && walk->hide_cb)
+		hide = walk->hide_cb(&commit->oid, walk->hide_cb_payload);
+
+	if (hide && mark_uninteresting(walk, commit) < 0)
 		return -1;
 
 	if (commit->seen)
@@ -92,7 +92,10 @@ static int process_commit(git_revwalk *walk, git_commit_list_node *commit, int h
 	if ((error = git_commit_list_parse(walk, commit)) < 0)
 		return error;
 
-	return walk->enqueue(walk, commit);
+	if (!hide)
+		return walk->enqueue(walk, commit);
+
+	return 0;
 }
 
 static int process_commit_parents(git_revwalk *walk, git_commit_list_node *commit)
@@ -110,24 +113,34 @@ static int process_commit_parents(git_revwalk *walk, git_commit_list_node *commi
 	return error;
 }
 
-static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting)
+static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting, int from_glob)
 {
-	git_object *obj;
-	git_otype type;
+	git_oid commit_id;
+	int error;
+	git_object *obj, *oobj;
 	git_commit_list_node *commit;
 
-	if (git_object_lookup(&obj, walk->repo, oid, GIT_OBJ_ANY) < 0)
-		return -1;
+	if ((error = git_object_lookup(&oobj, walk->repo, oid, GIT_OBJ_ANY)) < 0)
+		return error;
 
-	type = git_object_type(obj);
-	git_object_free(obj);
+	error = git_object_peel(&obj, oobj, GIT_OBJ_COMMIT);
+	git_object_free(oobj);
+
+	if (error == GIT_ENOTFOUND) {
+		/* If this comes from e.g. push_glob("tags"), ignore this */
+		if (from_glob)
+			return 0;
 
-	if (type != GIT_OBJ_COMMIT) {
-		giterr_set(GITERR_INVALID, "Object is no commit object");
+		giterr_set(GITERR_INVALID, "Object is not a committish");
 		return -1;
 	}
+	if (error < 0)
+		return error;
+
+	git_oid_cpy(&commit_id, git_object_id(obj));
+	git_object_free(obj);
 
-	commit = git_revwalk__commit_lookup(walk, oid);
+	commit = git_revwalk__commit_lookup(walk, &commit_id);
 	if (commit == NULL)
 		return -1; /* error already reported by failed lookup */
 
@@ -145,43 +158,32 @@ static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting)
 int git_revwalk_push(git_revwalk *walk, const git_oid *oid)
 {
 	assert(walk && oid);
-	return push_commit(walk, oid, 0);
+	return push_commit(walk, oid, 0, false);
 }
 
 
 int git_revwalk_hide(git_revwalk *walk, const git_oid *oid)
 {
 	assert(walk && oid);
-	return push_commit(walk, oid, 1);
+	return push_commit(walk, oid, 1, false);
 }
 
-static int push_ref(git_revwalk *walk, const char *refname, int hide)
+static int push_ref(git_revwalk *walk, const char *refname, int hide, int from_glob)
 {
 	git_oid oid;
 
 	if (git_reference_name_to_id(&oid, walk->repo, refname) < 0)
 		return -1;
 
-	return push_commit(walk, &oid, hide);
-}
-
-struct push_cb_data {
-	git_revwalk *walk;
-	int hide;
-};
-
-static int push_glob_cb(const char *refname, void *data_)
-{
-	struct push_cb_data *data = (struct push_cb_data *)data_;
-
-	return push_ref(data->walk, refname, data->hide);
+	return push_commit(walk, &oid, hide, from_glob);
 }
 
 static int push_glob(git_revwalk *walk, const char *glob, int hide)
 {
 	int error = 0;
 	git_buf buf = GIT_BUF_INIT;
-	struct push_cb_data data;
+	git_reference *ref;
+	git_reference_iterator *iter;
 	size_t wildcard;
 
 	assert(walk && glob);
@@ -191,21 +193,28 @@ static int push_glob(git_revwalk *walk, const char *glob, int hide)
 		git_buf_joinpath(&buf, GIT_REFS_DIR, glob);
 	else
 		git_buf_puts(&buf, glob);
+	if (git_buf_oom(&buf))
+		return -1;
 
 	/* If no '?', '*' or '[' exist, we append '/ *' to the glob */
 	wildcard = strcspn(glob, "?*[");
 	if (!glob[wildcard])
 		git_buf_put(&buf, "/*", 2);
 
-	data.walk = walk;
-	data.hide = hide;
+	if ((error = git_reference_iterator_glob_new(&iter, walk->repo, buf.ptr)) < 0)
+		goto out;
 
-	if (git_buf_oom(&buf))
-		error = -1;
-	else
-		error = git_reference_foreach_glob(
-			walk->repo, git_buf_cstr(&buf), push_glob_cb, &data);
+	while ((error = git_reference_next(&ref, iter)) == 0) {
+		error = push_ref(walk, git_reference_name(ref), hide, true);
+		git_reference_free(ref);
+		if (error < 0)
+			break;
+	}
+	git_reference_iterator_free(iter);
 
+	if (error == GIT_ITEROVER)
+		error = 0;
+out:
 	git_buf_free(&buf);
 	return error;
 }
@@ -225,19 +234,19 @@ int git_revwalk_hide_glob(git_revwalk *walk, const char *glob)
 int git_revwalk_push_head(git_revwalk *walk)
 {
 	assert(walk);
-	return push_ref(walk, GIT_HEAD_FILE, 0);
+	return push_ref(walk, GIT_HEAD_FILE, 0, false);
 }
 
 int git_revwalk_hide_head(git_revwalk *walk)
 {
 	assert(walk);
-	return push_ref(walk, GIT_HEAD_FILE, 1);
+	return push_ref(walk, GIT_HEAD_FILE, 1, false);
 }
 
 int git_revwalk_push_ref(git_revwalk *walk, const char *refname)
 {
 	assert(walk && refname);
-	return push_ref(walk, refname, 0);
+	return push_ref(walk, refname, 0, false);
 }
 
 int git_revwalk_push_range(git_revwalk *walk, const char *range)
@@ -254,10 +263,10 @@ int git_revwalk_push_range(git_revwalk *walk, const char *range)
 		return GIT_EINVALIDSPEC;
 	}
 
-	if ((error = push_commit(walk, git_object_id(revspec.from), 1)))
+	if ((error = push_commit(walk, git_object_id(revspec.from), 1, false)))
 		goto out;
 
-	error = push_commit(walk, git_object_id(revspec.to), 0);
+	error = push_commit(walk, git_object_id(revspec.to), 0, false);
 
 out:
 	git_object_free(revspec.from);
@@ -268,7 +277,7 @@ out:
 int git_revwalk_hide_ref(git_revwalk *walk, const char *refname)
 {
 	assert(walk && refname);
-	return push_ref(walk, refname, 1);
+	return push_ref(walk, refname, 1, false);
 }
 
 static int revwalk_enqueue_timesort(git_revwalk *walk, git_commit_list_node *commit)
@@ -286,15 +295,14 @@ static int revwalk_next_timesort(git_commit_list_node **object_out, git_revwalk
 	int error;
 	git_commit_list_node *next;
 
-	while ((next = git_pqueue_pop(&walk->iterator_time)) != NULL) {
-		if ((error = process_commit_parents(walk, next)) < 0)
-			return error;
-
+	while ((next = git_pqueue_pop(&walk->iterator_time)) != NULL)
 		if (!next->uninteresting) {
+			if ((error = process_commit_parents(walk, next)) < 0)
+				return error;
+
 			*object_out = next;
 			return 0;
 		}
-	}
 
 	giterr_clear();
 	return GIT_ITEROVER;
@@ -305,15 +313,14 @@ static int revwalk_next_unsorted(git_commit_list_node **object_out, git_revwalk
 	int error;
 	git_commit_list_node *next;
 
-	while ((next = git_commit_list_pop(&walk->iterator_rand)) != NULL) {
-		if ((error = process_commit_parents(walk, next)) < 0)
-			return error;
-
+	while ((next = git_commit_list_pop(&walk->iterator_rand)) != NULL)
 		if (!next->uninteresting) {
+			if ((error = process_commit_parents(walk, next)) < 0)
+				return error;
+
 			*object_out = next;
 			return 0;
 		}
-	}
 
 	giterr_clear();
 	return GIT_ITEROVER;
@@ -368,7 +375,6 @@ static int prepare_walk(git_revwalk *walk)
 	int error;
 	unsigned int i;
 	git_commit_list_node *next, *two;
-	git_commit_list *bases = NULL;
 
 	/*
 	 * If walk->one is NULL, there were no positive references,
@@ -379,11 +385,6 @@ static int prepare_walk(git_revwalk *walk)
 		return GIT_ITEROVER;
 	}
 
-	/* first figure out what the merge bases are */
-	if (git_merge__bases_many(&bases, walk, walk->one, &walk->twos) < 0)
-		return -1;
-
-	git_commit_list_free(&bases);
 	if (process_commit(walk, walk->one, walk->one->uninteresting) < 0)
 		return -1;
 
@@ -440,7 +441,8 @@ int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo)
 	walk->commits = git_oidmap_alloc();
 	GITERR_CHECK_ALLOC(walk->commits);
 
-	if (git_pqueue_init(&walk->iterator_time, 8, git_commit_list_time_cmp) < 0 ||
+	if (git_pqueue_init(
+			&walk->iterator_time, 0, 8, git_commit_list_time_cmp) < 0 ||
 		git_vector_init(&walk->twos, 4, NULL) < 0 ||
 		git_pool_init(&walk->commit_pool, 1,
 			git_pool__suggest_items_per_page(COMMIT_ALLOC) * COMMIT_ALLOC) < 0)
@@ -553,3 +555,25 @@ void git_revwalk_reset(git_revwalk *walk)
 	git_vector_clear(&walk->twos);
 }
 
+int git_revwalk_add_hide_cb(
+	git_revwalk *walk,
+	git_revwalk_hide_cb hide_cb,
+	void *payload)
+{
+	assert(walk);
+
+	if (walk->walking)
+		git_revwalk_reset(walk);
+
+	if (walk->hide_cb) {
+		/* There is already a callback added */
+		giterr_set(GITERR_INVALID, "There is already a callback added to hide commits in revision walker.");
+		return -1;
+	}
+
+	walk->hide_cb = hide_cb;
+	walk->hide_cb_payload = payload;
+
+	return 0;
+}
+
diff --git a/src/revwalk.h b/src/revwalk.h
index 8c821d098..d81f97c01 100644
--- a/src/revwalk.h
+++ b/src/revwalk.h
@@ -38,6 +38,10 @@ struct git_revwalk {
 	/* merge base calculation */
 	git_commit_list_node *one;
 	git_vector twos;
+
+	/* hide callback */
+	git_revwalk_hide_cb hide_cb;
+	void *hide_cb_payload;
 };
 
 git_commit_list_node *git_revwalk__commit_lookup(git_revwalk *walk, const git_oid *oid);
diff --git a/src/settings.c b/src/settings.c
new file mode 100644
index 000000000..1a21ea024
--- /dev/null
+++ b/src/settings.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include <git2.h>
+#include "common.h"
+#include "sysdir.h"
+#include "cache.h"
+
+void git_libgit2_version(int *major, int *minor, int *rev)
+{
+	*major = LIBGIT2_VER_MAJOR;
+	*minor = LIBGIT2_VER_MINOR;
+	*rev = LIBGIT2_VER_REVISION;
+}
+
+int git_libgit2_features()
+{
+	return 0
+#ifdef GIT_THREADS
+		| GIT_FEATURE_THREADS
+#endif
+#if defined(GIT_SSL) || defined(GIT_WINHTTP)
+		| GIT_FEATURE_HTTPS
+#endif
+#if defined(GIT_SSH)
+		| GIT_FEATURE_SSH
+#endif
+	;
+}
+
+/* Declarations for tuneable settings */
+extern size_t git_mwindow__window_size;
+extern size_t git_mwindow__mapped_limit;
+
+static int config_level_to_sysdir(int config_level)
+{
+	int val = -1;
+
+	switch (config_level) {
+	case GIT_CONFIG_LEVEL_SYSTEM: val = GIT_SYSDIR_SYSTEM; break;
+	case GIT_CONFIG_LEVEL_XDG:    val = GIT_SYSDIR_XDG; break;
+	case GIT_CONFIG_LEVEL_GLOBAL: val = GIT_SYSDIR_GLOBAL; break;
+	default:
+		giterr_set(
+			GITERR_INVALID, "Invalid config path selector %d", config_level);
+	}
+
+	return val;
+}
+
+int git_libgit2_opts(int key, ...)
+{
+	int error = 0;
+	va_list ap;
+
+	va_start(ap, key);
+
+	switch (key) {
+	case GIT_OPT_SET_MWINDOW_SIZE:
+		git_mwindow__window_size = va_arg(ap, size_t);
+		break;
+
+	case GIT_OPT_GET_MWINDOW_SIZE:
+		*(va_arg(ap, size_t *)) = git_mwindow__window_size;
+		break;
+
+	case GIT_OPT_SET_MWINDOW_MAPPED_LIMIT:
+		git_mwindow__mapped_limit = va_arg(ap, size_t);
+		break;
+
+	case GIT_OPT_GET_MWINDOW_MAPPED_LIMIT:
+		*(va_arg(ap, size_t *)) = git_mwindow__mapped_limit;
+		break;
+
+	case GIT_OPT_GET_SEARCH_PATH:
+		if ((error = config_level_to_sysdir(va_arg(ap, int))) >= 0) {
+			git_buf *out = va_arg(ap, git_buf *);
+			const git_buf *tmp;
+
+			git_buf_sanitize(out);
+			if ((error = git_sysdir_get(&tmp, error)) < 0)
+				break;
+
+			error = git_buf_sets(out, tmp->ptr);
+		}
+		break;
+
+	case GIT_OPT_SET_SEARCH_PATH:
+		if ((error = config_level_to_sysdir(va_arg(ap, int))) >= 0)
+			error = git_sysdir_set(error, va_arg(ap, const char *));
+		break;
+
+	case GIT_OPT_SET_CACHE_OBJECT_LIMIT:
+		{
+			git_otype type = (git_otype)va_arg(ap, int);
+			size_t size = va_arg(ap, size_t);
+			error = git_cache_set_max_object_size(type, size);
+			break;
+		}
+
+	case GIT_OPT_SET_CACHE_MAX_SIZE:
+		git_cache__max_storage = va_arg(ap, ssize_t);
+		break;
+
+	case GIT_OPT_ENABLE_CACHING:
+		git_cache__enabled = (va_arg(ap, int) != 0);
+		break;
+
+	case GIT_OPT_GET_CACHED_MEMORY:
+		*(va_arg(ap, ssize_t *)) = git_cache__current_storage.val;
+		*(va_arg(ap, ssize_t *)) = git_cache__max_storage;
+		break;
+
+	case GIT_OPT_GET_TEMPLATE_PATH:
+		{
+			git_buf *out = va_arg(ap, git_buf *);
+			const git_buf *tmp;
+
+			git_buf_sanitize(out);
+			if ((error = git_sysdir_get(&tmp, GIT_SYSDIR_TEMPLATE)) < 0)
+				break;
+
+			error = git_buf_sets(out, tmp->ptr);
+		}
+		break;
+
+	case GIT_OPT_SET_TEMPLATE_PATH:
+		error = git_sysdir_set(GIT_SYSDIR_TEMPLATE, va_arg(ap, const char *));
+		break;
+	}
+
+	va_end(ap);
+
+	return error;
+}
+
diff --git a/src/signature.c b/src/signature.c
index ec51a42e9..2545b7519 100644
--- a/src/signature.c
+++ b/src/signature.c
@@ -82,23 +82,28 @@ int git_signature_new(git_signature **sig_out, const char *name, const char *ema
 	return 0;
 }
 
-git_signature *git_signature_dup(const git_signature *sig)
+int git_signature_dup(git_signature **dest, const git_signature *source)
 {
-	git_signature *new;
+	git_signature *signature;
 
-	if (sig == NULL)
-		return NULL;
+	if (source == NULL)
+		return 0;
+
+	signature = git__calloc(1, sizeof(git_signature));
+	GITERR_CHECK_ALLOC(signature);
+
+	signature->name = git__strdup(source->name);
+	GITERR_CHECK_ALLOC(signature->name);
 
-	new = git__calloc(1, sizeof(git_signature));
-	if (new == NULL)
-		return NULL;
+	signature->email = git__strdup(source->email);
+	GITERR_CHECK_ALLOC(signature->email);
 
-	new->name = git__strdup(sig->name);
-	new->email = git__strdup(sig->email);
-	new->when.time = sig->when.time;
-	new->when.offset = sig->when.offset;
+	signature->when.time = source->when.time;
+	signature->when.offset = source->when.offset;
 
-	return new;
+	*dest = signature;
+
+	return 0;
 }
 
 int git_signature_now(git_signature **sig_out, const char *name, const char *email)
@@ -139,7 +144,7 @@ int git_signature_default(git_signature **out, git_repository *repo)
 	git_config *cfg;
 	const char *user_name, *user_email;
 
-	if ((error = git_repository_config(&cfg, repo)) < 0)
+	if ((error = git_repository_config_snapshot(&cfg, repo)) < 0)
 		return error;
 
 	if (!(error = git_config_get_string(&user_name, cfg, "user.name")) &&
@@ -225,6 +230,8 @@ void git_signature__writebuf(git_buf *buf, const char *header, const git_signatu
 	int offset, hours, mins;
 	char sign;
 
+	assert(buf && sig);
+
 	offset = sig->when.offset;
 	sign = (sig->when.offset < 0) ? '-' : '+';
 
diff --git a/src/sortedcache.c b/src/sortedcache.c
index 466e55dbe..c6b226153 100644
--- a/src/sortedcache.c
+++ b/src/sortedcache.c
@@ -20,7 +20,7 @@ int git_sortedcache_new(
 
 	if (git_pool_init(&sc->pool, 1, 0) < 0 ||
 		git_vector_init(&sc->items, 4, item_cmp) < 0 ||
-		(sc->map = git_strmap_alloc()) == NULL)
+		git_strmap_alloc(&sc->map) < 0)
 		goto fail;
 
 	if (git_rwlock_init(&sc->lock)) {
@@ -39,8 +39,7 @@ int git_sortedcache_new(
 	return 0;
 
 fail:
-	if (sc->map)
-		git_strmap_free(sc->map);
+	git_strmap_free(sc->map);
 	git_vector_free(&sc->items);
 	git_pool_clear(&sc->pool);
 	git__free(sc);
@@ -233,9 +232,8 @@ unlock:
 
 void git_sortedcache_updated(git_sortedcache *sc)
 {
-	 /* update filestamp to latest value */
-	if (git_futils_filestamp_check(&sc->stamp, sc->path) < 0)
-		giterr_clear();
+	/* update filestamp to latest value */
+	git_futils_filestamp_check(&sc->stamp, sc->path);
 }
 
 /* release all items in sorted cache */
@@ -321,7 +319,7 @@ size_t git_sortedcache_entrycount(const git_sortedcache *sc)
 void *git_sortedcache_entry(git_sortedcache *sc, size_t pos)
 {
 	/* make sure the items are sorted so this gets the correct item */
-	if (!sc->items.sorted)
+	if (!git_vector_is_sorted(&sc->items))
 		git_vector_sort(&sc->items);
 
 	return git_vector_get(&sc->items, pos);
diff --git a/src/stash.c b/src/stash.c
index 083c2a4cd..86e0a627c 100644
--- a/src/stash.c
+++ b/src/stash.c
@@ -178,7 +178,8 @@ static int stash_update_index_from_diff(
 			break;
 
 		case GIT_DELTA_UNTRACKED:
-			if (data->include_untracked)
+			if (data->include_untracked &&
+				delta->new_file.mode != GIT_FILEMODE_TREE)
 				add_path = delta->new_file.path;
 			break;
 
@@ -412,25 +413,15 @@ static int update_reflog(
 	const char *message)
 {
 	git_reference *stash;
-	git_reflog *reflog = NULL;
 	int error;
 
-	if ((error = git_reference_create(&stash, repo, GIT_REFS_STASH_FILE, w_commit_oid, 1)) < 0)
-		goto cleanup;
-
-	git_reference_free(stash);
-
-	if ((error = git_reflog_read(&reflog, repo, GIT_REFS_STASH_FILE) < 0))
-		goto cleanup;
+	if ((error = git_reference_ensure_log(repo, GIT_REFS_STASH_FILE)) < 0)
+		return error;
 
-	if ((error = git_reflog_append(reflog, w_commit_oid, stasher, message)) < 0)
-		goto cleanup;
+	error = git_reference_create(&stash, repo, GIT_REFS_STASH_FILE, w_commit_oid, 1, stasher, message);
 
-	if ((error = git_reflog_write(reflog)) < 0)
-		goto cleanup;
+	git_reference_free(stash);
 
-cleanup:
-	git_reflog_free(reflog);
 	return error;
 }
 
@@ -440,7 +431,7 @@ static int is_dirty_cb(const char *path, unsigned int status, void *payload)
 	GIT_UNUSED(status);
 	GIT_UNUSED(payload);
 
-	return 1;
+	return GIT_PASSTHROUGH;
 }
 
 static int ensure_there_are_changes_to_stash(
@@ -463,7 +454,7 @@ static int ensure_there_are_changes_to_stash(
 
 	error = git_status_foreach_ext(repo, &opts, is_dirty_cb, NULL);
 
-	if (error == GIT_EUSER)
+	if (error == GIT_PASSTHROUGH)
 		return 0;
 
 	if (!error)
@@ -475,15 +466,19 @@ static int ensure_there_are_changes_to_stash(
 static int reset_index_and_workdir(
 	git_repository *repo,
 	git_commit *commit,
-	bool remove_untracked)
+	bool remove_untracked,
+	bool remove_ignored)
 {
-	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT;
 
 	opts.checkout_strategy = GIT_CHECKOUT_FORCE;
 
 	if (remove_untracked)
 		opts.checkout_strategy |= GIT_CHECKOUT_REMOVE_UNTRACKED;
 
+	if (remove_ignored)
+		opts.checkout_strategy |= GIT_CHECKOUT_REMOVE_IGNORED;
+
 	return git_checkout_tree(repo, (git_object *)commit, &opts);
 }
 
@@ -542,7 +537,8 @@ int git_stash_save(
 	if ((error = reset_index_and_workdir(
 		repo,
 		((flags & GIT_STASH_KEEP_INDEX) != 0) ? i_commit : b_commit,
-		(flags & GIT_STASH_INCLUDE_UNTRACKED) != 0)) < 0)
+		(flags & GIT_STASH_INCLUDE_UNTRACKED) != 0,
+		(flags & GIT_STASH_INCLUDE_IGNORED) != 0)) < 0)
 		goto cleanup;
 
 cleanup:
@@ -582,12 +578,14 @@ int git_stash_foreach(
 	for (i = 0; i < max; i++) {
 		entry = git_reflog_entry_byindex(reflog, i);
 
-		if (callback(i,
+		error = callback(i,
 			git_reflog_entry_message(entry),
 			git_reflog_entry_id_new(entry),
-			payload)) {
-				error = GIT_EUSER;
-				break;
+			payload);
+
+		if (error) {
+			giterr_set_after_callback(error);
+			break;
 		}
 	}
 
@@ -636,7 +634,11 @@ int git_stash_drop(
 		entry = git_reflog_entry_byindex(reflog, 0);
 
 		git_reference_free(stash);
-		error = git_reference_create(&stash, repo, GIT_REFS_STASH_FILE, &entry->oid_cur, 1);
+		if ((error = git_reference_create(&stash, repo, GIT_REFS_STASH_FILE, &entry->oid_cur, 1, NULL, NULL) < 0))
+			goto cleanup;
+
+		/* We need to undo the writing that we just did */
+		error = git_reflog_write(reflog);
 	}
 
 cleanup:
diff --git a/src/status.c b/src/status.c
index 07fdcb5c3..8d7612f72 100644
--- a/src/status.c
+++ b/src/status.c
@@ -38,7 +38,7 @@ static unsigned int index_delta2status(const git_diff_delta *head2idx)
 	case GIT_DELTA_RENAMED:
 		st = GIT_STATUS_INDEX_RENAMED;
 
-		if (!git_oid_equal(&head2idx->old_file.oid, &head2idx->new_file.oid))
+		if (!git_oid_equal(&head2idx->old_file.id, &head2idx->new_file.id))
 			st |= GIT_STATUS_INDEX_MODIFIED;
 		break;
 	case GIT_DELTA_TYPECHANGE:
@@ -74,25 +74,25 @@ static unsigned int workdir_delta2status(
 	case GIT_DELTA_RENAMED:
 		st = GIT_STATUS_WT_RENAMED;
 
-		if (!git_oid_equal(&idx2wd->old_file.oid, &idx2wd->new_file.oid)) {
+		if (!git_oid_equal(&idx2wd->old_file.id, &idx2wd->new_file.id)) {
 			/* if OIDs don't match, we might need to calculate them now to
 			 * discern between RENAMED vs RENAMED+MODIFED
 			 */
-			if (git_oid_iszero(&idx2wd->old_file.oid) &&
+			if (git_oid_iszero(&idx2wd->old_file.id) &&
 				diff->old_src == GIT_ITERATOR_TYPE_WORKDIR &&
 				!git_diff__oid_for_file(
-					diff->repo, idx2wd->old_file.path, idx2wd->old_file.mode,
-					idx2wd->old_file.size, &idx2wd->old_file.oid))
-			idx2wd->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+					&idx2wd->old_file.id, diff, idx2wd->old_file.path,
+					idx2wd->old_file.mode, idx2wd->old_file.size))
+			idx2wd->old_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
-			if (git_oid_iszero(&idx2wd->new_file.oid) &&
+			if (git_oid_iszero(&idx2wd->new_file.id) &&
 				diff->new_src == GIT_ITERATOR_TYPE_WORKDIR &&
 				!git_diff__oid_for_file(
-					diff->repo, idx2wd->new_file.path, idx2wd->new_file.mode,
-					idx2wd->new_file.size, &idx2wd->new_file.oid))
-				idx2wd->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+					&idx2wd->new_file.id, diff, idx2wd->new_file.path,
+					idx2wd->new_file.mode, idx2wd->new_file.size))
+				idx2wd->new_file.flags |= GIT_DIFF_FLAG_VALID_ID;
 
-			if (!git_oid_equal(&idx2wd->old_file.oid, &idx2wd->new_file.oid))
+			if (!git_oid_equal(&idx2wd->old_file.id, &idx2wd->new_file.id))
 				st |= GIT_STATUS_WT_MODIFIED;
 		}
 		break;
@@ -225,6 +225,28 @@ static git_status_list *git_status_list_alloc(git_index *index)
 	return status;
 }
 
+static int status_validate_options(const git_status_options *opts)
+{
+	if (!opts)
+		return 0;
+
+	GITERR_CHECK_VERSION(opts, GIT_STATUS_OPTIONS_VERSION, "git_status_options");
+
+	if (opts->show > GIT_STATUS_SHOW_WORKDIR_ONLY) {
+		giterr_set(GITERR_INVALID, "Unknown status 'show' option");
+		return -1;
+	}
+
+	if ((opts->flags & GIT_STATUS_OPT_NO_REFRESH) != 0 &&
+		(opts->flags & GIT_STATUS_OPT_UPDATE_INDEX) != 0) {
+		giterr_set(GITERR_INVALID, "Updating index from status "
+			"is not allowed when index refresh is disabled");
+		return -1;
+	}
+
+	return 0;
+}
+
 int git_status_list_new(
 	git_status_list **out,
 	git_repository *repo,
@@ -240,11 +262,10 @@ int git_status_list_new(
 	int error = 0;
 	unsigned int flags = opts ? opts->flags : GIT_STATUS_OPT_DEFAULTS;
 
-	assert(show <= GIT_STATUS_SHOW_WORKDIR_ONLY);
-
 	*out = NULL;
 
-	GITERR_CHECK_VERSION(opts, GIT_STATUS_OPTIONS_VERSION, "git_status_options");
+	if (status_validate_options(opts) < 0)
+		return -1;
 
 	if ((error = git_repository__ensure_not_bare(repo, "status")) < 0 ||
 		(error = git_repository_index(&index, repo)) < 0)
@@ -287,6 +308,8 @@ int git_status_list_new(
 		diffopt.flags = diffopt.flags | GIT_DIFF_RECURSE_IGNORED_DIRS;
 	if ((flags & GIT_STATUS_OPT_EXCLUDE_SUBMODULES) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_IGNORE_SUBMODULES;
+	if ((flags & GIT_STATUS_OPT_UPDATE_INDEX) != 0)
+		diffopt.flags = diffopt.flags | GIT_DIFF_UPDATE_INDEX;
 
 	if ((flags & GIT_STATUS_OPT_RENAMES_FROM_REWRITES) != 0)
 		findopt.flags = findopt.flags |
@@ -314,8 +337,9 @@ int git_status_list_new(
 			goto done;
 	}
 
-	if ((error = git_diff__paired_foreach(
-			status->head2idx, status->idx2wd, status_collect, status)) < 0)
+	error = git_diff__paired_foreach(
+		status->head2idx, status->idx2wd, status_collect, status);
+	if (error < 0)
 		goto done;
 
 	if (flags & GIT_STATUS_OPT_SORT_CASE_SENSITIVELY)
@@ -360,19 +384,13 @@ const git_status_entry *git_status_byindex(git_status_list *status, size_t i)
 
 void git_status_list_free(git_status_list *status)
 {
-	git_status_entry *status_entry;
-	size_t i;
-
 	if (status == NULL)
 		return;
 
 	git_diff_free(status->head2idx);
 	git_diff_free(status->idx2wd);
 
-	git_vector_foreach(&status->paired, i, status_entry)
-		git__free(status_entry);
-
-	git_vector_free(&status->paired);
+	git_vector_free_deep(&status->paired);
 
 	git__memzero(status, sizeof(*status));
 	git__free(status);
@@ -397,9 +415,8 @@ int git_status_foreach_ext(
 			status_entry->head_to_index->old_file.path :
 			status_entry->index_to_workdir->old_file.path;
 
-		if (cb(path, status_entry->status, payload) != 0) {
-			error = GIT_EUSER;
-			giterr_clear();
+		if ((error = cb(path, status_entry->status, payload)) != 0) {
+			giterr_set_after_callback(error);
 			break;
 		}
 	}
@@ -501,3 +518,31 @@ int git_status_should_ignore(
 	return git_ignore_path_is_ignored(ignored, repo, path);
 }
 
+int git_status_init_options(git_status_options *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_status_options, GIT_STATUS_OPTIONS_INIT);
+	return 0;
+}
+
+int git_status_list_get_perfdata(
+	git_diff_perfdata *out, const git_status_list *status)
+{
+	assert(out);
+	GITERR_CHECK_VERSION(out, GIT_DIFF_PERFDATA_VERSION, "git_diff_perfdata");
+
+	out->stat_calls = 0;
+	out->oid_calculations = 0;
+
+	if (status->head2idx) {
+		out->stat_calls += status->head2idx->perf.stat_calls;
+		out->oid_calculations += status->head2idx->perf.oid_calculations;
+	}
+	if (status->idx2wd) {
+		out->stat_calls += status->idx2wd->perf.stat_calls;
+		out->oid_calculations += status->idx2wd->perf.oid_calculations;
+	}
+
+	return 0;
+}
+
diff --git a/src/strmap.h b/src/strmap.h
index 8276ab468..8985aaf7e 100644
--- a/src/strmap.h
+++ b/src/strmap.h
@@ -22,7 +22,9 @@ typedef khiter_t git_strmap_iter;
 #define GIT__USE_STRMAP \
 	__KHASH_IMPL(str, static kh_inline, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal)
 
-#define git_strmap_alloc()  kh_init(str)
+#define git_strmap_alloc(hp) \
+	((*(hp) = kh_init(str)) == NULL) ? giterr_set_oom(), -1 : 0
+
 #define git_strmap_free(h)  kh_destroy(str, h), h = NULL
 #define git_strmap_clear(h) kh_clear(str, h)
 
diff --git a/src/strnlen.h b/src/strnlen.h
new file mode 100644
index 000000000..fdd7fe39c
--- /dev/null
+++ b/src/strnlen.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_strlen_h__
+#define INCLUDE_strlen_h__
+
+#if defined(__MINGW32__) || defined(__sun) || defined(__APPLE__) || defined(__MidnightBSD__)
+#   define NO_STRNLEN
+#endif
+
+#ifdef NO_STRNLEN
+GIT_INLINE(size_t) p_strnlen(const char *s, size_t maxlen) {
+	const char *end = memchr(s, 0, maxlen);
+	return end ? (size_t)(end - s) : maxlen;
+}
+#else
+#   define p_strnlen strnlen
+#endif
+
+#endif
diff --git a/src/submodule.c b/src/submodule.c
index 586494fed..b1291df8e 100644
--- a/src/submodule.c
+++ b/src/submodule.c
@@ -43,6 +43,22 @@ static git_cvar_map _sm_ignore_map[] = {
 	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_IGNORE_ALL},
 };
 
+static git_cvar_map _sm_recurse_map[] = {
+	{GIT_CVAR_STRING, "on-demand", GIT_SUBMODULE_RECURSE_ONDEMAND},
+	{GIT_CVAR_FALSE, NULL, GIT_SUBMODULE_RECURSE_NO},
+	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_RECURSE_YES},
+};
+
+enum {
+	CACHE_OK = 0,
+	CACHE_REFRESH = 1,
+	CACHE_FLUSH = 2
+};
+enum {
+	GITMODULES_EXISTING = 0,
+	GITMODULES_CREATE = 1,
+};
+
 static kh_inline khint_t str_hash_no_trailing_slash(const char *s)
 {
 	khint_t h;
@@ -71,13 +87,15 @@ __KHASH_IMPL(
 	str, static kh_inline, const char *, void *, 1,
 	str_hash_no_trailing_slash, str_equal_no_trailing_slash);
 
-static int load_submodule_config(git_repository *repo);
-static git_config_backend *open_gitmodules(git_repository *, bool, const git_oid *);
-static int lookup_head_remote(git_buf *url, git_repository *repo);
-static int submodule_get(git_submodule **, git_repository *, const char *, const char *);
+static int submodule_cache_init(git_repository *repo, int refresh);
+static void submodule_cache_free(git_submodule_cache *cache);
+
+static git_config_backend *open_gitmodules(git_submodule_cache *, int gitmod);
+static int get_url_base(git_buf *url, git_repository *repo);
+static int lookup_head_remote_key(git_buf *remote_key, git_repository *repo);
+static int submodule_get(git_submodule **, git_submodule_cache *, const char *, const char *);
 static int submodule_load_from_config(const git_config_entry *, void *);
-static int submodule_load_from_wd_lite(git_submodule *, const char *, void *);
-static int submodule_update_config(git_submodule *, const char *, const char *, bool, bool);
+static int submodule_load_from_wd_lite(git_submodule *);
 static void submodule_get_index_status(unsigned int *, git_submodule *);
 static void submodule_get_wd_status(unsigned int *, git_submodule *, git_repository *, git_submodule_ignore_t);
 
@@ -93,52 +111,134 @@ static int submodule_config_key_trunc_puts(git_buf *key, const char *suffix)
 	return git_buf_puts(key, suffix);
 }
 
+/* lookup submodule or return ENOTFOUND if it doesn't exist */
+static int submodule_lookup(
+	git_submodule **out,
+	git_submodule_cache *cache,
+	const char *name,
+	const char *alternate)
+{
+	khiter_t pos;
+
+	/* lock cache */
+
+	pos = git_strmap_lookup_index(cache->submodules, name);
+
+	if (!git_strmap_valid_index(cache->submodules, pos) && alternate)
+		pos = git_strmap_lookup_index(cache->submodules, alternate);
+
+	if (!git_strmap_valid_index(cache->submodules, pos)) {
+		/* unlock cache */
+		return GIT_ENOTFOUND; /* don't set error - caller will cope */
+	}
+
+	if (out != NULL) {
+		git_submodule *sm = git_strmap_value_at(cache->submodules, pos);
+		GIT_REFCOUNT_INC(sm);
+		*out = sm;
+	}
+
+	/* unlock cache */
+
+	return 0;
+}
+
+/* clear a set of flags on all submodules */
+static void submodule_cache_clear_flags(
+	git_submodule_cache *cache, uint32_t mask)
+{
+	git_submodule *sm;
+	uint32_t inverted_mask = ~mask;
+
+	git_strmap_foreach_value(cache->submodules, sm, {
+		sm->flags &= inverted_mask;
+	});
+}
+
 /*
  * PUBLIC APIS
  */
 
-int git_submodule_lookup(
-	git_submodule **sm_ptr, /* NULL if user only wants to test existence */
+bool git_submodule__is_submodule(git_repository *repo, const char *name)
+{
+	git_strmap *map;
+
+	if (submodule_cache_init(repo, CACHE_OK) < 0) {
+		giterr_clear();
+		return false;
+	}
+
+	if (!repo->_submodules || !(map = repo->_submodules->submodules))
+		return false;
+
+	return git_strmap_valid_index(map, git_strmap_lookup_index(map, name));
+}
+
+static void submodule_set_lookup_error(int error, const char *name)
+{
+	if (!error)
+		return;
+
+	giterr_set(GITERR_SUBMODULE, (error == GIT_ENOTFOUND) ?
+		"No submodule named '%s'" :
+		"Submodule '%s' has not been added yet", name);
+}
+
+int git_submodule__lookup(
+	git_submodule **out, /* NULL if user only wants to test existence */
 	git_repository *repo,
-	const char *name)       /* trailing slash is allowed */
+	const char *name)    /* trailing slash is allowed */
 {
 	int error;
-	khiter_t pos;
 
 	assert(repo && name);
 
-	if ((error = load_submodule_config(repo)) < 0)
+	if ((error = submodule_cache_init(repo, CACHE_OK)) < 0)
 		return error;
 
-	pos = git_strmap_lookup_index(repo->submodules, name);
+	if ((error = submodule_lookup(out, repo->_submodules, name, NULL)) < 0)
+		submodule_set_lookup_error(error, name);
+
+	return error;
+}
+
+int git_submodule_lookup(
+	git_submodule **out, /* NULL if user only wants to test existence */
+	git_repository *repo,
+	const char *name)    /* trailing slash is allowed */
+{
+	int error;
+
+	assert(repo && name);
+
+	if ((error = submodule_cache_init(repo, CACHE_REFRESH)) < 0)
+		return error;
 
-	if (!git_strmap_valid_index(repo->submodules, pos)) {
-		error = GIT_ENOTFOUND;
+	if ((error = submodule_lookup(out, repo->_submodules, name, NULL)) < 0) {
 
 		/* check if a plausible submodule exists at path */
 		if (git_repository_workdir(repo)) {
 			git_buf path = GIT_BUF_INIT;
 
-			if (git_buf_joinpath(&path, git_repository_workdir(repo), name) < 0)
+			if (git_buf_join3(&path,
+					'/', git_repository_workdir(repo), name, DOT_GIT) < 0)
 				return -1;
 
-			if (git_path_contains_dir(&path, DOT_GIT))
+			if (git_path_exists(path.ptr))
 				error = GIT_EEXISTS;
 
 			git_buf_free(&path);
 		}
 
-		giterr_set(GITERR_SUBMODULE, (error == GIT_ENOTFOUND) ?
-			"No submodule named '%s'" :
-			"Submodule '%s' has not been added yet", name);
-
-		return error;
+		submodule_set_lookup_error(error, name);
 	}
 
-	if (sm_ptr)
-		*sm_ptr = git_strmap_value_at(repo->submodules, pos);
+	return error;
+}
 
-	return 0;
+static void submodule_free_dup(void *sm)
+{
+	git_submodule_free(sm);
 }
 
 int git_submodule_foreach(
@@ -147,58 +247,67 @@ int git_submodule_foreach(
 	void *payload)
 {
 	int error;
+	size_t i;
 	git_submodule *sm;
-	git_vector seen = GIT_VECTOR_INIT;
-	git_vector_set_cmp(&seen, submodule_cmp);
+	git_submodule_cache *cache;
+	git_vector snapshot = GIT_VECTOR_INIT;
 
 	assert(repo && callback);
 
-	if ((error = load_submodule_config(repo)) < 0)
+	if ((error = submodule_cache_init(repo, CACHE_REFRESH)) < 0)
 		return error;
 
-	git_strmap_foreach_value(repo->submodules, sm, {
-		/* Usually the following will not come into play - it just prevents
-		 * us from issuing a callback twice for a submodule where the name
-		 * and path are not the same.
-		 */
-		if (GIT_REFCOUNT_VAL(sm) > 1) {
-			if (git_vector_bsearch(NULL, &seen, sm) != GIT_ENOTFOUND)
-				continue;
-			if ((error = git_vector_insert(&seen, sm)) < 0)
+	cache = repo->_submodules;
+
+	if (git_mutex_lock(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire lock on submodule cache");
+		return -1;
+	}
+
+	if (!(error = git_vector_init(
+			&snapshot, kh_size(cache->submodules), submodule_cmp))) {
+
+		git_strmap_foreach_value(cache->submodules, sm, {
+			if ((error = git_vector_insert(&snapshot, sm)) < 0)
 				break;
-		}
+			GIT_REFCOUNT_INC(sm);
+		});
+	}
 
-		if (callback(sm, sm->name, payload)) {
-			giterr_clear();
-			error = GIT_EUSER;
+	git_mutex_unlock(&cache->lock);
+
+	if (error < 0)
+		goto done;
+
+	git_vector_uniq(&snapshot, submodule_free_dup);
+
+	git_vector_foreach(&snapshot, i, sm) {
+		if ((error = callback(sm, sm->name, payload)) != 0) {
+			giterr_set_after_callback(error);
 			break;
 		}
-	});
+	}
 
-	git_vector_free(&seen);
+done:
+	git_vector_foreach(&snapshot, i, sm)
+		git_submodule_free(sm);
+	git_vector_free(&snapshot);
 
 	return error;
 }
 
-void git_submodule_config_free(git_repository *repo)
+void git_submodule_cache_free(git_repository *repo)
 {
-	git_strmap *smcfg;
-	git_submodule *sm;
+	git_submodule_cache *cache;
 
 	assert(repo);
 
-	smcfg = repo->submodules;
-	repo->submodules = NULL;
-
-	if (smcfg == NULL)
-		return;
-
-	git_strmap_foreach_value(smcfg, sm, { git_submodule_free(sm); });
-	git_strmap_free(smcfg);
+	if ((cache = git__swap(repo->_submodules, NULL)) != NULL)
+		submodule_cache_free(cache);
 }
 
 int git_submodule_add_setup(
-	git_submodule **submodule,
+	git_submodule **out,
 	git_repository *repo,
 	const char *url,
 	const char *path,
@@ -206,7 +315,7 @@ int git_submodule_add_setup(
 {
 	int error = 0;
 	git_config_backend *mods = NULL;
-	git_submodule *sm;
+	git_submodule *sm = NULL;
 	git_buf name = GIT_BUF_INIT, real_url = GIT_BUF_INIT;
 	git_repository_init_options initopt = GIT_REPOSITORY_INIT_OPTIONS_INIT;
 	git_repository *subrepo = NULL;
@@ -215,26 +324,16 @@ int git_submodule_add_setup(
 
 	/* see if there is already an entry for this submodule */
 
-	if (git_submodule_lookup(&sm, repo, path) < 0)
+	if (git_submodule_lookup(NULL, repo, path) < 0)
 		giterr_clear();
 	else {
 		giterr_set(GITERR_SUBMODULE,
-			"Attempt to add a submodule that already exists");
+			"Attempt to add submodule '%s' that already exists", path);
 		return GIT_EEXISTS;
 	}
 
 	/* resolve parameters */
-
-	if (url[0] == '.' && (url[1] == '/' || (url[1] == '.' && url[2] == '/'))) {
-		if (!(error = lookup_head_remote(&real_url, repo)))
-			error = git_path_apply_relative(&real_url, url);
-	} else if (strchr(url, ':') != NULL || url[0] == '/') {
-		error = git_buf_sets(&real_url, url);
-	} else {
-		giterr_set(GITERR_SUBMODULE, "Invalid format for submodule URL");
-		error = -1;
-	}
-	if (error)
+	if ((error = git_submodule_resolve_url(&real_url, repo, url)) < 0)
 		goto cleanup;
 
 	/* validate and normalize path */
@@ -250,9 +349,9 @@ int git_submodule_add_setup(
 
 	/* update .gitmodules */
 
-	if ((mods = open_gitmodules(repo, true, NULL)) == NULL) {
+	if (!(mods = open_gitmodules(repo->_submodules, GITMODULES_CREATE))) {
 		giterr_set(GITERR_SUBMODULE,
-			"Adding submodules to a bare repository is not supported (for now)");
+			"Adding submodules to a bare repository is not supported");
 		return -1;
 	}
 
@@ -290,8 +389,8 @@ int git_submodule_add_setup(
 	else if (use_gitlink) {
 		git_buf repodir = GIT_BUF_INIT;
 
-		error = git_buf_join_n(
-			&repodir, '/', 3, git_repository_path(repo), "modules", path);
+		error = git_buf_join3(
+			&repodir, '/', git_repository_path(repo), "modules", path);
 		if (error < 0)
 			goto cleanup;
 
@@ -310,16 +409,27 @@ int git_submodule_add_setup(
 
 	/* add submodule to hash and "reload" it */
 
-	if (!(error = submodule_get(&sm, repo, path, NULL)) &&
-		!(error = git_submodule_reload(sm)))
+	if (git_mutex_lock(&repo->_submodules->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire lock on submodule cache");
+		error = -1;
+		goto cleanup;
+	}
+
+	if (!(error = submodule_get(&sm, repo->_submodules, path, NULL)) &&
+		!(error = git_submodule_reload(sm, false)))
 		error = git_submodule_init(sm, false);
 
+	git_mutex_unlock(&repo->_submodules->lock);
+
 cleanup:
-	if (submodule != NULL)
-		*submodule = !error ? sm : NULL;
+	if (error && sm) {
+		git_submodule_free(sm);
+		sm = NULL;
+	}
+	if (out != NULL)
+		*out = sm;
 
-	if (mods != NULL)
-		git_config_file_free(mods);
+	git_config_file_free(mods);
 	git_repository_free(subrepo);
 	git_buf_free(&real_url);
 	git_buf_free(&name);
@@ -382,7 +492,7 @@ int git_submodule_add_to_index(git_submodule *sm, int write_index)
 		error = -1;
 		goto cleanup;
 	}
-	git_oid_cpy(&entry.oid, &sm->wd_oid);
+	git_oid_cpy(&entry.id, &sm->wd_oid);
 
 	if ((error = git_commit_lookup(&head, sm_repo, &sm->wd_oid)) < 0)
 		goto cleanup;
@@ -429,6 +539,15 @@ const char *git_submodule_update_to_str(git_submodule_update_t update)
 	return NULL;
 }
 
+const char *git_submodule_recurse_to_str(git_submodule_recurse_t recurse)
+{
+	int i;
+	for (i = 0; i < (int)ARRAY_SIZE(_sm_recurse_map); ++i)
+		if (_sm_recurse_map[i].map_value == recurse)
+			return _sm_recurse_map[i].str_match;
+	return NULL;
+}
+
 int git_submodule_save(git_submodule *submodule)
 {
 	int error = 0;
@@ -438,10 +557,10 @@ int git_submodule_save(git_submodule *submodule)
 
 	assert(submodule);
 
-	mods = open_gitmodules(submodule->repo, true, NULL);
+	mods = open_gitmodules(submodule->repo->_submodules, GITMODULES_CREATE);
 	if (!mods) {
 		giterr_set(GITERR_SUBMODULE,
-			"Adding submodules to a bare repository is not supported (for now)");
+			"Adding submodules to a bare repository is not supported");
 		return -1;
 	}
 
@@ -458,6 +577,10 @@ int git_submodule_save(git_submodule *submodule)
 		(error = git_config_file_set_string(mods, key.ptr, submodule->url)) < 0)
 		goto cleanup;
 
+	if ((error = submodule_config_key_trunc_puts(&key, "branch")) < 0 ||
+		(error = git_config_file_set_string(mods, key.ptr, submodule->branch)) < 0)
+		goto cleanup;
+
 	if (!(error = submodule_config_key_trunc_puts(&key, "update")) &&
 		(val = git_submodule_update_to_str(submodule->update)) != NULL)
 		error = git_config_file_set_string(mods, key.ptr, val);
@@ -470,21 +593,21 @@ int git_submodule_save(git_submodule *submodule)
 	if (error < 0)
 		goto cleanup;
 
-	if ((error = submodule_config_key_trunc_puts(
-			&key, "fetchRecurseSubmodules")) < 0 ||
-		(error = git_config_file_set_string(
-			mods, key.ptr, submodule->fetch_recurse ? "true" : "false")) < 0)
+	if (!(error = submodule_config_key_trunc_puts(&key, "fetchRecurseSubmodules")) &&
+		(val = git_submodule_recurse_to_str(submodule->fetch_recurse)) != NULL)
+		error = git_config_file_set_string(mods, key.ptr, val);
+	if (error < 0)
 		goto cleanup;
 
 	/* update internal defaults */
 
 	submodule->ignore_default = submodule->ignore;
 	submodule->update_default = submodule->update;
+	submodule->fetch_recurse_default = submodule->fetch_recurse;
 	submodule->flags |= GIT_SUBMODULE_STATUS_IN_CONFIG;
 
 cleanup:
-	if (mods != NULL)
-		git_config_file_free(mods);
+	git_config_file_free(mods);
 	git_buf_free(&key);
 
 	return error;
@@ -514,6 +637,33 @@ const char *git_submodule_url(git_submodule *submodule)
 	return submodule->url;
 }
 
+int git_submodule_resolve_url(git_buf *out, git_repository *repo, const char *url)
+{
+	int error = 0;
+
+	assert(out && repo && url);
+
+	git_buf_sanitize(out);
+
+	if (git_path_is_relative(url)) {
+		if (!(error = get_url_base(out, repo)))
+			error = git_path_apply_relative(out, url);
+	} else if (strchr(url, ':') != NULL || url[0] == '/') {
+		error = git_buf_sets(out, url);
+	} else {
+		giterr_set(GITERR_SUBMODULE, "Invalid format for submodule URL");
+		error = -1;
+	}
+
+	return error;
+}
+
+const char *git_submodule_branch(git_submodule *submodule)
+{
+	assert(submodule);
+	return submodule->branch;
+}
+
 int git_submodule_set_url(git_submodule *submodule, const char *url)
 {
 	assert(submodule && url);
@@ -611,66 +761,118 @@ git_submodule_update_t git_submodule_set_update(
 	return old;
 }
 
-int git_submodule_fetch_recurse_submodules(
+git_submodule_recurse_t git_submodule_fetch_recurse_submodules(
 	git_submodule *submodule)
 {
 	assert(submodule);
 	return submodule->fetch_recurse;
 }
 
-int git_submodule_set_fetch_recurse_submodules(
+git_submodule_recurse_t git_submodule_set_fetch_recurse_submodules(
 	git_submodule *submodule,
-	int fetch_recurse_submodules)
+	git_submodule_recurse_t fetch_recurse_submodules)
 {
-	int old;
+	git_submodule_recurse_t old;
 
 	assert(submodule);
 
+	if (fetch_recurse_submodules == GIT_SUBMODULE_RECURSE_RESET)
+		fetch_recurse_submodules = submodule->fetch_recurse_default;
+
 	old = submodule->fetch_recurse;
-	submodule->fetch_recurse = (fetch_recurse_submodules != 0);
+	submodule->fetch_recurse = fetch_recurse_submodules;
 	return old;
 }
 
-int git_submodule_init(git_submodule *submodule, int overwrite)
+int git_submodule_init(git_submodule *sm, int overwrite)
 {
 	int error;
 	const char *val;
+	git_buf key = GIT_BUF_INIT;
+	git_config *cfg = NULL;
 
-	/* write "submodule.NAME.url" */
-
-	if (!submodule->url) {
+	if (!sm->url) {
 		giterr_set(GITERR_SUBMODULE,
-			"No URL configured for submodule '%s'", submodule->name);
+			"No URL configured for submodule '%s'", sm->name);
 		return -1;
 	}
 
-	error = submodule_update_config(
-		submodule, "url", submodule->url, overwrite != 0, false);
-	if (error < 0)
+	if ((error = git_repository_config(&cfg, sm->repo)) < 0)
 		return error;
 
+	/* write "submodule.NAME.url" */
+
+	if ((error = git_buf_printf(&key, "submodule.%s.url", sm->name)) < 0 ||
+		(error = git_config__update_entry(
+			cfg, key.ptr, sm->url, overwrite != 0, false)) < 0)
+		goto cleanup;
+
 	/* write "submodule.NAME.update" if not default */
 
-	val = (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
-		NULL : git_submodule_update_to_str(submodule->update);
-	error = submodule_update_config(
-		submodule, "update", val, (overwrite != 0), false);
+	val = (sm->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
+		NULL : git_submodule_update_to_str(sm->update);
+
+	if ((error = git_buf_printf(&key, "submodule.%s.update", sm->name)) < 0 ||
+		(error = git_config__update_entry(
+			cfg, key.ptr, val, overwrite != 0, false)) < 0)
+		goto cleanup;
+
+	/* success */
+
+cleanup:
+	git_config_free(cfg);
+	git_buf_free(&key);
 
 	return error;
 }
 
-int git_submodule_sync(git_submodule *submodule)
+int git_submodule_sync(git_submodule *sm)
 {
-	if (!submodule->url) {
+	int error = 0;
+	git_config *cfg = NULL;
+	git_buf key = GIT_BUF_INIT;
+	git_repository *smrepo = NULL;
+
+	if (!sm->url) {
 		giterr_set(GITERR_SUBMODULE,
-			"No URL configured for submodule '%s'", submodule->name);
+			"No URL configured for submodule '%s'", sm->name);
 		return -1;
 	}
 
 	/* copy URL over to config only if it already exists */
 
-	return submodule_update_config(
-		submodule, "url", submodule->url, true, true);
+	if (!(error = git_repository_config__weakptr(&cfg, sm->repo)) &&
+		!(error = git_buf_printf(&key, "submodule.%s.url", sm->name)))
+		error = git_config__update_entry(cfg, key.ptr, sm->url, true, true);
+
+	/* if submodule exists in the working directory, update remote url */
+
+	if (!error &&
+		(sm->flags & GIT_SUBMODULE_STATUS_IN_WD) != 0 &&
+		!(error = git_submodule_open(&smrepo, sm)))
+	{
+		git_buf remote_name = GIT_BUF_INIT;
+
+		if ((error = git_repository_config__weakptr(&cfg, smrepo)) < 0)
+			/* return error from reading submodule config */;
+		else if ((error = lookup_head_remote_key(&remote_name, smrepo)) < 0) {
+			giterr_clear();
+			error = git_buf_sets(&key, "branch.origin.remote");
+		} else {
+			error = git_buf_join3(
+				&key, '.', "branch", remote_name.ptr, "remote");
+			git_buf_free(&remote_name);
+		}
+
+		if (!error)
+			error = git_config__update_entry(cfg, key.ptr, sm->url, true, false);
+
+		git_repository_free(smrepo);
+	}
+
+	git_buf_free(&key);
+
+	return error;
 }
 
 static int git_submodule__open(
@@ -737,11 +939,9 @@ int git_submodule_open(git_repository **subrepo, git_submodule *sm)
 	return git_submodule__open(subrepo, sm, false);
 }
 
-int git_submodule_reload_all(git_repository *repo)
+int git_submodule_reload_all(git_repository *repo, int force)
 {
-	assert(repo);
-	git_submodule_config_free(repo);
-	return load_submodule_config(repo);
+	return submodule_cache_init(repo, force ? CACHE_FLUSH : CACHE_REFRESH);
 }
 
 static void submodule_update_from_index_entry(
@@ -756,7 +956,7 @@ static void submodule_update_from_index_entry(
 		if (already_found)
 			sm->flags |= GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
 		else
-			git_oid_cpy(&sm->index_oid, &ie->oid);
+			git_oid_cpy(&sm->index_oid, &ie->id);
 
 		sm->flags |= GIT_SUBMODULE_STATUS_IN_INDEX |
 			GIT_SUBMODULE_STATUS__INDEX_OID_VALID;
@@ -817,54 +1017,59 @@ static int submodule_update_head(git_submodule *submodule)
 	return 0;
 }
 
-int git_submodule_reload(git_submodule *submodule)
+
+int git_submodule_reload(git_submodule *sm, int force)
 {
 	int error = 0;
 	git_config_backend *mods;
+	git_submodule_cache *cache;
 
-	assert(submodule);
+	GIT_UNUSED(force);
 
-	/* refresh index data */
+	assert(sm);
 
-	if (submodule_update_index(submodule) < 0)
-		return -1;
+	cache = sm->repo->_submodules;
+
+	/* refresh index data */
+	if ((error = submodule_update_index(sm)) < 0)
+		return error;
 
 	/* refresh HEAD tree data */
+	if ((error = submodule_update_head(sm)) < 0)
+		return error;
 
-	if (submodule_update_head(submodule) < 0)
-		return -1;
+	/* done if bare */
+	if (git_repository_is_bare(sm->repo))
+		return error;
 
 	/* refresh config data */
-
-	mods = open_gitmodules(submodule->repo, false, NULL);
+	mods = open_gitmodules(cache, GITMODULES_EXISTING);
 	if (mods != NULL) {
 		git_buf path = GIT_BUF_INIT;
 
 		git_buf_sets(&path, "submodule\\.");
-		git_buf_text_puts_escape_regex(&path, submodule->name);
+		git_buf_text_puts_escape_regex(&path, sm->name);
 		git_buf_puts(&path, ".*");
 
 		if (git_buf_oom(&path))
 			error = -1;
 		else
 			error = git_config_file_foreach_match(
-				mods, path.ptr, submodule_load_from_config, submodule->repo);
+				mods, path.ptr, submodule_load_from_config, cache);
 
 		git_buf_free(&path);
 		git_config_file_free(mods);
-	}
 
-	if (error < 0)
-		return error;
+		if (error < 0)
+			return error;
+	}
 
 	/* refresh wd data */
+	sm->flags &=
+		~(GIT_SUBMODULE_STATUS_IN_WD | GIT_SUBMODULE_STATUS__WD_OID_VALID |
+		  GIT_SUBMODULE_STATUS__WD_FLAGS);
 
-	submodule->flags = submodule->flags &
-		~(GIT_SUBMODULE_STATUS_IN_WD | GIT_SUBMODULE_STATUS__WD_OID_VALID);
-
-	error = submodule_load_from_wd_lite(submodule, submodule->path, NULL);
-
-	return error;
+	return submodule_load_from_wd_lite(sm);
 }
 
 static void submodule_copy_oid_maybe(
@@ -958,32 +1163,69 @@ int git_submodule_location(unsigned int *location, git_submodule *sm)
  * INTERNAL FUNCTIONS
  */
 
-static git_submodule *submodule_alloc(git_repository *repo, const char *name)
+static int submodule_alloc(
+	git_submodule **out, git_submodule_cache *cache, const char *name)
 {
 	size_t namelen;
 	git_submodule *sm;
 
 	if (!name || !(namelen = strlen(name))) {
 		giterr_set(GITERR_SUBMODULE, "Invalid submodule name");
-		return NULL;
+		return -1;
 	}
 
 	sm = git__calloc(1, sizeof(git_submodule));
-	if (sm == NULL)
-		return NULL;
+	GITERR_CHECK_ALLOC(sm);
 
 	sm->name = sm->path = git__strdup(name);
 	if (!sm->name) {
 		git__free(sm);
-		return NULL;
+		return -1;
 	}
 
 	GIT_REFCOUNT_INC(sm);
 	sm->ignore = sm->ignore_default = GIT_SUBMODULE_IGNORE_NONE;
 	sm->update = sm->update_default = GIT_SUBMODULE_UPDATE_CHECKOUT;
-	sm->repo   = repo;
+	sm->fetch_recurse = sm->fetch_recurse_default = GIT_SUBMODULE_RECURSE_NO;
+	sm->repo   = cache->repo;
+	sm->branch = NULL;
+
+	*out = sm;
+	return 0;
+}
+
+static void submodule_cache_remove_item(
+	git_submodule_cache *cache,
+	git_submodule *item,
+	bool free_after_remove)
+{
+	git_strmap *map;
+	const char *name, *alt;
+
+	if (!cache || !(map = cache->submodules) || !item)
+		return;
 
-	return sm;
+	name = item->name;
+	alt  = (item->path != item->name) ? item->path : NULL;
+
+	for (; name; name = alt, alt = NULL) {
+		khiter_t pos = git_strmap_lookup_index(map, name);
+		git_submodule *found;
+
+		if (!git_strmap_valid_index(map, pos))
+			continue;
+
+		found = git_strmap_value_at(map, pos);
+
+		if (found != item)
+			continue;
+
+		git_strmap_set_value_at(map, pos, NULL);
+		git_strmap_delete_at(map, pos);
+
+		if (free_after_remove)
+			git_submodule_free(found);
+	}
 }
 
 static void submodule_release(git_submodule *sm)
@@ -991,10 +1233,17 @@ static void submodule_release(git_submodule *sm)
 	if (!sm)
 		return;
 
+	if (sm->repo) {
+		git_submodule_cache *cache = sm->repo->_submodules;
+		sm->repo = NULL;
+		submodule_cache_remove_item(cache, sm, false);
+	}
+
 	if (sm->path != sm->name)
 		git__free(sm->path);
 	git__free(sm->name);
 	git__free(sm->url);
+	git__free(sm->branch);
 	git__memzero(sm, sizeof(*sm));
 	git__free(sm);
 }
@@ -1007,48 +1256,51 @@ void git_submodule_free(git_submodule *sm)
 }
 
 static int submodule_get(
-	git_submodule **sm_ptr,
-	git_repository *repo,
+	git_submodule **out,
+	git_submodule_cache *cache,
 	const char *name,
 	const char *alternate)
 {
-	git_strmap *smcfg = repo->submodules;
+	int error = 0;
 	khiter_t pos;
 	git_submodule *sm;
-	int error;
 
-	assert(repo && name);
-
-	pos = git_strmap_lookup_index(smcfg, name);
+	pos = git_strmap_lookup_index(cache->submodules, name);
 
-	if (!git_strmap_valid_index(smcfg, pos) && alternate)
-		pos = git_strmap_lookup_index(smcfg, alternate);
+	if (!git_strmap_valid_index(cache->submodules, pos) && alternate)
+		pos = git_strmap_lookup_index(cache->submodules, alternate);
 
-	if (!git_strmap_valid_index(smcfg, pos)) {
-		sm = submodule_alloc(repo, name);
-		GITERR_CHECK_ALLOC(sm);
+	if (!git_strmap_valid_index(cache->submodules, pos)) {
+		if ((error = submodule_alloc(&sm, cache, name)) < 0)
+			return error;
 
 		/* insert value at name - if another thread beats us to it, then use
 		 * their record and release our own.
 		 */
-		pos = kh_put(str, smcfg, sm->name, &error);
+		pos = kh_put(str, cache->submodules, sm->name, &error);
 
-		if (error < 0) {
-			git_submodule_free(sm);
-			sm = NULL;
-		} else if (error == 0) {
+		if (error < 0)
+			goto done;
+		else if (error == 0) {
 			git_submodule_free(sm);
-			sm = git_strmap_value_at(smcfg, pos);
+			sm = git_strmap_value_at(cache->submodules, pos);
 		} else {
-			git_strmap_set_value_at(smcfg, pos, sm);
+			error = 0;
+			git_strmap_set_value_at(cache->submodules, pos, sm);
 		}
 	} else {
-		sm = git_strmap_value_at(smcfg, pos);
+		sm = git_strmap_value_at(cache->submodules, pos);
 	}
 
-	*sm_ptr = sm;
+done:
+	if (error < 0)
+		git_submodule_free(sm);
+	else if (out) {
+		GIT_REFCOUNT_INC(sm);
+		*out = sm;
+	}
 
-	return (sm != NULL) ? 0 : -1;
+	return error;
 }
 
 static int submodule_config_error(const char *property, const char *value)
@@ -1086,16 +1338,29 @@ int git_submodule_parse_update(git_submodule_update_t *out, const char *value)
 	return 0;
 }
 
+int git_submodule_parse_recurse(git_submodule_recurse_t *out, const char *value)
+{
+	int val;
+
+	if (git_config_lookup_map_value(
+			&val, _sm_recurse_map, ARRAY_SIZE(_sm_recurse_map), value) < 0) {
+		*out = GIT_SUBMODULE_RECURSE_YES;
+		return submodule_config_error("recurse", value);
+	}
+
+	*out = (git_submodule_recurse_t)val;
+	return 0;
+}
+
 static int submodule_load_from_config(
-	const git_config_entry *entry, void *data)
+	const git_config_entry *entry, void *payload)
 {
-	git_repository *repo = data;
-	git_strmap *smcfg = repo->submodules;
-	const char *namestart, *property, *alternate = NULL;
-	const char *key = entry->name, *value = entry->value;
+	git_submodule_cache *cache = payload;
+	const char *namestart, *property;
+	const char *key = entry->name, *value = entry->value, *path;
+	char *alternate = NULL, *replaced = NULL;
 	git_buf name = GIT_BUF_INIT;
-	git_submodule *sm;
-	bool is_path;
+	git_submodule *sm = NULL;
 	int error = 0;
 
 	if (git__prefixcmp(key, "submodule.") != 0)
@@ -1108,15 +1373,11 @@ static int submodule_load_from_config(
 		return 0;
 
 	property++;
-	is_path = (strcasecmp(property, "path") == 0);
-
-	if (git_buf_set(&name, namestart, property - namestart - 1) < 0)
-		return -1;
+	path = !strcasecmp(property, "path") ? value : NULL;
 
-	if (submodule_get(&sm, repo, name.ptr, is_path ? value : NULL) < 0) {
-		git_buf_free(&name);
-		return -1;
-	}
+	if ((error = git_buf_set(&name, namestart, property - namestart - 1)) < 0 ||
+		(error = submodule_get(&sm, cache, name.ptr, path)) < 0)
+		goto done;
 
 	sm->flags |= GIT_SUBMODULE_STATUS_IN_CONFIG;
 
@@ -1128,19 +1389,49 @@ static int submodule_load_from_config(
 	 * should be strcasecmp
 	 */
 
-	if (strcmp(sm->name, name.ptr) != 0) {
-		alternate = sm->name = git_buf_detach(&name);
-	} else if (is_path && value && strcmp(sm->path, value) != 0) {
-		alternate = sm->path = git__strdup(value);
-		if (!sm->path)
-			error = -1;
+	if (strcmp(sm->name, name.ptr) != 0) { /* name changed */
+		if (!strcmp(sm->path, name.ptr)) { /* already set as path */
+			replaced = sm->name;
+			sm->name = sm->path;
+		} else {
+			if (sm->name != sm->path)
+				replaced = sm->name;
+			alternate = sm->name = git_buf_detach(&name);
+		}
 	}
+	else if (path && strcmp(path, sm->path) != 0) { /* path changed */
+		if (!strcmp(sm->name, value)) { /* already set as name */
+			replaced = sm->path;
+			sm->path = sm->name;
+		} else {
+			if (sm->path != sm->name)
+				replaced = sm->path;
+			if ((alternate = git__strdup(value)) == NULL) {
+				error = -1;
+				goto done;
+			}
+			sm->path = alternate;
+		}
+	}
+
+	/* Deregister under name being replaced */
+	if (replaced) {
+		git_strmap_delete(cache->submodules, replaced);
+		git_submodule_free(sm);
+		git__free(replaced);
+	}
+
+	/* Insert under alternate key */
 	if (alternate) {
 		void *old_sm = NULL;
-		git_strmap_insert2(smcfg, alternate, sm, old_sm, error);
+		git_strmap_insert2(cache->submodules, alternate, sm, old_sm, error);
+
+		if (error < 0)
+			goto done;
+		if (error > 0)
+			error = 0;
 
-		if (error >= 0)
-			GIT_REFCOUNT_INC(sm); /* inserted under a new key */
+		GIT_REFCOUNT_INC(sm); /* increase refcount for new key */
 
 		/* if we replaced an old module under this key, release the old one */
 		if (old_sm && ((git_submodule *)old_sm) != sm) {
@@ -1149,55 +1440,59 @@ static int submodule_load_from_config(
 		}
 	}
 
-	git_buf_free(&name);
-	if (error < 0)
-		return error;
-
 	/* TODO: Look up path in index and if it is present but not a GITLINK
 	 * then this should be deleted (at least to match git's behavior)
 	 */
 
-	if (is_path)
-		return 0;
+	if (path)
+		goto done;
 
 	/* copy other properties into submodule entry */
 	if (strcasecmp(property, "url") == 0) {
 		git__free(sm->url);
 		sm->url = NULL;
 
-		if (value != NULL && (sm->url = git__strdup(value)) == NULL)
-			return -1;
+		if (value != NULL && (sm->url = git__strdup(value)) == NULL) {
+			error = -1;
+			goto done;
+		}
+	}
+	else if (strcasecmp(property, "branch") == 0) {
+		git__free(sm->branch);
+		sm->branch = NULL;
+
+		if (value != NULL && (sm->branch = git__strdup(value)) == NULL) {
+			error = -1;
+			goto done;
+		}
 	}
 	else if (strcasecmp(property, "update") == 0) {
-		if (git_submodule_parse_update(&sm->update, value) < 0)
-			return -1;
+		if ((error = git_submodule_parse_update(&sm->update, value)) < 0)
+			goto done;
 		sm->update_default = sm->update;
 	}
 	else if (strcasecmp(property, "fetchRecurseSubmodules") == 0) {
-		if (git__parse_bool(&sm->fetch_recurse, value) < 0)
-			return submodule_config_error("fetchRecurseSubmodules", value);
+		if ((error = git_submodule_parse_recurse(&sm->fetch_recurse, value)) < 0)
+			goto done;
+		sm->fetch_recurse_default = sm->fetch_recurse;
 	}
 	else if (strcasecmp(property, "ignore") == 0) {
-		if (git_submodule_parse_ignore(&sm->ignore, value) < 0)
-			return -1;
+		if ((error = git_submodule_parse_ignore(&sm->ignore, value)) < 0)
+			goto done;
 		sm->ignore_default = sm->ignore;
 	}
 	/* ignore other unknown submodule properties */
 
-	return 0;
+done:
+	git_submodule_free(sm); /* offset refcount inc from submodule_get() */
+	git_buf_free(&name);
+	return error;
 }
 
-static int submodule_load_from_wd_lite(
-	git_submodule *sm, const char *name, void *payload)
+static int submodule_load_from_wd_lite(git_submodule *sm)
 {
 	git_buf path = GIT_BUF_INIT;
 
-	GIT_UNUSED(name);
-	GIT_UNUSED(payload);
-
-	if (git_repository_is_bare(sm->repo))
-		return 0;
-
 	if (git_buf_joinpath(&path, git_repository_workdir(sm->repo), sm->path) < 0)
 		return -1;
 
@@ -1208,38 +1503,36 @@ static int submodule_load_from_wd_lite(
 		sm->flags |= GIT_SUBMODULE_STATUS_IN_WD;
 
 	git_buf_free(&path);
-
 	return 0;
 }
 
-static int load_submodule_config_from_index(
-	git_repository *repo, git_oid *gitmodules_oid)
+static int submodule_cache_refresh_from_index(
+	git_submodule_cache *cache, git_index *idx)
 {
 	int error;
-	git_index *index;
 	git_iterator *i;
 	const git_index_entry *entry;
 
-	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
-		(error = git_iterator_for_index(&i, index, 0, NULL, NULL)) < 0)
+	if ((error = git_iterator_for_index(&i, idx, 0, NULL, NULL)) < 0)
 		return error;
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		khiter_t pos = git_strmap_lookup_index(cache->submodules, entry->path);
 		git_submodule *sm;
 
-		if (git_strmap_valid_index(repo->submodules, pos)) {
-			sm = git_strmap_value_at(repo->submodules, pos);
+		if (git_strmap_valid_index(cache->submodules, pos)) {
+			sm = git_strmap_value_at(cache->submodules, pos);
 
 			if (S_ISGITLINK(entry->mode))
 				submodule_update_from_index_entry(sm, entry);
 			else
 				sm->flags |= GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE;
 		} else if (S_ISGITLINK(entry->mode)) {
-			if (!submodule_get(&sm, repo, entry->path, NULL))
+			if (!submodule_get(&sm, cache, entry->path, NULL)) {
 				submodule_update_from_index_entry(sm, entry);
-		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0)
-			git_oid_cpy(gitmodules_oid, &entry->oid);
+				git_submodule_free(sm);
+			}
+		}
 	}
 
 	if (error == GIT_ITEROVER)
@@ -1250,44 +1543,33 @@ static int load_submodule_config_from_index(
 	return error;
 }
 
-static int load_submodule_config_from_head(
-	git_repository *repo, git_oid *gitmodules_oid)
+static int submodule_cache_refresh_from_head(
+	git_submodule_cache *cache, git_tree *head)
 {
 	int error;
-	git_tree *head;
 	git_iterator *i;
 	const git_index_entry *entry;
 
-	/* if we can't look up current head, then there's no submodule in it */
-	if (git_repository_head_tree(&head, repo) < 0) {
-		giterr_clear();
-		return 0;
-	}
-
-	if ((error = git_iterator_for_tree(&i, head, 0, NULL, NULL)) < 0) {
-		git_tree_free(head);
+	if ((error = git_iterator_for_tree(&i, head, 0, NULL, NULL)) < 0)
 		return error;
-	}
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		khiter_t pos = git_strmap_lookup_index(cache->submodules, entry->path);
 		git_submodule *sm;
 
-		if (git_strmap_valid_index(repo->submodules, pos)) {
-			sm = git_strmap_value_at(repo->submodules, pos);
+		if (git_strmap_valid_index(cache->submodules, pos)) {
+			sm = git_strmap_value_at(cache->submodules, pos);
 
 			if (S_ISGITLINK(entry->mode))
-				submodule_update_from_head_data(
-					sm, entry->mode, &entry->oid);
+				submodule_update_from_head_data(sm, entry->mode, &entry->id);
 			else
 				sm->flags |= GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE;
 		} else if (S_ISGITLINK(entry->mode)) {
-			if (!submodule_get(&sm, repo, entry->path, NULL))
+			if (!submodule_get(&sm, cache, entry->path, NULL)) {
 				submodule_update_from_head_data(
-					sm, entry->mode, &entry->oid);
-		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0 &&
-				   git_oid_iszero(gitmodules_oid)) {
-			git_oid_cpy(gitmodules_oid, &entry->oid);
+					sm, entry->mode, &entry->id);
+				git_submodule_free(sm);
+			}
 		}
 	}
 
@@ -1295,17 +1577,15 @@ static int load_submodule_config_from_head(
 		error = 0;
 
 	git_iterator_free(i);
-	git_tree_free(head);
 
 	return error;
 }
 
 static git_config_backend *open_gitmodules(
-	git_repository *repo,
-	bool okay_to_create,
-	const git_oid *gitmodules_oid)
+	git_submodule_cache *cache,
+	int okay_to_create)
 {
-	const char *workdir = git_repository_workdir(repo);
+	const char *workdir = git_repository_workdir(cache->repo);
 	git_buf path = GIT_BUF_INIT;
 	git_config_backend *mods = NULL;
 
@@ -1325,178 +1605,278 @@ static git_config_backend *open_gitmodules(
 		}
 	}
 
-	if (!mods && gitmodules_oid && !git_oid_iszero(gitmodules_oid)) {
-		/* TODO: Retrieve .gitmodules content from ODB */
+	git_buf_free(&path);
 
-		/* Should we actually do this?  Core git does not, but it means you
-		 * can't really get much information about submodules on bare repos.
-		 */
+	return mods;
+}
+
+static void submodule_cache_free(git_submodule_cache *cache)
+{
+	git_submodule *sm;
+
+	if (!cache)
+		return;
+
+	git_strmap_foreach_value(cache->submodules, sm, {
+		sm->repo = NULL; /* disconnect from repo */
+		git_submodule_free(sm);
+	});
+	git_strmap_free(cache->submodules);
+
+	git_buf_free(&cache->gitmodules_path);
+	git_mutex_free(&cache->lock);
+	git__free(cache);
+}
+
+static int submodule_cache_alloc(
+	git_submodule_cache **out, git_repository *repo)
+{
+	git_submodule_cache *cache = git__calloc(1, sizeof(git_submodule_cache));
+	GITERR_CHECK_ALLOC(cache);
+
+	if (git_mutex_init(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to initialize submodule cache lock");
+		git__free(cache);
+		return -1;
 	}
 
-	git_buf_free(&path);
+	if (git_strmap_alloc(&cache->submodules) < 0) {
+		submodule_cache_free(cache);
+		return -1;
+	}
 
-	return mods;
+	cache->repo = repo;
+	git_buf_init(&cache->gitmodules_path, 0);
+
+	*out = cache;
+	return 0;
 }
 
-static int load_submodule_config(git_repository *repo)
+static int submodule_cache_refresh(git_submodule_cache *cache, int refresh)
 {
-	int error;
-	git_oid gitmodules_oid;
+	int error = 0, update_index, update_head, update_gitmod;
+	git_index *idx = NULL;
+	git_tree *head = NULL;
+	const char *wd = NULL;
 	git_buf path = GIT_BUF_INIT;
+	git_submodule *sm;
 	git_config_backend *mods = NULL;
+	uint32_t mask;
 
-	if (repo->submodules)
+	if (!cache || !cache->repo || !refresh)
 		return 0;
 
-	memset(&gitmodules_oid, 0, sizeof(gitmodules_oid));
-
-	/* Submodule data is kept in a hashtable keyed by both name and path.
-	 * These are usually the same, but that is not guaranteed.
-	 */
-	if (!repo->submodules) {
-		repo->submodules = git_strmap_alloc();
-		GITERR_CHECK_ALLOC(repo->submodules);
+	if (git_mutex_lock(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire lock on submodule cache");
+		return -1;
 	}
 
-	/* add submodule information from index */
+	/* get sources that we will need to check */
+
+	if (git_repository_index(&idx, cache->repo) < 0)
+		giterr_clear();
+	if (git_repository_head_tree(&head, cache->repo) < 0)
+		giterr_clear();
 
-	if ((error = load_submodule_config_from_index(repo, &gitmodules_oid)) < 0)
+	wd = git_repository_workdir(cache->repo);
+	if (wd && (error = git_buf_joinpath(&path, wd, GIT_MODULES_FILE)) < 0)
 		goto cleanup;
 
+	/* check for invalidation */
+
+	if (refresh == CACHE_FLUSH)
+		update_index = update_head = update_gitmod = true;
+	else {
+		update_index =
+			!idx || git_index__changed_relative_to(idx, &cache->index_stamp);
+		update_head =
+			!head || !git_oid_equal(&cache->head_id, git_tree_id(head));
+
+		update_gitmod = (wd != NULL) ?
+			git_futils_filestamp_check(&cache->gitmodules_stamp, path.ptr) :
+			(cache->gitmodules_stamp.mtime != 0);
+	}
+
+	/* clear submodule flags that are to be refreshed */
+
+	mask = 0;
+	if (!idx || update_index)
+		mask |= GIT_SUBMODULE_STATUS_IN_INDEX |
+			GIT_SUBMODULE_STATUS__INDEX_FLAGS |
+			GIT_SUBMODULE_STATUS__INDEX_OID_VALID |
+			GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
+	if (!head || update_head)
+		mask |= GIT_SUBMODULE_STATUS_IN_HEAD |
+			GIT_SUBMODULE_STATUS__HEAD_OID_VALID;
+	if (update_gitmod)
+		mask |= GIT_SUBMODULE_STATUS_IN_CONFIG;
+	if (mask != 0)
+		mask |= GIT_SUBMODULE_STATUS_IN_WD |
+			GIT_SUBMODULE_STATUS__WD_SCANNED |
+			GIT_SUBMODULE_STATUS__WD_FLAGS |
+			GIT_SUBMODULE_STATUS__WD_OID_VALID;
+	else
+		goto cleanup; /* nothing to do */
+
+	submodule_cache_clear_flags(cache, mask);
+
+	/* add back submodule information from index */
+
+	if (idx && update_index) {
+		if ((error = submodule_cache_refresh_from_index(cache, idx)) < 0)
+			goto cleanup;
+
+		git_futils_filestamp_set(
+			&cache->index_stamp, git_index__filestamp(idx));
+	}
+
 	/* add submodule information from HEAD */
 
-	if ((error = load_submodule_config_from_head(repo, &gitmodules_oid)) < 0)
-		goto cleanup;
+	if (head && update_head) {
+		if ((error = submodule_cache_refresh_from_head(cache, head)) < 0)
+			goto cleanup;
+
+		git_oid_cpy(&cache->head_id, git_tree_id(head));
+	}
 
 	/* add submodule information from .gitmodules */
 
-	if ((mods = open_gitmodules(repo, false, &gitmodules_oid)) != NULL)
-		error = git_config_file_foreach(mods, submodule_load_from_config, repo);
+	if (wd && update_gitmod > 0) {
+		if ((mods = open_gitmodules(cache, false)) != NULL &&
+			(error = git_config_file_foreach(
+				mods, submodule_load_from_config, cache)) < 0)
+			goto cleanup;
+	}
 
-	if (error != 0)
-		goto cleanup;
+	/* shallow scan submodules in work tree as needed */
 
-	/* shallow scan submodules in work tree */
+	if (wd && mask != 0) {
+		git_strmap_foreach_value(cache->submodules, sm, {
+			submodule_load_from_wd_lite(sm);
+		});
+	}
+
+	/* remove submodules that no longer exist */
 
-	if (!git_repository_is_bare(repo))
-		error = git_submodule_foreach(repo, submodule_load_from_wd_lite, NULL);
+	git_strmap_foreach_value(cache->submodules, sm, {
+		/* purge unless in HEAD, index, or .gitmodules; no sm for wd only */
+		if (sm != NULL &&
+			!(sm->flags &
+			 (GIT_SUBMODULE_STATUS_IN_HEAD |
+			  GIT_SUBMODULE_STATUS_IN_INDEX |
+			  GIT_SUBMODULE_STATUS_IN_CONFIG)))
+			submodule_cache_remove_item(cache, sm, true);
+	});
 
 cleanup:
-	git_buf_free(&path);
+	git_config_file_free(mods);
 
-	if (mods != NULL)
-		git_config_file_free(mods);
+	/* TODO: if we got an error, mark submodule config as invalid? */
+
+	git_mutex_unlock(&cache->lock);
 
-	if (error)
-		git_submodule_config_free(repo);
+	git_index_free(idx);
+	git_tree_free(head);
+	git_buf_free(&path);
 
 	return error;
 }
 
-static int lookup_head_remote(git_buf *url, git_repository *repo)
+static int submodule_cache_init(git_repository *repo, int cache_refresh)
 {
-	int error;
-	git_config *cfg;
-	git_reference *head = NULL, *remote = NULL;
-	const char *tgt, *scan;
-	git_buf key = GIT_BUF_INIT;
+	int error = 0;
+	git_submodule_cache *cache = NULL;
 
-	/* 1. resolve HEAD -> refs/heads/BRANCH
-	 * 2. lookup config branch.BRANCH.remote -> ORIGIN
-	 * 3. lookup remote.ORIGIN.url
-	 */
+	/* if submodules already exist, just refresh as requested */
+	if (repo->_submodules)
+		return submodule_cache_refresh(repo->_submodules, cache_refresh);
 
-	if ((error = git_repository_config__weakptr(&cfg, repo)) < 0)
-		return error;
+	/* otherwise create a new cache, load it, and atomically swap it in */
+	if (!(error = submodule_cache_alloc(&cache, repo)) &&
+		!(error = submodule_cache_refresh(cache, CACHE_FLUSH)))
+		cache = git__compare_and_swap(&repo->_submodules, NULL, cache);
 
-	if (git_reference_lookup(&head, repo, GIT_HEAD_FILE) < 0) {
-		giterr_set(GITERR_SUBMODULE,
-			"Cannot resolve relative URL when HEAD cannot be resolved");
-		error = GIT_ENOTFOUND;
-		goto cleanup;
-	}
+	/* might have raced with another thread to set cache, so free if needed */
+	if (cache)
+		submodule_cache_free(cache);
 
-	if (git_reference_type(head) != GIT_REF_SYMBOLIC) {
-		giterr_set(GITERR_SUBMODULE,
-			"Cannot resolve relative URL when HEAD is not symbolic");
-		error = GIT_ENOTFOUND;
-		goto cleanup;
-	}
+	return error;
+}
 
-	if ((error = git_branch_upstream(&remote, head)) < 0)
-		goto cleanup;
+/* Lookup name of remote of the local tracking branch HEAD points to */
+static int lookup_head_remote_key(git_buf *remote_name, git_repository *repo)
+{
+	int error;
+	git_reference *head = NULL;
+	git_buf upstream_name = GIT_BUF_INIT;
 
-	/* remote should refer to something like refs/remotes/ORIGIN/BRANCH */
+	/* lookup and dereference HEAD */
+	if ((error = git_repository_head(&head, repo)) < 0)
+		return error;
 
-	if (git_reference_type(remote) != GIT_REF_SYMBOLIC ||
-		git__prefixcmp(git_reference_symbolic_target(remote), GIT_REFS_REMOTES_DIR) != 0)
+	/* lookup remote tracking branch of HEAD */
+	if (!(error = git_branch_upstream_name(
+			&upstream_name, repo, git_reference_name(head))))
 	{
-		giterr_set(GITERR_SUBMODULE,
-			"Cannot resolve relative URL when HEAD is not symbolic");
-		error = GIT_ENOTFOUND;
-		goto cleanup;
+		/* lookup remote of remote tracking branch */
+		error = git_branch_remote_name(remote_name, repo, upstream_name.ptr);
+
+		git_buf_free(&upstream_name);
 	}
 
-	scan = tgt = git_reference_symbolic_target(remote) + strlen(GIT_REFS_REMOTES_DIR);
-	while (*scan && (*scan != '/' || (scan > tgt && scan[-1] != '\\')))
-		scan++; /* find non-escaped slash to end ORIGIN name */
+	git_reference_free(head);
 
-	error = git_buf_printf(&key, "remote.%.*s.url", (int)(scan - tgt), tgt);
-	if (error < 0)
-		goto cleanup;
+	return error;
+}
 
-	if ((error = git_config_get_string(&tgt, cfg, key.ptr)) < 0)
-		goto cleanup;
+/* Lookup the remote of the local tracking branch HEAD points to */
+static int lookup_head_remote(git_remote **remote, git_repository *repo)
+{
+	int error;
+	git_buf remote_name = GIT_BUF_INIT;
 
-	error = git_buf_sets(url, tgt);
+	/* lookup remote of remote tracking branch name */
+	if (!(error = lookup_head_remote_key(&remote_name, repo)))
+		error = git_remote_load(remote, repo, remote_name.ptr);
 
-cleanup:
-	git_buf_free(&key);
-	git_reference_free(head);
-	git_reference_free(remote);
+	git_buf_free(&remote_name);
 
 	return error;
 }
 
-static int submodule_update_config(
-	git_submodule *submodule,
-	const char *attr,
-	const char *value,
-	bool overwrite,
-	bool only_existing)
+/* Lookup remote, either from HEAD or fall back on origin */
+static int lookup_default_remote(git_remote **remote, git_repository *repo)
 {
-	int error;
-	git_config *config;
-	git_buf key = GIT_BUF_INIT;
-	const char *old = NULL;
+	int error = lookup_head_remote(remote, repo);
 
-	assert(submodule);
-
-	error = git_repository_config__weakptr(&config, submodule->repo);
-	if (error < 0)
-		return error;
+	/* if that failed, use 'origin' instead */
+	if (error == GIT_ENOTFOUND)
+		error = git_remote_load(remote, repo, "origin");
 
-	error = git_buf_printf(&key, "submodule.%s.%s", submodule->name, attr);
-	if (error < 0)
-		goto cleanup;
+	if (error == GIT_ENOTFOUND)
+		giterr_set(
+			GITERR_SUBMODULE,
+			"Cannot get default remote for submodule - no local tracking "
+			"branch for HEAD and origin does not exist");
 
-	if (git_config_get_string(&old, config, key.ptr) < 0)
-		giterr_clear();
+	return error;
+}
 
-	if (!old && only_existing)
-		goto cleanup;
-	if (old && !overwrite)
-		goto cleanup;
-	if ((!old && !value) || (old && value && strcmp(old, value) == 0))
-		goto cleanup;
+static int get_url_base(git_buf *url, git_repository *repo)
+{
+	int error;
+	git_remote *remote = NULL;
 
-	if (!value)
-		error = git_config_delete_entry(config, key.ptr);
-	else
-		error = git_config_set_string(config, key.ptr, value);
+	if (!(error = lookup_default_remote(&remote, repo))) {
+		error = git_buf_sets(url, git_remote_url(remote));
+		git_remote_free(remote);
+	}
+	else if (error == GIT_ENOTFOUND) {
+		/* if repository does not have a default remote, use workdir instead */
+		giterr_clear();
+		error = git_buf_sets(url, git_repository_workdir(repo));
+	}
 
-cleanup:
-	git_buf_free(&key);
 	return error;
 }
 
diff --git a/src/submodule.h b/src/submodule.h
index b05937503..a6182beca 100644
--- a/src/submodule.h
+++ b/src/submodule.h
@@ -60,7 +60,9 @@
  * - `update_default` is the update value from the config
  * - `ignore` is a git_submodule_ignore_t value - see gitmodules(5) ignore.
  * - `ignore_default` is the ignore value from the config
- * - `fetch_recurse` is 0 or 1 - see gitmodules(5) fetchRecurseSubmodules.
+ * - `fetch_recurse` is a git_submodule_recurse_t value - see gitmodules(5)
+ *    fetchRecurseSubmodules.
+ * - `fetch_recurse_default` is the recurse value from the config
  *
  * - `repo` is the parent repository that contains this submodule.
  * - `flags` after for internal use, tracking where this submodule has been
@@ -81,11 +83,13 @@ struct git_submodule {
 	char *name;
 	char *path; /* important: may just point to "name" string */
 	char *url;
+	char *branch;
 	git_submodule_update_t update;
 	git_submodule_update_t update_default;
 	git_submodule_ignore_t ignore;
 	git_submodule_ignore_t ignore_default;
-	int fetch_recurse;
+	git_submodule_recurse_t fetch_recurse;
+	git_submodule_recurse_t fetch_recurse_default;
 
 	/* internal information */
 	git_repository *repo;
@@ -95,6 +99,29 @@ struct git_submodule {
 	git_oid wd_oid;
 };
 
+/**
+ * The git_submodule_cache stores known submodules along with timestamps,
+ * etc. about when they were loaded
+ */
+typedef struct {
+	git_repository *repo;
+	git_strmap *submodules;
+	git_mutex  lock;
+
+	/* cache invalidation data */
+	git_oid head_id;
+	git_futils_filestamp index_stamp;
+	git_buf gitmodules_path;
+	git_futils_filestamp gitmodules_stamp;
+	git_futils_filestamp config_stamp;
+} git_submodule_cache;
+
+/* Force revalidation of submodule data cache (alloc as needed) */
+extern int git_submodule_cache_refresh(git_repository *repo);
+
+/* Release all submodules */
+extern void git_submodule_cache_free(git_repository *repo);
+
 /* Additional flags on top of public GIT_SUBMODULE_STATUS values */
 enum {
 	GIT_SUBMODULE_STATUS__WD_SCANNED          = (1u << 20),
@@ -110,6 +137,13 @@ enum {
 #define GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(S) \
 	((S) & ~(0xFFFFFFFFu << 20))
 
+/* Internal submodule check does not attempt to refresh cached data */
+extern bool git_submodule__is_submodule(git_repository *repo, const char *name);
+
+/* Internal lookup does not attempt to refresh cached data */
+extern int git_submodule__lookup(
+	git_submodule **out, git_repository *repo, const char *path);
+
 /* Internal status fn returns status and optionally the various OIDs */
 extern int git_submodule__status(
 	unsigned int *out_status,
@@ -124,9 +158,6 @@ extern int git_submodule_open_bare(
 	git_repository **repo,
 	git_submodule *submodule);
 
-/* Release reference to submodule object - not currently for external use */
-extern void git_submodule_free(git_submodule *sm);
-
 extern int git_submodule_parse_ignore(
 	git_submodule_ignore_t *out, const char *value);
 extern int git_submodule_parse_update(
@@ -134,5 +165,6 @@ extern int git_submodule_parse_update(
 
 extern const char *git_submodule_ignore_to_str(git_submodule_ignore_t);
 extern const char *git_submodule_update_to_str(git_submodule_update_t);
+extern const char *git_submodule_recurse_to_str(git_submodule_recurse_t);
 
 #endif
diff --git a/src/sysdir.c b/src/sysdir.c
new file mode 100644
index 000000000..cd94a8b57
--- /dev/null
+++ b/src/sysdir.c
@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "sysdir.h"
+#include "global.h"
+#include "buffer.h"
+#include "path.h"
+#include <ctype.h>
+#if GIT_WIN32
+#include "win32/findfile.h"
+#endif
+
+static int git_sysdir_guess_system_dirs(git_buf *out)
+{
+#ifdef GIT_WIN32
+	return git_win32__find_system_dirs(out, L"etc\\");
+#else
+	return git_buf_sets(out, "/etc");
+#endif
+}
+
+static int git_sysdir_guess_global_dirs(git_buf *out)
+{
+#ifdef GIT_WIN32
+	return git_win32__find_global_dirs(out);
+#else
+	return git_buf_sets(out, getenv("HOME"));
+#endif
+}
+
+static int git_sysdir_guess_xdg_dirs(git_buf *out)
+{
+#ifdef GIT_WIN32
+	return git_win32__find_xdg_dirs(out);
+#else
+	const char *env = NULL;
+
+	if ((env = getenv("XDG_CONFIG_HOME")) != NULL)
+		return git_buf_joinpath(out, env, "git");
+	else if ((env = getenv("HOME")) != NULL)
+		return git_buf_joinpath(out, env, ".config/git");
+
+	git_buf_clear(out);
+	return 0;
+#endif
+}
+
+static int git_sysdir_guess_template_dirs(git_buf *out)
+{
+#ifdef GIT_WIN32
+	return git_win32__find_system_dirs(out, L"share\\git-core\\templates");
+#else
+	return git_buf_sets(out, "/usr/share/git-core/templates");
+#endif
+}
+
+typedef int (*git_sysdir_guess_cb)(git_buf *out);
+
+static git_buf git_sysdir__dirs[GIT_SYSDIR__MAX] =
+	{ GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT };
+
+static git_sysdir_guess_cb git_sysdir__dir_guess[GIT_SYSDIR__MAX] = {
+	git_sysdir_guess_system_dirs,
+	git_sysdir_guess_global_dirs,
+	git_sysdir_guess_xdg_dirs,
+	git_sysdir_guess_template_dirs,
+};
+
+static int git_sysdir__dirs_shutdown_set = 0;
+
+int git_sysdir_global_init(void)
+{
+	git_sysdir_t i;
+	const git_buf *path;
+	int error = 0;
+
+	for (i = 0; !error && i < GIT_SYSDIR__MAX; i++)
+		error = git_sysdir_get(&path, i);
+
+	return error;
+}
+
+void git_sysdir_global_shutdown(void)
+{
+	int i;
+	for (i = 0; i < GIT_SYSDIR__MAX; ++i)
+		git_buf_free(&git_sysdir__dirs[i]);
+
+	git_sysdir__dirs_shutdown_set = 0;
+}
+
+static int git_sysdir_check_selector(git_sysdir_t which)
+{
+	if (which < GIT_SYSDIR__MAX)
+		return 0;
+
+	giterr_set(GITERR_INVALID, "config directory selector out of range");
+	return -1;
+}
+
+
+int git_sysdir_get(const git_buf **out, git_sysdir_t which)
+{
+	assert(out);
+
+	*out = NULL;
+
+	GITERR_CHECK_ERROR(git_sysdir_check_selector(which));
+
+	if (!git_buf_len(&git_sysdir__dirs[which])) {
+		/* prepare shutdown if we're going to need it */
+		if (!git_sysdir__dirs_shutdown_set) {
+			git__on_shutdown(git_sysdir_global_shutdown);
+			git_sysdir__dirs_shutdown_set = 1;
+		}
+
+		GITERR_CHECK_ERROR(
+			git_sysdir__dir_guess[which](&git_sysdir__dirs[which]));
+	}
+
+	*out = &git_sysdir__dirs[which];
+	return 0;
+}
+
+int git_sysdir_get_str(
+	char *out,
+	size_t outlen,
+	git_sysdir_t which)
+{
+	const git_buf *path = NULL;
+
+	GITERR_CHECK_ERROR(git_sysdir_check_selector(which));
+	GITERR_CHECK_ERROR(git_sysdir_get(&path, which));
+
+	if (!out || path->size >= outlen) {
+		giterr_set(GITERR_NOMEMORY, "Buffer is too short for the path");
+		return GIT_EBUFS;
+	}
+
+	git_buf_copy_cstr(out, outlen, path);
+	return 0;
+}
+
+#define PATH_MAGIC "$PATH"
+
+int git_sysdir_set(git_sysdir_t which, const char *search_path)
+{
+	const char *expand_path = NULL;
+	git_buf merge = GIT_BUF_INIT;
+
+	GITERR_CHECK_ERROR(git_sysdir_check_selector(which));
+
+	if (search_path != NULL)
+		expand_path = strstr(search_path, PATH_MAGIC);
+
+	/* init with default if not yet done and needed (ignoring error) */
+	if ((!search_path || expand_path) &&
+		!git_buf_len(&git_sysdir__dirs[which]))
+		git_sysdir__dir_guess[which](&git_sysdir__dirs[which]);
+
+	/* if $PATH is not referenced, then just set the path */
+	if (!expand_path)
+		return git_buf_sets(&git_sysdir__dirs[which], search_path);
+
+	/* otherwise set to join(before $PATH, old value, after $PATH) */
+	if (expand_path > search_path)
+		git_buf_set(&merge, search_path, expand_path - search_path);
+
+	if (git_buf_len(&git_sysdir__dirs[which]))
+		git_buf_join(&merge, GIT_PATH_LIST_SEPARATOR,
+			merge.ptr, git_sysdir__dirs[which].ptr);
+
+	expand_path += strlen(PATH_MAGIC);
+	if (*expand_path)
+		git_buf_join(&merge, GIT_PATH_LIST_SEPARATOR, merge.ptr, expand_path);
+
+	git_buf_swap(&git_sysdir__dirs[which], &merge);
+	git_buf_free(&merge);
+
+	return git_buf_oom(&git_sysdir__dirs[which]) ? -1 : 0;
+}
+
+static int git_sysdir_find_in_dirlist(
+	git_buf *path,
+	const char *name,
+	git_sysdir_t which,
+	const char *label)
+{
+	size_t len;
+	const char *scan, *next = NULL;
+	const git_buf *syspath;
+
+	GITERR_CHECK_ERROR(git_sysdir_get(&syspath, which));
+	if (!syspath || !git_buf_len(syspath))
+		goto done;
+
+	for (scan = git_buf_cstr(syspath); scan; scan = next) {
+		/* find unescaped separator or end of string */
+		for (next = scan; *next; ++next) {
+			if (*next == GIT_PATH_LIST_SEPARATOR &&
+				(next <= scan || next[-1] != '\\'))
+				break;
+		}
+
+		len = (size_t)(next - scan);
+		next = (*next ? next + 1 : NULL);
+		if (!len)
+			continue;
+
+		GITERR_CHECK_ERROR(git_buf_set(path, scan, len));
+		if (name)
+			GITERR_CHECK_ERROR(git_buf_joinpath(path, path->ptr, name));
+
+		if (git_path_exists(path->ptr))
+			return 0;
+	}
+
+done:
+	git_buf_free(path);
+	giterr_set(GITERR_OS, "The %s file '%s' doesn't exist", label, name);
+	return GIT_ENOTFOUND;
+}
+
+int git_sysdir_find_system_file(git_buf *path, const char *filename)
+{
+	return git_sysdir_find_in_dirlist(
+		path, filename, GIT_SYSDIR_SYSTEM, "system");
+}
+
+int git_sysdir_find_global_file(git_buf *path, const char *filename)
+{
+	return git_sysdir_find_in_dirlist(
+		path, filename, GIT_SYSDIR_GLOBAL, "global");
+}
+
+int git_sysdir_find_xdg_file(git_buf *path, const char *filename)
+{
+	return git_sysdir_find_in_dirlist(
+		path, filename, GIT_SYSDIR_XDG, "global/xdg");
+}
+
+int git_sysdir_find_template_dir(git_buf *path)
+{
+	return git_sysdir_find_in_dirlist(
+		path, NULL, GIT_SYSDIR_TEMPLATE, "template");
+}
+
diff --git a/src/sysdir.h b/src/sysdir.h
new file mode 100644
index 000000000..f1bbf0bae
--- /dev/null
+++ b/src/sysdir.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_sysdir_h__
+#define INCLUDE_sysdir_h__
+
+#include "common.h"
+#include "posix.h"
+#include "buffer.h"
+
+/**
+ * Find a "global" file (i.e. one in a user's home directory).
+ *
+ * @param path buffer to write the full path into
+ * @param filename name of file to find in the home directory
+ * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
+ */
+extern int git_sysdir_find_global_file(git_buf *path, const char *filename);
+
+/**
+ * Find an "XDG" file (i.e. one in user's XDG config path).
+ *
+ * @param path buffer to write the full path into
+ * @param filename name of file to find in the home directory
+ * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
+ */
+extern int git_sysdir_find_xdg_file(git_buf *path, const char *filename);
+
+/**
+ * Find a "system" file (i.e. one shared for all users of the system).
+ *
+ * @param path buffer to write the full path into
+ * @param filename name of file to find in the home directory
+ * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
+ */
+extern int git_sysdir_find_system_file(git_buf *path, const char *filename);
+
+/**
+ * Find template directory.
+ *
+ * @param path buffer to write the full path into
+ * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
+ */
+extern int git_sysdir_find_template_dir(git_buf *path);
+
+typedef enum {
+	GIT_SYSDIR_SYSTEM = 0,
+	GIT_SYSDIR_GLOBAL = 1,
+	GIT_SYSDIR_XDG    = 2,
+	GIT_SYSDIR_TEMPLATE = 3,
+	GIT_SYSDIR__MAX   = 4,
+} git_sysdir_t;
+
+/**
+ * Configures global data for configuration file search paths.
+ *
+ * @return 0 on success, <0 on failure
+ */
+extern int git_sysdir_global_init(void);
+
+/**
+ * Get the search path for global/system/xdg files
+ *
+ * @param out pointer to git_buf containing search path
+ * @param which which list of paths to return
+ * @return 0 on success, <0 on failure
+ */
+extern int git_sysdir_get(const git_buf **out, git_sysdir_t which);
+
+/**
+ * Get search path into a preallocated buffer
+ *
+ * @param out String buffer to write into
+ * @param outlen Size of string buffer
+ * @param which Which search path to return
+ * @return 0 on success, GIT_EBUFS if out is too small, <0 on other failure
+ */
+
+extern int git_sysdir_get_str(char *out, size_t outlen, git_sysdir_t which);
+
+/**
+ * Set search paths for global/system/xdg files
+ *
+ * The first occurrence of the magic string "$PATH" in the new value will
+ * be replaced with the old value of the search path.
+ *
+ * @param which Which search path to modify
+ * @param paths New search path (separated by GIT_PATH_LIST_SEPARATOR)
+ * @return 0 on success, <0 on failure (allocation error)
+ */
+extern int git_sysdir_set(git_sysdir_t which, const char *paths);
+
+/**
+ * Free the configuration file search paths.
+ */
+extern void git_sysdir_global_shutdown(void);
+
+#endif /* INCLUDE_sysdir_h__ */
diff --git a/src/tag.c b/src/tag.c
index 31a3c8b80..d7b531d34 100644
--- a/src/tag.c
+++ b/src/tag.c
@@ -271,7 +271,7 @@ static int git_tag_create__internal(
 	} else
 		git_oid_cpy(oid, git_object_id(target));
 
-	error = git_reference_create(&new_ref, repo, ref_name.ptr, oid, allow_ref_overwrite);
+	error = git_reference_create(&new_ref, repo, ref_name.ptr, oid, allow_ref_overwrite, NULL, NULL);
 
 cleanup:
 	git_reference_free(new_ref);
@@ -363,20 +363,22 @@ int git_tag_create_frombuffer(git_oid *oid, git_repository *repo, const char *bu
 	}
 
 	/* write the buffer */
-	if (git_odb_open_wstream(&stream, odb, strlen(buffer), GIT_OBJ_TAG) < 0)
-		return -1;
+	if ((error = git_odb_open_wstream(
+			&stream, odb, strlen(buffer), GIT_OBJ_TAG)) < 0)
+		return error;
 
-	git_odb_stream_write(stream, buffer, strlen(buffer));
+	if (!(error = git_odb_stream_write(stream, buffer, strlen(buffer))))
+		error = git_odb_stream_finalize_write(oid, stream);
 
-	error = git_odb_stream_finalize_write(oid, stream);
 	git_odb_stream_free(stream);
 
 	if (error < 0) {
 		git_buf_free(&ref_name);
-		return -1;
+		return error;
 	}
 
-	error = git_reference_create(&new_ref, repo, ref_name.ptr, oid, allow_ref_overwrite);
+	error = git_reference_create(
+		&new_ref, repo, ref_name.ptr, oid, allow_ref_overwrite, NULL, NULL);
 
 	git_reference_free(new_ref);
 	git_buf_free(&ref_name);
@@ -418,16 +420,19 @@ typedef struct {
 
 static int tags_cb(const char *ref, void *data)
 {
+	int error;
 	git_oid oid;
 	tag_cb_data *d = (tag_cb_data *)data;
 
 	if (git__prefixcmp(ref, GIT_REFS_TAGS_DIR) != 0)
 		return 0; /* no tag */
 
-	if (git_reference_name_to_id(&oid, d->repo, ref) < 0)
-		return -1;
+	if (!(error = git_reference_name_to_id(&oid, d->repo, ref))) {
+		if ((error = d->cb(ref, &oid, d->cb_data)) != 0)
+			giterr_set_after_callback_function(error, "git_tag_foreach");
+	}
 
-	return d->cb(ref, &oid, d->cb_data);
+	return error;
 }
 
 int git_tag_foreach(git_repository *repo, git_tag_foreach_cb cb, void *cb_data)
@@ -455,8 +460,14 @@ static int tag_list_cb(const char *tag_name, git_oid *oid, void *data)
 	tag_filter_data *filter = (tag_filter_data *)data;
 	GIT_UNUSED(oid);
 
-	if (!*filter->pattern || p_fnmatch(filter->pattern, tag_name + GIT_REFS_TAGS_DIR_LEN, 0) == 0)
-		return git_vector_insert(filter->taglist, git__strdup(tag_name + GIT_REFS_TAGS_DIR_LEN));
+	if (!*filter->pattern ||
+		p_fnmatch(filter->pattern, tag_name + GIT_REFS_TAGS_DIR_LEN, 0) == 0)
+	{
+		char *matched = git__strdup(tag_name + GIT_REFS_TAGS_DIR_LEN);
+		GITERR_CHECK_ALLOC(matched);
+
+		return git_vector_insert(filter->taglist, matched);
+	}
 
 	return 0;
 }
@@ -469,20 +480,20 @@ int git_tag_list_match(git_strarray *tag_names, const char *pattern, git_reposit
 
 	assert(tag_names && repo && pattern);
 
-	if (git_vector_init(&taglist, 8, NULL) < 0)
-		return -1;
+	if ((error = git_vector_init(&taglist, 8, NULL)) < 0)
+		return error;
 
 	filter.taglist = &taglist;
 	filter.pattern = pattern;
 
 	error = git_tag_foreach(repo, &tag_list_cb, (void *)&filter);
-	if (error < 0) {
+
+	if (error < 0)
 		git_vector_free(&taglist);
-		return -1;
-	}
 
-	tag_names->strings = (char **)taglist.contents;
-	tag_names->count = taglist.length;
+	tag_names->strings =
+		(char **)git_vector_detach(&tag_names->count, NULL, &taglist);
+
 	return 0;
 }
 
diff --git a/src/thread-utils.h b/src/thread-utils.h
index 914c1357d..daec14eeb 100644
--- a/src/thread-utils.h
+++ b/src/thread-utils.h
@@ -40,12 +40,24 @@ typedef git_atomic git_atomic_ssize;
 
 #ifdef GIT_THREADS
 
-#define git_thread pthread_t
-#define git_thread_create(thread, attr, start_routine, arg) \
-	pthread_create(thread, attr, start_routine, arg)
-#define git_thread_kill(thread) pthread_cancel(thread)
-#define git_thread_exit(status)	pthread_exit(status)
-#define git_thread_join(id, status) pthread_join(id, status)
+#if !defined(GIT_WIN32)
+
+typedef struct {
+	pthread_t thread;
+} git_thread;
+
+#define git_thread_create(git_thread_ptr, attr, start_routine, arg) \
+	pthread_create(&(git_thread_ptr)->thread, attr, start_routine, arg)
+#define git_thread_join(git_thread_ptr, status) \
+	pthread_join((git_thread_ptr)->thread, status)
+
+#endif
+
+#if defined(GIT_WIN32)
+#define git_thread_yield() Sleep(0)
+#else
+#define git_thread_yield() sched_yield()
+#endif
 
 /* Pthreads Mutex */
 #define git_mutex pthread_mutex_t
@@ -173,9 +185,8 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 
 #define git_thread unsigned int
 #define git_thread_create(thread, attr, start_routine, arg) 0
-#define git_thread_kill(thread) (void)0
-#define git_thread_exit(status) (void)0
 #define git_thread_join(id, status) (void)0
+#define git_thread_yield() (void)0
 
 /* Pthreads Mutex */
 #define git_mutex unsigned int
diff --git a/src/trace.h b/src/trace.h
index 77b1e03ef..4d4e3bf53 100644
--- a/src/trace.h
+++ b/src/trace.h
@@ -7,8 +7,6 @@
 #ifndef INCLUDE_trace_h__
 #define INCLUDE_trace_h__
 
-#include <stdarg.h>
-
 #include <git2/trace.h>
 #include "buffer.h"
 
diff --git a/src/transport.c b/src/transport.c
index ff926b1be..2194b1864 100644
--- a/src/transport.c
+++ b/src/transport.c
@@ -79,12 +79,8 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
 	/* Check to see if the path points to a file on the local file system */
 	if (!definition && git_path_exists(url) && git_path_isdir(url))
 		definition = &local_transport_definition;
+#endif
 
-	/* It could be a SSH remote path. Check to see if there's a :
-	 * SSH is an unsupported transport mechanism in this version of libgit2 */
-	if (!definition && strrchr(url, ':'))
-		definition = &dummy_transport_definition;
-#else
 	/* For other systems, perform the SSH check first, to avoid going to the
 	 * filesystem if it is not necessary */
 
@@ -97,6 +93,7 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
         definition = &dummy_transport_definition;
 #endif
 
+#ifndef GIT_WIN32
 	/* Check to see if the path points to a file on the local file system */
 	if (!definition && git_path_exists(url) && git_path_isdir(url))
 		definition = &local_transport_definition;
@@ -220,3 +217,10 @@ int git_remote_supported_url(const char* url)
 
 	return fn != &git_transport_dummy;
 }
+
+int git_transport_init(git_transport *opts, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		opts, version, git_transport, GIT_TRANSPORT_INIT);
+	return 0;
+}
diff --git a/src/transports/cred.c b/src/transports/cred.c
index 05d2c8dc6..913ec36cc 100644
--- a/src/transports/cred.c
+++ b/src/transports/cred.c
@@ -11,31 +11,10 @@
 
 int git_cred_has_username(git_cred *cred)
 {
-	int ret = 0;
+	if (cred->credtype == GIT_CREDTYPE_DEFAULT)
+		return 0;
 
-	switch (cred->credtype) {
-	case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
-		git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
-		ret = !!c->username;
-		break;
-	}
-	case GIT_CREDTYPE_SSH_KEY: {
-		git_cred_ssh_key *c = (git_cred_ssh_key *)cred;
-		ret = !!c->username;
-		break;
-	}
-	case GIT_CREDTYPE_SSH_CUSTOM: {
-		git_cred_ssh_custom *c = (git_cred_ssh_custom *)cred;
-		ret = !!c->username;
-		break;
-	}
-	case GIT_CREDTYPE_DEFAULT: {
-		ret = 0;
-		break;
-	}
-	}
-
-	return ret;
+	return 1;
 }
 
 static void plaintext_free(struct git_cred *cred)
@@ -51,7 +30,6 @@ static void plaintext_free(struct git_cred *cred)
 		git__free(c->password);
 	}
 
-	git__memzero(c, sizeof(*c));
 	git__free(c);
 }
 
@@ -94,8 +72,13 @@ static void ssh_key_free(struct git_cred *cred)
 		(git_cred_ssh_key *)cred;
 
 	git__free(c->username);
-	git__free(c->publickey);
-	git__free(c->privatekey);
+
+	if (c->privatekey) {
+		/* Zero the memory which previously held the private key */
+		size_t key_len = strlen(c->privatekey);
+		git__memzero(c->privatekey, key_len);
+		git__free(c->privatekey);
+	}
 
 	if (c->passphrase) {
 		/* Zero the memory which previously held the passphrase */
@@ -104,7 +87,22 @@ static void ssh_key_free(struct git_cred *cred)
 		git__free(c->passphrase);
 	}
 
-	git__memzero(c, sizeof(*c));
+	if (c->publickey) {
+		/* Zero the memory which previously held the public key */
+		size_t key_len = strlen(c->publickey);
+		git__memzero(c->publickey, key_len);
+		git__free(c->publickey);
+	}
+
+	git__free(c);
+}
+
+static void ssh_interactive_free(struct git_cred *cred)
+{
+	git_cred_ssh_interactive *c = (git_cred_ssh_interactive *)cred;
+
+	git__free(c->username);
+
 	git__free(c);
 }
 
@@ -113,9 +111,14 @@ static void ssh_custom_free(struct git_cred *cred)
 	git_cred_ssh_custom *c = (git_cred_ssh_custom *)cred;
 
 	git__free(c->username);
-	git__free(c->publickey);
 
-	git__memzero(c, sizeof(*c));
+	if (c->publickey) {
+		/* Zero the memory which previously held the publickey */
+		size_t key_len = strlen(c->publickey);
+		git__memzero(c->publickey, key_len);
+		git__free(c->publickey);
+	}
+
 	git__free(c);
 }
 
@@ -135,7 +138,7 @@ int git_cred_ssh_key_new(
 {
 	git_cred_ssh_key *c;
 
-	assert(cred && privatekey);
+	assert(username && cred && privatekey);
 
 	c = git__calloc(1, sizeof(git_cred_ssh_key));
 	GITERR_CHECK_ALLOC(c);
@@ -143,10 +146,8 @@ int git_cred_ssh_key_new(
 	c->parent.credtype = GIT_CREDTYPE_SSH_KEY;
 	c->parent.free = ssh_key_free;
 
-	if (username) {
-		c->username = git__strdup(username);
-		GITERR_CHECK_ALLOC(c->username);
-	}
+	c->username = git__strdup(username);
+	GITERR_CHECK_ALLOC(c->username);
 
 	c->privatekey = git__strdup(privatekey);
 	GITERR_CHECK_ALLOC(c->privatekey);
@@ -165,17 +166,63 @@ int git_cred_ssh_key_new(
 	return 0;
 }
 
+int git_cred_ssh_interactive_new(
+	git_cred **out,
+	const char *username,
+	git_cred_ssh_interactive_callback prompt_callback,
+	void *payload)
+{
+	git_cred_ssh_interactive *c;
+
+	assert(out && username && prompt_callback);
+
+	c = git__calloc(1, sizeof(git_cred_ssh_interactive));
+	GITERR_CHECK_ALLOC(c);
+
+	c->parent.credtype = GIT_CREDTYPE_SSH_INTERACTIVE;
+	c->parent.free = ssh_interactive_free;
+
+	c->username = git__strdup(username);
+	GITERR_CHECK_ALLOC(c->username);
+
+	c->prompt_callback = prompt_callback;
+	c->payload = payload;
+
+	*out = &c->parent;
+	return 0;
+}
+
+int git_cred_ssh_key_from_agent(git_cred **cred, const char *username) {
+	git_cred_ssh_key *c;
+
+	assert(username && cred);
+
+	c = git__calloc(1, sizeof(git_cred_ssh_key));
+	GITERR_CHECK_ALLOC(c);
+
+	c->parent.credtype = GIT_CREDTYPE_SSH_KEY;
+	c->parent.free = ssh_key_free;
+
+	c->username = git__strdup(username);
+	GITERR_CHECK_ALLOC(c->username);
+
+	c->privatekey = NULL;
+
+	*cred = &c->parent;
+	return 0;
+}
+
 int git_cred_ssh_custom_new(
 	git_cred **cred,
 	const char *username,
 	const char *publickey,
 	size_t publickey_len,
 	git_cred_sign_callback sign_callback,
-	void *sign_data)
+	void *payload)
 {
 	git_cred_ssh_custom *c;
 
-	assert(cred);
+	assert(username && cred);
 
 	c = git__calloc(1, sizeof(git_cred_ssh_custom));
 	GITERR_CHECK_ALLOC(c);
@@ -183,10 +230,8 @@ int git_cred_ssh_custom_new(
 	c->parent.credtype = GIT_CREDTYPE_SSH_CUSTOM;
 	c->parent.free = ssh_custom_free;
 
-	if (username) {
-		c->username = git__strdup(username);
-		GITERR_CHECK_ALLOC(c->username);
-	}
+	c->username = git__strdup(username);
+	GITERR_CHECK_ALLOC(c->username);
 
 	if (publickey_len > 0) {
 		c->publickey = git__malloc(publickey_len);
@@ -197,7 +242,7 @@ int git_cred_ssh_custom_new(
 
 	c->publickey_len = publickey_len;
 	c->sign_callback = sign_callback;
-	c->sign_data = sign_data;
+	c->payload = payload;
 
 	*cred = &c->parent;
 	return 0;
diff --git a/src/transports/git.c b/src/transports/git.c
index 5dcd4eff7..21507c1c7 100644
--- a/src/transports/git.c
+++ b/src/transports/git.c
@@ -93,18 +93,19 @@ static int git_stream_read(
 	size_t buf_size,
 	size_t *bytes_read)
 {
+	int error;
 	git_stream *s = (git_stream *)stream;
 	gitno_buffer buf;
 
 	*bytes_read = 0;
 
-	if (!s->sent_command && send_command(s) < 0)
-		return -1;
+	if (!s->sent_command && (error = send_command(s)) < 0)
+		return error;
 
 	gitno_buffer_setup(&s->socket, &buf, buffer, buf_size);
 
-	if (gitno_recv(&buf) < 0)
-		return -1;
+	if ((error = gitno_recv(&buf)) < 0)
+		return error;
 
 	*bytes_read = buf.offset;
 
@@ -116,10 +117,11 @@ static int git_stream_write(
 	const char *buffer,
 	size_t len)
 {
+	int error;
 	git_stream *s = (git_stream *)stream;
 
-	if (!s->sent_command && send_command(s) < 0)
-		return -1;
+	if (!s->sent_command && (error = send_command(s)) < 0)
+		return error;
 
 	return gitno_send(&s->socket, buffer, len, 0);
 }
@@ -140,7 +142,7 @@ static void git_stream_free(git_smart_subtransport_stream *stream)
 	}
 
 	git__free(s->url);
-	git__free(s);	
+	git__free(s);
 }
 
 static int git_stream_alloc(
@@ -182,18 +184,21 @@ static int _git_uploadpack_ls(
 	char *host=NULL, *port=NULL, *path=NULL, *user=NULL, *pass=NULL;
 	const char *stream_url = url;
 	git_stream *s;
-	int error = -1;
+	int error;
 
 	*stream = NULL;
+
 	if (!git__prefixcmp(url, prefix_git))
 		stream_url += strlen(prefix_git);
 
-	if (git_stream_alloc(t, stream_url, cmd_uploadpack, stream) < 0)
-		return -1;
+	if ((error = git_stream_alloc(t, stream_url, cmd_uploadpack, stream)) < 0)
+		return error;
 
 	s = (git_stream *)*stream;
 
-	if (!(error = gitno_extract_url_parts(&host, &port, &path, &user, &pass, url, GIT_DEFAULT_PORT))) {
+	if (!(error = gitno_extract_url_parts(
+			&host, &port, &path, &user, &pass, url, GIT_DEFAULT_PORT))) {
+
 		if (!(error = gitno_connect(&s->socket, host, port, 0)))
 			t->current_stream = s;
 
diff --git a/src/transports/http.c b/src/transports/http.c
index ace0d97d0..ae608ab3d 100644
--- a/src/transports/http.c
+++ b/src/transports/http.c
@@ -248,6 +248,7 @@ static int on_headers_complete(http_parser *parser)
 	http_subtransport *t = ctx->t;
 	http_stream *s = ctx->s;
 	git_buf buf = GIT_BUF_INIT;
+	int error = 0, no_callback = 0;
 
 	/* Both parse_header_name and parse_header_value are populated
 	 * and ready for consumption. */
@@ -256,29 +257,43 @@ static int on_headers_complete(http_parser *parser)
 			return t->parse_error = PARSE_ERROR_GENERIC;
 
 	/* Check for an authentication failure. */
+
 	if (parser->status_code == 401 &&
-		get_verb == s->verb &&
-		t->owner->cred_acquire_cb) {
-		int allowed_types = 0;
+	    get_verb == s->verb) {
+		if (!t->owner->cred_acquire_cb) {
+			no_callback = 1;
+		} else {
+			int allowed_types = 0;
+
+			if (parse_unauthorized_response(&t->www_authenticate,
+							&allowed_types, &t->auth_mechanism) < 0)
+				return t->parse_error = PARSE_ERROR_GENERIC;
+
+			if (allowed_types &&
+			    (!t->cred || 0 == (t->cred->credtype & allowed_types))) {
+
+				error = t->owner->cred_acquire_cb(&t->cred,
+								  t->owner->url,
+								  t->connection_data.user,
+								  allowed_types,
+								  t->owner->cred_acquire_payload);
+
+				if (error == GIT_PASSTHROUGH) {
+					no_callback = 1;
+				} else if (error < 0) {
+					return PARSE_ERROR_GENERIC;
+				} else {
+					assert(t->cred);
+
+					/* Successfully acquired a credential. */
+					return t->parse_error = PARSE_ERROR_REPLAY;
+				}
+			}
+		}
 
-		if (parse_unauthorized_response(&t->www_authenticate,
-			&allowed_types, &t->auth_mechanism) < 0)
+		if (no_callback) {
+			giterr_set(GITERR_NET, "authentication required but no callback set");
 			return t->parse_error = PARSE_ERROR_GENERIC;
-
-		if (allowed_types &&
-			(!t->cred || 0 == (t->cred->credtype & allowed_types))) {
-
-			if (t->owner->cred_acquire_cb(&t->cred,
-					t->owner->url,
-					t->connection_data.user,
-					allowed_types,
-					t->owner->cred_acquire_payload) < 0)
-				return PARSE_ERROR_GENERIC;
-
-			assert(t->cred);
-
-			/* Successfully acquired a credential. */
-			return t->parse_error = PARSE_ERROR_REPLAY;
 		}
 	}
 
@@ -382,9 +397,6 @@ static int on_body_fill_buffer(http_parser *parser, const char *str, size_t len)
 
 static void clear_parser_state(http_subtransport *t)
 {
-	unsigned i;
-	char *entry;
-
 	http_parser_init(&t->parser, HTTP_RESPONSE);
 	gitno_buffer_setup(&t->socket,
 		&t->parse_buffer,
@@ -407,10 +419,7 @@ static void clear_parser_state(http_subtransport *t)
 	git__free(t->location);
 	t->location = NULL;
 
-	git_vector_foreach(&t->www_authenticate, i, entry)
-		git__free(entry);
-
-	git_vector_free(&t->www_authenticate);
+	git_vector_free_deep(&t->www_authenticate);
 }
 
 static int write_chunk(gitno_socket *socket, const char *buffer, size_t len)
diff --git a/src/transports/local.c b/src/transports/local.c
index 4502f0202..f859f0b70 100644
--- a/src/transports/local.c
+++ b/src/transports/local.c
@@ -40,46 +40,66 @@ typedef struct {
 		have_refs : 1;
 } transport_local;
 
+static void free_head(git_remote_head *head)
+{
+	git__free(head->name);
+	git__free(head->symref_target);
+	git__free(head);
+}
+
 static int add_ref(transport_local *t, const char *name)
 {
 	const char peeled[] = "^{}";
+	git_reference *ref, *resolved;
 	git_remote_head *head;
+	git_oid obj_id;
 	git_object *obj = NULL, *target = NULL;
 	git_buf buf = GIT_BUF_INIT;
 	int error;
 
-	head = git__calloc(1, sizeof(git_remote_head));
-	GITERR_CHECK_ALLOC(head);
-
-	head->name = git__strdup(name);
-	GITERR_CHECK_ALLOC(head->name);
+	if ((error = git_reference_lookup(&ref, t->repo, name)) < 0)
+		return error;
 
-	error = git_reference_name_to_id(&head->oid, t->repo, name);
+	error = git_reference_resolve(&resolved, ref);
 	if (error < 0) {
-		git__free(head->name);
-		git__free(head);
+		git_reference_free(ref);
 		if (!strcmp(name, GIT_HEAD_FILE) && error == GIT_ENOTFOUND) {
-			/* This is actually okay.  Empty repos often have a HEAD that points to
-			 * a nonexistent "refs/heads/master". */
+			/* This is actually okay.  Empty repos often have a HEAD that
+			 * points to a nonexistent "refs/heads/master". */
 			giterr_clear();
 			return 0;
 		}
 		return error;
 	}
 
-	if (git_vector_insert(&t->refs, head) < 0)
-	{
-		git__free(head->name);
-		git__free(head);
-		return -1;
+	git_oid_cpy(&obj_id, git_reference_target(resolved));
+	git_reference_free(resolved);
+
+	head = git__calloc(1, sizeof(git_remote_head));
+	GITERR_CHECK_ALLOC(head);
+
+	head->name = git__strdup(name);
+	GITERR_CHECK_ALLOC(head->name);
+
+	git_oid_cpy(&head->oid, &obj_id);
+
+	if (git_reference_type(ref) == GIT_REF_SYMBOLIC) {
+		head->symref_target = git__strdup(git_reference_symbolic_target(ref));
+		GITERR_CHECK_ALLOC(head->symref_target);
+	}
+	git_reference_free(ref);
+
+	if ((error = git_vector_insert(&t->refs, head)) < 0) {
+		free_head(head);
+		return error;
 	}
 
 	/* If it's not a tag, we don't need to try to peel it */
 	if (git__prefixcmp(name, GIT_REFS_TAGS_DIR))
 		return 0;
 
-	if (git_object_lookup(&obj, t->repo, &head->oid, GIT_OBJ_ANY) < 0)
-		return -1;
+	if ((error = git_object_lookup(&obj, t->repo, &head->oid, GIT_OBJ_ANY)) < 0)
+		return error;
 
 	head = NULL;
 
@@ -94,27 +114,23 @@ static int add_ref(transport_local *t, const char *name)
 	/* And if it's a tag, peel it, and add it to the list */
 	head = git__calloc(1, sizeof(git_remote_head));
 	GITERR_CHECK_ALLOC(head);
+
 	if (git_buf_join(&buf, 0, name, peeled) < 0)
 		return -1;
-
 	head->name = git_buf_detach(&buf);
 
-	if (git_tag_peel(&target, (git_tag *) obj) < 0)
-		goto on_error;
-
-	git_oid_cpy(&head->oid, git_object_id(target));
-	git_object_free(obj);
-	git_object_free(target);
-
-	if (git_vector_insert(&t->refs, head) < 0)
-		return -1;
+	if (!(error = git_tag_peel(&target, (git_tag *)obj))) {
+		git_oid_cpy(&head->oid, git_object_id(target));
 
-	return 0;
+		if ((error = git_vector_insert(&t->refs, head)) < 0) {
+			free_head(head);
+		}
+	}
 
-on_error:
 	git_object_free(obj);
 	git_object_free(target);
-	return -1;
+
+	return error;
 }
 
 static int store_refs(transport_local *t)
@@ -161,7 +177,7 @@ on_error:
 
 /*
  * Try to open the url as a git directory. The direction doesn't
- * matter in this case because we're calulating the heads ourselves.
+ * matter in this case because we're calculating the heads ourselves.
  */
 static int local_connect(
 	git_transport *transport,
@@ -179,22 +195,20 @@ static int local_connect(
 	GIT_UNUSED(cred_acquire_cb);
 	GIT_UNUSED(cred_acquire_payload);
 
+	if (t->connected)
+		return 0;
+
 	t->url = git__strdup(url);
 	GITERR_CHECK_ALLOC(t->url);
 	t->direction = direction;
 	t->flags = flags;
 
-	/* The repo layer doesn't want the prefix */
-	if (!git__prefixcmp(t->url, "file://")) {
-		if (git_path_fromurl(&buf, t->url) < 0) {
-			git_buf_free(&buf);
-			return -1;
-		}
-		path = git_buf_cstr(&buf);
-
-	} else { /* We assume transport->url is already a path */
-		path = t->url;
+	/* 'url' may be a url or path; convert to a path */
+	if ((error = git_path_from_url_or_path(&buf, url)) < 0) {
+		git_buf_free(&buf);
+		return error;
 	}
+	path = git_buf_cstr(&buf);
 
 	error = git_repository_open(&repo, path);
 
@@ -222,7 +236,7 @@ static int local_ls(const git_remote_head ***out, size_t *size, git_transport *t
 		return -1;
 	}
 
-	*out = (const git_remote_head **) t->refs.contents;
+	*out = (const git_remote_head **)t->refs.contents;
 	*size = t->refs.length;
 
 	return 0;
@@ -250,8 +264,9 @@ static int local_negotiate_fetch(
 			git_oid_cpy(&rhead->loid, git_object_id(obj));
 		else if (error != GIT_ENOTFOUND)
 			return error;
+		else
+			giterr_clear();
 		git_object_free(obj);
-		giterr_clear();
 	}
 
 	return 0;
@@ -317,7 +332,7 @@ static int local_push_update_remote_ref(
 	if (lref) {
 		/* Create or update a ref */
 		if ((error = git_reference_create(NULL, remote_repo, rref, loid,
-				!git_oid_iszero(roid))) < 0)
+				!git_oid_iszero(roid), NULL, NULL)) < 0)
 			return error;
 	} else {
 		/* Delete a ref */
@@ -346,11 +361,24 @@ static int local_push(
 	git_repository *remote_repo = NULL;
 	push_spec *spec;
 	char *url = NULL;
+	const char *path;
+	git_buf buf = GIT_BUF_INIT;
 	int error;
 	unsigned int i;
 	size_t j;
 
-	if ((error = git_repository_open(&remote_repo, push->remote->url)) < 0)
+	/* 'push->remote->url' may be a url or path; convert to a path */
+	if ((error = git_path_from_url_or_path(&buf, push->remote->url)) < 0) {
+		git_buf_free(&buf);
+		return error;
+	}
+	path = git_buf_cstr(&buf);
+
+	error = git_repository_open(&remote_repo, path);
+
+	git_buf_free(&buf);
+
+	if (error < 0)
 		return error;
 
 	/* We don't currently support pushing locally to non-bare repos. Proper
@@ -445,7 +473,7 @@ on_error:
 
 typedef struct foreach_data {
 	git_transfer_progress *stats;
-	git_transfer_progress_callback progress_cb;
+	git_transfer_progress_cb progress_cb;
 	void *progress_payload;
 	git_odb_writepack *writepack;
 } foreach_data;
@@ -462,7 +490,7 @@ static int local_download_pack(
 		git_transport *transport,
 		git_repository *repo,
 		git_transfer_progress *stats,
-		git_transfer_progress_callback progress_cb,
+		git_transfer_progress_cb progress_cb,
 		void *progress_payload)
 {
 	transport_local *t = (transport_local*)transport;
@@ -528,7 +556,7 @@ static int local_download_pack(
 		}
 	}
 
-	if ((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) < 0)
+	if ((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) != 0)
 		goto cleanup;
 
 	/* Write the data to the ODB */
@@ -539,7 +567,7 @@ static int local_download_pack(
 		data.progress_payload = progress_payload;
 		data.writepack = writepack;
 
-		if ((error = git_packbuilder_foreach(pack, foreach_cb, &data)) < 0)
+		if ((error = git_packbuilder_foreach(pack, foreach_cb, &data)) != 0)
 			goto cleanup;
 	}
 	error = writepack->commit(writepack, stats);
@@ -599,10 +627,8 @@ static void local_free(git_transport *transport)
 	size_t i;
 	git_remote_head *head;
 
-	git_vector_foreach(&t->refs, i, head) {
-		git__free(head->name);
-		git__free(head);
-	}
+	git_vector_foreach(&t->refs, i, head)
+		free_head(head);
 
 	git_vector_free(&t->refs);
 
diff --git a/src/transports/smart.c b/src/transports/smart.c
index 5242beb65..a5c3e82dc 100644
--- a/src/transports/smart.c
+++ b/src/transports/smart.c
@@ -7,6 +7,7 @@
 #include "git2.h"
 #include "smart.h"
 #include "refs.h"
+#include "refspec.h"
 
 static int git_smart__recv_cb(gitno_buffer *buf)
 {
@@ -23,13 +24,13 @@ static int git_smart__recv_cb(gitno_buffer *buf)
 
 	buf->offset += bytes_read;
 
-	if (t->packetsize_cb && !t->cancelled.val)
-		if (t->packetsize_cb(bytes_read, t->packetsize_payload)) {
+	if (t->packetsize_cb && !t->cancelled.val) {
+		error = t->packetsize_cb(bytes_read, t->packetsize_payload);
+		if (error) {
 			git_atomic_set(&t->cancelled, 1);
-
-			giterr_clear();
 			return GIT_EUSER;
 		}
+	}
 
 	return (int)(buf->offset - old_len);
 }
@@ -63,7 +64,7 @@ static int git_smart__set_callbacks(
 	return 0;
 }
 
-int git_smart__update_heads(transport_smart *t)
+int git_smart__update_heads(transport_smart *t, git_vector *symrefs)
 {
 	size_t i;
 	git_pkt *pkt;
@@ -74,6 +75,25 @@ int git_smart__update_heads(transport_smart *t)
 		if (pkt->type != GIT_PKT_REF)
 			continue;
 
+		if (symrefs) {
+			git_refspec *spec;
+			git_buf buf = GIT_BUF_INIT;
+			size_t j;
+			int error = 0;
+
+			git_vector_foreach(symrefs, j, spec) {
+				git_buf_clear(&buf);
+				if (git_refspec_src_matches(spec, ref->head.name) &&
+				    !(error = git_refspec_transform(&buf, spec, ref->head.name)))
+					ref->head.symref_target = git_buf_detach(&buf);
+			}
+
+			git_buf_free(&buf);
+
+			if (error < 0)
+				return error;
+		}
+
 		if (git_vector_insert(&t->heads, &ref->head) < 0)
 			return -1;
 	}
@@ -81,6 +101,19 @@ int git_smart__update_heads(transport_smart *t)
 	return 0;
 }
 
+static void free_symrefs(git_vector *symrefs)
+{
+	git_refspec *spec;
+	size_t i;
+
+	git_vector_foreach(symrefs, i, spec) {
+		git_refspec__free(spec);
+		git__free(spec);
+	}
+
+	git_vector_free(symrefs);
+}
+
 static int git_smart__connect(
 	git_transport *transport,
 	const char *url,
@@ -94,6 +127,7 @@ static int git_smart__connect(
 	int error;
 	git_pkt *pkt;
 	git_pkt_ref *first;
+	git_vector symrefs;
 	git_smart_service_t service;
 
 	if (git_smart__reset_stream(t, true) < 0)
@@ -147,8 +181,11 @@ static int git_smart__connect(
 
 	first = (git_pkt_ref *)git_vector_get(&t->refs, 0);
 
+	if ((error = git_vector_init(&symrefs, 1, NULL)) < 0)
+		return error;
+
 	/* Detect capabilities */
-	if (git_smart__detect_caps(first, &t->caps) < 0)
+	if (git_smart__detect_caps(first, &t->caps, &symrefs) < 0)
 		return -1;
 
 	/* If the only ref in the list is capabilities^{} with OID_ZERO, remove it */
@@ -159,7 +196,9 @@ static int git_smart__connect(
 	}
 
 	/* Keep a list of heads for _ls */
-	git_smart__update_heads(t);
+	git_smart__update_heads(t, &symrefs);
+
+	free_symrefs(&symrefs);
 
 	if (t->rpc && git_smart__reset_stream(t, false) < 0)
 		return -1;
@@ -272,6 +311,18 @@ static int git_smart__close(git_transport *transport)
 	unsigned int i;
 	git_pkt *p;
 	int ret;
+	git_smart_subtransport_stream *stream;
+	const char flush[] = "0000";
+
+	/*
+	 * If we're still connected at this point and not using RPC,
+	 * we should say goodbye by sending a flush, or git-daemon
+	 * will complain that we disconnected unexpectedly.
+	 */
+	if (t->connected && !t->rpc &&
+	    !t->wrapped->action(&stream, t->wrapped, t->url, GIT_SERVICE_UPLOADPACK)) {
+		t->current_stream->write(t->current_stream, flush, 4);
+	}
 
 	ret = git_smart__reset_stream(t, true);
 
@@ -342,7 +393,7 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param)
 	t->parent.is_connected = git_smart__is_connected;
 	t->parent.read_flags = git_smart__read_flags;
 	t->parent.cancel = git_smart__cancel;
-	
+
 	t->owner = owner;
 	t->rpc = definition->rpc;
 
@@ -359,7 +410,7 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param)
 	if (definition->callback(&t->wrapped, &t->parent) < 0) {
 		git__free(t);
 		return -1;
-	}	
+	}
 
 	*out = (git_transport *) t;
 	return 0;
diff --git a/src/transports/smart.h b/src/transports/smart.h
index 32f0be7f2..f1fc29520 100644
--- a/src/transports/smart.h
+++ b/src/transports/smart.h
@@ -23,6 +23,7 @@
 #define GIT_CAP_DELETE_REFS "delete-refs"
 #define GIT_CAP_REPORT_STATUS "report-status"
 #define GIT_CAP_THIN_PACK "thin-pack"
+#define GIT_CAP_SYMREF "symref"
 
 enum git_pkt_type {
 	GIT_PKT_CMD,
@@ -154,7 +155,7 @@ typedef struct {
 
 /* smart_protocol.c */
 int git_smart__store_refs(transport_smart *t, int flushes);
-int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps);
+int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vector *symrefs);
 int git_smart__push(git_transport *transport, git_push *push);
 
 int git_smart__negotiate_fetch(
@@ -167,14 +168,14 @@ int git_smart__download_pack(
 	git_transport *transport,
 	git_repository *repo,
 	git_transfer_progress *stats,
-	git_transfer_progress_callback progress_cb,
+	git_transfer_progress_cb progress_cb,
 	void *progress_payload);
 
 /* smart.c */
 int git_smart__negotiation_step(git_transport *transport, void *data, size_t len);
 int git_smart__get_push_stream(transport_smart *t, git_smart_subtransport_stream **out);
 
-int git_smart__update_heads(transport_smart *t);
+int git_smart__update_heads(transport_smart *t, git_vector *symrefs);
 
 /* smart_pkt.c */
 int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_t len);
diff --git a/src/transports/smart_pkt.c b/src/transports/smart_pkt.c
index 2bb09c750..b5f9d6dbe 100644
--- a/src/transports/smart_pkt.c
+++ b/src/transports/smart_pkt.c
@@ -153,7 +153,7 @@ static int data_pkt(git_pkt **out, const char *line, size_t len)
 	return 0;
 }
 
-static int progress_pkt(git_pkt **out, const char *line, size_t len)
+static int sideband_progress_pkt(git_pkt **out, const char *line, size_t len)
 {
 	git_pkt_progress *pkt;
 
@@ -403,7 +403,7 @@ int git_pkt_parse_line(
 	if (*line == GIT_SIDE_BAND_DATA)
 		ret = data_pkt(head, line, len);
 	else if (*line == GIT_SIDE_BAND_PROGRESS)
-		ret = progress_pkt(head, line, len);
+		ret = sideband_progress_pkt(head, line, len);
 	else if (*line == GIT_SIDE_BAND_ERROR)
 		ret = sideband_error_pkt(head, line, len);
 	else if (!git__prefixcmp(line, "ACK"))
@@ -433,6 +433,7 @@ void git_pkt_free(git_pkt *pkt)
 	if (pkt->type == GIT_PKT_REF) {
 		git_pkt_ref *p = (git_pkt_ref *) pkt;
 		git__free(p->head.name);
+		git__free(p->head.symref_target);
 	}
 
 	if (pkt->type == GIT_PKT_OK) {
diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c
index 3bf1f9329..a52aacc60 100644
--- a/src/transports/smart_protocol.c
+++ b/src/transports/smart_protocol.c
@@ -26,17 +26,16 @@ int git_smart__store_refs(transport_smart *t, int flushes)
 	int error, flush = 0, recvd;
 	const char *line_end = NULL;
 	git_pkt *pkt = NULL;
-	git_pkt_ref *ref;
 	size_t i;
 
 	/* Clear existing refs in case git_remote_connect() is called again
 	 * after git_remote_disconnect().
 	 */
-	git_vector_foreach(refs, i, ref) {
-		git__free(ref->head.name);
-		git__free(ref);
+	git_vector_foreach(refs, i, pkt) {
+		git_pkt_free(pkt);
 	}
 	git_vector_clear(refs);
+	pkt = NULL;
 
 	do {
 		if (buf->offset > 0)
@@ -45,7 +44,7 @@ int git_smart__store_refs(transport_smart *t, int flushes)
 			error = GIT_EBUFS;
 
 		if (error < 0 && error != GIT_EBUFS)
-			return -1;
+			return error;
 
 		if (error == GIT_EBUFS) {
 			if ((recvd = gitno_recv(buf)) < 0)
@@ -78,7 +77,52 @@ int git_smart__store_refs(transport_smart *t, int flushes)
 	return flush;
 }
 
-int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps)
+static int append_symref(const char **out, git_vector *symrefs, const char *ptr)
+{
+	int error;
+	const char *end;
+	git_buf buf = GIT_BUF_INIT;
+	git_refspec *mapping;
+
+	ptr += strlen(GIT_CAP_SYMREF);
+	if (*ptr != '=')
+		goto on_invalid;
+
+	ptr++;
+	if (!(end = strchr(ptr, ' ')) &&
+	    !(end = strchr(ptr, '\0')))
+		goto on_invalid;
+
+	if ((error = git_buf_put(&buf, ptr, end - ptr)) < 0)
+		return error;
+
+	/* symref mapping has refspec format */
+	mapping = git__malloc(sizeof(git_refspec));
+	GITERR_CHECK_ALLOC(mapping);
+
+	error = git_refspec__parse(mapping, git_buf_cstr(&buf), true);
+	git_buf_free(&buf);
+
+	/* if the error isn't OOM, then it's a parse error; let's use a nicer message */
+	if (error < 0) {
+		if (giterr_last()->klass != GITERR_NOMEMORY)
+			goto on_invalid;
+
+		return error;
+	}
+
+	if ((error = git_vector_insert(symrefs, mapping)) < 0)
+		return error;
+
+	*out = end;
+	return 0;
+
+on_invalid:
+	giterr_set(GITERR_NET, "remote sent invalid symref");
+	return -1;
+}
+
+int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vector *symrefs)
 {
 	const char *ptr;
 
@@ -141,6 +185,15 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps)
 			continue;
 		}
 
+		if (!git__prefixcmp(ptr, GIT_CAP_SYMREF)) {
+			int error;
+
+			if ((error = append_symref(&ptr, symrefs, ptr)) < 0)
+				return error;
+
+			continue;
+		}
+
 		/* We don't know this capability, so skip it */
 		ptr = strchr(ptr, ' ');
 	}
@@ -209,12 +262,13 @@ static int fetch_setup_walk(git_revwalk **out, git_repository *repo)
 	git_strarray refs;
 	unsigned int i;
 	git_reference *ref;
+	int error;
 
-	if (git_reference_list(&refs, repo) < 0)
-		return -1;
+	if ((error = git_reference_list(&refs, repo)) < 0)
+		return error;
 
-	if (git_revwalk_new(&walk, repo) < 0)
-		return -1;
+	if ((error = git_revwalk_new(&walk, repo)) < 0)
+		return error;
 
 	git_revwalk_sorting(walk, GIT_SORT_TIME);
 
@@ -223,13 +277,13 @@ static int fetch_setup_walk(git_revwalk **out, git_repository *repo)
 		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
 			continue;
 
-		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
+		if ((error = git_reference_lookup(&ref, repo, refs.strings[i])) < 0)
 			goto on_error;
 
 		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
 			continue;
 
-		if (git_revwalk_push(walk, git_reference_target(ref)) < 0)
+		if ((error = git_revwalk_push(walk, git_reference_target(ref))) < 0)
 			goto on_error;
 
 		git_reference_free(ref);
@@ -242,7 +296,7 @@ static int fetch_setup_walk(git_revwalk **out, git_repository *repo)
 on_error:
 	git_reference_free(ref);
 	git_strarray_free(&refs);
-	return -1;
+	return error;
 }
 
 static int wait_while_ack(gitno_buffer *buf)
@@ -449,7 +503,7 @@ static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack,
 
 struct network_packetsize_payload
 {
-	git_transfer_progress_callback callback;
+	git_transfer_progress_cb callback;
 	void *payload;
 	git_transfer_progress *stats;
 	size_t last_fired_bytes;
@@ -477,7 +531,7 @@ int git_smart__download_pack(
 	git_transport *transport,
 	git_repository *repo,
 	git_transfer_progress *stats,
-	git_transfer_progress_callback progress_cb,
+	git_transfer_progress_cb transfer_progress_cb,
 	void *progress_payload)
 {
 	transport_smart *t = (transport_smart *)transport;
@@ -489,8 +543,8 @@ int git_smart__download_pack(
 
 	memset(stats, 0, sizeof(git_transfer_progress));
 
-	if (progress_cb) {
-		npp.callback = progress_cb;
+	if (transfer_progress_cb) {
+		npp.callback = transfer_progress_cb;
 		npp.payload = progress_payload;
 		npp.stats = stats;
 		t->packetsize_cb = &network_packetsize;
@@ -503,7 +557,7 @@ int git_smart__download_pack(
 	}
 
 	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
-		((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) < 0))
+		((error = git_odb_write_pack(&writepack, odb, transfer_progress_cb, progress_payload)) != 0))
 		goto done;
 
 	/*
@@ -517,57 +571,65 @@ int git_smart__download_pack(
 	}
 
 	do {
-		git_pkt *pkt;
+		git_pkt *pkt = NULL;
 
 		/* Check cancellation before network call */
 		if (t->cancelled.val) {
-			giterr_set(GITERR_NET, "The fetch was cancelled by the user");
+			giterr_clear();
 			error = GIT_EUSER;
 			goto done;
 		}
 
-		if ((error = recv_pkt(&pkt, buf)) < 0)
-			goto done;
-
-		/* Check cancellation after network call */
-		if (t->cancelled.val) {
-			giterr_set(GITERR_NET, "The fetch was cancelled by the user");
-			error = GIT_EUSER;
-			goto done;
-		}
-
-		if (pkt->type == GIT_PKT_PROGRESS) {
-			if (t->progress_cb) {
-				git_pkt_progress *p = (git_pkt_progress *) pkt;
-				if (t->progress_cb(p->data, p->len, t->message_cb_payload)) {
-					giterr_set(GITERR_NET, "The fetch was cancelled by the user");
-					return GIT_EUSER;
+		if ((error = recv_pkt(&pkt, buf)) >= 0) {
+			/* Check cancellation after network call */
+			if (t->cancelled.val) {
+				giterr_clear();
+				error = GIT_EUSER;
+			} else if (pkt->type == GIT_PKT_PROGRESS) {
+				if (t->progress_cb) {
+					git_pkt_progress *p = (git_pkt_progress *) pkt;
+					error = t->progress_cb(p->data, p->len, t->message_cb_payload);
 				}
+			} else if (pkt->type == GIT_PKT_DATA) {
+				git_pkt_data *p = (git_pkt_data *) pkt;
+				error = writepack->append(writepack, p->data, p->len, stats);
+			} else if (pkt->type == GIT_PKT_FLUSH) {
+				/* A flush indicates the end of the packfile */
+				git__free(pkt);
+				break;
 			}
-			git__free(pkt);
-		} else if (pkt->type == GIT_PKT_DATA) {
-			git_pkt_data *p = (git_pkt_data *) pkt;
-			error = writepack->append(writepack, p->data, p->len, stats);
-
-			git__free(pkt);
-			if (error < 0)
-				goto done;
-		} else if (pkt->type == GIT_PKT_FLUSH) {
-			/* A flush indicates the end of the packfile */
-			git__free(pkt);
-			break;
 		}
+
+		git__free(pkt);
+		if (error < 0)
+			goto done;
+
 	} while (1);
 
+	/*
+	 * Trailing execution of transfer_progress_cb, if necessary...
+	 * Only the callback through the npp datastructure currently
+	 * updates the last_fired_bytes value. It is possible that
+	 * progress has already been reported with the correct
+	 * "received_bytes" value, but until (if?) this is unified
+	 * then we will report progress again to be sure that the
+	 * correct last received_bytes value is reported.
+	 */
+	if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes) {
+		error = npp.callback(npp.stats, npp.payload);
+		if (error != 0)
+			goto done;
+	}
+
 	error = writepack->commit(writepack, stats);
 
 done:
 	if (writepack)
 		writepack->free(writepack);
-
-	/* Trailing execution of progress_cb, if necessary */
-	if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes)
-		npp.callback(npp.stats, npp.payload);
+	if (transfer_progress_cb) {
+		t->packetsize_cb = NULL;
+		t->packetsize_payload = NULL;
+	}
 
 	return error;
 }
@@ -619,7 +681,7 @@ static int add_push_report_pkt(git_push *push, git_pkt *pkt)
 
 	switch (pkt->type) {
 		case GIT_PKT_OK:
-			status = git__calloc(1, sizeof(push_status));
+			status = git__calloc(sizeof(push_status), 1);
 			GITERR_CHECK_ALLOC(status);
 			status->msg = NULL;
 			status->ref = git__strdup(((git_pkt_ok *)pkt)->ref);
@@ -681,10 +743,11 @@ static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt)
 	return 0;
 }
 
-static int parse_report(gitno_buffer *buf, git_push *push)
+static int parse_report(transport_smart *transport, git_push *push)
 {
 	git_pkt *pkt = NULL;
 	const char *line_end = NULL;
+	gitno_buffer *buf = &transport->buffer;
 	int error, recvd;
 
 	for (;;) {
@@ -723,6 +786,10 @@ static int parse_report(gitno_buffer *buf, git_push *push)
 				error = -1;
 				break;
 			case GIT_PKT_PROGRESS:
+				if (transport->progress_cb) {
+					git_pkt_progress *p = (git_pkt_progress *) pkt;
+					error = transport->progress_cb(p->data, p->len, transport->message_cb_payload);
+				}
 				break;
 			default:
 				error = add_push_report_pkt(push, pkt);
@@ -868,10 +935,7 @@ static int stream_thunk(void *buf, size_t size, void *data)
 
 		if ((current_time - payload->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
 			payload->last_progress_report_time = current_time;
-			if (payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload)) {
-				giterr_clear();
-				error = GIT_EUSER;
-			}
+			error = payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload);
 		}
 	}
 
@@ -938,12 +1002,19 @@ int git_smart__push(git_transport *transport, git_push *push)
 	 * we consider the pack to have been unpacked successfully */
 	if (!push->specs.length || !push->report_status)
 		push->unpack_ok = 1;
-	else if ((error = parse_report(&t->buffer, push)) < 0)
+	else if ((error = parse_report(t, push)) < 0)
 		goto done;
 
 	/* If progress is being reported write the final report */
 	if (push->transfer_progress_cb) {
-		push->transfer_progress_cb(push->pb->nr_written, push->pb->nr_objects, packbuilder_payload.last_bytes, push->transfer_progress_cb_payload);
+		error = push->transfer_progress_cb(
+					push->pb->nr_written,
+					push->pb->nr_objects,
+					packbuilder_payload.last_bytes,
+					push->transfer_progress_cb_payload);
+
+		if (error < 0)
+			goto done;
 	}
 
 	if (push->status.length) {
@@ -951,7 +1022,7 @@ int git_smart__push(git_transport *transport, git_push *push)
 		if (error < 0)
 			goto done;
 
-		error = git_smart__update_heads(t);
+		error = git_smart__update_heads(t, NULL);
 	}
 
 done:
diff --git a/src/transports/ssh.c b/src/transports/ssh.c
index 4a905e3c9..b403727c9 100644
--- a/src/transports/ssh.c
+++ b/src/transports/ssh.c
@@ -53,6 +53,7 @@ static void ssh_error(LIBSSH2_SESSION *session, const char *errmsg)
 static int gen_proto(git_buf *request, const char *cmd, const char *url)
 {
 	char *repo;
+	int len;
 
 	if (!git__prefixcmp(url, prefix_ssh)) {
 		url = url + strlen(prefix_ssh);
@@ -67,7 +68,7 @@ static int gen_proto(git_buf *request, const char *cmd, const char *url)
 		return -1;
 	}
 
-	int len = strlen(cmd) + 1 /* Space */ + 1 /* Quote */ + strlen(repo) + 1 /* Quote */ + 1;
+	len = strlen(cmd) + 1 /* Space */ + 1 /* Quote */ + strlen(repo) + 1 /* Quote */ + 1;
 
 	git_buf_grow(request, len);
 	git_buf_printf(request, "%s '%s'", cmd, repo);
@@ -235,9 +236,52 @@ static int git_ssh_extract_url_parts(
 	return 0;
 }
 
+static int ssh_agent_auth(LIBSSH2_SESSION *session, git_cred_ssh_key *c) {
+	int rc = LIBSSH2_ERROR_NONE;
+
+	struct libssh2_agent_publickey *curr, *prev = NULL;
+
+	LIBSSH2_AGENT *agent = libssh2_agent_init(session);
+
+	if (agent == NULL)
+		return -1;
+
+	rc = libssh2_agent_connect(agent);
+
+	if (rc != LIBSSH2_ERROR_NONE)
+		goto shutdown;
+
+	rc = libssh2_agent_list_identities(agent);
+
+	if (rc != LIBSSH2_ERROR_NONE)
+		goto shutdown;
+
+	while (1) {
+		rc = libssh2_agent_get_identity(agent, &curr, prev);
+
+		if (rc < 0)
+			goto shutdown;
+
+		if (rc == 1)
+			goto shutdown;
+
+		rc = libssh2_agent_userauth(agent, c->username, curr);
+
+		if (rc == 0)
+			break;
+
+		prev = curr;
+	}
+
+shutdown:
+	libssh2_agent_disconnect(agent);
+	libssh2_agent_free(agent);
+
+	return rc;
+}
+
 static int _git_ssh_authenticate_session(
 	LIBSSH2_SESSION* session,
-	const char *user,
 	git_cred* cred)
 {
 	int rc;
@@ -246,24 +290,48 @@ static int _git_ssh_authenticate_session(
 		switch (cred->credtype) {
 		case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
 			git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
-			user = c->username ? c->username : user;
-			rc = libssh2_userauth_password(session, user, c->password);
+			rc = libssh2_userauth_password(session, c->username, c->password);
 			break;
 		}
 		case GIT_CREDTYPE_SSH_KEY: {
 			git_cred_ssh_key *c = (git_cred_ssh_key *)cred;
-			user = c->username ? c->username : user;
-			rc = libssh2_userauth_publickey_fromfile(
-				session, c->username, c->publickey, c->privatekey, c->passphrase);
+
+			if (c->privatekey)
+				rc = libssh2_userauth_publickey_fromfile(
+					session, c->username, c->publickey,
+					c->privatekey, c->passphrase);
+			else
+				rc = ssh_agent_auth(session, c);
+
 			break;
 		}
 		case GIT_CREDTYPE_SSH_CUSTOM: {
 			git_cred_ssh_custom *c = (git_cred_ssh_custom *)cred;
 
-			user = c->username ? c->username : user;
 			rc = libssh2_userauth_publickey(
 				session, c->username, (const unsigned char *)c->publickey,
-				c->publickey_len, c->sign_callback, &c->sign_data);
+				c->publickey_len, c->sign_callback, &c->payload);
+			break;
+		}
+		case GIT_CREDTYPE_SSH_INTERACTIVE: {
+			void **abstract = libssh2_session_abstract(session);
+			git_cred_ssh_interactive *c = (git_cred_ssh_interactive *)cred;
+
+			/* ideally, we should be able to set this by calling
+			 * libssh2_session_init_ex() instead of libssh2_session_init().
+			 * libssh2's API is inconsistent here i.e. libssh2_userauth_publickey()
+			 * allows you to pass the `abstract` as part of the call, whereas
+			 * libssh2_userauth_keyboard_interactive() does not!
+			 *
+			 * The only way to set the `abstract` pointer is by calling
+			 * libssh2_session_abstract(), which will replace the existing
+			 * pointer as is done below. This is safe for now (at time of writing),
+			 * but may not be valid in future.
+			 */
+			*abstract = c->payload;
+
+			rc = libssh2_userauth_keyboard_interactive(
+				session, c->username, c->prompt_callback);
 			break;
 		}
 		default:
@@ -319,6 +387,7 @@ static int _git_ssh_setup_conn(
 {
 	char *host=NULL, *port=NULL, *path=NULL, *user=NULL, *pass=NULL;
 	const char *default_port="22";
+	int no_callback = 0;
 	ssh_stream *s;
 	LIBSSH2_SESSION* session=NULL;
 	LIBSSH2_CHANNEL* channel=NULL;
@@ -345,34 +414,37 @@ static int _git_ssh_setup_conn(
 	if (user && pass) {
 		if (git_cred_userpass_plaintext_new(&t->cred, user, pass) < 0)
 			goto on_error;
-	} else if (t->owner->cred_acquire_cb) {
-		if (t->owner->cred_acquire_cb(
-				&t->cred, t->owner->url, user,
-				GIT_CREDTYPE_USERPASS_PLAINTEXT |
-				GIT_CREDTYPE_SSH_KEY |
-				GIT_CREDTYPE_SSH_CUSTOM,
-				t->owner->cred_acquire_payload) < 0)
+	} else if (!t->owner->cred_acquire_cb) {
+		no_callback = 1;
+	} else {
+		int error;
+		error = t->owner->cred_acquire_cb(&t->cred, t->owner->url, user,
+			GIT_CREDTYPE_USERPASS_PLAINTEXT |
+			GIT_CREDTYPE_SSH_KEY | GIT_CREDTYPE_SSH_CUSTOM |
+			GIT_CREDTYPE_SSH_INTERACTIVE,
+			t->owner->cred_acquire_payload);
+
+		if (error == GIT_PASSTHROUGH)
+			no_callback = 1;
+		else if (error < 0)
 			goto on_error;
-
-		if (!t->cred) {
+		else if (!t->cred) {
 			giterr_set(GITERR_SSH, "Callback failed to initialize SSH credentials");
 			goto on_error;
 		}
-	} else {
-		giterr_set(GITERR_SSH, "Cannot set up SSH connection without credentials");
-		goto on_error;
 	}
-	assert(t->cred);
 
-	if (!user && !git_cred_has_username(t->cred)) {
-		giterr_set_str(GITERR_NET, "Cannot authenticate without a username");
+	if (no_callback) {
+		giterr_set(GITERR_SSH, "authentication required but no callback set");
 		goto on_error;
 	}
 
+	assert(t->cred);
+
 	if (_git_ssh_session_create(&session, s->socket) < 0)
 		goto on_error;
 
-	if (_git_ssh_authenticate_session(session, user, t->cred) < 0)
+	if (_git_ssh_authenticate_session(session, t->cred) < 0)
 		goto on_error;
 
 	channel = libssh2_channel_open_session(session);
diff --git a/src/transports/winhttp.c b/src/transports/winhttp.c
index 673cd0faf..bd9509cd4 100644
--- a/src/transports/winhttp.c
+++ b/src/transports/winhttp.c
@@ -21,6 +21,10 @@
 
 #include <strsafe.h>
 
+/* For IInternetSecurityManager zone check */
+#include <objbase.h>
+#include <urlmon.h>
+
 /* For UuidCreate */
 #pragma comment(lib, "rpcrt4")
 
@@ -87,7 +91,7 @@ static int apply_basic_credential(HINTERNET request, git_cred *cred)
 	git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
 	git_buf buf = GIT_BUF_INIT, raw = GIT_BUF_INIT;
 	wchar_t *wide = NULL;
-	int error = -1, wide_len = 0;
+	int error = -1, wide_len;
 
 	git_buf_printf(&raw, "%s:%s", c->username, c->password);
 
@@ -96,21 +100,7 @@ static int apply_basic_credential(HINTERNET request, git_cred *cred)
 		git_buf_put_base64(&buf, git_buf_cstr(&raw), raw.size) < 0)
 		goto on_error;
 
-	wide_len = MultiByteToWideChar(CP_UTF8,	MB_ERR_INVALID_CHARS,
-		git_buf_cstr(&buf),	-1, NULL, 0);
-
-	if (!wide_len) {
-		giterr_set(GITERR_OS, "Failed to measure string for wide conversion");
-		goto on_error;
-	}
-
-	wide = git__malloc(wide_len * sizeof(wchar_t));
-
-	if (!wide)
-		goto on_error;
-
-	if (!MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
-		git_buf_cstr(&buf), -1, wide, wide_len)) {
+	if ((wide_len = git__utf8_to_16_alloc(&wide, git_buf_cstr(&buf))) < 0) {
 		giterr_set(GITERR_OS, "Failed to convert string to wide form");
 		goto on_error;
 	}
@@ -141,12 +131,12 @@ on_error:
 
 static int apply_default_credentials(HINTERNET request)
 {
-	/* If we are explicitly asked to deliver default credentials, turn set
-	 * the security level to low which will guarantee they are delivered.
-	 * The default is "medium" which applies to the intranet and sounds
-	 * like it would correspond to Internet Explorer security zones, but
-	 * in fact does not.
-	 */
+	/* Either the caller explicitly requested that default credentials be passed,
+	 * or our fallback credential callback was invoked and checked that the target
+	 * URI was in the appropriate Internet Explorer security zone. By setting this
+	 * flag, we guarantee that the credentials are delivered by WinHTTP. The default
+	 * is "medium" which applies to the intranet and sounds like it would correspond
+	 * to Internet Explorer security zones, but in fact does not. */
 	DWORD data = WINHTTP_AUTOLOGON_SECURITY_LEVEL_LOW;
 
 	if (!WinHttpSetOption(request, WINHTTP_OPTION_AUTOLOGON_POLICY, &data, sizeof(DWORD)))
@@ -155,6 +145,59 @@ static int apply_default_credentials(HINTERNET request)
 	return 0;
 }
 
+static int fallback_cred_acquire_cb(
+	git_cred **cred,
+	const char *url,
+	const char *username_from_url,
+	unsigned int allowed_types,
+	void *payload)
+{
+	int error = 1;
+
+	/* If the target URI supports integrated Windows authentication
+	 * as an authentication mechanism */
+	if (GIT_CREDTYPE_DEFAULT & allowed_types) {
+		wchar_t *wide_url;
+
+		/* Convert URL to wide characters */
+		if (git__utf8_to_16_alloc(&wide_url, url) < 0) {
+			giterr_set(GITERR_OS, "Failed to convert string to wide form");
+			return -1;
+		}
+
+		if (SUCCEEDED(CoInitializeEx(NULL, COINIT_MULTITHREADED))) {
+			IInternetSecurityManager* pISM;
+
+			/* And if the target URI is in the My Computer, Intranet, or Trusted zones */
+			if (SUCCEEDED(CoCreateInstance(&CLSID_InternetSecurityManager, NULL,
+				CLSCTX_ALL, &IID_IInternetSecurityManager, (void **)&pISM))) {
+				DWORD dwZone;
+
+				if (SUCCEEDED(pISM->lpVtbl->MapUrlToZone(pISM, wide_url, &dwZone, 0)) &&
+					(URLZONE_LOCAL_MACHINE == dwZone ||
+					URLZONE_INTRANET == dwZone ||
+					URLZONE_TRUSTED == dwZone)) {
+					git_cred *existing = *cred;
+
+					if (existing)
+						existing->free(existing);
+
+					/* Then use default Windows credentials to authenticate this request */
+					error = git_cred_default_new(cred);
+				}
+
+				pISM->lpVtbl->Release(pISM);
+			}
+
+			CoUninitialize();
+		}
+
+		git__free(wide_url);
+	}
+
+	return error;
+}
+
 static int winhttp_stream_connect(winhttp_stream *s)
 {
 	winhttp_subtransport *t = OWNING_SUBTRANSPORT(s);
@@ -163,7 +206,7 @@ static int winhttp_stream_connect(winhttp_stream *s)
 	wchar_t ct[MAX_CONTENT_TYPE_LEN];
 	wchar_t *types[] = { L"*/*", NULL };
 	BOOL peerdist = FALSE;
-	int error = -1, wide_len;
+	int error = -1;
 	unsigned long disable_redirects = WINHTTP_DISABLE_REDIRECTS;
 
 	/* Prepare URL */
@@ -173,21 +216,7 @@ static int winhttp_stream_connect(winhttp_stream *s)
 		return -1;
 
 	/* Convert URL to wide characters */
-	wide_len = MultiByteToWideChar(CP_UTF8,	MB_ERR_INVALID_CHARS,
-		git_buf_cstr(&buf),	-1, NULL, 0);
-
-	if (!wide_len) {
-		giterr_set(GITERR_OS, "Failed to measure string for wide conversion");
-		goto on_error;
-	}
-
-	s->request_uri = git__malloc(wide_len * sizeof(wchar_t));
-
-	if (!s->request_uri)
-		goto on_error;
-
-	if (!MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
-		git_buf_cstr(&buf), -1, s->request_uri, wide_len)) {
+	if (git__utf8_to_16_alloc(&s->request_uri, git_buf_cstr(&buf)) < 0) {
 		giterr_set(GITERR_OS, "Failed to convert string to wide form");
 		goto on_error;
 	}
@@ -216,30 +245,17 @@ static int winhttp_stream_connect(winhttp_stream *s)
 		wchar_t *proxy_wide;
 
 		/* Convert URL to wide characters */
-		wide_len = MultiByteToWideChar(CP_UTF8,	MB_ERR_INVALID_CHARS,
-			proxy_url, -1, NULL, 0);
-
-		if (!wide_len) {
-			giterr_set(GITERR_OS, "Failed to measure string for wide conversion");
-			goto on_error;
-		}
-
-		proxy_wide = git__malloc(wide_len * sizeof(wchar_t));
+		int proxy_wide_len = git__utf8_to_16_alloc(&proxy_wide, proxy_url);
 
-		if (!proxy_wide)
-			goto on_error;
-
-		if (!MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
-			proxy_url, -1, proxy_wide, wide_len)) {
+		if (proxy_wide_len < 0) {
 			giterr_set(GITERR_OS, "Failed to convert string to wide form");
-			git__free(proxy_wide);
 			goto on_error;
 		}
 
 		/* Strip any trailing forward slash on the proxy URL;
 		 * WinHTTP doesn't like it if one is present */
-		if (wide_len > 1 && L'/' == proxy_wide[wide_len - 2])
-			proxy_wide[wide_len - 2] = L'\0';
+		if (proxy_wide_len > 1 && L'/' == proxy_wide[proxy_wide_len - 2])
+			proxy_wide[proxy_wide_len - 2] = L'\0';
 
 		proxy_info.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY;
 		proxy_info.lpszProxy = proxy_wide;
@@ -290,7 +306,10 @@ static int winhttp_stream_connect(winhttp_stream *s)
 			s->service) < 0)
 			goto on_error;
 
-		git__utf8_to_16(ct, MAX_CONTENT_TYPE_LEN, git_buf_cstr(&buf));
+		if (git__utf8_to_16(ct, MAX_CONTENT_TYPE_LEN, git_buf_cstr(&buf)) < 0) {
+			giterr_set(GITERR_OS, "Failed to convert content-type to wide characters");
+			goto on_error;
+		}
 
 		if (!WinHttpAddRequestHeaders(s->request, ct, (ULONG)-1L,
 			WINHTTP_ADDREQ_FLAG_ADD | WINHTTP_ADDREQ_FLAG_REPLACE)) {
@@ -304,7 +323,10 @@ static int winhttp_stream_connect(winhttp_stream *s)
 			s->service) < 0)
 			goto on_error;
 
-		git__utf8_to_16(ct, MAX_CONTENT_TYPE_LEN, git_buf_cstr(&buf));
+		if (git__utf8_to_16(ct, MAX_CONTENT_TYPE_LEN, git_buf_cstr(&buf)) < 0) {
+			giterr_set(GITERR_OS, "Failed to convert accept header to wide characters");
+			goto on_error;
+		}
 
 		if (!WinHttpAddRequestHeaders(s->request, ct, (ULONG)-1L,
 			WINHTTP_ADDREQ_FLAG_ADD | WINHTTP_ADDREQ_FLAG_REPLACE)) {
@@ -437,16 +459,20 @@ static int winhttp_connect(
 	const char *url)
 {
 	wchar_t *ua = L"git/1.0 (libgit2 " WIDEN(LIBGIT2_VERSION) L")";
-	git_win32_path host;
+	wchar_t *wide_host;
 	int32_t port;
 	const char *default_port = "80";
+	int error = -1;
 
 	/* Prepare port */
 	if (git__strtol32(&port, t->connection_data.port, NULL, 10) < 0)
 		return -1;
 
 	/* Prepare host */
-	git_win32_path_from_c(host, t->connection_data.host);
+	if (git__utf8_to_16_alloc(&wide_host, t->connection_data.host) < 0) {
+		giterr_set(GITERR_OS, "Unable to convert host to wide characters");
+		return -1;
+	}
 
 	/* Establish session */
 	t->session = WinHttpOpen(
@@ -458,22 +484,27 @@ static int winhttp_connect(
 
 	if (!t->session) {
 		giterr_set(GITERR_OS, "Failed to init WinHTTP");
-		return -1;
+		goto on_error;
 	}
 
 	/* Establish connection */
 	t->connection = WinHttpConnect(
 		t->session,
-		host,
+		wide_host,
 		(INTERNET_PORT) port,
 		0);
 
 	if (!t->connection) {
 		giterr_set(GITERR_OS, "Failed to connect to host");
-		return -1;
+		goto on_error;
 	}
 
-	return 0;
+	error = 0;
+
+on_error:
+	git__free(wide_host);
+
+	return error;
 }
 
 static int winhttp_stream_read(
@@ -624,7 +655,6 @@ replay:
 			}
 
 			location = git__malloc(location_length);
-			location8 = git__malloc(location_length);
 			GITERR_CHECK_ALLOC(location);
 
 			if (!WinHttpQueryHeaders(s->request,
@@ -637,7 +667,14 @@ replay:
 				git__free(location);
 				return -1;
 			}
-			git__utf16_to_8(location8, location_length, location);
+
+			/* Convert the Location header to UTF-8 */
+			if (git__utf16_to_8_alloc(&location8, location) < 0) {
+				giterr_set(GITERR_OS, "Failed to convert Location header to UTF-8");
+				git__free(location);
+				return -1;
+			}
+
 			git__free(location);
 
 			/* Replay the request */
@@ -647,8 +684,11 @@ replay:
 
 			if (!git__prefixcmp_icase(location8, prefix_https)) {
 				/* Upgrade to secure connection; disconnect and start over */
-				if (gitno_connection_data_from_url(&t->connection_data, location8, s->service_url) < 0)
+				if (gitno_connection_data_from_url(&t->connection_data, location8, s->service_url) < 0) {
+					git__free(location8);
 					return -1;
+				}
+
 				winhttp_connect(t, location8);
 			}
 
@@ -657,8 +697,7 @@ replay:
 		}
 
 		/* Handle authentication failures */
-		if (HTTP_STATUS_DENIED == status_code &&
-			get_verb == s->verb && t->owner->cred_acquire_cb) {
+		if (HTTP_STATUS_DENIED == status_code && get_verb == s->verb) {
 			int allowed_types;
 
 			if (parse_unauthorized_response(s->request, &allowed_types, &t->auth_mechanism) < 0)
@@ -666,19 +705,36 @@ replay:
 
 			if (allowed_types &&
 				(!t->cred || 0 == (t->cred->credtype & allowed_types))) {
+				int cred_error = 1;
+
+				/* Start with the user-supplied credential callback, if present */
+				if (t->owner->cred_acquire_cb) {
+					cred_error = t->owner->cred_acquire_cb(&t->cred, t->owner->url,
+						t->connection_data.user, allowed_types,	t->owner->cred_acquire_payload);
+
+					if (cred_error < 0)
+						return cred_error;
+				}
+
+				/* Invoke the fallback credentials acquisition callback if necessary */
+				if (cred_error > 0) {
+					cred_error = fallback_cred_acquire_cb(&t->cred, t->owner->url,
+						t->connection_data.user, allowed_types, NULL);
 
-				if (t->owner->cred_acquire_cb(&t->cred, t->owner->url, t->connection_data.user, allowed_types, 
-					t->owner->cred_acquire_payload) < 0)
-					return GIT_EUSER;
+					if (cred_error < 0)
+						return cred_error;
+				}
 
-				assert(t->cred);
+				if (!cred_error) {
+					assert(t->cred);
 
-				WinHttpCloseHandle(s->request);
-				s->request = NULL;
-				s->sent_request = 0;
+					WinHttpCloseHandle(s->request);
+					s->request = NULL;
+					s->sent_request = 0;
 
-				/* Successfully acquired a credential */
-				goto replay;
+					/* Successfully acquired a credential */
+					goto replay;
+				}
 			}
 		}
 
@@ -693,7 +749,11 @@ replay:
 		else
 			snprintf(expected_content_type_8, MAX_CONTENT_TYPE_LEN, "application/x-git-%s-advertisement", s->service);
 
-		git__utf8_to_16(expected_content_type, MAX_CONTENT_TYPE_LEN, expected_content_type_8);
+		if (git__utf8_to_16(expected_content_type, MAX_CONTENT_TYPE_LEN, expected_content_type_8) < 0) {
+			giterr_set(GITERR_OS, "Failed to convert expected content-type to wide characters");
+			return -1;
+		}
+
 		content_type_length = sizeof(content_type);
 
 		if (!WinHttpQueryHeaders(s->request,
diff --git a/src/tree-cache.c b/src/tree-cache.c
index 1d3997154..49afd6e49 100644
--- a/src/tree-cache.c
+++ b/src/tree-cache.c
@@ -7,23 +7,16 @@
 
 #include "tree-cache.h"
 
-static git_tree_cache *find_child(const git_tree_cache *tree, const char *path)
+static git_tree_cache *find_child(
+	const git_tree_cache *tree, const char *path, const char *end)
 {
-	size_t i, dirlen;
-	const char *end;
-
-	end = strchr(path, '/');
-	if (end == NULL) {
-		end = strrchr(path, '\0');
-	}
-
-	dirlen = end - path;
+	size_t i, dirlen = end ? (size_t)(end - path) : strlen(path);
 
 	for (i = 0; i < tree->children_count; ++i) {
-		const char *childname = tree->children[i]->name;
+		git_tree_cache *child = tree->children[i];
 
-		if (strlen(childname) == dirlen && !memcmp(path, childname, dirlen))
-			return tree->children[i];
+		if (child->namelen == dirlen && !memcmp(path, child->name, dirlen))
+			return child;
 	}
 
 	return NULL;
@@ -44,7 +37,7 @@ void git_tree_cache_invalidate_path(git_tree_cache *tree, const char *path)
 		if (end == NULL) /* End of path */
 			break;
 
-		tree = find_child(tree, ptr);
+		tree = find_child(tree, ptr, end);
 		if (tree == NULL) /* We don't have that tree */
 			return;
 
@@ -64,10 +57,9 @@ const git_tree_cache *git_tree_cache_get(const git_tree_cache *tree, const char
 	while (1) {
 		end = strchr(ptr, '/');
 
-		tree = find_child(tree, ptr);
-		if (tree == NULL) { /* Can't find it */
+		tree = find_child(tree, ptr, end);
+		if (tree == NULL) /* Can't find it */
 			return NULL;
-		}
 
 		if (end == NULL || *end + 1 == '\0')
 			return tree;
@@ -100,6 +92,7 @@ static int read_tree_internal(git_tree_cache **out,
 	tree->parent = parent;
 
 	/* NUL-terminated tree name */
+	tree->namelen = name_len;
 	memcpy(tree->name, name_start, name_len);
 	tree->name[name_len] = '\0';
 
diff --git a/src/tree-cache.h b/src/tree-cache.h
index 805483a78..90c82dbbf 100644
--- a/src/tree-cache.h
+++ b/src/tree-cache.h
@@ -18,6 +18,7 @@ struct git_tree_cache {
 
 	ssize_t entries;
 	git_oid oid;
+	size_t namelen;
 	char name[GIT_FLEX_ARRAY];
 };
 
diff --git a/src/tree.c b/src/tree.c
index bb59ff82b..b64efe460 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -204,22 +204,22 @@ void git_tree_entry_free(git_tree_entry *entry)
 	git__free(entry);
 }
 
-git_tree_entry *git_tree_entry_dup(const git_tree_entry *entry)
+int git_tree_entry_dup(git_tree_entry **dest, const git_tree_entry *source)
 {
 	size_t total_size;
 	git_tree_entry *copy;
 
-	assert(entry);
+	assert(source);
 
-	total_size = sizeof(git_tree_entry) + entry->filename_len + 1;
+	total_size = sizeof(git_tree_entry) + source->filename_len + 1;
 
 	copy = git__malloc(total_size);
-	if (!copy)
-		return NULL;
+	GITERR_CHECK_ALLOC(copy);
 
-	memcpy(copy, entry, total_size);
+	memcpy(copy, source, total_size);
 
-	return copy;
+	*dest = copy;
+	return 0;
 }
 
 void git_tree__free(void *_tree)
@@ -283,7 +283,8 @@ static const git_tree_entry *entry_fromname(
 {
 	size_t idx;
 
-	assert(tree->entries.sorted); /* be safe when we cast away constness */
+	/* be safe when we cast away constness - i.e. don't trigger a sort */
+	assert(git_vector_is_sorted(&tree->entries));
 
 	if (tree_key_search(&idx, (git_vector *)&tree->entries, name, name_len) < 0)
 		return NULL;
@@ -305,8 +306,8 @@ const git_tree_entry *git_tree_entry_byindex(
 	return git_vector_get(&tree->entries, idx);
 }
 
-const git_tree_entry *git_tree_entry_byoid(
-	const git_tree *tree, const git_oid *oid)
+const git_tree_entry *git_tree_entry_byid(
+	const git_tree *tree, const git_oid *id)
 {
 	size_t i;
 	const git_tree_entry *e;
@@ -314,7 +315,7 @@ const git_tree_entry *git_tree_entry_byoid(
 	assert(tree);
 
 	git_vector_foreach(&tree->entries, i, e) {
-		if (memcmp(&e->oid.id, &oid->id, sizeof(oid->id)) == 0)
+		if (memcmp(&e->oid.id, &id->id, sizeof(id->id)) == 0)
 			return e;
 	}
 
@@ -333,7 +334,8 @@ int git_tree__prefix_position(const git_tree *tree, const char *path)
 	ksearch.filename = path;
 	ksearch.filename_len = strlen(path);
 
-	assert(tree->entries.sorted); /* be safe when we cast away constness */
+	/* be safe when we cast away constness - i.e. don't trigger a sort */
+	assert(git_vector_is_sorted(&tree->entries));
 
 	/* Find tree entry with appropriate prefix */
 	git_vector_bsearch2(
@@ -458,7 +460,7 @@ static int append_entry(
 	git_oid_cpy(&entry->oid, id);
 	entry->attr = (uint16_t)filemode;
 
-	if (git_vector_insert(&bld->entries, entry) < 0) {
+	if (git_vector_insert_sorted(&bld->entries, entry, NULL) < 0) {
 		git__free(entry);
 		return -1;
 	}
@@ -551,7 +553,7 @@ static int write_tree(
 			if (error < 0)
 				goto on_error;
 		} else {
-			error = append_entry(bld, filename, &entry->oid, entry->mode);
+			error = append_entry(bld, filename, &entry->id, entry->mode);
 			if (error < 0)
 				goto on_error;
 		}
@@ -669,7 +671,7 @@ int git_treebuilder_insert(
 		entry = alloc_entry(filename);
 		GITERR_CHECK_ALLOC(entry);
 
-		if (git_vector_insert(&bld->entries, entry) < 0) {
+		if (git_vector_insert_sorted(&bld->entries, entry, NULL) < 0) {
 			git__free(entry);
 			return -1;
 		}
@@ -853,8 +855,7 @@ int git_tree_entry_bypath(
 	case '\0':
 		/* If there are no more components in the path, return
 		 * this entry */
-		*entry_out = git_tree_entry_dup(entry);
-		return 0;
+		return git_tree_entry_dup(entry_out, entry);
 	}
 
 	if (git_tree_lookup(&subtree, root->object.repo, &entry->oid) < 0)
@@ -884,22 +885,22 @@ static int tree_walk(
 	git_vector_foreach(&tree->entries, i, entry) {
 		if (preorder) {
 			error = callback(path->ptr, entry, payload);
-			if (error > 0) {
+			if (error < 0) { /* negative value stops iteration */
+				giterr_set_after_callback_function(error, "git_tree_walk");
+				break;
+			}
+			if (error > 0) { /* positive value skips this entry */
 				error = 0;
 				continue;
 			}
-			if (error < 0) {
-				giterr_clear();
-				return GIT_EUSER;
-			}
 		}
 
 		if (git_tree_entry__is_tree(entry)) {
 			git_tree *subtree;
 			size_t path_len = git_buf_len(path);
 
-			if ((error = git_tree_lookup(
-				&subtree, tree->object.repo, &entry->oid)) < 0)
+			error = git_tree_lookup(&subtree, tree->object.repo, &entry->oid);
+			if (error < 0)
 				break;
 
 			/* append the next entry to the path */
@@ -907,21 +908,24 @@ static int tree_walk(
 			git_buf_putc(path, '/');
 
 			if (git_buf_oom(path))
-				return -1;
+				error = -1;
+			else
+				error = tree_walk(subtree, callback, path, payload, preorder);
 
-			error = tree_walk(subtree, callback, path, payload, preorder);
 			git_tree_free(subtree);
-
 			if (error != 0)
 				break;
 
 			git_buf_truncate(path, path_len);
 		}
 
-		if (!preorder && callback(path->ptr, entry, payload) < 0) {
-			giterr_clear();
-			error = GIT_EUSER;
-			break;
+		if (!preorder) {
+			error = callback(path->ptr, entry, payload);
+			if (error < 0) { /* negative value stops iteration */
+				giterr_set_after_callback_function(error, "git_tree_walk");
+				break;
+			}
+			error = 0;
 		}
 	}
 
diff --git a/src/unix/map.c b/src/unix/map.c
index 7de99c99d..3d0cbbaf8 100644
--- a/src/unix/map.c
+++ b/src/unix/map.c
@@ -6,12 +6,18 @@
  */
 #include <git2/common.h>
 
-#ifndef GIT_WIN32
+#if !defined(GIT_WIN32) && !defined(NO_MMAP)
 
 #include "map.h"
 #include <sys/mman.h>
+#include <unistd.h>
 #include <errno.h>
 
+long git__page_size(void)
+{
+	return sysconf(_SC_PAGE_SIZE);
+}
+
 int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
 {
 	int mprot = 0;
diff --git a/src/unix/posix.h b/src/unix/posix.h
index 9c9f837b9..1e41bcf18 100644
--- a/src/unix/posix.h
+++ b/src/unix/posix.h
@@ -28,7 +28,6 @@ char *p_realpath(const char *, char *);
 #define p_vsnprintf(b, c, f, a) vsnprintf(b, c, f, a)
 #define p_snprintf(b, c, f, ...) snprintf(b, c, f, __VA_ARGS__)
 #define p_mkstemp(p) mkstemp(p)
-#define p_setenv(n,v,o) setenv(n,v,o)
 #define p_inet_pton(a, b, c) inet_pton(a, b, c)
 
 /* see win32/posix.h for explanation about why this exists */
diff --git a/src/userdiff.h b/src/userdiff.h
new file mode 100644
index 000000000..523f2f8d4
--- /dev/null
+++ b/src/userdiff.h
@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_userdiff_h__
+#define INCLUDE_userdiff_h__
+
+/*
+ * This file isolates the built in diff driver function name patterns.
+ * Most of these patterns are taken from Git (with permission from the
+ * original authors for relicensing to libgit2).
+ */
+
+typedef struct {
+	const char *name;
+	const char *fns;
+	const char *words;
+	int flags;
+} git_diff_driver_definition;
+
+#define WORD_DEFAULT "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+"
+
+/*
+ * These builtin driver definition macros have same signature as in core
+ * git userdiff.c so that the data can be extracted verbatim
+ */
+#define PATTERNS(NAME, FN_PATS, WORD_PAT) \
+	{ NAME, FN_PATS, WORD_PAT WORD_DEFAULT, 0 }
+#define IPATTERN(NAME, FN_PATS, WORD_PAT) \
+	{ NAME, FN_PATS, WORD_PAT WORD_DEFAULT, REG_ICASE }
+
+/*
+ * The table of diff driver patterns
+ *
+ * Function name patterns are a list of newline separated patterns that
+ * match a function declaration (i.e. the line you want in the hunk header),
+ * or a negative pattern prefixed with a '!' to reject a pattern (such as
+ * rejecting goto labels in C code).
+ *
+ * Word boundary patterns are just a simple pattern that will be OR'ed with
+ * the default value above (i.e. whitespace or non-ASCII characters).
+ */
+static git_diff_driver_definition builtin_defs[] = {
+
+IPATTERN("ada",
+	 "!^(.*[ \t])?(is[ \t]+new|renames|is[ \t]+separate)([ \t].*)?$\n"
+	 "!^[ \t]*with[ \t].*$\n"
+	 "^[ \t]*((procedure|function)[ \t]+.*)$\n"
+	 "^[ \t]*((package|protected|task)[ \t]+.*)$",
+	 /* -- */
+	 "[a-zA-Z][a-zA-Z0-9_]*"
+	 "|[-+]?[0-9][0-9#_.aAbBcCdDeEfF]*([eE][+-]?[0-9_]+)?"
+	 "|=>|\\.\\.|\\*\\*|:=|/=|>=|<=|<<|>>|<>"),
+
+IPATTERN("fortran",
+	 "!^([C*]|[ \t]*!)\n"
+	 "!^[ \t]*MODULE[ \t]+PROCEDURE[ \t]\n"
+	 "^[ \t]*((END[ \t]+)?(PROGRAM|MODULE|BLOCK[ \t]+DATA"
+		"|([^'\" \t]+[ \t]+)*(SUBROUTINE|FUNCTION))[ \t]+[A-Z].*)$",
+	 /* -- */
+	 "[a-zA-Z][a-zA-Z0-9_]*"
+	 "|\\.([Ee][Qq]|[Nn][Ee]|[Gg][TtEe]|[Ll][TtEe]|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]|[Aa][Nn][Dd]|[Oo][Rr]|[Nn]?[Ee][Qq][Vv]|[Nn][Oo][Tt])\\."
+	 /* numbers and format statements like 2E14.4, or ES12.6, 9X.
+	  * Don't worry about format statements without leading digits since
+	  * they would have been matched above as a variable anyway. */
+	 "|[-+]?[0-9.]+([AaIiDdEeFfLlTtXx][Ss]?[-+]?[0-9.]*)?(_[a-zA-Z0-9][a-zA-Z0-9_]*)?"
+	 "|//|\\*\\*|::|[/<>=]="),
+
+PATTERNS("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$",
+	 "[^<>= \t]+"),
+
+PATTERNS("java",
+	 "!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n"
+	 "^[ \t]*(([A-Za-z_][A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]="
+	 "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"),
+
+PATTERNS("matlab",
+	 "^[[:space:]]*((classdef|function)[[:space:]].*)$|^%%[[:space:]].*$",
+	 "[a-zA-Z_][a-zA-Z0-9_]*|[-+0-9.e]+|[=~<>]=|\\.[*/\\^']|\\|\\||&&"),
+
+PATTERNS("objc",
+	 /* Negate C statements that can look like functions */
+	 "!^[ \t]*(do|for|if|else|return|switch|while)\n"
+	 /* Objective-C methods */
+	 "^[ \t]*([-+][ \t]*\\([ \t]*[A-Za-z_][A-Za-z_0-9* \t]*\\)[ \t]*[A-Za-z_].*)$\n"
+	 /* C functions */
+	 "^[ \t]*(([A-Za-z_][A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$\n"
+	 /* Objective-C class/protocol definitions */
+	 "^(@(implementation|interface|protocol)[ \t].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
+
+PATTERNS("pascal",
+	 "^(((class[ \t]+)?(procedure|function)|constructor|destructor|interface|"
+		"implementation|initialization|finalization)[ \t]*.*)$"
+	 "\n"
+	 "^(.*=[ \t]*(class|record).*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
+	 "|<>|<=|>=|:=|\\.\\."),
+
+PATTERNS("perl",
+	 "^package .*\n"
+	 "^sub [[:alnum:]_':]+[ \t]*"
+		"(\\([^)]*\\)[ \t]*)?" /* prototype */
+		/*
+		 * Attributes.  A regex can't count nested parentheses,
+		 * so just slurp up whatever we see, taking care not
+		 * to accept lines like "sub foo; # defined elsewhere".
+		 *
+		 * An attribute could contain a semicolon, but at that
+		 * point it seems reasonable enough to give up.
+		 */
+		"(:[^;#]*)?"
+		"(\\{[ \t]*)?" /* brace can come here or on the next line */
+		"(#.*)?$\n" /* comment */
+	 "^(BEGIN|END|INIT|CHECK|UNITCHECK|AUTOLOAD|DESTROY)[ \t]*"
+		"(\\{[ \t]*)?" /* brace can come here or on the next line */
+		"(#.*)?$\n"
+	 "^=head[0-9] .*",	/* POD */
+	 /* -- */
+	 "[[:alpha:]_'][[:alnum:]_']*"
+	 "|0[xb]?[0-9a-fA-F_]*"
+	 /* taking care not to interpret 3..5 as (3.)(.5) */
+	 "|[0-9a-fA-F_]+(\\.[0-9a-fA-F_]+)?([eE][-+]?[0-9_]+)?"
+	 "|=>|-[rwxoRWXOezsfdlpSugkbctTBMAC>]|~~|::"
+	 "|&&=|\\|\\|=|//=|\\*\\*="
+	 "|&&|\\|\\||//|\\+\\+|--|\\*\\*|\\.\\.\\.?"
+	 "|[-+*/%.^&<>=!|]="
+	 "|=~|!~"
+	 "|<<|<>|<=>|>>"),
+
+PATTERNS("python", "^[ \t]*((class|def)[ \t].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"),
+
+PATTERNS("ruby", "^[ \t]*((class|module|def)[ \t].*)$",
+	 /* -- */
+	 "(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?."
+	 "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"),
+
+PATTERNS("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$",
+	 "[={}\"]|[^={}\" \t]+"),
+
+PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$",
+	 "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+"),
+
+PATTERNS("cpp",
+	 /* Jump targets or access declarations */
+	 "!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:[[:space:]]*($|/[/*])\n"
+	 /* functions/methods, variables, and compounds at top level */
+	 "^((::[[:space:]]*)?[A-Za-z_].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lLuU]*"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->\\*?|\\.\\*"),
+
+PATTERNS("csharp",
+	 /* Keywords */
+	 "!^[ \t]*(do|while|for|if|else|instanceof|new|return|switch|case|throw|catch|using)\n"
+	 /* Methods and constructors */
+	 "^[ \t]*(((static|public|internal|private|protected|new|virtual|sealed|override|unsafe)[ \t]+)*[][<>@.~_[:alnum:]]+[ \t]+[<>@._[:alnum:]]+[ \t]*\\(.*\\))[ \t]*$\n"
+	 /* Properties */
+	 "^[ \t]*(((static|public|internal|private|protected|new|virtual|sealed|override|unsafe)[ \t]+)*[][<>@.~_[:alnum:]]+[ \t]+[@._[:alnum:]]+)[ \t]*$\n"
+	 /* Type definitions */
+	 "^[ \t]*(((static|public|internal|private|protected|new|unsafe|sealed|abstract|partial)[ \t]+)*(class|enum|interface|struct)[ \t]+.*)$\n"
+	 /* Namespace */
+	 "^[ \t]*(namespace[ \t]+.*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
+
+PATTERNS("php",
+	 "^[ \t]*(((public|private|protected|static|final)[ \t]+)*((class|function)[ \t].*))$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
+
+PATTERNS("javascript",
+	 "^[ \t]*(function[ \t][a-zA-Z_][^\\{]*)\n"
+	 "^[ \t]*(var[ \t]+[a-zA-Z_][a-zA-Z0-9_]*[ \t]*=[ \t]*function[ \t\\(][^\\{]*)\n"
+	 "^[ \t]*([a-zA-Z_][a-zA-Z0-9_]*[ \t]*:[ \t]*function[ \t\\(][^\\{]*)",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
+};
+
+#undef IPATTERN
+#undef PATTERNS
+#undef WORD_DEFAULT
+
+#endif
+
diff --git a/src/util.c b/src/util.c
index 47516a8f7..f9d37e4f4 100644
--- a/src/util.c
+++ b/src/util.c
@@ -6,140 +6,21 @@
  */
 #include <git2.h>
 #include "common.h"
-#include <stdarg.h>
 #include <stdio.h>
 #include <ctype.h>
 #include "posix.h"
-#include "fileops.h"
-#include "cache.h"
 
 #ifdef _MSC_VER
 # include <Shlwapi.h>
 #endif
 
-void git_libgit2_version(int *major, int *minor, int *rev)
-{
-	*major = LIBGIT2_VER_MAJOR;
-	*minor = LIBGIT2_VER_MINOR;
-	*rev = LIBGIT2_VER_REVISION;
-}
-
-int git_libgit2_capabilities()
-{
-	return 0
-#ifdef GIT_THREADS
-		| GIT_CAP_THREADS
-#endif
-#if defined(GIT_SSL) || defined(GIT_WINHTTP)
-		| GIT_CAP_HTTPS
-#endif
-#if defined(GIT_SSH)
-		| GIT_CAP_SSH
-#endif
-	;
-}
-
-/* Declarations for tuneable settings */
-extern size_t git_mwindow__window_size;
-extern size_t git_mwindow__mapped_limit;
-
-static int config_level_to_futils_dir(int config_level)
-{
-	int val = -1;
-
-	switch (config_level) {
-	case GIT_CONFIG_LEVEL_SYSTEM: val = GIT_FUTILS_DIR_SYSTEM; break;
-	case GIT_CONFIG_LEVEL_XDG:    val = GIT_FUTILS_DIR_XDG; break;
-	case GIT_CONFIG_LEVEL_GLOBAL: val = GIT_FUTILS_DIR_GLOBAL; break;
-	default:
-		giterr_set(
-			GITERR_INVALID, "Invalid config path selector %d", config_level);
-	}
-
-	return val;
-}
-
-int git_libgit2_opts(int key, ...)
-{
-	int error = 0;
-	va_list ap;
-
-	va_start(ap, key);
-
-	switch (key) {
-	case GIT_OPT_SET_MWINDOW_SIZE:
-		git_mwindow__window_size = va_arg(ap, size_t);
-		break;
-
-	case GIT_OPT_GET_MWINDOW_SIZE:
-		*(va_arg(ap, size_t *)) = git_mwindow__window_size;
-		break;
-
-	case GIT_OPT_SET_MWINDOW_MAPPED_LIMIT:
-		git_mwindow__mapped_limit = va_arg(ap, size_t);
-		break;
-
-	case GIT_OPT_GET_MWINDOW_MAPPED_LIMIT:
-		*(va_arg(ap, size_t *)) = git_mwindow__mapped_limit;
-		break;
-
-	case GIT_OPT_GET_SEARCH_PATH:
-		if ((error = config_level_to_futils_dir(va_arg(ap, int))) >= 0) {
-			char *out = va_arg(ap, char *);
-			size_t outlen = va_arg(ap, size_t);
-
-			error = git_futils_dirs_get_str(out, outlen, error);
-		}
-		break;
-
-	case GIT_OPT_SET_SEARCH_PATH:
-		if ((error = config_level_to_futils_dir(va_arg(ap, int))) >= 0)
-			error = git_futils_dirs_set(error, va_arg(ap, const char *));
-		break;
-
-	case GIT_OPT_SET_CACHE_OBJECT_LIMIT:
-		{
-			git_otype type = (git_otype)va_arg(ap, int);
-			size_t size = va_arg(ap, size_t);
-			error = git_cache_set_max_object_size(type, size);
-			break;
-		}
-
-	case GIT_OPT_SET_CACHE_MAX_SIZE:
-		git_cache__max_storage = va_arg(ap, ssize_t);
-		break;
-
-	case GIT_OPT_ENABLE_CACHING:
-		git_cache__enabled = (va_arg(ap, int) != 0);
-		break;
-
-	case GIT_OPT_GET_CACHED_MEMORY:
-		*(va_arg(ap, ssize_t *)) = git_cache__current_storage.val;
-		*(va_arg(ap, ssize_t *)) = git_cache__max_storage;
-		break;
-
-	case GIT_OPT_GET_TEMPLATE_PATH:
-		{
-			char *out = va_arg(ap, char *);
-			size_t outlen = va_arg(ap, size_t);
-
-			error = git_futils_dirs_get_str(out, outlen, GIT_FUTILS_DIR_TEMPLATE);
-		}
-		break;
-
-	case GIT_OPT_SET_TEMPLATE_PATH:
-		error = git_futils_dirs_set(GIT_FUTILS_DIR_TEMPLATE, va_arg(ap, const char *));
-		break;
-	}
-
-	va_end(ap);
-
-	return error;
-}
-
 void git_strarray_free(git_strarray *array)
 {
 	size_t i;
+
+	if (array == NULL)
+		return;
+
 	for (i = 0; i < array->count; ++i)
 		git__free(array->strings[i]);
 
@@ -661,10 +542,12 @@ int git__bsearch_r(
  */
 int git__strcmp_cb(const void *a, const void *b)
 {
-	const char *stra = (const char *)a;
-	const char *strb = (const char *)b;
+	return strcmp((const char *)a, (const char *)b);
+}
 
-	return strcmp(stra, strb);
+int git__strcasecmp_cb(const void *a, const void *b)
+{
+	return strcasecmp((const char *)a, (const char *)b);
 }
 
 int git__parse_bool(int *out, const char *value)
@@ -729,6 +612,7 @@ void git__qsort_r(
 #if defined(__MINGW32__) || defined(AMIGA) || \
 	defined(__OpenBSD__) || defined(__NetBSD__) || \
 	defined(__gnu_hurd__) || defined(__ANDROID_API__) || \
+	defined(__sun) || defined(__CYGWIN__) || \
 	(__GLIBC__ == 2 && __GLIBC_MINOR__ < 8)
 	git__insertsort_r(els, nel, elsize, NULL, cmp, payload);
 #elif defined(GIT_WIN32)
diff --git a/src/util.h b/src/util.h
index f9de909e9..ca676c039 100644
--- a/src/util.h
+++ b/src/util.h
@@ -8,6 +8,7 @@
 #define INCLUDE_util_h__
 
 #include "common.h"
+#include "strnlen.h"
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
 #define bitsizeof(x) (CHAR_BIT * sizeof(x))
@@ -19,6 +20,17 @@
 # define max(a,b) ((a) > (b) ? (a) : (b))
 #endif
 
+#define GIT_DATE_RFC2822_SZ  32
+
+/**
+ * Return the length of a constant string.
+ * We are aware that `strlen` performs the same task and is usually
+ * optimized away by the compiler, whilst being safer because it returns
+ * valid values when passed a pointer instead of a constant string; however
+ * this macro will transparently work with wide-char and single-char strings.
+ */
+#define CONST_STRLEN(x) ((sizeof(x)/sizeof(x[0])) - 1)
+
 /*
  * Custom memory allocation wrappers
  * that set error code and error message
@@ -50,8 +62,7 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n)
 	size_t length = 0;
 	char *ptr;
 
-	while (length < n && str[length])
-		++length;
+	length = p_strnlen(str, n);
 
 	ptr = (char*)git__malloc(length + 1);
 
@@ -122,6 +133,13 @@ GIT_INLINE(int) git__is_uint32(size_t p)
 	return p == (size_t)r;
 }
 
+/** @return true if p fits into the range of an unsigned long */
+GIT_INLINE(int) git__is_ulong(git_off_t p)
+{
+	unsigned long r = (unsigned long)p;
+	return p == (git_off_t)r;
+}
+
 /* 32-bit cross-platform rotl */
 #ifdef _MSC_VER /* use built-in method in MSVC */
 #	define git__rotl(v, s) (uint32_t)_rotl(v, s)
@@ -194,6 +212,7 @@ extern int git__bsearch_r(
 	size_t *position);
 
 extern int git__strcmp_cb(const void *a, const void *b);
+extern int git__strcasecmp_cb(const void *a, const void *b);
 
 extern int git__strcmp(const char *a, const char *b);
 extern int git__strcasecmp(const char *a, const char *b);
@@ -329,6 +348,16 @@ extern int git__parse_bool(int *out, const char *value);
 extern int git__date_parse(git_time_t *out, const char *date);
 
 /*
+ * Format a git_time as a RFC2822 string
+ *
+ * @param out buffer to store formatted date; a '\\0' terminator will automatically be added.
+ * @param len size of the buffer; should be atleast `GIT_DATE_RFC2822_SZ` in size;
+ * @param date the date to be formatted
+ * @return 0 if successful; -1 on error
+ */
+extern int git__date_rfc2822_fmt(char *out, size_t len, const git_time *date);
+
+/*
  * Unescapes a string in-place.
  *
  * Edge cases behavior:
diff --git a/src/vector.c b/src/vector.c
index 362e7b0c0..c769b696a 100644
--- a/src/vector.c
+++ b/src/vector.c
@@ -6,7 +6,6 @@
  */
 
 #include "common.h"
-#include "repository.h"
 #include "vector.h"
 
 /* In elements, not bytes */
@@ -55,9 +54,11 @@ int git_vector_dup(git_vector *v, const git_vector *src, git_vector_cmp cmp)
 	bytes = src->length * sizeof(void *);
 
 	v->_alloc_size = src->length;
-	v->_cmp = cmp;
+	v->_cmp = cmp ? cmp : src->_cmp;
 	v->length = src->length;
-	v->sorted = src->sorted && cmp == src->_cmp;
+	v->flags  = src->flags;
+	if (cmp != src->_cmp)
+		git_vector_set_sorted(v, 0);
 	v->contents = git__malloc(bytes);
 	GITERR_CHECK_ALLOC(v->contents);
 
@@ -77,6 +78,20 @@ void git_vector_free(git_vector *v)
 	v->_alloc_size = 0;
 }
 
+void git_vector_free_deep(git_vector *v)
+{
+	size_t i;
+
+	assert(v);
+
+	for (i = 0; i < v->length; ++i) {
+		git__free(v->contents[i]);
+		v->contents[i] = NULL;
+	}
+
+	git_vector_free(v);
+}
+
 int git_vector_init(git_vector *v, size_t initial_size, git_vector_cmp cmp)
 {
 	assert(v);
@@ -84,12 +99,28 @@ int git_vector_init(git_vector *v, size_t initial_size, git_vector_cmp cmp)
 	v->_alloc_size = 0;
 	v->_cmp = cmp;
 	v->length = 0;
-	v->sorted = 1;
+	v->flags = GIT_VECTOR_SORTED;
 	v->contents = NULL;
 
 	return resize_vector(v, max(initial_size, MIN_ALLOCSIZE));
 }
 
+void **git_vector_detach(size_t *size, size_t *asize, git_vector *v)
+{
+	void **data = v->contents;
+
+	if (size)
+		*size = v->length;
+	if (asize)
+		*asize = v->_alloc_size;
+
+	v->_alloc_size = 0;
+	v->length   = 0;
+	v->contents = NULL;
+
+	return data;
+}
+
 int git_vector_insert(git_vector *v, void *element)
 {
 	assert(v);
@@ -99,7 +130,8 @@ int git_vector_insert(git_vector *v, void *element)
 		return -1;
 
 	v->contents[v->length++] = element;
-	v->sorted = 0;
+
+	git_vector_set_sorted(v, v->length <= 1);
 
 	return 0;
 }
@@ -112,7 +144,7 @@ int git_vector_insert_sorted(
 
 	assert(v && v->_cmp);
 
-	if (!v->sorted)
+	if (!git_vector_is_sorted(v))
 		git_vector_sort(v);
 
 	if (v->length >= v->_alloc_size &&
@@ -142,11 +174,12 @@ void git_vector_sort(git_vector *v)
 {
 	assert(v);
 
-	if (v->sorted || !v->_cmp)
+	if (git_vector_is_sorted(v) || !v->_cmp)
 		return;
 
-	git__tsort(v->contents, v->length, v->_cmp);
-	v->sorted = 1;
+	if (v->length > 1)
+		git__tsort(v->contents, v->length, v->_cmp);
+	git_vector_set_sorted(v, 1);
 }
 
 int git_vector_bsearch2(
@@ -244,14 +277,16 @@ void git_vector_uniq(git_vector *v, void  (*git_free_cb)(void *))
 }
 
 void git_vector_remove_matching(
-	git_vector *v, int (*match)(const git_vector *v, size_t idx))
+	git_vector *v,
+	int (*match)(const git_vector *v, size_t idx, void *payload),
+	void *payload)
 {
 	size_t i, j;
 
 	for (i = 0, j = 0; j < v->length; ++j) {
 		v->contents[i] = v->contents[j];
 
-		if (!match(v, i))
+		if (!match(v, i, payload))
 			i++;
 	}
 
@@ -262,7 +297,7 @@ void git_vector_clear(git_vector *v)
 {
 	assert(v);
 	v->length = 0;
-	v->sorted = 1;
+	git_vector_set_sorted(v, 1);
 }
 
 void git_vector_swap(git_vector *a, git_vector *b)
@@ -295,8 +330,10 @@ int git_vector_resize_to(git_vector *v, size_t new_length)
 
 int git_vector_set(void **old, git_vector *v, size_t position, void *value)
 {
-	if (git_vector_resize_to(v, position + 1) < 0)
-		return -1;
+	if (position + 1 > v->length) {
+		if (git_vector_resize_to(v, position + 1) < 0)
+			return -1;
+	}
 
 	if (old != NULL)
 		*old = v->contents[position];
@@ -305,3 +342,18 @@ int git_vector_set(void **old, git_vector *v, size_t position, void *value)
 
 	return 0;
 }
+
+int git_vector_verify_sorted(const git_vector *v)
+{
+	size_t i;
+
+	if (!git_vector_is_sorted(v))
+		return -1;
+
+	for (i = 1; i < v->length; ++i) {
+		if (v->_cmp(v->contents[i - 1], v->contents[i]) > 0)
+			return -1;
+	}
+
+	return 0;
+}
diff --git a/src/vector.h b/src/vector.h
index 279f5c6ee..aac46c4b3 100644
--- a/src/vector.h
+++ b/src/vector.h
@@ -7,26 +7,34 @@
 #ifndef INCLUDE_vector_h__
 #define INCLUDE_vector_h__
 
-#include "git2/common.h"
+#include "common.h"
 
 typedef int (*git_vector_cmp)(const void *, const void *);
 
+enum {
+	GIT_VECTOR_SORTED = (1u << 0),
+	GIT_VECTOR_FLAG_MAX = (1u << 1),
+};
+
 typedef struct git_vector {
 	size_t _alloc_size;
 	git_vector_cmp _cmp;
 	void **contents;
 	size_t length;
-	int sorted;
+	uint32_t flags;
 } git_vector;
 
 #define GIT_VECTOR_INIT {0}
 
 int git_vector_init(git_vector *v, size_t initial_size, git_vector_cmp cmp);
 void git_vector_free(git_vector *v);
+void git_vector_free_deep(git_vector *v); /* free each entry and self */
 void git_vector_clear(git_vector *v);
 int git_vector_dup(git_vector *v, const git_vector *src, git_vector_cmp cmp);
 void git_vector_swap(git_vector *a, git_vector *b);
 
+void **git_vector_detach(size_t *size, size_t *asize, git_vector *v);
+
 void git_vector_sort(git_vector *v);
 
 /** Linear search for matching entry using internal comparison function */
@@ -77,19 +85,33 @@ int git_vector_insert_sorted(git_vector *v, void *element,
 int git_vector_remove(git_vector *v, size_t idx);
 void git_vector_pop(git_vector *v);
 void git_vector_uniq(git_vector *v, void  (*git_free_cb)(void *));
+
 void git_vector_remove_matching(
-	git_vector *v, int (*match)(const git_vector *v, size_t idx));
+	git_vector *v,
+	int (*match)(const git_vector *v, size_t idx, void *payload),
+	void *payload);
 
 int git_vector_resize_to(git_vector *v, size_t new_length);
 int git_vector_set(void **old, git_vector *v, size_t position, void *value);
 
+/** Check if vector is sorted */
+#define git_vector_is_sorted(V) (((V)->flags & GIT_VECTOR_SORTED) != 0)
+
+/** Directly set sorted state of vector */
+#define git_vector_set_sorted(V,S) do { \
+	(V)->flags = (S) ? ((V)->flags | GIT_VECTOR_SORTED) : \
+		((V)->flags & ~GIT_VECTOR_SORTED); } while (0)
+
 /** Set the comparison function used for sorting the vector */
 GIT_INLINE(void) git_vector_set_cmp(git_vector *v, git_vector_cmp cmp)
 {
 	if (cmp != v->_cmp) {
 		v->_cmp = cmp;
-		v->sorted = 0;
+		git_vector_set_sorted(v, 0);
 	}
 }
 
+/* Just use this in tests, not for realz. returns -1 if not sorted */
+int git_vector_verify_sorted(const git_vector *v);
+
 #endif
diff --git a/src/win32/dir.c b/src/win32/dir.c
index f7859b73f..c7427ea54 100644
--- a/src/win32/dir.c
+++ b/src/win32/dir.c
@@ -7,29 +7,13 @@
 #define GIT__WIN32_NO_WRAP_DIR
 #include "posix.h"
 
-static int init_filter(char *filter, size_t n, const char *dir)
-{
-	size_t len = strlen(dir);
-
-	if (len+3 >= n)
-		return 0;
-
-	strcpy(filter, dir);
-	if (len && dir[len-1] != '/')
-		strcat(filter, "/");
-	strcat(filter, "*");
-
-	return 1;
-}
-
 git__DIR *git__opendir(const char *dir)
 {
-	git_win32_path_as_utf8 filter;
 	git_win32_path filter_w;
 	git__DIR *new = NULL;
 	size_t dirlen;
 
-	if (!dir || !init_filter(filter, sizeof(filter), dir))
+	if (!dir || !git_win32__findfirstfile_filter(filter_w, dir))
 		return NULL;
 
 	dirlen = strlen(dir);
@@ -39,7 +23,6 @@ git__DIR *git__opendir(const char *dir)
 		return NULL;
 	memcpy(new->dir, dir, dirlen);
 
-	git_win32_path_from_c(filter_w, filter);
 	new->h = FindFirstFileW(filter_w, &new->f);
 
 	if (new->h == INVALID_HANDLE_VALUE) {
@@ -72,10 +55,10 @@ int git__readdir_ext(
 		return -1;
 	}
 
-	if (wcslen(d->f.cFileName) >= sizeof(entry->d_name))
+	/* Convert the path to UTF-8 */
+	if (git_win32_path_to_utf8(entry->d_name, d->f.cFileName) < 0)
 		return -1;
 
-	git_win32_path_to_c(entry->d_name, d->f.cFileName);
 	entry->d_ino = 0;
 
 	*result = entry;
@@ -96,7 +79,6 @@ struct git__dirent *git__readdir(git__DIR *d)
 
 void git__rewinddir(git__DIR *d)
 {
-	git_win32_path_as_utf8 filter;
 	git_win32_path filter_w;
 
 	if (!d)
@@ -108,10 +90,9 @@ void git__rewinddir(git__DIR *d)
 		d->first = 0;
 	}
 
-	if (!init_filter(filter, sizeof(filter), d->dir))
+	if (!git_win32__findfirstfile_filter(filter_w, d->dir))
 		return;
 
-	git_win32_path_from_c(filter_w, filter);
 	d->h = FindFirstFileW(filter_w, &d->f);
 
 	if (d->h == INVALID_HANDLE_VALUE)
diff --git a/src/win32/dir.h b/src/win32/dir.h
index 24d48f6ba..bef39d774 100644
--- a/src/win32/dir.h
+++ b/src/win32/dir.h
@@ -8,10 +8,11 @@
 #define INCLUDE_dir_h__
 
 #include "common.h"
+#include "w32_util.h"
 
 struct git__dirent {
 	int d_ino;
-	git_win32_path_as_utf8 d_name;
+	git_win32_utf8_path d_name;
 };
 
 typedef struct {
diff --git a/src/win32/error.c b/src/win32/error.c
index bc598ae32..6b450093f 100644
--- a/src/win32/error.c
+++ b/src/win32/error.c
@@ -7,21 +7,17 @@
 
 #include "common.h"
 #include "error.h"
+#include "utf-conv.h"
 
 #ifdef GIT_WINHTTP
 # include <winhttp.h>
 #endif
 
-#ifndef WC_ERR_INVALID_CHARS
-#define WC_ERR_INVALID_CHARS	0x80
-#endif
-
 char *git_win32_get_error_message(DWORD error_code)
 {
 	LPWSTR lpMsgBuf = NULL;
 	HMODULE hModule = NULL;
 	char *utf8_msg = NULL;
-	int utf8_size;
 	DWORD dwFlags =
 		FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_IGNORE_INSERTS;
 
@@ -45,33 +41,11 @@ char *git_win32_get_error_message(DWORD error_code)
 	if (FormatMessageW(dwFlags, hModule, error_code,
 		MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
 		(LPWSTR)&lpMsgBuf, 0, NULL)) {
+		/* Convert the message to UTF-8. If this fails, we will
+		 * return NULL, which is a condition expected by the caller */
+		if (git__utf16_to_8_alloc(&utf8_msg, lpMsgBuf) < 0)
+			utf8_msg = NULL;
 
-		/* Invalid code point check supported on Vista+ only */
-		if (git_has_win32_version(6, 0, 0))
-			dwFlags = WC_ERR_INVALID_CHARS;
-		else
-			dwFlags = 0;
-
-		utf8_size = WideCharToMultiByte(CP_UTF8, dwFlags,
-			lpMsgBuf, -1, NULL, 0, NULL, NULL);
-
-		if (!utf8_size) {
-			assert(0);
-			goto on_error;
-		}
-
-		utf8_msg = git__malloc(utf8_size);
-
-		if (!utf8_msg)
-			goto on_error;
-
-		if (!WideCharToMultiByte(CP_UTF8, dwFlags,
-			lpMsgBuf, -1, utf8_msg, utf8_size, NULL, NULL)) {
-			git__free(utf8_msg);
-			goto on_error;
-		}
-
-on_error:
 		LocalFree(lpMsgBuf);
 	}
 
diff --git a/src/win32/findfile.c b/src/win32/findfile.c
index a9e812e28..86d4ef5bd 100644
--- a/src/win32/findfile.c
+++ b/src/win32/findfile.c
@@ -17,54 +17,34 @@
 #define REG_MSYSGIT_INSTALL L"SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Git_is1"
 #endif
 
-int git_win32__expand_path(struct git_win32__path *s_root, const wchar_t *templ)
-{
-	s_root->len = ExpandEnvironmentStringsW(templ, s_root->path, MAX_PATH);
-	return s_root->len ? 0 : -1;
-}
+typedef struct {
+	git_win32_path path;
+	DWORD len;
+} _findfile_path;
 
-static int win32_path_to_8(git_buf *path_utf8, const wchar_t *path)
+static int git_win32__expand_path(_findfile_path *dest, const wchar_t *src)
 {
-	char temp_utf8[GIT_PATH_MAX];
+	dest->len = ExpandEnvironmentStringsW(src, dest->path, ARRAY_SIZE(dest->path));
 
-	git__utf16_to_8(temp_utf8, GIT_PATH_MAX, path);
-	git_path_mkposix(temp_utf8);
+	if (!dest->len || dest->len > ARRAY_SIZE(dest->path))
+		return -1;
 
-	return git_buf_sets(path_utf8, temp_utf8);
+	return 0;
 }
 
-int git_win32__find_file(
-	git_buf *path, const struct git_win32__path *root, const char *filename)
+static int win32_path_to_8(git_buf *dest, const wchar_t *src)
 {
-	size_t len, alloc_len;
-	wchar_t *file_utf16 = NULL;
-
-	if (!root || !filename || (len = strlen(filename)) == 0)
-		return GIT_ENOTFOUND;
-
-	/* allocate space for wchar_t path to file */
-	alloc_len = root->len + len + 2;
-	file_utf16 = git__calloc(alloc_len, sizeof(wchar_t));
-	GITERR_CHECK_ALLOC(file_utf16);
+	git_win32_utf8_path utf8_path;
 
-	/* append root + '\\' + filename as wchar_t */
-	memcpy(file_utf16, root->path, root->len * sizeof(wchar_t));
-
-	if (*filename == '/' || *filename == '\\')
-		filename++;
-
-	git__utf8_to_16(file_utf16 + root->len - 1, alloc_len - root->len, filename);
-
-	/* check access */
-	if (_waccess(file_utf16, F_OK) < 0) {
-		git__free(file_utf16);
-		return GIT_ENOTFOUND;
+	if (git_win32_path_to_utf8(utf8_path, src) < 0) {
+		giterr_set(GITERR_OS, "Unable to convert path to UTF-8");
+		return -1;
 	}
 
-	win32_path_to_8(path, file_utf16);
-	git__free(file_utf16);
+	/* Convert backslashes to forward slashes */
+	git_path_mkposix(utf8_path);
 
-	return 0;
+	return git_buf_sets(dest, utf8_path);
 }
 
 static wchar_t* win32_walkpath(wchar_t *path, wchar_t *buf, size_t buflen)
@@ -89,7 +69,7 @@ static wchar_t* win32_walkpath(wchar_t *path, wchar_t *buf, size_t buflen)
 static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe, const wchar_t *subdir)
 {
 	wchar_t *env = _wgetenv(L"PATH"), lastch;
-	struct git_win32__path root;
+	_findfile_path root;
 	size_t gitexe_len = wcslen(gitexe);
 
 	if (!env)
@@ -122,43 +102,44 @@ static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe, const wch
 }
 
 static int win32_find_git_in_registry(
-	git_buf *buf, const HKEY hieve, const wchar_t *key, const wchar_t *subdir)
+	git_buf *buf, const HKEY hive, const wchar_t *key, const wchar_t *subdir)
 {
 	HKEY hKey;
-	DWORD dwType = REG_SZ;
-	struct git_win32__path path16;
+	int error = GIT_ENOTFOUND;
 
 	assert(buf);
 
-	path16.len = MAX_PATH;
+	if (!RegOpenKeyExW(hive, key, 0, KEY_READ, &hKey)) {
+		DWORD dwType, cbData;
+		git_win32_path path;
 
-	if (RegOpenKeyExW(hieve, key, 0, KEY_READ, &hKey) == ERROR_SUCCESS) {
-		if (RegQueryValueExW(hKey, L"InstallLocation", NULL, &dwType,
-			(LPBYTE)&path16.path, &path16.len) == ERROR_SUCCESS)
-		{
-			/* InstallLocation points to the root of the git directory */
+		/* Ensure that the buffer is big enough to have the suffix attached
+		 * after we receive the result. */
+		cbData = (DWORD)(sizeof(path) - wcslen(subdir) * sizeof(wchar_t));
 
-			if (path16.len + 4 > MAX_PATH) { /* 4 = wcslen(L"etc\\") */
-				giterr_set(GITERR_OS, "Cannot locate git - path too long");
-				return -1;
-			}
+		/* InstallLocation points to the root of the git directory */
+		if (!RegQueryValueExW(hKey, L"InstallLocation", NULL, &dwType, (LPBYTE)path, &cbData) &&
+			dwType == REG_SZ) {
 
-			wcscat(path16.path, subdir);
-			path16.len += 4;
+			/* Append the suffix */
+			wcscat(path, subdir);
 
-			win32_path_to_8(buf, path16.path);
+			/* Convert to UTF-8, with forward slashes, and output the path
+			 * to the provided buffer */
+			if (!win32_path_to_8(buf, path))
+				error = 0;
 		}
 
 		RegCloseKey(hKey);
 	}
 
-	return path16.len ? 0 : GIT_ENOTFOUND;
+	return error;
 }
 
 static int win32_find_existing_dirs(
 	git_buf *out, const wchar_t *tmpl[])
 {
-	struct git_win32__path path16;
+	_findfile_path path16;
 	git_buf buf = GIT_BUF_INIT;
 
 	git_buf_clear(out);
diff --git a/src/win32/findfile.h b/src/win32/findfile.h
index 11bf7e620..a50319b9a 100644
--- a/src/win32/findfile.h
+++ b/src/win32/findfile.h
@@ -8,17 +8,6 @@
 #ifndef INCLUDE_git_findfile_h__
 #define INCLUDE_git_findfile_h__
 
-struct git_win32__path {
-	wchar_t path[MAX_PATH];
-	DWORD len;
-};
-
-extern int git_win32__expand_path(
-	struct git_win32__path *s_root, const wchar_t *templ);
-
-extern int git_win32__find_file(
-	git_buf *path, const struct git_win32__path *root, const char *filename);
-
 extern int git_win32__find_system_dirs(git_buf *out, const wchar_t *subpath);
 extern int git_win32__find_global_dirs(git_buf *out);
 extern int git_win32__find_xdg_dirs(git_buf *out);
diff --git a/src/win32/map.c b/src/win32/map.c
index 44c6c4e2e..ef83f882e 100644
--- a/src/win32/map.c
+++ b/src/win32/map.c
@@ -8,6 +8,7 @@
 #include "map.h"
 #include <errno.h>
 
+#ifndef NO_MMAP
 
 static DWORD get_page_size(void)
 {
@@ -22,6 +23,11 @@ static DWORD get_page_size(void)
 	return page_size;
 }
 
+long git__page_size(void)
+{
+	return (long)get_page_size();
+}
+
 int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
 {
 	HANDLE fh = (HANDLE)_get_osfhandle(fd);
@@ -112,4 +118,4 @@ int p_munmap(git_map *map)
 	return error;
 }
 
-
+#endif
diff --git a/src/win32/mingw-compat.h b/src/win32/mingw-compat.h
index fe0abfb54..059e39cbc 100644
--- a/src/win32/mingw-compat.h
+++ b/src/win32/mingw-compat.h
@@ -10,8 +10,11 @@
 #if defined(__MINGW32__)
 
 /* use a 64-bit file offset type */
+# undef lseek
 # define lseek _lseeki64
+# undef stat
 # define stat _stati64
+# undef fstat
 # define fstat _fstati64
 
 /* stat: file mode type testing macros */
@@ -19,11 +22,6 @@
 # define S_IFLNK _S_IFLNK
 # define S_ISLNK(m) (((m) & _S_IFMT) == _S_IFLNK)
 
-GIT_INLINE(size_t) p_strnlen(const char *s, size_t maxlen) {
-	const char *end = memchr(s, 0, maxlen);
-	return end ? (size_t)(end - s) : maxlen;
-}
-
 #endif
 
 #endif /* INCLUDE_mingw_compat__ */
diff --git a/src/win32/posix.h b/src/win32/posix.h
index 24cba23e0..2cbea1807 100644
--- a/src/win32/posix.h
+++ b/src/win32/posix.h
@@ -19,6 +19,12 @@
 # define EAFNOSUPPORT (INT_MAX-1)
 #endif
 
+#ifdef _MSC_VER
+# define p_ftruncate(fd, sz) _chsize_s(fd, sz)
+#else  /* MinGW */
+# define p_ftruncate(fd, sz) _chsize(fd, sz)
+#endif
+
 GIT_INLINE(int) p_link(const char *old, const char *new)
 {
 	GIT_UNUSED(old);
@@ -27,24 +33,15 @@ GIT_INLINE(int) p_link(const char *old, const char *new)
 	return -1;
 }
 
-GIT_INLINE(int) p_mkdir(const char *path, mode_t mode)
-{
-	git_win32_path buf;
-	GIT_UNUSED(mode);
-	git_win32_path_from_c(buf, path);
-	return _wmkdir(buf);
-}
-
+extern int p_mkdir(const char *path, mode_t mode);
 extern int p_unlink(const char *path);
 extern int p_lstat(const char *file_name, struct stat *buf);
-extern int p_readlink(const char *link, char *target, size_t target_len);
+extern int p_readlink(const char *path, char *buf, size_t bufsiz);
 extern int p_symlink(const char *old, const char *new);
-extern int p_hide_directory__w32(const char *path);
 extern char *p_realpath(const char *orig_path, char *buffer);
 extern int p_vsnprintf(char *buffer, size_t count, const char *format, va_list argptr);
 extern int p_snprintf(char *buffer, size_t count, const char *format, ...) GIT_FORMAT_PRINTF(3, 4);
 extern int p_mkstemp(char *tmp_path);
-extern int p_setenv(const char* name, const char* value, int overwrite);
 extern int p_stat(const char* path, struct stat* buf);
 extern int p_chdir(const char* path);
 extern int p_chmod(const char* path, mode_t mode);
diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c
index 18f717b0f..34938431a 100644
--- a/src/win32/posix_w32.c
+++ b/src/win32/posix_w32.c
@@ -9,17 +9,74 @@
 #include "path.h"
 #include "utf-conv.h"
 #include "repository.h"
+#include "reparse.h"
 #include <errno.h>
 #include <io.h>
 #include <fcntl.h>
 #include <ws2tcpip.h>
 
+#ifndef FILE_NAME_NORMALIZED
+# define FILE_NAME_NORMALIZED 0
+#endif
+
+/* Options which we always provide to _wopen.
+ *
+ * _O_BINARY - Raw access; no translation of CR or LF characters
+ * _O_NOINHERIT - Do not mark the created handle as inheritable by child processes.
+ *    The Windows default is 'not inheritable', but the CRT's default (following
+ *    POSIX convention) is 'inheritable'. We have no desire for our handles to be
+ *    inheritable on Windows, so specify the flag to get default behavior back. */
+#define STANDARD_OPEN_FLAGS (_O_BINARY | _O_NOINHERIT)
+
+/* GetFinalPathNameByHandleW signature */
+typedef DWORD(WINAPI *PFGetFinalPathNameByHandleW)(HANDLE, LPWSTR, DWORD, DWORD);
+
+/* Helper function which converts UTF-8 paths to UTF-16.
+ * On failure, errno is set. */
+static int utf8_to_16_with_errno(git_win32_path dest, const char *src)
+{
+	int len = git_win32_path_from_utf8(dest, src);
+
+	if (len < 0) {
+		if (GetLastError() == ERROR_INSUFFICIENT_BUFFER)
+			errno = ENAMETOOLONG;
+		else
+			errno = EINVAL; /* Bad code point, presumably */
+	}
+
+	return len;
+}
+
+int p_mkdir(const char *path, mode_t mode)
+{
+	git_win32_path buf;
+
+	GIT_UNUSED(mode);
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
+
+	return _wmkdir(buf);
+}
+
 int p_unlink(const char *path)
 {
 	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
-	_wchmod(buf, 0666);
-	return _wunlink(buf);
+	int error;
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
+
+	error = _wunlink(buf);
+
+	/* If the file could not be deleted because it was
+	 * read-only, clear the bit and try again */
+	if (error == -1 && errno == EACCES) {
+		_wchmod(buf, 0666);
+		error = _wunlink(buf);
+	}
+
+	return error;
 }
 
 int p_fsync(int fd)
@@ -53,28 +110,79 @@ GIT_INLINE(time_t) filetime_to_time_t(const FILETIME *ft)
 	return (time_t)winTime;
 }
 
+/* On success, returns the length, in characters, of the path stored in dest.
+ * On failure, returns a negative value. */
+static int readlink_w(
+	git_win32_path dest,
+	const git_win32_path path)
+{
+	BYTE buf[MAXIMUM_REPARSE_DATA_BUFFER_SIZE];
+	GIT_REPARSE_DATA_BUFFER *reparse_buf = (GIT_REPARSE_DATA_BUFFER *)buf;
+	HANDLE handle = NULL;
+	DWORD ioctl_ret;
+	wchar_t *target;
+	size_t target_len;
+
+	int error = -1;
+
+	handle = CreateFileW(path, GENERIC_READ,
+		FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
+		FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, NULL);
+
+	if (handle == INVALID_HANDLE_VALUE) {
+		errno = ENOENT;
+		return -1;
+	}
+
+	if (!DeviceIoControl(handle, FSCTL_GET_REPARSE_POINT, NULL, 0,
+		reparse_buf, sizeof(buf), &ioctl_ret, NULL)) {
+		errno = EINVAL;
+		goto on_error;
+	}
+
+	switch (reparse_buf->ReparseTag) {
+	case IO_REPARSE_TAG_SYMLINK:
+		target = reparse_buf->SymbolicLinkReparseBuffer.PathBuffer +
+			(reparse_buf->SymbolicLinkReparseBuffer.SubstituteNameOffset / sizeof(WCHAR));
+		target_len = reparse_buf->SymbolicLinkReparseBuffer.SubstituteNameLength / sizeof(WCHAR);
+		break;
+	case IO_REPARSE_TAG_MOUNT_POINT:
+		target = reparse_buf->MountPointReparseBuffer.PathBuffer +
+			(reparse_buf->MountPointReparseBuffer.SubstituteNameOffset / sizeof(WCHAR));
+		target_len = reparse_buf->MountPointReparseBuffer.SubstituteNameLength / sizeof(WCHAR);
+		break;
+	default:
+		errno = EINVAL;
+		goto on_error;
+	}
+
+	if (target_len) {
+		/* The path may need to have a prefix removed. */
+		target_len = git_win32__canonicalize_path(target, target_len);
+
+		/* Need one additional character in the target buffer
+		 * for the terminating NULL. */
+		if (GIT_WIN_PATH_UTF16 > target_len) {
+			wcscpy(dest, target);
+			error = (int)target_len;
+		}
+	}
+
+on_error:
+	CloseHandle(handle);
+	return error;
+}
+
 #define WIN32_IS_WSEP(CH) ((CH) == L'/' || (CH) == L'\\')
 
-static int do_lstat(
-	const char *file_name, struct stat *buf, int posix_enotdir)
+static int lstat_w(
+	wchar_t *path,
+	struct stat *buf,
+	bool posix_enotdir)
 {
 	WIN32_FILE_ATTRIBUTE_DATA fdata;
-	git_win32_path fbuf;
-	wchar_t lastch;
-	int flen;
-
-	flen = git_win32_path_from_c(fbuf, file_name);
-
-	/* truncate trailing slashes */
-	for (; flen > 0; --flen) {
-		lastch = fbuf[flen - 1];
-		if (WIN32_IS_WSEP(lastch))
-			fbuf[flen - 1] = L'\0';
-		else if (lastch != L'\0')
-			break;
-	}
 
-	if (GetFileAttributesExW(fbuf, GetFileExInfoStandard, &fdata)) {
+	if (GetFileAttributesExW(path, GetFileExInfoStandard, &fdata)) {
 		int fMode = S_IREAD;
 
 		if (!buf)
@@ -88,12 +196,6 @@ static int do_lstat(
 		if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_READONLY))
 			fMode |= S_IWRITE;
 
-		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
-			fMode |= S_IFLNK;
-
-		if ((fMode & (S_IFDIR | S_IFLNK)) == (S_IFDIR | S_IFLNK)) // junction
-			fMode ^= S_IFLNK;
-
 		buf->st_ino = 0;
 		buf->st_gid = 0;
 		buf->st_uid = 0;
@@ -105,19 +207,17 @@ static int do_lstat(
 		buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime));
 		buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime));
 
-		/* Windows symlinks have zero file size, call readlink to determine
-		 * the length of the path pointed to, which we expect everywhere else
-		 */
-		if (S_ISLNK(fMode)) {
-			git_win32_path_as_utf8 target;
-			int readlink_result;
-
-			readlink_result = p_readlink(file_name, target, sizeof(target));
+		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+			git_win32_path target;
 
-			if (readlink_result == -1)
-				return -1;
+			if (readlink_w(target, path) >= 0) {
+				buf->st_mode = (buf->st_mode & ~S_IFMT) | S_IFLNK;
 
-			buf->st_size = strlen(target);
+				/* st_size gets the UTF-8 length of the target name, in bytes,
+				 * not counting the NULL terminator */
+				if ((buf->st_size = git__utf16_to_8(NULL, 0, target)) < 0)
+					return -1;
+			}
 		}
 
 		return 0;
@@ -129,18 +229,23 @@ static int do_lstat(
 	 * file path is a regular file, otherwise set ENOENT.
 	 */
 	if (posix_enotdir) {
+		size_t path_len = wcslen(path);
+
 		/* scan up path until we find an existing item */
 		while (1) {
+			DWORD attrs;
+
 			/* remove last directory component */
-			for (--flen; flen > 0 && !WIN32_IS_WSEP(fbuf[flen]); --flen);
+			for (path_len--; path_len > 0 && !WIN32_IS_WSEP(path[path_len]); path_len--);
 
-			if (flen <= 0)
+			if (path_len <= 0)
 				break;
 
-			fbuf[flen] = L'\0';
+			path[path_len] = L'\0';
+			attrs = GetFileAttributesW(path);
 
-			if (GetFileAttributesExW(fbuf, GetFileExInfoStandard, &fdata)) {
-				if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
+			if (attrs != INVALID_FILE_ATTRIBUTES) {
+				if (!(attrs & FILE_ATTRIBUTE_DIRECTORY))
 					errno = ENOTDIR;
 				break;
 			}
@@ -150,108 +255,51 @@ static int do_lstat(
 	return -1;
 }
 
+static int do_lstat(const char *path, struct stat *buf, bool posixly_correct)
+{
+	git_win32_path path_w;
+	int len;
+
+	if ((len = utf8_to_16_with_errno(path_w, path)) < 0)
+		return -1;
+
+	git_win32__path_trim_end(path_w, len);
+
+	return lstat_w(path_w, buf, posixly_correct);
+}
+
 int p_lstat(const char *filename, struct stat *buf)
 {
-	return do_lstat(filename, buf, 0);
+	return do_lstat(filename, buf, false);
 }
 
 int p_lstat_posixly(const char *filename, struct stat *buf)
 {
-	return do_lstat(filename, buf, 1);
+	return do_lstat(filename, buf, true);
 }
 
-
-/*
- * Parts of the The p_readlink function are heavily inspired by the php 
- * readlink function in link_win32.c
- *
- * Copyright (c) 1999 - 2012 The PHP Group. All rights reserved.
- *
- * For details of the PHP license see http://www.php.net/license/3_01.txt
- */
-int p_readlink(const char *link, char *target, size_t target_len)
+int p_readlink(const char *path, char *buf, size_t bufsiz)
 {
-	typedef DWORD (WINAPI *fpath_func)(HANDLE, LPWSTR, DWORD, DWORD);
-	static fpath_func pGetFinalPath = NULL;
-	HANDLE hFile;
-	DWORD dwRet;
-	git_win32_path link_w;
-	wchar_t* target_w;
-	int error = 0;
-
-	assert(link && target && target_len > 0);
-
-	/*
-	 * Try to load the pointer to pGetFinalPath dynamically, because
-	 * it is not available in platforms older than Vista
-	 */
-	if (pGetFinalPath == NULL) {
-		HMODULE module = GetModuleHandle("kernel32");
-
-		if (module != NULL)
-			pGetFinalPath = (fpath_func)GetProcAddress(module, "GetFinalPathNameByHandleW");
-
-		if (pGetFinalPath == NULL) {
-			giterr_set(GITERR_OS,
-				"'GetFinalPathNameByHandleW' is not available in this platform");
-			return -1;
-		}
-	}
-
-	git_win32_path_from_c(link_w, link);
+	git_win32_path path_w, target_w;
+	git_win32_utf8_path target;
+	int len;
 
-	hFile = CreateFileW(link_w,			// file to open
-			GENERIC_READ,			// open for reading
-			FILE_SHARE_READ,		// share for reading
-			NULL,					// default security
-			OPEN_EXISTING,			// existing file only
-			FILE_FLAG_BACKUP_SEMANTICS, // normal file
-			NULL);					// no attr. template
+	/* readlink(2) does not NULL-terminate the string written
+	 * to the target buffer. Furthermore, the target buffer need
+	 * not be large enough to hold the entire result. A truncated
+	 * result should be written in this case. Since this truncation
+	 * could occur in the middle of the encoding of a code point,
+	 * we need to buffer the result on the stack. */
 
-	if (hFile == INVALID_HANDLE_VALUE) {
-		giterr_set(GITERR_OS, "Cannot open '%s' for reading", link);
+	if (utf8_to_16_with_errno(path_w, path) < 0 ||
+		readlink_w(target_w, path_w) < 0 ||
+		(len = git_win32_path_to_utf8(target, target_w)) < 0)
 		return -1;
-	}
-
-	target_w = (wchar_t*)git__malloc(target_len * sizeof(wchar_t));
-	GITERR_CHECK_ALLOC(target_w);
 
-	dwRet = pGetFinalPath(hFile, target_w, (DWORD)target_len, 0x0);
-	if (dwRet == 0 ||
-		dwRet >= target_len ||
-		!WideCharToMultiByte(CP_UTF8, 0, target_w, -1, target,
-			(int)(target_len * sizeof(char)), NULL, NULL))
-		error = -1;
+	bufsiz = min((size_t)len, bufsiz);
+	memcpy(buf, target, bufsiz);
 
-	git__free(target_w);
-	CloseHandle(hFile);
-
-	if (error)
-		return error;
-
-	/* Skip first 4 characters if they are "\\?\" */
-	if (dwRet > 4 &&
-		target[0] == '\\' && target[1] == '\\' &&
-		target[2] == '?' && target[3] == '\\')
-	{
-		unsigned int offset = 4;
-		dwRet -= 4;
-
-		/* \??\UNC\ */
-		if (dwRet > 7 &&
-			target[4] == 'U' && target[5] == 'N' && target[6] == 'C')
-		{
-			offset += 2;
-			dwRet -= 2;
-			target[offset] = '\\';
-		}
-
-		memmove(target, target + offset, dwRet);
-	}
-
-	target[dwRet] = '\0';
-
-	return dwRet;
+	return (int)bufsiz;
 }
 
 int p_symlink(const char *old, const char *new)
@@ -267,7 +315,8 @@ int p_open(const char *path, int flags, ...)
 	git_win32_path buf;
 	mode_t mode = 0;
 
-	git_win32_path_from_c(buf, path);
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
 
 	if (flags & O_CREAT) {
 		va_list arg_list;
@@ -277,131 +326,217 @@ int p_open(const char *path, int flags, ...)
 		va_end(arg_list);
 	}
 
-	return _wopen(buf, flags | _O_BINARY, mode);
+	return _wopen(buf, flags | STANDARD_OPEN_FLAGS, mode);
 }
 
 int p_creat(const char *path, mode_t mode)
 {
 	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
-	return _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, mode);
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
+
+	return _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | STANDARD_OPEN_FLAGS, mode);
 }
 
 int p_getcwd(char *buffer_out, size_t size)
 {
-	int ret;
-	wchar_t *buf;
+	git_win32_path buf;
+	wchar_t *cwd = _wgetcwd(buf, GIT_WIN_PATH_UTF16);
 
-	if ((size_t)((int)size) != size)
+	if (!cwd)
 		return -1;
 
-	buf = (wchar_t*)git__malloc(sizeof(wchar_t) * (int)size);
-	GITERR_CHECK_ALLOC(buf);
+	/* Convert the working directory back to UTF-8 */
+	if (git__utf16_to_8(buffer_out, size, cwd) < 0) {
+		DWORD code = GetLastError();
 
-	_wgetcwd(buf, (int)size);
+		if (code == ERROR_INSUFFICIENT_BUFFER)
+			errno = ERANGE;
+		else
+			errno = EINVAL;
 
-	ret = WideCharToMultiByte(
-		CP_UTF8, 0, buf, -1, buffer_out, (int)size, NULL, NULL);
+		return -1;
+	}
 
-	git__free(buf);
-	return !ret ? -1 : 0;
+	return 0;
+}
+
+/*
+ * Returns the address of the GetFinalPathNameByHandleW function.
+ * This function is available on Windows Vista and higher.
+ */
+static PFGetFinalPathNameByHandleW get_fpnbyhandle(void)
+{
+	static PFGetFinalPathNameByHandleW pFunc = NULL;
+	PFGetFinalPathNameByHandleW toReturn = pFunc;
+
+	if (!toReturn) {
+		HMODULE hModule = GetModuleHandleW(L"kernel32");
+
+		if (hModule)
+			toReturn = (PFGetFinalPathNameByHandleW)GetProcAddress(hModule, "GetFinalPathNameByHandleW");
+
+		pFunc = toReturn;
+	}
+
+	assert(toReturn);
+
+	return toReturn;
+}
+
+static int getfinalpath_w(
+	git_win32_path dest,
+	const wchar_t *path)
+{
+	PFGetFinalPathNameByHandleW pgfp = get_fpnbyhandle();
+	HANDLE hFile;
+	DWORD dwChars;
+
+	if (!pgfp)
+		return -1;
+
+	/* Use FILE_FLAG_BACKUP_SEMANTICS so we can open a directory. Do not
+	* specify FILE_FLAG_OPEN_REPARSE_POINT; we want to open a handle to the
+	* target of the link. */
+	hFile = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE,
+		NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+
+	if (INVALID_HANDLE_VALUE == hFile)
+		return -1;
+
+	/* Call GetFinalPathNameByHandle */
+	dwChars = pgfp(hFile, dest, GIT_WIN_PATH_UTF16, FILE_NAME_NORMALIZED);
+	CloseHandle(hFile);
+
+	if (!dwChars || dwChars >= GIT_WIN_PATH_UTF16)
+		return -1;
+
+	/* The path may be delivered to us with a prefix; canonicalize */
+	return (int)git_win32__canonicalize_path(dest, dwChars);
+}
+
+static int follow_and_lstat_link(git_win32_path path, struct stat* buf)
+{
+	git_win32_path target_w;
+
+	if (getfinalpath_w(target_w, path) < 0)
+		return -1;
+
+	return lstat_w(target_w, buf, false);
 }
 
 int p_stat(const char* path, struct stat* buf)
 {
-	git_win32_path_as_utf8 target;
-	int error = 0;
+	git_win32_path path_w;
+	int len;
 
-	error = do_lstat(path, buf, 0);
+	if ((len = utf8_to_16_with_errno(path_w, path)) < 0)
+		return -1;
 
-	/* We need not do this in a loop to unwind chains of symlinks since
-	 * p_readlink calls GetFinalPathNameByHandle which does it for us. */
-	if (error >= 0 && S_ISLNK(buf->st_mode) &&
-		(error = p_readlink(path, target, sizeof(target))) >= 0)
-		error = do_lstat(target, buf, 0);
+	git_win32__path_trim_end(path_w, len);
 
-	return error;
+	if (lstat_w(path_w, buf, false) < 0)
+		return -1;
+
+	/* The item is a symbolic link or mount point. No need to iterate
+	 * to follow multiple links; use GetFinalPathNameFromHandle. */
+	if (S_ISLNK(buf->st_mode))
+		return follow_and_lstat_link(path_w, buf);
+
+	return 0;
 }
 
 int p_chdir(const char* path)
 {
 	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
+
 	return _wchdir(buf);
 }
 
 int p_chmod(const char* path, mode_t mode)
 {
 	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
+
 	return _wchmod(buf, mode);
 }
 
 int p_rmdir(const char* path)
 {
-	int error;
 	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
+	int error;
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
 
 	error = _wrmdir(buf);
 
-	/* _wrmdir() is documented to return EACCES if "A program has an open
-	 * handle to the directory."  This sounds like what everybody else calls
-	 * EBUSY.  Let's convert appropriate error codes.
-	 */
-	if (GetLastError() == ERROR_SHARING_VIOLATION)
-		errno = EBUSY;
+	if (error == -1) {
+		switch (GetLastError()) {
+			/* _wrmdir() is documented to return EACCES if "A program has an open
+			 * handle to the directory."  This sounds like what everybody else calls
+			 * EBUSY.  Let's convert appropriate error codes.
+			 */
+			case ERROR_SHARING_VIOLATION:
+				errno = EBUSY;
+				break;
 
-	return error;
-}
+			/* This error can be returned when trying to rmdir an extant file. */
+			case ERROR_DIRECTORY:
+				errno = ENOTDIR;
+				break;
+		}
+	}
 
-int p_hide_directory__w32(const char *path)
-{
-	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
-	return (SetFileAttributesW(buf, FILE_ATTRIBUTE_HIDDEN) != 0) ? 0 : -1;
+	return error;
 }
 
 char *p_realpath(const char *orig_path, char *buffer)
 {
-	int ret;
-	git_win32_path orig_path_w;
-	git_win32_path buffer_w;
+	git_win32_path orig_path_w, buffer_w;
 
-	git_win32_path_from_c(orig_path_w, orig_path);
+	if (utf8_to_16_with_errno(orig_path_w, orig_path) < 0)
+		return NULL;
 
-	/* Implicitly use GetCurrentDirectory which can be a threading issue */
-	ret = GetFullPathNameW(orig_path_w, GIT_WIN_PATH_UTF16, buffer_w, NULL);
+	/* Note that if the path provided is a relative path, then the current directory
+	 * is used to resolve the path -- which is a concurrency issue because the current
+	 * directory is a process-wide variable. */
+	if (!GetFullPathNameW(orig_path_w, GIT_WIN_PATH_UTF16, buffer_w, NULL)) {
+		if (GetLastError() == ERROR_INSUFFICIENT_BUFFER)
+			errno = ENAMETOOLONG;
+		else
+			errno = EINVAL;
 
-	/* According to MSDN, a return value equals to zero means a failure. */
-	if (ret == 0 || ret > GIT_WIN_PATH_UTF16)
-		buffer = NULL;
+		return NULL;
+	}
 
-	else if (GetFileAttributesW(buffer_w) == INVALID_FILE_ATTRIBUTES) {
-		buffer = NULL;
+	/* The path must exist. */
+	if (GetFileAttributesW(buffer_w) == INVALID_FILE_ATTRIBUTES) {
 		errno = ENOENT;
+		return NULL;
 	}
 
-	else if (buffer == NULL) {
-		int buffer_sz = WideCharToMultiByte(
-			CP_UTF8, 0, buffer_w, -1, NULL, 0, NULL, NULL);
-
-		if (!buffer_sz ||
-			!(buffer = (char *)git__malloc(buffer_sz)) ||
-			!WideCharToMultiByte(
-				CP_UTF8, 0, buffer_w, -1, buffer, buffer_sz, NULL, NULL))
-		{
-			git__free(buffer);
-			buffer = NULL;
-		}
+	/* Convert the path to UTF-8. */
+	if (buffer) {
+		/* If the caller provided a buffer, then it is assumed to be GIT_WIN_PATH_UTF8
+		 * characters in size. If it isn't, then we may overflow. */
+		if (git__utf16_to_8(buffer, GIT_WIN_PATH_UTF8, buffer_w) < 0)
+			return NULL;
+	} else {
+		/* If the caller did not provide a buffer, then we allocate one for the caller
+		 * from the heap. */
+		if (git__utf16_to_8_alloc(&buffer, buffer_w) < 0)
+			return NULL;
 	}
 
-	else if (!WideCharToMultiByte(
-		CP_UTF8, 0, buffer_w, -1, buffer, GIT_PATH_MAX, NULL, NULL))
-		buffer = NULL;
-
-	if (buffer)
-		git_path_mkposix(buffer);
+	/* Convert backslashes to forward slashes */
+	git_path_mkposix(buffer);
 
 	return buffer;
 }
@@ -433,8 +568,6 @@ int p_snprintf(char *buffer, size_t count, const char *format, ...)
 	return r;
 }
 
-extern int p_creat(const char *path, mode_t mode);
-
 int p_mkstemp(char *tmp_path)
 {
 #if defined(_MSC_VER)
@@ -445,21 +578,16 @@ int p_mkstemp(char *tmp_path)
 		return -1;
 #endif
 
-	return p_creat(tmp_path, 0744); //-V536
-}
-
-int p_setenv(const char* name, const char* value, int overwrite)
-{
-	if (overwrite != 1)
-		return -1;
-
-	return (SetEnvironmentVariableA(name, value) == 0 ? -1 : 0);
+	return p_open(tmp_path, O_RDWR | O_CREAT | O_EXCL, 0744); //-V536
 }
 
 int p_access(const char* path, mode_t mode)
 {
 	git_win32_path buf;
-	git_win32_path_from_c(buf, path);
+
+	if (utf8_to_16_with_errno(buf, path) < 0)
+		return -1;
+
 	return _waccess(buf, mode);
 }
 
@@ -467,10 +595,32 @@ int p_rename(const char *from, const char *to)
 {
 	git_win32_path wfrom;
 	git_win32_path wto;
+	int rename_tries;
+	int rename_succeeded;
+	int error;
 
-	git_win32_path_from_c(wfrom, from);
-	git_win32_path_from_c(wto, to);
-	return MoveFileExW(wfrom, wto, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? 0 : -1;
+	if (utf8_to_16_with_errno(wfrom, from) < 0 ||
+		utf8_to_16_with_errno(wto, to) < 0)
+		return -1;
+	
+	/* wait up to 50ms if file is locked by another thread or process */
+	rename_tries = 0;
+	rename_succeeded = 0;
+	while (rename_tries < 10) {
+		if (MoveFileExW(wfrom, wto, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) != 0) {
+			rename_succeeded = 1;
+			break;
+		}
+		
+		error = GetLastError();
+		if (error == ERROR_SHARING_VIOLATION || error == ERROR_ACCESS_DENIED) {
+			Sleep(5);
+			rename_tries++;
+		} else
+			break;
+	}
+	
+	return rename_succeeded ? 0 : -1;
 }
 
 int p_recv(GIT_SOCKET socket, void *buffer, size_t length, int flags)
diff --git a/src/win32/precompiled.h b/src/win32/precompiled.h
index cbfe98812..33ce106d3 100644
--- a/src/win32/precompiled.h
+++ b/src/win32/precompiled.h
@@ -1,6 +1,3 @@
-#include "git2.h"
-#include "common.h"
-
 #include <assert.h>
 #include <errno.h>
 #include <limits.h>
@@ -9,6 +6,7 @@
 #include <string.h>
 #include <fcntl.h>
 #include <time.h>
+#include <stdarg.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -20,3 +18,6 @@
 #ifdef GIT_THREADS
  #include "win32/pthread.h"
 #endif
+
+#include "git2.h"
+#include "common.h"
diff --git a/src/win32/pthread.c b/src/win32/pthread.c
index db8927471..ec45ecbe5 100644
--- a/src/win32/pthread.c
+++ b/src/win32/pthread.c
@@ -8,32 +8,64 @@
 #include "pthread.h"
 #include "../global.h"
 
-int pthread_create(
-	pthread_t *GIT_RESTRICT thread,
+#define CLEAN_THREAD_EXIT 0x6F012842
+
+/* The thread procedure stub used to invoke the caller's procedure
+ * and capture the return value for later collection. Windows will
+ * only hold a DWORD, but we need to be able to store an entire
+ * void pointer. This requires the indirection. */
+static DWORD WINAPI git_win32__threadproc(LPVOID lpParameter)
+{
+	git_win32_thread *thread = lpParameter;
+
+	thread->result = thread->proc(thread->param);
+
+	return CLEAN_THREAD_EXIT;
+}
+
+int git_win32__thread_create(
+	git_win32_thread *GIT_RESTRICT thread,
 	const pthread_attr_t *GIT_RESTRICT attr,
 	void *(*start_routine)(void*),
 	void *GIT_RESTRICT arg)
 {
 	GIT_UNUSED(attr);
-	*thread = CreateThread(
-		NULL, 0, (LPTHREAD_START_ROUTINE)start_routine, arg, 0, NULL);
-	return *thread ? 0 : -1;
+
+	thread->result = NULL;
+	thread->param = arg;
+	thread->proc = start_routine;
+	thread->thread = CreateThread(
+		NULL, 0, git_win32__threadproc, thread, 0, NULL);
+
+	return thread->thread ? 0 : -1;
 }
 
-int pthread_join(pthread_t thread, void **value_ptr)
+int git_win32__thread_join(
+	git_win32_thread *thread,
+	void **value_ptr)
 {
-	DWORD ret = WaitForSingleObject(thread, INFINITE);
+	DWORD exit;
+
+	if (WaitForSingleObject(thread->thread, INFINITE) != WAIT_OBJECT_0)
+		return -1;
+
+	if (!GetExitCodeThread(thread->thread, &exit)) {
+		CloseHandle(thread->thread);
+		return -1;
+	}
 
-	if (ret == WAIT_OBJECT_0) {
-		if (value_ptr != NULL) {
-			*value_ptr = NULL;
-			GetExitCodeThread(thread, (void *)value_ptr);
-		}
-		CloseHandle(thread);
-		return 0;
+	/* Check for the thread having exited uncleanly. If exit was unclean,
+	 * then we don't have a return value to give back to the caller. */
+	if (exit != CLEAN_THREAD_EXIT) {
+		assert(false);
+		thread->result = NULL;
 	}
 
-	return -1;
+	if (value_ptr)
+		*value_ptr = thread->result;
+
+	CloseHandle(thread->thread);
+	return 0;
 }
 
 int pthread_mutex_init(
@@ -144,9 +176,6 @@ int pthread_num_processors_np(void)
 	return n ? n : 1;
 }
 
-
-static HINSTANCE win32_kernel32_dll;
-
 typedef void (WINAPI *win32_srwlock_fn)(GIT_SRWLOCK *);
 
 static win32_srwlock_fn win32_srwlock_initialize;
@@ -159,7 +188,7 @@ int pthread_rwlock_init(
 	pthread_rwlock_t *GIT_RESTRICT lock,
 	const pthread_rwlockattr_t *GIT_RESTRICT attr)
 {
-	(void)attr;
+	GIT_UNUSED(attr);
 
 	if (win32_srwlock_initialize)
 		win32_srwlock_initialize(&lock->native.srwl);
@@ -217,38 +246,22 @@ int pthread_rwlock_destroy(pthread_rwlock_t *lock)
 	return 0;
 }
 
-
-static void win32_pthread_shutdown(void)
-{
-	if (win32_kernel32_dll) {
-		FreeLibrary(win32_kernel32_dll);
-		win32_kernel32_dll = NULL;
-	}
-}
-
 int win32_pthread_initialize(void)
 {
-	if (win32_kernel32_dll)
-		return 0;
-
-	win32_kernel32_dll = LoadLibrary("Kernel32.dll");
-	if (!win32_kernel32_dll) {
-		giterr_set(GITERR_OS, "Could not load Kernel32.dll!");
-		return -1;
+	HMODULE hModule = GetModuleHandleW(L"kernel32");
+
+	if (hModule) {
+		win32_srwlock_initialize = (win32_srwlock_fn)
+			GetProcAddress(hModule, "InitializeSRWLock");
+		win32_srwlock_acquire_shared = (win32_srwlock_fn)
+			GetProcAddress(hModule, "AcquireSRWLockShared");
+		win32_srwlock_release_shared = (win32_srwlock_fn)
+			GetProcAddress(hModule, "ReleaseSRWLockShared");
+		win32_srwlock_acquire_exclusive = (win32_srwlock_fn)
+			GetProcAddress(hModule, "AcquireSRWLockExclusive");
+		win32_srwlock_release_exclusive = (win32_srwlock_fn)
+			GetProcAddress(hModule, "ReleaseSRWLockExclusive");
 	}
 
-	win32_srwlock_initialize = (win32_srwlock_fn)
-		GetProcAddress(win32_kernel32_dll, "InitializeSRWLock");
-	win32_srwlock_acquire_shared = (win32_srwlock_fn)
-		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockShared");
-	win32_srwlock_release_shared = (win32_srwlock_fn)
-		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockShared");
-	win32_srwlock_acquire_exclusive = (win32_srwlock_fn)
-		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockExclusive");
-	win32_srwlock_release_exclusive = (win32_srwlock_fn)
-		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockExclusive");
-
-	git__on_shutdown(win32_pthread_shutdown);
-
 	return 0;
 }
diff --git a/src/win32/pthread.h b/src/win32/pthread.h
index af5b121f0..e4826ca7f 100644
--- a/src/win32/pthread.h
+++ b/src/win32/pthread.h
@@ -16,13 +16,19 @@
 #	define GIT_RESTRICT __restrict__
 #endif
 
+typedef struct {
+	HANDLE thread;
+	void *(*proc)(void *);
+	void *param;
+	void *result;
+} git_win32_thread;
+
 typedef int pthread_mutexattr_t;
 typedef int pthread_condattr_t;
 typedef int pthread_attr_t;
 typedef int pthread_rwlockattr_t;
 
 typedef CRITICAL_SECTION pthread_mutex_t;
-typedef HANDLE pthread_t;
 typedef HANDLE pthread_cond_t;
 
 typedef struct { void *Ptr; } GIT_SRWLOCK;
@@ -36,13 +42,26 @@ typedef struct {
 
 #define PTHREAD_MUTEX_INITIALIZER  {(void*)-1}
 
-int pthread_create(
-	pthread_t *GIT_RESTRICT thread,
-	const pthread_attr_t *GIT_RESTRICT attr,
-	void *(*start_routine)(void*),
-	void *GIT_RESTRICT arg);
+int git_win32__thread_create(
+	git_win32_thread *GIT_RESTRICT,
+	const pthread_attr_t *GIT_RESTRICT,
+	void *(*) (void *),
+	void *GIT_RESTRICT);
+
+int git_win32__thread_join(
+	git_win32_thread *,
+	void **);
+
+#ifdef GIT_THREADS
 
-int pthread_join(pthread_t, void **);
+typedef git_win32_thread git_thread;
+
+#define git_thread_create(git_thread_ptr, attr, start_routine, arg) \
+	git_win32__thread_create(git_thread_ptr, attr, start_routine, arg)
+#define git_thread_join(git_thread_ptr, status) \
+	git_win32__thread_join(git_thread_ptr, status)
+
+#endif
 
 int pthread_mutex_init(
 	pthread_mutex_t *GIT_RESTRICT mutex,
diff --git a/src/win32/reparse.h b/src/win32/reparse.h
new file mode 100644
index 000000000..70f9fd652
--- /dev/null
+++ b/src/win32/reparse.h
@@ -0,0 +1,57 @@
+/*
+* Copyright (C) the libgit2 contributors. All rights reserved.
+*
+* This file is part of libgit2, distributed under the GNU GPL v2 with
+* a Linking Exception. For full terms see the included COPYING file.
+*/
+
+#ifndef INCLUDE_git_win32_reparse_h__
+#define INCLUDE_git_win32_reparse_h__
+
+/* This structure is defined on MSDN at
+* http://msdn.microsoft.com/en-us/library/windows/hardware/ff552012(v=vs.85).aspx
+*
+* It was formerly included in the Windows 2000 SDK and remains defined in
+* MinGW, so we must define it with a silly name to avoid conflicting.
+*/
+typedef struct _GIT_REPARSE_DATA_BUFFER {
+	ULONG  ReparseTag;
+	USHORT ReparseDataLength;
+	USHORT Reserved;
+	union {
+		struct {
+			USHORT SubstituteNameOffset;
+			USHORT SubstituteNameLength;
+			USHORT PrintNameOffset;
+			USHORT PrintNameLength;
+			ULONG  Flags;
+			WCHAR  PathBuffer[1];
+		} SymbolicLinkReparseBuffer;
+		struct {
+			USHORT SubstituteNameOffset;
+			USHORT SubstituteNameLength;
+			USHORT PrintNameOffset;
+			USHORT PrintNameLength;
+			WCHAR  PathBuffer[1];
+		} MountPointReparseBuffer;
+		struct {
+			UCHAR DataBuffer[1];
+		} GenericReparseBuffer;
+	};
+} GIT_REPARSE_DATA_BUFFER;
+
+#define REPARSE_DATA_HEADER_SIZE			8
+#define REPARSE_DATA_MOUNTPOINT_HEADER_SIZE	8
+#define REPARSE_DATA_UNION_SIZE				12
+
+/* Missing in MinGW */
+#ifndef FSCTL_GET_REPARSE_POINT
+# define FSCTL_GET_REPARSE_POINT			0x000900a8
+#endif
+
+/* Missing in MinGW */
+#ifndef FSCTL_SET_REPARSE_POINT
+# define FSCTL_SET_REPARSE_POINT			0x000900a4
+#endif
+
+#endif
diff --git a/src/win32/utf-conv.c b/src/win32/utf-conv.c
index d4dbfbab9..b9ccfb5e5 100644
--- a/src/win32/utf-conv.c
+++ b/src/win32/utf-conv.c
@@ -8,74 +8,131 @@
 #include "common.h"
 #include "utf-conv.h"
 
-#define U16_LEAD(c) (wchar_t)(((c)>>10)+0xd7c0)
-#define U16_TRAIL(c) (wchar_t)(((c)&0x3ff)|0xdc00)
+#ifndef WC_ERR_INVALID_CHARS
+# define WC_ERR_INVALID_CHARS	0x80
+#endif
 
-#if 0
-void git__utf8_to_16(wchar_t *dest, size_t length, const char *src)
+GIT_INLINE(DWORD) get_wc_flags(void)
 {
-	wchar_t *pDest = dest;
-	uint32_t ch;
-	const uint8_t* pSrc = (uint8_t*) src;
-
-	assert(dest && src && length);
-
-	length--;
-
-	while(*pSrc && length > 0) {
-		ch = *pSrc++;
-		length--;
-
-		if(ch < 0xc0) {
-			/*
-			 * ASCII, or a trail byte in lead position which is treated like
-			 * a single-byte sequence for better character boundary
-			 * resynchronization after illegal sequences.
-			 */
-			*pDest++ = (wchar_t)ch;
-			continue;
-		} else if(ch < 0xe0) { /* U+0080..U+07FF */
-			if (pSrc[0]) {
-				/* 0x3080 = (0xc0 << 6) + 0x80 */
-				*pDest++ = (wchar_t)((ch << 6) + *pSrc++ - 0x3080);
-				continue;
-			}
-		} else if(ch < 0xf0) { /* U+0800..U+FFFF */
-			if (pSrc[0] && pSrc[1]) {
-				/* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */
-				/* 0x2080 = (0x80 << 6) + 0x80 */
-				ch = (ch << 12) + (*pSrc++ << 6);
-				*pDest++ = (wchar_t)(ch + *pSrc++ - 0x2080);
-				continue;
-			}
-		} else /* f0..f4 */ { /* U+10000..U+10FFFF */
-			if (length >= 1 && pSrc[0] && pSrc[1] && pSrc[2]) {
-				/* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */
-				ch = (ch << 18) + (*pSrc++ << 12);
-				ch += *pSrc++ << 6;
-				ch += *pSrc++ - 0x3c82080;
-				*(pDest++) = U16_LEAD(ch);
-				*(pDest++) = U16_TRAIL(ch);
-				length--; /* two bytes for this character */
-				continue;
-			}
-		}
-
-		/* truncated character at the end */
-		*pDest++ = 0xfffd;
-		break;
+	static char inited = 0;
+	static DWORD flags;
+
+	/* Invalid code point check supported on Vista+ only */
+	if (!inited) {
+		flags = git_has_win32_version(6, 0, 0) ? WC_ERR_INVALID_CHARS : 0;
+		inited = 1;
 	}
 
-	*pDest++ = 0x0;
+	return flags;
 }
-#endif
 
-int git__utf8_to_16(wchar_t * dest, size_t dest_size, const char *src)
+/**
+ * Converts a UTF-8 string to wide characters.
+ *
+ * @param dest The buffer to receive the wide string.
+ * @param dest_size The size of the buffer, in characters.
+ * @param src The UTF-8 string to convert.
+ * @return The length of the wide string, in characters (not counting the NULL terminator), or < 0 for failure
+ */
+int git__utf8_to_16(wchar_t *dest, size_t dest_size, const char *src)
 {
-	return MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)dest_size);
+	/* Length of -1 indicates NULL termination of the input string. Subtract 1 from the result to
+	* turn 0 into -1 (an error code) and to not count the NULL terminator as part of the string's
+	* length. MultiByteToWideChar never returns int's minvalue, so underflow is not possible */
+	return MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, -1, dest, (int)dest_size) - 1;
 }
 
+/**
+ * Converts a wide string to UTF-8.
+ *
+ * @param dest The buffer to receive the UTF-8 string.
+ * @param dest_size The size of the buffer, in bytes.
+ * @param src The wide string to convert.
+ * @return The length of the UTF-8 string, in bytes (not counting the NULL terminator), or < 0 for failure
+ */
 int git__utf16_to_8(char *dest, size_t dest_size, const wchar_t *src)
 {
-	return WideCharToMultiByte(CP_UTF8, 0, src, -1, dest, (int)dest_size, NULL, NULL);
+	/* Length of -1 indicates NULL termination of the input string. Subtract 1 from the result to
+	 * turn 0 into -1 (an error code) and to not count the NULL terminator as part of the string's
+	 * length. WideCharToMultiByte never returns int's minvalue, so underflow is not possible */
+	return WideCharToMultiByte(CP_UTF8, get_wc_flags(), src, -1, dest, (int)dest_size, NULL, NULL) - 1;
+}
+
+/**
+ * Converts a UTF-8 string to wide characters.
+ * Memory is allocated to hold the converted string.
+ * The caller is responsible for freeing the string with git__free.
+ *
+ * @param dest Receives a pointer to the wide string.
+ * @param src The UTF-8 string to convert.
+ * @return The length of the wide string, in characters (not counting the NULL terminator), or < 0 for failure
+ */
+int git__utf8_to_16_alloc(wchar_t **dest, const char *src)
+{
+	int utf16_size;
+
+	*dest = NULL;
+
+	/* Length of -1 indicates NULL termination of the input string */
+	utf16_size = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, -1, NULL, 0);
+
+	if (!utf16_size)
+		return -1;
+
+	*dest = git__malloc(utf16_size * sizeof(wchar_t));
+
+	if (!*dest)
+		return -1;
+
+	utf16_size = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, -1, *dest, utf16_size);
+
+	if (!utf16_size) {
+		git__free(*dest);
+		*dest = NULL;
+	}
+
+	/* Subtract 1 from the result to turn 0 into -1 (an error code) and to not count the NULL
+	 * terminator as part of the string's length. MultiByteToWideChar never returns int's minvalue,
+	 * so underflow is not possible */
+	return utf16_size - 1;
+}
+
+/**
+ * Converts a wide string to UTF-8.
+ * Memory is allocated to hold the converted string.
+ * The caller is responsible for freeing the string with git__free.
+ *
+ * @param dest Receives a pointer to the UTF-8 string.
+ * @param src The wide string to convert.
+ * @return The length of the UTF-8 string, in bytes (not counting the NULL terminator), or < 0 for failure
+ */
+int git__utf16_to_8_alloc(char **dest, const wchar_t *src)
+{
+	int utf8_size;
+	DWORD dwFlags = get_wc_flags();
+
+	*dest = NULL;
+
+	/* Length of -1 indicates NULL termination of the input string */
+	utf8_size = WideCharToMultiByte(CP_UTF8, dwFlags, src, -1, NULL, 0, NULL, NULL);
+
+	if (!utf8_size)
+		return -1;
+
+	*dest = git__malloc(utf8_size);
+
+	if (!*dest)
+		return -1;
+
+	utf8_size = WideCharToMultiByte(CP_UTF8, dwFlags, src, -1, *dest, utf8_size, NULL, NULL);
+
+	if (!utf8_size) {
+		git__free(*dest);
+		*dest = NULL;
+	}
+
+	/* Subtract 1 from the result to turn 0 into -1 (an error code) and to not count the NULL
+	 * terminator as part of the string's length. MultiByteToWideChar never returns int's minvalue,
+	 * so underflow is not possible */
+	return utf8_size - 1;
 }
diff --git a/src/win32/utf-conv.h b/src/win32/utf-conv.h
index 3af77580e..a480cd93e 100644
--- a/src/win32/utf-conv.h
+++ b/src/win32/utf-conv.h
@@ -10,27 +10,83 @@
 #include <wchar.h>
 #include "common.h"
 
-/* Maximum characters in a Windows path plus one for NUL byte */
-#define GIT_WIN_PATH_UTF16 (260 + 1)
+/* Equal to the Win32 MAX_PATH constant. The maximum path length is 259
+ * characters plus a NULL terminator. */
+#define GIT_WIN_PATH_UTF16		260
 
-/* Maximum bytes necessary to convert a full-length UTF16 path to UTF8 */
-#define GIT_WIN_PATH_UTF8  (260 * 4 + 1)
+/* Maximum size of a UTF-8 Win32 path. UTF-8 does have 4-byte sequences,
+ * but they are encoded in UTF-16 using surrogate pairs, which takes up
+ * the space of two characters. Two characters in the range U+0800 ->
+ * U+FFFF take up more space in UTF-8 (6 bytes) than one surrogate pair
+ * (4 bytes). */
+#define GIT_WIN_PATH_UTF8		(259 * 3 + 1)
 
+/* Win32 path types */
 typedef wchar_t git_win32_path[GIT_WIN_PATH_UTF16];
+typedef char git_win32_utf8_path[GIT_WIN_PATH_UTF8];
 
-typedef char git_win32_path_as_utf8[GIT_WIN_PATH_UTF8];
+/**
+ * Converts a UTF-8 string to wide characters.
+ *
+ * @param dest The buffer to receive the wide string.
+ * @param dest_size The size of the buffer, in characters.
+ * @param src The UTF-8 string to convert.
+ * @return The length of the wide string, in characters (not counting the NULL terminator), or < 0 for failure
+ */
+int git__utf8_to_16(wchar_t *dest, size_t dest_size, const char *src);
 
-/* dest_size is the size of dest in wchar_t's */
-int git__utf8_to_16(wchar_t * dest, size_t dest_size, const char *src);
-/* dest_size is the size of dest in char's */
+/**
+ * Converts a wide string to UTF-8.
+ *
+ * @param dest The buffer to receive the UTF-8 string.
+ * @param dest_size The size of the buffer, in bytes.
+ * @param src The wide string to convert.
+ * @return The length of the UTF-8 string, in bytes (not counting the NULL terminator), or < 0 for failure
+ */
 int git__utf16_to_8(char *dest, size_t dest_size, const wchar_t *src);
 
-GIT_INLINE(int) git_win32_path_from_c(git_win32_path dest, const char *src)
+/**
+ * Converts a UTF-8 string to wide characters.
+ * Memory is allocated to hold the converted string.
+ * The caller is responsible for freeing the string with git__free.
+ *
+ * @param dest Receives a pointer to the wide string.
+ * @param src The UTF-8 string to convert.
+ * @return The length of the wide string, in characters (not counting the NULL terminator), or < 0 for failure
+ */
+int git__utf8_to_16_alloc(wchar_t **dest, const char *src);
+
+/**
+ * Converts a wide string to UTF-8.
+ * Memory is allocated to hold the converted string.
+ * The caller is responsible for freeing the string with git__free.
+ *
+ * @param dest Receives a pointer to the UTF-8 string.
+ * @param src The wide string to convert.
+ * @return The length of the UTF-8 string, in bytes (not counting the NULL terminator), or < 0 for failure
+ */
+int git__utf16_to_8_alloc(char **dest, const wchar_t *src);
+
+/**
+ * Converts a UTF-8 Win32 path to wide characters.
+ *
+ * @param dest The buffer to receive the wide string.
+ * @param src The UTF-8 string to convert.
+ * @return The length of the wide string, in characters (not counting the NULL terminator), or < 0 for failure
+ */
+GIT_INLINE(int) git_win32_path_from_utf8(git_win32_path dest, const char *src)
 {
 	return git__utf8_to_16(dest, GIT_WIN_PATH_UTF16, src);
 }
 
-GIT_INLINE(int) git_win32_path_to_c(git_win32_path_as_utf8 dest, const wchar_t *src)
+/**
+ * Converts a wide Win32 path to UTF-8.
+ *
+ * @param dest The buffer to receive the UTF-8 string.
+ * @param src The wide string to convert.
+ * @return The length of the UTF-8 string, in bytes (not counting the NULL terminator), or < 0 for failure
+ */
+GIT_INLINE(int) git_win32_path_to_utf8(git_win32_utf8_path dest, const wchar_t *src)
 {
 	return git__utf16_to_8(dest, GIT_WIN_PATH_UTF8, src);
 }
diff --git a/src/win32/w32_util.c b/src/win32/w32_util.c
new file mode 100644
index 000000000..2e52525d5
--- /dev/null
+++ b/src/win32/w32_util.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "w32_util.h"
+
+/**
+ * Creates a FindFirstFile(Ex) filter string from a UTF-8 path.
+ * The filter string enumerates all items in the directory.
+ *
+ * @param dest The buffer to receive the filter string.
+ * @param src The UTF-8 path of the directory to enumerate.
+ * @return True if the filter string was created successfully; false otherwise
+ */
+bool git_win32__findfirstfile_filter(git_win32_path dest, const char *src)
+{
+	static const wchar_t suffix[] = L"\\*";
+	int len = git_win32_path_from_utf8(dest, src);
+
+	/* Ensure the path was converted */
+	if (len < 0)
+		return false;
+
+	/* Ensure that the path does not end with a trailing slash,
+	 * because we're about to add one. Don't rely our trim_end
+	 * helper, because we want to remove the backslash even for
+	 * drive letter paths, in this case. */
+	if (len > 0 &&
+		(dest[len - 1] == L'/' || dest[len - 1] == L'\\')) {
+		dest[len - 1] = L'\0';
+		len--;
+	}
+
+	/* Ensure we have enough room to add the suffix */
+	if ((size_t)len >= GIT_WIN_PATH_UTF16 - CONST_STRLEN(suffix))
+		return false;
+
+	wcscat(dest, suffix);
+	return true;
+}
+
+/**
+ * Ensures the given path (file or folder) has the +H (hidden) attribute set.
+ *
+ * @param path The path which should receive the +H bit.
+ * @return 0 on success; -1 on failure
+ */
+int git_win32__sethidden(const char *path)
+{
+	git_win32_path buf;
+	DWORD attrs;
+
+	if (git_win32_path_from_utf8(buf, path) < 0)
+		return -1;
+
+	attrs = GetFileAttributesW(buf);
+
+	/* Ensure the path exists */
+	if (attrs == INVALID_FILE_ATTRIBUTES)
+		return -1;
+
+	/* If the item isn't already +H, add the bit */
+	if ((attrs & FILE_ATTRIBUTE_HIDDEN) == 0 &&
+		!SetFileAttributesW(buf, attrs | FILE_ATTRIBUTE_HIDDEN))
+		return -1;
+
+	return 0;
+}
+
+/**
+ * Removes any trailing backslashes from a path, except in the case of a drive
+ * letter path (C:\, D:\, etc.). This function cannot fail.
+ *
+ * @param path The path which should be trimmed.
+ * @return The length of the modified string (<= the input length)
+ */
+size_t git_win32__path_trim_end(wchar_t *str, size_t len)
+{
+	while (1) {
+		if (!len || str[len - 1] != L'\\')
+			break;
+
+		/* Don't trim backslashes from drive letter paths, which
+		 * are 3 characters long and of the form C:\, D:\, etc. */
+		if (len == 3 && git_win32__isalpha(str[0]) && str[1] == ':')
+			break;
+
+		len--;
+	}
+
+	str[len] = L'\0';
+
+	return len;
+}
+
+/**
+ * Removes any of the following namespace prefixes from a path,
+ * if found: "\??\", "\\?\", "\\?\UNC\". This function cannot fail.
+ *
+ * @param path The path which should be converted.
+ * @return The length of the modified string (<= the input length)
+ */
+size_t git_win32__canonicalize_path(wchar_t *str, size_t len)
+{
+	static const wchar_t dosdevices_prefix[] = L"\\\?\?\\";
+	static const wchar_t nt_prefix[] = L"\\\\?\\";
+	static const wchar_t unc_prefix[] = L"UNC\\";
+	size_t to_advance = 0;
+
+	/* "\??\" -- DOS Devices prefix */
+	if (len >= CONST_STRLEN(dosdevices_prefix) &&
+		!wcsncmp(str, dosdevices_prefix, CONST_STRLEN(dosdevices_prefix))) {
+		to_advance += CONST_STRLEN(dosdevices_prefix);
+		len -= CONST_STRLEN(dosdevices_prefix);
+	}
+	/* "\\?\" -- NT namespace prefix */
+	else if (len >= CONST_STRLEN(nt_prefix) &&
+		!wcsncmp(str, nt_prefix, CONST_STRLEN(nt_prefix))) {
+		to_advance += CONST_STRLEN(nt_prefix);
+		len -= CONST_STRLEN(nt_prefix);
+	}
+
+	/* "\??\UNC\", "\\?\UNC\" -- UNC prefix */
+	if (to_advance && len >= CONST_STRLEN(unc_prefix) &&
+		!wcsncmp(str + to_advance, unc_prefix, CONST_STRLEN(unc_prefix))) {
+		to_advance += CONST_STRLEN(unc_prefix);
+		len -= CONST_STRLEN(unc_prefix);
+	}
+
+	if (to_advance) {
+		memmove(str, str + to_advance, len * sizeof(wchar_t));
+		str[len] = L'\0';
+	}
+
+	return git_win32__path_trim_end(str, len);
+}
diff --git a/src/win32/w32_util.h b/src/win32/w32_util.h
new file mode 100644
index 000000000..a1d388af5
--- /dev/null
+++ b/src/win32/w32_util.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_w32_util_h__
+#define INCLUDE_w32_util_h__
+
+#include "utf-conv.h"
+
+GIT_INLINE(bool) git_win32__isalpha(wchar_t c)
+{
+	return ((c >= L'A' && c <= L'Z') || (c >= L'a' && c <= L'z'));
+}
+
+/**
+ * Creates a FindFirstFile(Ex) filter string from a UTF-8 path.
+ * The filter string enumerates all items in the directory.
+ *
+ * @param dest The buffer to receive the filter string.
+ * @param src The UTF-8 path of the directory to enumerate.
+ * @return True if the filter string was created successfully; false otherwise
+ */
+bool git_win32__findfirstfile_filter(git_win32_path dest, const char *src);
+
+/**
+ * Ensures the given path (file or folder) has the +H (hidden) attribute set.
+ *
+ * @param path The path which should receive the +H bit.
+ * @return 0 on success; -1 on failure
+ */
+int git_win32__sethidden(const char *path);
+
+/**
+ * Removes any trailing backslashes from a path, except in the case of a drive
+ * letter path (C:\, D:\, etc.). This function cannot fail.
+ *
+ * @param path The path which should be trimmed.
+ * @return The length of the modified string (<= the input length)
+ */
+size_t git_win32__path_trim_end(wchar_t *str, size_t len);
+
+/**
+ * Removes any of the following namespace prefixes from a path,
+ * if found: "\??\", "\\?\", "\\?\UNC\". This function cannot fail.
+ *
+ * @param path The path which should be converted.
+ * @return The length of the modified string (<= the input length)
+ */
+size_t git_win32__canonicalize_path(wchar_t *str, size_t len);
+
+#endif
diff --git a/src/zstream.c b/src/zstream.c
new file mode 100644
index 000000000..e75fb265e
--- /dev/null
+++ b/src/zstream.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include <zlib.h>
+
+#include "zstream.h"
+#include "buffer.h"
+
+#define ZSTREAM_BUFFER_SIZE (1024 * 1024)
+#define ZSTREAM_BUFFER_MIN_EXTRA 8
+
+static int zstream_seterr(git_zstream *zs)
+{
+	if (zs->zerr == Z_OK || zs->zerr == Z_STREAM_END)
+		return 0;
+
+	if (zs->zerr == Z_MEM_ERROR)
+		giterr_set_oom();
+	else if (zs->z.msg)
+		giterr_set(GITERR_ZLIB, zs->z.msg);
+	else
+		giterr_set(GITERR_ZLIB, "Unknown compression error");
+
+	return -1;
+}
+
+int git_zstream_init(git_zstream *zstream)
+{
+	zstream->zerr = deflateInit(&zstream->z, Z_DEFAULT_COMPRESSION);
+	return zstream_seterr(zstream);
+}
+
+void git_zstream_free(git_zstream *zstream)
+{
+	deflateEnd(&zstream->z);
+}
+
+void git_zstream_reset(git_zstream *zstream)
+{
+	deflateReset(&zstream->z);
+	zstream->in = NULL;
+	zstream->in_len = 0;
+	zstream->zerr = Z_STREAM_END;
+}
+
+int git_zstream_set_input(git_zstream *zstream, const void *in, size_t in_len)
+{
+	zstream->in = in;
+	zstream->in_len = in_len;
+	zstream->zerr = Z_OK;
+	return 0;
+}
+
+bool git_zstream_done(git_zstream *zstream)
+{
+	return (!zstream->in_len && zstream->zerr == Z_STREAM_END);
+}
+
+size_t git_zstream_suggest_output_len(git_zstream *zstream)
+{
+	if (zstream->in_len > ZSTREAM_BUFFER_SIZE)
+		return ZSTREAM_BUFFER_SIZE;
+	else if (zstream->in_len > ZSTREAM_BUFFER_MIN_EXTRA)
+		return zstream->in_len;
+	else
+		return ZSTREAM_BUFFER_MIN_EXTRA;
+}
+
+int git_zstream_get_output(void *out, size_t *out_len, git_zstream *zstream)
+{
+	int zflush = Z_FINISH;
+	size_t out_remain = *out_len;
+
+	while (out_remain > 0 && zstream->zerr != Z_STREAM_END) {
+		size_t out_queued, in_queued, out_used, in_used;
+
+		/* set up in data */
+		zstream->z.next_in  = (Bytef *)zstream->in;
+		zstream->z.avail_in = (uInt)zstream->in_len;
+		if ((size_t)zstream->z.avail_in != zstream->in_len) {
+			zstream->z.avail_in = INT_MAX;
+			zflush = Z_NO_FLUSH;
+		} else {
+			zflush = Z_FINISH;
+		}
+		in_queued = (size_t)zstream->z.avail_in;
+
+		/* set up out data */
+		zstream->z.next_out = out;
+		zstream->z.avail_out = (uInt)out_remain;
+		if ((size_t)zstream->z.avail_out != out_remain)
+			zstream->z.avail_out = INT_MAX;
+		out_queued = (size_t)zstream->z.avail_out;
+
+		/* compress next chunk */
+		zstream->zerr = deflate(&zstream->z, zflush);
+
+		if (zstream->zerr == Z_STREAM_ERROR)
+			return zstream_seterr(zstream);
+
+		out_used = (out_queued - zstream->z.avail_out);
+		out_remain -= out_used;
+		out = ((char *)out) + out_used;
+
+		in_used = (in_queued - zstream->z.avail_in);
+		zstream->in_len -= in_used;
+		zstream->in += in_used;
+	}
+
+	/* either we finished the input or we did not flush the data */
+	assert(zstream->in_len > 0 || zflush == Z_FINISH);
+
+	/* set out_size to number of bytes actually written to output */
+	*out_len = *out_len - out_remain;
+
+	return 0;
+}
+
+int git_zstream_deflatebuf(git_buf *out, const void *in, size_t in_len)
+{
+	git_zstream zs = GIT_ZSTREAM_INIT;
+	int error = 0;
+
+	if ((error = git_zstream_init(&zs)) < 0)
+		return error;
+
+	if ((error = git_zstream_set_input(&zs, in, in_len)) < 0)
+		goto done;
+
+	while (!git_zstream_done(&zs)) {
+		size_t step = git_zstream_suggest_output_len(&zs), written;
+
+		if ((error = git_buf_grow(out, out->size + step)) < 0)
+			goto done;
+
+		written = out->asize - out->size;
+
+		if ((error = git_zstream_get_output(
+				out->ptr + out->size, &written, &zs)) < 0)
+			goto done;
+
+		out->size += written;
+	}
+
+	/* NULL terminate for consistency if possible */
+	if (out->size < out->asize)
+		out->ptr[out->size] = '\0';
+
+done:
+	git_zstream_free(&zs);
+	return error;
+}
diff --git a/src/zstream.h b/src/zstream.h
new file mode 100644
index 000000000..9b5bf6ace
--- /dev/null
+++ b/src/zstream.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_zstream_h__
+#define INCLUDE_zstream_h__
+
+#include <zlib.h>
+
+#include "common.h"
+#include "buffer.h"
+
+typedef struct {
+	z_stream z;
+	const char *in;
+	size_t in_len;
+	int zerr;
+} git_zstream;
+
+#define GIT_ZSTREAM_INIT {{0}}
+
+int git_zstream_init(git_zstream *zstream);
+void git_zstream_free(git_zstream *zstream);
+
+int git_zstream_set_input(git_zstream *zstream, const void *in, size_t in_len);
+
+size_t git_zstream_suggest_output_len(git_zstream *zstream);
+
+int git_zstream_get_output(void *out, size_t *out_len, git_zstream *zstream);
+
+bool git_zstream_done(git_zstream *zstream);
+
+void git_zstream_reset(git_zstream *zstream);
+
+int git_zstream_deflatebuf(git_buf *out, const void *in, size_t in_len);
+
+#endif /* INCLUDE_zstream_h__ */