Merge pull request #2215 from libgit2/rb/submodule-cache-fixes

Improve submodule cache management

15 files changed, 733 insertions, 410 deletions

diff --git a/src/buffer.c b/src/buffer.c
index a83ca8792..83960e912 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -467,6 +467,59 @@ int git_buf_join(
 	return 0;
 }
 
+int git_buf_join3(
+	git_buf *buf,
+	char separator,
+	const char *str_a,
+	const char *str_b,
+	const char *str_c)
+{
+	size_t len_a = strlen(str_a), len_b = strlen(str_b), len_c = strlen(str_c);
+	int sep_a = 0, sep_b = 0;
+	char *tgt;
+
+	/* for this function, disallow pointers into the existing buffer */
+	assert(str_a < buf->ptr || str_a >= buf->ptr + buf->size);
+	assert(str_b < buf->ptr || str_b >= buf->ptr + buf->size);
+	assert(str_c < buf->ptr || str_c >= buf->ptr + buf->size);
+
+	if (separator) {
+		if (len_a > 0) {
+			while (*str_b == separator) { str_b++; len_b--; }
+			sep_a = (str_a[len_a - 1] != separator);
+		}
+		if (len_a > 0 || len_b > 0)
+			while (*str_c == separator) { str_c++; len_c--; }
+		if (len_b > 0)
+			sep_b = (str_b[len_b - 1] != separator);
+	}
+
+	if (git_buf_grow(buf, len_a + sep_a + len_b + sep_b + len_c + 1) < 0)
+		return -1;
+
+	tgt = buf->ptr;
+
+	if (len_a) {
+		memcpy(tgt, str_a, len_a);
+		tgt += len_a;
+	}
+	if (sep_a)
+		*tgt++ = separator;
+	if (len_b) {
+		memcpy(tgt, str_b, len_b);
+		tgt += len_b;
+	}
+	if (sep_b)
+		*tgt++ = separator;
+	if (len_c)
+		memcpy(tgt, str_c, len_c);
+
+	buf->size = len_a + sep_a + len_b + sep_b + len_c;
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
 void git_buf_rtrim(git_buf *buf)
 {
 	while (buf->size > 0) {
diff --git a/src/buffer.h b/src/buffer.h
index 4c852b3cb..dba594d97 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -98,8 +98,12 @@ void git_buf_truncate(git_buf *buf, size_t len);
 void git_buf_shorten(git_buf *buf, size_t amount);
 void git_buf_rtruncate_at_char(git_buf *path, char separator);
 
+/** General join with separator */
 int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...);
+/** Fast join of two strings - first may legally point into `buf` data */
 int git_buf_join(git_buf *buf, char separator, const char *str_a, const char *str_b);
+/** Fast join of three strings - cannot reference `buf` data */
+int git_buf_join3(git_buf *buf, char separator, const char *str_a, const char *str_b, const char *str_c);
 
 /**
  * Join two strings as paths, inserting a slash between as needed.
diff --git a/src/config.c b/src/config.c
index 1a205fe13..b3168f735 100644
--- a/src/config.c
+++ b/src/config.c
@@ -615,6 +615,36 @@ int git_config_set_string(git_config *cfg, const char *name, const char *value)
 	return error;
 }
 
+int git_config__update_entry(
+	git_config *config,
+	const char *key,
+	const char *value,
+	bool overwrite_existing,
+	bool only_if_existing)
+{
+	int error = 0;
+	const git_config_entry *ce = NULL;
+
+	if ((error = git_config__lookup_entry(&ce, config, key, false)) < 0)
+		return error;
+
+	if (!ce && only_if_existing) /* entry doesn't exist */
+		return 0;
+	if (ce && !overwrite_existing) /* entry would be overwritten */
+		return 0;
+	if (value && ce && ce->value && !strcmp(ce->value, value)) /* no change */
+		return 0;
+	if (!value && (!ce || !ce->value)) /* asked to delete absent entry */
+		return 0;
+
+	if (!value)
+		error = git_config_delete_entry(config, key);
+	else
+		error = git_config_set_string(config, key, value);
+
+	return error;
+}
+
 /***********
  * Getters
  ***********/
diff --git a/src/config.h b/src/config.h
index 03d910616..00b6063e7 100644
--- a/src/config.h
+++ b/src/config.h
@@ -53,6 +53,14 @@ extern int git_config__lookup_entry(
 	const char *key,
 	bool no_errors);
 
+/* internal only: update and/or delete entry string with constraints */
+extern int git_config__update_entry(
+	git_config *cfg,
+	const char *key,
+	const char *value,
+	bool overwrite_existing,
+	bool only_if_existing);
+
 /*
  * Lookup functions that cannot fail.  These functions look up a config
  * value and return a fallback value if the value is missing or if any
diff --git a/src/config_file.h b/src/config_file.h
index d4a1a4061..fcccbd5cc 100644
--- a/src/config_file.h
+++ b/src/config_file.h
@@ -16,7 +16,8 @@ GIT_INLINE(int) git_config_file_open(git_config_backend *cfg, unsigned int level
 
 GIT_INLINE(void) git_config_file_free(git_config_backend *cfg)
 {
-	cfg->free(cfg);
+	if (cfg)
+		cfg->free(cfg);
 }
 
 GIT_INLINE(int) git_config_file_get_string(
diff --git a/src/index.c b/src/index.c
index 3fcd21115..b0b5eae9d 100644
--- a/src/index.c
+++ b/src/index.c
@@ -517,6 +517,18 @@ int git_index_read(git_index *index, int force)
 	return error;
 }
 
+int git_index__changed_relative_to(
+	git_index *index, const git_futils_filestamp *fs)
+{
+	/* attempt to update index (ignoring errors) */
+	if (git_index_read(index, false) < 0)
+		giterr_clear();
+
+	return (index->stamp.mtime != fs->mtime ||
+			index->stamp.size != fs->size ||
+			index->stamp.ino != fs->ino);
+}
+
 int git_index_write(git_index *index)
 {
 	git_filebuf file = GIT_FILEBUF_INIT;
diff --git a/src/index.h b/src/index.h
index 259a3149f..17f04f0ad 100644
--- a/src/index.h
+++ b/src/index.h
@@ -62,4 +62,11 @@ extern void git_index__set_ignore_case(git_index *index, bool ignore_case);
 
 extern unsigned int git_index__create_mode(unsigned int mode);
 
+GIT_INLINE(const git_futils_filestamp *) git_index__filestamp(git_index *index)
+{
+   return &index->stamp;
+}
+
+extern int git_index__changed_relative_to(git_index *index, const git_futils_filestamp *fs);
+
 #endif
diff --git a/src/path.h b/src/path.h
index f26175d15..2367d707b 100644
--- a/src/path.h
+++ b/src/path.h
@@ -119,6 +119,14 @@ GIT_INLINE(void) git_path_mkposix(char *path)
 #	define git_path_mkposix(p) /* blank */
 #endif
 
+/**
+ * Check if string is a relative path (i.e. starts with "./" or "../")
+ */
+GIT_INLINE(int) git_path_is_relative(const char *p)
+{
+	return (p[0] == '.' && (p[1] == '/' || (p[1] == '.' && p[2] == '/')));
+}
+
 extern int git__percent_decode(git_buf *decoded_out, const char *input);
 
 /**
diff --git a/src/refdb_fs.c b/src/refdb_fs.c
index 9120a3e87..2550b7e26 100644
--- a/src/refdb_fs.c
+++ b/src/refdb_fs.c
@@ -1432,7 +1432,7 @@ static int create_new_reflog_file(const char *filepath)
 
 GIT_INLINE(int) retrieve_reflog_path(git_buf *path, git_repository *repo, const char *name)
 {
-	return git_buf_join_n(path, '/', 3, repo->path_repository, GIT_REFLOG_DIR, name);
+	return git_buf_join3(path, '/', repo->path_repository, GIT_REFLOG_DIR, name);
 }
 
 static int refdb_reflog_fs__ensure_log(git_refdb_backend *_backend, const char *name)
diff --git a/src/remote.c b/src/remote.c
index d3e81b973..243086bf9 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -495,9 +495,10 @@ cleanup:
 int git_remote_save(const git_remote *remote)
 {
 	int error;
-	git_config *config;
+	git_config *cfg;
 	const char *tagopt = NULL;
 	git_buf buf = GIT_BUF_INIT;
+	const git_config_entry *existing;
 
 	assert(remote);
 
@@ -509,43 +510,31 @@ int git_remote_save(const git_remote *remote)
 	if ((error = ensure_remote_name_is_valid(remote->name)) < 0)
 		return error;
 
-	if (git_repository_config__weakptr(&config, remote->repo) < 0)
-		return -1;
+	if ((error = git_repository_config__weakptr(&cfg, remote->repo)) < 0)
+		return error;
 
-	if (git_buf_printf(&buf, "remote.%s.url", remote->name) < 0)
-		return -1;
+	if ((error = git_buf_printf(&buf, "remote.%s.url", remote->name)) < 0)
+		return error;
 
-	if (git_config_set_string(config, git_buf_cstr(&buf), remote->url) < 0) {
-		git_buf_free(&buf);
-		return -1;
-	}
+	/* after this point, buffer is allocated so end with cleanup */
+
+	if ((error = git_config_set_string(
+			cfg, git_buf_cstr(&buf), remote->url)) < 0)
+		goto cleanup;
 
 	git_buf_clear(&buf);
-	if (git_buf_printf(&buf, "remote.%s.pushurl", remote->name) < 0)
-		return -1;
+	if ((error = git_buf_printf(&buf, "remote.%s.pushurl", remote->name)) < 0)
+		goto cleanup;
 
-	if (remote->pushurl) {
-		if (git_config_set_string(config, git_buf_cstr(&buf), remote->pushurl) < 0) {
-			git_buf_free(&buf);
-			return -1;
-		}
-	} else {
-		int error = git_config_delete_entry(config, git_buf_cstr(&buf));
-		if (error == GIT_ENOTFOUND) {
-			error = 0;
-			giterr_clear();
-		}
-		if (error < 0) {
-			git_buf_free(&buf);
-			return error;
-		}
-	}
+	if ((error = git_config__update_entry(
+			cfg, git_buf_cstr(&buf), remote->pushurl, true, false)) < 0)
+		goto cleanup;
 
-	if (update_config_refspec(remote, config, GIT_DIRECTION_FETCH) < 0)
-		goto on_error;
+	if ((error = update_config_refspec(remote, cfg, GIT_DIRECTION_FETCH)) < 0)
+		goto cleanup;
 
-	if (update_config_refspec(remote, config, GIT_DIRECTION_PUSH) < 0)
-		goto on_error;
+	if ((error = update_config_refspec(remote, cfg, GIT_DIRECTION_PUSH)) < 0)
+		goto cleanup;
 
 	/*
 	 * What action to take depends on the old and new values. This
@@ -561,31 +550,26 @@ int git_remote_save(const git_remote *remote)
 	 */
 
 	git_buf_clear(&buf);
-	if (git_buf_printf(&buf, "remote.%s.tagopt", remote->name) < 0)
-		goto on_error;
-
-	error = git_config_get_string(&tagopt, config, git_buf_cstr(&buf));
-	if (error < 0 && error != GIT_ENOTFOUND)
-		goto on_error;
+	if ((error = git_buf_printf(&buf, "remote.%s.tagopt", remote->name)) < 0)
+		goto cleanup;
 
-	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
-		if (git_config_set_string(config, git_buf_cstr(&buf), "--tags") < 0)
-			goto on_error;
-	} else if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_NONE) {
-		if (git_config_set_string(config, git_buf_cstr(&buf), "--no-tags") < 0)
-			goto on_error;
-	} else if (tagopt) {
-		if (git_config_delete_entry(config, git_buf_cstr(&buf)) < 0)
-			goto on_error;
-	}
+	if ((error = git_config__lookup_entry(
+			&existing, cfg, git_buf_cstr(&buf), false)) < 0)
+		goto cleanup;
 
-	git_buf_free(&buf);
+	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL)
+		tagopt = "--tags";
+	else if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_NONE)
+		tagopt = "--no-tags";
+	else if (existing != NULL)
+		tagopt = NULL;
 
-	return 0;
+	error = git_config__update_entry(
+		cfg, git_buf_cstr(&buf), tagopt, true, false);
 
-on_error:
+cleanup:
 	git_buf_free(&buf);
-	return -1;
+	return error;
 }
 
 const char *git_remote_name(const git_remote *remote)
diff --git a/src/repository.c b/src/repository.c
index fccc16faa..49d1bc63e 100644
--- a/src/repository.c
+++ b/src/repository.c
@@ -93,6 +93,7 @@ void git_repository__cleanup(git_repository *repo)
 
 	git_cache_clear(&repo->objects);
 	git_attr_cache_flush(repo);
+	git_submodule_cache_free(repo);
 
 	set_config(repo, NULL);
 	set_index(repo, NULL);
@@ -108,7 +109,6 @@ void git_repository_free(git_repository *repo)
 	git_repository__cleanup(repo);
 
 	git_cache_free(&repo->objects);
-	git_submodule_config_free(repo);
 
 	git_diff_driver_registry_free(repo->diff_drivers);
 	repo->diff_drivers = NULL;
diff --git a/src/repository.h b/src/repository.h
index 4e79714f1..99923b63b 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -19,7 +19,7 @@
 #include "buffer.h"
 #include "object.h"
 #include "attrcache.h"
-#include "strmap.h"
+#include "submodule.h"
 #include "diff_driver.h"
 
 #define DOT_GIT ".git"
@@ -105,10 +105,10 @@ struct git_repository {
 	git_refdb *_refdb;
 	git_config *_config;
 	git_index *_index;
+	git_submodule_cache *_submodules;
 
 	git_cache objects;
 	git_attr_cache attrcache;
-	git_strmap *submodules;
 	git_diff_driver_registry *diff_drivers;
 
 	char *path_repository;
@@ -149,11 +149,6 @@ int git_repository_index__weakptr(git_index **out, git_repository *repo);
 int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar);
 void git_repository__cvar_cache_clear(git_repository *repo);
 
-/*
- * Submodule cache
- */
-extern void git_submodule_config_free(git_repository *repo);
-
 GIT_INLINE(int) git_repository__ensure_not_bare(
 	git_repository *repo,
 	const char *operation_name)
diff --git a/src/submodule.c b/src/submodule.c
index fdcc2251a..bea096df5 100644
--- a/src/submodule.c
+++ b/src/submodule.c
@@ -49,6 +49,16 @@ static git_cvar_map _sm_recurse_map[] = {
 	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_RECURSE_YES},
 };
 
+enum {
+	CACHE_OK = 0,
+	CACHE_REFRESH = 1,
+	CACHE_FLUSH = 2
+};
+enum {
+	GITMODULES_EXISTING = 0,
+	GITMODULES_CREATE = 1,
+};
+
 static kh_inline khint_t str_hash_no_trailing_slash(const char *s)
 {
 	khint_t h;
@@ -77,13 +87,15 @@ __KHASH_IMPL(
 	str, static kh_inline, const char *, void *, 1,
 	str_hash_no_trailing_slash, str_equal_no_trailing_slash);
 
-static int load_submodule_config(git_repository *repo, bool reload);
-static git_config_backend *open_gitmodules(git_repository *, bool, const git_oid *);
-static int lookup_head_remote(git_buf *url, git_repository *repo);
-static int submodule_get(git_submodule **, git_repository *, const char *, const char *);
+static int submodule_cache_init(git_repository *repo, int refresh);
+static void submodule_cache_free(git_submodule_cache *cache);
+
+static git_config_backend *open_gitmodules(git_submodule_cache *, int gitmod);
+static int get_url_base(git_buf *url, git_repository *repo);
+static int lookup_head_remote_key(git_buf *remote_key, git_repository *repo);
+static int submodule_get(git_submodule **, git_submodule_cache *, const char *, const char *);
 static int submodule_load_from_config(const git_config_entry *, void *);
 static int submodule_load_from_wd_lite(git_submodule *);
-static int submodule_update_config(git_submodule *, const char *, const char *, bool, bool);
 static void submodule_get_index_status(unsigned int *, git_submodule *);
 static void submodule_get_wd_status(unsigned int *, git_submodule *, git_repository *, git_submodule_ignore_t);
 
@@ -102,7 +114,7 @@ static int submodule_config_key_trunc_puts(git_buf *key, const char *suffix)
 /* lookup submodule or return ENOTFOUND if it doesn't exist */
 static int submodule_lookup(
 	git_submodule **out,
-	git_strmap *cache,
+	git_submodule_cache *cache,
 	const char *name,
 	const char *alternate)
 {
@@ -110,18 +122,18 @@ static int submodule_lookup(
 
 	/* lock cache */
 
-	pos = git_strmap_lookup_index(cache, name);
+	pos = git_strmap_lookup_index(cache->submodules, name);
 
-	if (!git_strmap_valid_index(cache, pos) && alternate)
-		pos = git_strmap_lookup_index(cache, alternate);
+	if (!git_strmap_valid_index(cache->submodules, pos) && alternate)
+		pos = git_strmap_lookup_index(cache->submodules, alternate);
 
-	if (!git_strmap_valid_index(cache, pos)) {
+	if (!git_strmap_valid_index(cache->submodules, pos)) {
 		/* unlock cache */
 		return GIT_ENOTFOUND; /* don't set error - caller will cope */
 	}
 
 	if (out != NULL) {
-		git_submodule *sm = git_strmap_value_at(cache, pos);
+		git_submodule *sm = git_strmap_value_at(cache->submodules, pos);
 		GIT_REFCOUNT_INC(sm);
 		*out = sm;
 	}
@@ -131,6 +143,18 @@ static int submodule_lookup(
 	return 0;
 }
 
+/* clear a set of flags on all submodules */
+static void submodule_cache_clear_flags(
+	git_submodule_cache *cache, uint32_t mask)
+{
+	git_submodule *sm;
+	uint32_t inverted_mask = ~mask;
+
+	git_strmap_foreach_value(cache->submodules, sm, {
+		sm->flags &= inverted_mask;
+	});
+}
+
 /*
  * PUBLIC APIS
  */
@@ -139,17 +163,45 @@ bool git_submodule__is_submodule(git_repository *repo, const char *name)
 {
 	git_strmap *map;
 
-	if (load_submodule_config(repo, false) < 0) {
+	if (submodule_cache_init(repo, CACHE_OK) < 0) {
 		giterr_clear();
 		return false;
 	}
 
-	if (!(map = repo->submodules))
+	if (!repo->_submodules || !(map = repo->_submodules->submodules))
 		return false;
 
 	return git_strmap_valid_index(map, git_strmap_lookup_index(map, name));
 }
 
+static void submodule_set_lookup_error(int error, const char *name)
+{
+	if (!error)
+		return;
+
+	giterr_set(GITERR_SUBMODULE, (error == GIT_ENOTFOUND) ?
+		"No submodule named '%s'" :
+		"Submodule '%s' has not been added yet", name);
+}
+
+int git_submodule__lookup(
+	git_submodule **out, /* NULL if user only wants to test existence */
+	git_repository *repo,
+	const char *name)    /* trailing slash is allowed */
+{
+	int error;
+
+	assert(repo && name);
+
+	if ((error = submodule_cache_init(repo, CACHE_OK)) < 0)
+		return error;
+
+	if ((error = submodule_lookup(out, repo->_submodules, name, NULL)) < 0)
+		submodule_set_lookup_error(error, name);
+
+	return error;
+}
+
 int git_submodule_lookup(
 	git_submodule **out, /* NULL if user only wants to test existence */
 	git_repository *repo,
@@ -159,88 +211,99 @@ int git_submodule_lookup(
 
 	assert(repo && name);
 
-	if ((error = load_submodule_config(repo, false)) < 0)
+	if ((error = submodule_cache_init(repo, CACHE_REFRESH)) < 0)
 		return error;
 
-	if ((error = submodule_lookup(out, repo->submodules, name, NULL)) < 0) {
+	if ((error = submodule_lookup(out, repo->_submodules, name, NULL)) < 0) {
 
 		/* check if a plausible submodule exists at path */
 		if (git_repository_workdir(repo)) {
 			git_buf path = GIT_BUF_INIT;
 
-			if (git_buf_joinpath(&path, git_repository_workdir(repo), name) < 0)
+			if (git_buf_join3(&path,
+					'/', git_repository_workdir(repo), name, DOT_GIT) < 0)
 				return -1;
 
-			if (git_path_contains(&path, DOT_GIT))
+			if (git_path_exists(path.ptr))
 				error = GIT_EEXISTS;
 
 			git_buf_free(&path);
 		}
 
-		giterr_set(GITERR_SUBMODULE, (error == GIT_ENOTFOUND) ?
-			"No submodule named '%s'" :
-			"Submodule '%s' has not been added yet", name);
+		submodule_set_lookup_error(error, name);
 	}
 
 	return error;
 }
 
+static void submodule_free_dup(void *sm)
+{
+	git_submodule_free(sm);
+}
+
 int git_submodule_foreach(
 	git_repository *repo,
 	int (*callback)(git_submodule *sm, const char *name, void *payload),
 	void *payload)
 {
 	int error;
+	size_t i;
 	git_submodule *sm;
-	git_vector seen = GIT_VECTOR_INIT;
-	git_vector_set_cmp(&seen, submodule_cmp);
+	git_submodule_cache *cache;
+	git_vector snapshot = GIT_VECTOR_INIT;
 
 	assert(repo && callback);
 
-	if ((error = load_submodule_config(repo, true)) < 0)
+	if ((error = submodule_cache_init(repo, CACHE_REFRESH)) < 0)
 		return error;
 
-	git_strmap_foreach_value(repo->submodules, sm, {
-		/* Usually the following will not come into play - it just prevents
-		 * us from issuing a callback twice for a submodule where the name
-		 * and path are not the same.
-		 */
-		if (GIT_REFCOUNT_VAL(sm) > 1) {
-			if (git_vector_bsearch(NULL, &seen, sm) != GIT_ENOTFOUND)
-				continue;
-			if ((error = git_vector_insert(&seen, sm)) < 0)
+	cache = repo->_submodules;
+
+	if (git_mutex_lock(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire lock on submodule cache");
+		return -1;
+	}
+
+	if (!(error = git_vector_init(
+			&snapshot, kh_size(cache->submodules), submodule_cmp))) {
+
+		git_strmap_foreach_value(cache->submodules, sm, {
+			if ((error = git_vector_insert(&snapshot, sm)) < 0)
 				break;
-		}
+			GIT_REFCOUNT_INC(sm);
+		});
+	}
+
+	git_mutex_unlock(&cache->lock);
+
+	if (error < 0)
+		goto done;
+
+	git_vector_uniq(&snapshot, submodule_free_dup);
 
+	git_vector_foreach(&snapshot, i, sm) {
 		if ((error = callback(sm, sm->name, payload)) != 0) {
 			giterr_set_after_callback(error);
 			break;
 		}
-	});
+	}
 
-	git_vector_free(&seen);
+done:
+	git_vector_foreach(&snapshot, i, sm)
+		git_submodule_free(sm);
+	git_vector_free(&snapshot);
 
 	return error;
 }
 
-void git_submodule_config_free(git_repository *repo)
+void git_submodule_cache_free(git_repository *repo)
 {
-	git_strmap *smcfg;
-	git_submodule *sm;
+	git_submodule_cache *cache;
 
 	assert(repo);
 
-	smcfg = repo->submodules;
-	repo->submodules = NULL;
-
-	if (smcfg == NULL)
-		return;
-
-	git_strmap_foreach_value(smcfg, sm, {
-		sm->repo = NULL; /* disconnect from repo */;
-		git_submodule_free(sm);
-	});
-	git_strmap_free(smcfg);
+	if ((cache = git__swap(repo->_submodules, NULL)) != NULL)
+		submodule_cache_free(cache);
 }
 
 int git_submodule_add_setup(
@@ -261,18 +324,16 @@ int git_submodule_add_setup(
 
 	/* see if there is already an entry for this submodule */
 
-	if (git_submodule_lookup(&sm, repo, path) < 0)
+	if (git_submodule_lookup(NULL, repo, path) < 0)
 		giterr_clear();
 	else {
-		git_submodule_free(sm);
 		giterr_set(GITERR_SUBMODULE,
-			"Attempt to add a submodule that already exists");
+			"Attempt to add submodule '%s' that already exists", path);
 		return GIT_EEXISTS;
 	}
 
 	/* resolve parameters */
-	error = git_submodule_resolve_url(&real_url, repo, url);
-	if (error)
+	if ((error = git_submodule_resolve_url(&real_url, repo, url)) < 0)
 		goto cleanup;
 
 	/* validate and normalize path */
@@ -288,9 +349,9 @@ int git_submodule_add_setup(
 
 	/* update .gitmodules */
 
-	if ((mods = open_gitmodules(repo, true, NULL)) == NULL) {
+	if (!(mods = open_gitmodules(repo->_submodules, GITMODULES_CREATE))) {
 		giterr_set(GITERR_SUBMODULE,
-			"Adding submodules to a bare repository is not supported (for now)");
+			"Adding submodules to a bare repository is not supported");
 		return -1;
 	}
 
@@ -328,8 +389,8 @@ int git_submodule_add_setup(
 	else if (use_gitlink) {
 		git_buf repodir = GIT_BUF_INIT;
 
-		error = git_buf_join_n(
-			&repodir, '/', 3, git_repository_path(repo), "modules", path);
+		error = git_buf_join3(
+			&repodir, '/', git_repository_path(repo), "modules", path);
 		if (error < 0)
 			goto cleanup;
 
@@ -348,10 +409,18 @@ int git_submodule_add_setup(
 
 	/* add submodule to hash and "reload" it */
 
-	if (!(error = submodule_get(&sm, repo, path, NULL)) &&
+	if (git_mutex_lock(&repo->_submodules->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire lock on submodule cache");
+		error = -1;
+		goto cleanup;
+	}
+
+	if (!(error = submodule_get(&sm, repo->_submodules, path, NULL)) &&
 		!(error = git_submodule_reload(sm, false)))
 		error = git_submodule_init(sm, false);
 
+	git_mutex_unlock(&repo->_submodules->lock);
+
 cleanup:
 	if (error && sm) {
 		git_submodule_free(sm);
@@ -360,8 +429,7 @@ cleanup:
 	if (out != NULL)
 		*out = sm;
 
-	if (mods != NULL)
-		git_config_file_free(mods);
+	git_config_file_free(mods);
 	git_repository_free(subrepo);
 	git_buf_free(&real_url);
 	git_buf_free(&name);
@@ -489,10 +557,10 @@ int git_submodule_save(git_submodule *submodule)
 
 	assert(submodule);
 
-	mods = open_gitmodules(submodule->repo, true, NULL);
+	mods = open_gitmodules(submodule->repo->_submodules, GITMODULES_CREATE);
 	if (!mods) {
 		giterr_set(GITERR_SUBMODULE,
-			"Adding submodules to a bare repository is not supported (for now)");
+			"Adding submodules to a bare repository is not supported");
 		return -1;
 	}
 
@@ -539,8 +607,7 @@ int git_submodule_save(git_submodule *submodule)
 	submodule->flags |= GIT_SUBMODULE_STATUS_IN_CONFIG;
 
 cleanup:
-	if (mods != NULL)
-		git_config_file_free(mods);
+	git_config_file_free(mods);
 	git_buf_free(&key);
 
 	return error;
@@ -576,8 +643,8 @@ int git_submodule_resolve_url(git_buf *out, git_repository *repo, const char *ur
 
 	assert(url);
 
-	if (url[0] == '.' && (url[1] == '/' || (url[1] == '.' && url[2] == '/'))) {
-		if (!(error = lookup_head_remote(out, repo)))
+	if (git_path_is_relative(url)) {
+		if (!(error = get_url_base(out, repo)))
 			error = git_path_apply_relative(out, url);
 	} else if (strchr(url, ':') != NULL || url[0] == '/') {
 		error = git_buf_sets(out, url);
@@ -715,46 +782,95 @@ git_submodule_recurse_t git_submodule_set_fetch_recurse_submodules(
 	return old;
 }
 
-int git_submodule_init(git_submodule *submodule, int overwrite)
+int git_submodule_init(git_submodule *sm, int overwrite)
 {
 	int error;
 	const char *val;
+	git_buf key = GIT_BUF_INIT;
+	git_config *cfg = NULL;
 
-	/* write "submodule.NAME.url" */
-
-	if (!submodule->url) {
+	if (!sm->url) {
 		giterr_set(GITERR_SUBMODULE,
-			"No URL configured for submodule '%s'", submodule->name);
+			"No URL configured for submodule '%s'", sm->name);
 		return -1;
 	}
 
-	error = submodule_update_config(
-		submodule, "url", submodule->url, overwrite != 0, false);
-	if (error < 0)
+	if ((error = git_repository_config(&cfg, sm->repo)) < 0)
 		return error;
 
+	/* write "submodule.NAME.url" */
+
+	if ((error = git_buf_printf(&key, "submodule.%s.url", sm->name)) < 0 ||
+		(error = git_config__update_entry(
+			cfg, key.ptr, sm->url, overwrite != 0, false)) < 0)
+		goto cleanup;
+
 	/* write "submodule.NAME.update" if not default */
 
-	val = (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
-		NULL : git_submodule_update_to_str(submodule->update);
-	error = submodule_update_config(
-		submodule, "update", val, (overwrite != 0), false);
+	val = (sm->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
+		NULL : git_submodule_update_to_str(sm->update);
+
+	if ((error = git_buf_printf(&key, "submodule.%s.update", sm->name)) < 0 ||
+		(error = git_config__update_entry(
+			cfg, key.ptr, val, overwrite != 0, false)) < 0)
+		goto cleanup;
+
+	/* success */
+
+cleanup:
+	git_config_free(cfg);
+	git_buf_free(&key);
 
 	return error;
 }
 
-int git_submodule_sync(git_submodule *submodule)
+int git_submodule_sync(git_submodule *sm)
 {
-	if (!submodule->url) {
+	int error = 0;
+	git_config *cfg = NULL;
+	git_buf key = GIT_BUF_INIT;
+	git_repository *smrepo = NULL;
+
+	if (!sm->url) {
 		giterr_set(GITERR_SUBMODULE,
-			"No URL configured for submodule '%s'", submodule->name);
+			"No URL configured for submodule '%s'", sm->name);
 		return -1;
 	}
 
 	/* copy URL over to config only if it already exists */
 
-	return submodule_update_config(
-		submodule, "url", submodule->url, true, true);
+	if (!(error = git_repository_config__weakptr(&cfg, sm->repo)) &&
+		!(error = git_buf_printf(&key, "submodule.%s.url", sm->name)))
+		error = git_config__update_entry(cfg, key.ptr, sm->url, true, true);
+
+	/* if submodule exists in the working directory, update remote url */
+
+	if (!error &&
+		(sm->flags & GIT_SUBMODULE_STATUS_IN_WD) != 0 &&
+		!(error = git_submodule_open(&smrepo, sm)))
+	{
+		git_buf remote_name = GIT_BUF_INIT;
+
+		if ((error = git_repository_config__weakptr(&cfg, smrepo)) < 0)
+			/* return error from reading submodule config */;
+		else if ((error = lookup_head_remote_key(&remote_name, smrepo)) < 0) {
+			giterr_clear();
+			error = git_buf_sets(&key, "branch.origin.remote");
+		} else {
+			error = git_buf_join3(
+				&key, '.', "branch", remote_name.ptr, "remote");
+			git_buf_free(&remote_name);
+		}
+
+		if (!error)
+			error = git_config__update_entry(cfg, key.ptr, sm->url, true, false);
+
+		git_repository_free(smrepo);
+	}
+
+	git_buf_free(&key);
+
+	return error;
 }
 
 static int git_submodule__open(
@@ -821,64 +937,9 @@ int git_submodule_open(git_repository **subrepo, git_submodule *sm)
 	return git_submodule__open(subrepo, sm, false);
 }
 
-static void submodule_cache_remove_item(
-	git_strmap *cache,
-	const char *name,
-	git_submodule *expected,
-	bool free_after_remove)
-{
-	khiter_t pos;
-	git_submodule *found;
-
-	if (!cache)
-		return;
-
-	pos = git_strmap_lookup_index(cache, name);
-
-	if (!git_strmap_valid_index(cache, pos))
-		return;
-
-	found = git_strmap_value_at(cache, pos);
-
-	if (expected && found != expected)
-		return;
-
-	git_strmap_set_value_at(cache, pos, NULL);
-	git_strmap_delete_at(cache, pos);
-
-	if (free_after_remove)
-		git_submodule_free(found);
-}
-
 int git_submodule_reload_all(git_repository *repo, int force)
 {
-	int error = 0;
-	git_submodule *sm;
-
-	GIT_UNUSED(force);
-	assert(repo);
-
-	if (repo->submodules)
-		git_strmap_foreach_value(repo->submodules, sm, { sm->flags = 0; });
-
-	if ((error = load_submodule_config(repo, true)) < 0)
-		return error;
-
-	git_strmap_foreach_value(repo->submodules, sm, {
-		git_strmap *cache = repo->submodules;
-
-		if (sm && (sm->flags & GIT_SUBMODULE_STATUS__IN_FLAGS) == 0) {
-			/* we must check path != name before first remove, in case
-			 * that call frees the submodule */
-			bool free_as_path = (sm->path != sm->name);
-
-			submodule_cache_remove_item(cache, sm->name, sm, true);
-			if (free_as_path)
-				submodule_cache_remove_item(cache, sm->path, sm, true);
-		}
-	});
-
-	return error;
+	return submodule_cache_init(repo, force ? CACHE_FLUSH : CACHE_REFRESH);
 }
 
 static void submodule_update_from_index_entry(
@@ -955,41 +1016,44 @@ static int submodule_update_head(git_submodule *submodule)
 }
 
 
-int git_submodule_reload(git_submodule *submodule, int force)
+int git_submodule_reload(git_submodule *sm, int force)
 {
 	int error = 0;
 	git_config_backend *mods;
+	git_submodule_cache *cache;
 
 	GIT_UNUSED(force);
 
-	assert(submodule);
+	assert(sm);
+
+	cache = sm->repo->_submodules;
 
 	/* refresh index data */
-	if ((error = submodule_update_index(submodule)) < 0)
+	if ((error = submodule_update_index(sm)) < 0)
 		return error;
 
 	/* refresh HEAD tree data */
-	if ((error = submodule_update_head(submodule)) < 0)
+	if ((error = submodule_update_head(sm)) < 0)
 		return error;
 
 	/* done if bare */
-	if (git_repository_is_bare(submodule->repo))
+	if (git_repository_is_bare(sm->repo))
 		return error;
 
 	/* refresh config data */
-	mods = open_gitmodules(submodule->repo, false, NULL);
+	mods = open_gitmodules(cache, GITMODULES_EXISTING);
 	if (mods != NULL) {
 		git_buf path = GIT_BUF_INIT;
 
 		git_buf_sets(&path, "submodule\\.");
-		git_buf_text_puts_escape_regex(&path, submodule->name);
+		git_buf_text_puts_escape_regex(&path, sm->name);
 		git_buf_puts(&path, ".*");
 
 		if (git_buf_oom(&path))
 			error = -1;
 		else
 			error = git_config_file_foreach_match(
-				mods, path.ptr, submodule_load_from_config, submodule->repo);
+				mods, path.ptr, submodule_load_from_config, cache);
 
 		git_buf_free(&path);
 		git_config_file_free(mods);
@@ -999,9 +1063,11 @@ int git_submodule_reload(git_submodule *submodule, int force)
 	}
 
 	/* refresh wd data */
-	submodule->flags &= ~GIT_SUBMODULE_STATUS__ALL_WD_FLAGS;
+	sm->flags &=
+		~(GIT_SUBMODULE_STATUS_IN_WD | GIT_SUBMODULE_STATUS__WD_OID_VALID |
+		  GIT_SUBMODULE_STATUS__WD_FLAGS);
 
-	return submodule_load_from_wd_lite(submodule);
+	return submodule_load_from_wd_lite(sm);
 }
 
 static void submodule_copy_oid_maybe(
@@ -1095,34 +1161,69 @@ int git_submodule_location(unsigned int *location, git_submodule *sm)
  * INTERNAL FUNCTIONS
  */
 
-static git_submodule *submodule_alloc(git_repository *repo, const char *name)
+static int submodule_alloc(
+	git_submodule **out, git_submodule_cache *cache, const char *name)
 {
 	size_t namelen;
 	git_submodule *sm;
 
 	if (!name || !(namelen = strlen(name))) {
 		giterr_set(GITERR_SUBMODULE, "Invalid submodule name");
-		return NULL;
+		return -1;
 	}
 
 	sm = git__calloc(1, sizeof(git_submodule));
-	if (sm == NULL)
-		return NULL;
+	GITERR_CHECK_ALLOC(sm);
 
 	sm->name = sm->path = git__strdup(name);
 	if (!sm->name) {
 		git__free(sm);
-		return NULL;
+		return -1;
 	}
 
 	GIT_REFCOUNT_INC(sm);
 	sm->ignore = sm->ignore_default = GIT_SUBMODULE_IGNORE_NONE;
 	sm->update = sm->update_default = GIT_SUBMODULE_UPDATE_CHECKOUT;
 	sm->fetch_recurse = sm->fetch_recurse_default = GIT_SUBMODULE_RECURSE_NO;
-	sm->repo   = repo;
+	sm->repo   = cache->repo;
 	sm->branch = NULL;
 
-	return sm;
+	*out = sm;
+	return 0;
+}
+
+static void submodule_cache_remove_item(
+	git_submodule_cache *cache,
+	git_submodule *item,
+	bool free_after_remove)
+{
+	git_strmap *map;
+	const char *name, *alt;
+
+	if (!cache || !(map = cache->submodules) || !item)
+		return;
+
+	name = item->name;
+	alt  = (item->path != item->name) ? item->path : NULL;
+
+	for (; name; name = alt, alt = NULL) {
+		khiter_t pos = git_strmap_lookup_index(map, name);
+		git_submodule *found;
+
+		if (!git_strmap_valid_index(map, pos))
+			continue;
+
+		found = git_strmap_value_at(map, pos);
+
+		if (found != item)
+			continue;
+
+		git_strmap_set_value_at(map, pos, NULL);
+		git_strmap_delete_at(map, pos);
+
+		if (free_after_remove)
+			git_submodule_free(found);
+	}
 }
 
 static void submodule_release(git_submodule *sm)
@@ -1131,10 +1232,9 @@ static void submodule_release(git_submodule *sm)
 		return;
 
 	if (sm->repo) {
-		git_strmap *cache = sm->repo->submodules;
-		submodule_cache_remove_item(cache, sm->name, sm, false);
-		if (sm->path != sm->name)
-			submodule_cache_remove_item(cache, sm->path, sm, false);
+		git_submodule_cache *cache = sm->repo->_submodules;
+		sm->repo = NULL;
+		submodule_cache_remove_item(cache, sm, false);
 	}
 
 	if (sm->path != sm->name)
@@ -1155,40 +1255,39 @@ void git_submodule_free(git_submodule *sm)
 
 static int submodule_get(
 	git_submodule **out,
-	git_repository *repo,
+	git_submodule_cache *cache,
 	const char *name,
 	const char *alternate)
 {
 	int error = 0;
-	git_strmap *smcfg = repo->submodules;
 	khiter_t pos;
 	git_submodule *sm;
 
-	pos = git_strmap_lookup_index(smcfg, name);
+	pos = git_strmap_lookup_index(cache->submodules, name);
 
-	if (!git_strmap_valid_index(smcfg, pos) && alternate)
-		pos = git_strmap_lookup_index(smcfg, alternate);
+	if (!git_strmap_valid_index(cache->submodules, pos) && alternate)
+		pos = git_strmap_lookup_index(cache->submodules, alternate);
 
-	if (!git_strmap_valid_index(smcfg, pos)) {
-		sm = submodule_alloc(repo, name);
-		GITERR_CHECK_ALLOC(sm);
+	if (!git_strmap_valid_index(cache->submodules, pos)) {
+		if ((error = submodule_alloc(&sm, cache, name)) < 0)
+			return error;
 
 		/* insert value at name - if another thread beats us to it, then use
 		 * their record and release our own.
 		 */
-		pos = kh_put(str, smcfg, sm->name, &error);
+		pos = kh_put(str, cache->submodules, sm->name, &error);
 
 		if (error < 0)
 			goto done;
 		else if (error == 0) {
 			git_submodule_free(sm);
-			sm = git_strmap_value_at(smcfg, pos);
+			sm = git_strmap_value_at(cache->submodules, pos);
 		} else {
 			error = 0;
-			git_strmap_set_value_at(smcfg, pos, sm);
+			git_strmap_set_value_at(cache->submodules, pos, sm);
 		}
 	} else {
-		sm = git_strmap_value_at(smcfg, pos);
+		sm = git_strmap_value_at(cache->submodules, pos);
 	}
 
 done:
@@ -1254,10 +1353,10 @@ int git_submodule_parse_recurse(git_submodule_recurse_t *out, const char *value)
 static int submodule_load_from_config(
 	const git_config_entry *entry, void *payload)
 {
-	git_repository *repo = payload;
-	git_strmap *smcfg = repo->submodules;
-	const char *namestart, *property, *alternate = NULL;
+	git_submodule_cache *cache = payload;
+	const char *namestart, *property;
 	const char *key = entry->name, *value = entry->value, *path;
+	char *alternate = NULL, *replaced = NULL;
 	git_buf name = GIT_BUF_INIT;
 	git_submodule *sm = NULL;
 	int error = 0;
@@ -1275,7 +1374,7 @@ static int submodule_load_from_config(
 	path = !strcasecmp(property, "path") ? value : NULL;
 
 	if ((error = git_buf_set(&name, namestart, property - namestart - 1)) < 0 ||
-		(error = submodule_get(&sm, repo, name.ptr, path)) < 0)
+		(error = submodule_get(&sm, cache, name.ptr, path)) < 0)
 		goto done;
 
 	sm->flags |= GIT_SUBMODULE_STATUS_IN_CONFIG;
@@ -1288,20 +1387,42 @@ static int submodule_load_from_config(
 	 * should be strcasecmp
 	 */
 
-	if (strcmp(sm->name, name.ptr) != 0) {
-		alternate = sm->name = git_buf_detach(&name);
-	} else if (path && strcmp(path, sm->path) != 0) {
-		alternate = sm->path = git__strdup(value);
-		if (!sm->path) {
-			error = -1;
-			goto done;
+	if (strcmp(sm->name, name.ptr) != 0) { /* name changed */
+		if (!strcmp(sm->path, name.ptr)) { /* already set as path */
+			replaced = sm->name;
+			sm->name = sm->path;
+		} else {
+			if (sm->name != sm->path)
+				replaced = sm->name;
+			alternate = sm->name = git_buf_detach(&name);
+		}
+	}
+	else if (path && strcmp(path, sm->path) != 0) { /* path changed */
+		if (!strcmp(sm->name, value)) { /* already set as name */
+			replaced = sm->path;
+			sm->path = sm->name;
+		} else {
+			if (sm->path != sm->name)
+				replaced = sm->path;
+			if ((alternate = git__strdup(value)) == NULL) {
+				error = -1;
+				goto done;
+			}
+			sm->path = alternate;
 		}
 	}
 
-	/* Found a alternate key for the submodule */
+	/* Deregister under name being replaced */
+	if (replaced) {
+		git_strmap_delete(cache->submodules, replaced);
+		git_submodule_free(sm);
+		git__free(replaced);
+	}
+
+	/* Insert under alternate key */
 	if (alternate) {
 		void *old_sm = NULL;
-		git_strmap_insert2(smcfg, alternate, sm, old_sm, error);
+		git_strmap_insert2(cache->submodules, alternate, sm, old_sm, error);
 
 		if (error < 0)
 			goto done;
@@ -1383,36 +1504,33 @@ static int submodule_load_from_wd_lite(git_submodule *sm)
 	return 0;
 }
 
-static int load_submodule_config_from_index(
-	git_repository *repo, git_oid *gitmodules_oid)
+static int submodule_cache_refresh_from_index(
+	git_submodule_cache *cache, git_index *idx)
 {
 	int error;
-	git_index *index;
 	git_iterator *i;
 	const git_index_entry *entry;
 
-	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
-		(error = git_iterator_for_index(&i, index, 0, NULL, NULL)) < 0)
+	if ((error = git_iterator_for_index(&i, idx, 0, NULL, NULL)) < 0)
 		return error;
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		khiter_t pos = git_strmap_lookup_index(cache->submodules, entry->path);
 		git_submodule *sm;
 
-		if (git_strmap_valid_index(repo->submodules, pos)) {
-			sm = git_strmap_value_at(repo->submodules, pos);
+		if (git_strmap_valid_index(cache->submodules, pos)) {
+			sm = git_strmap_value_at(cache->submodules, pos);
 
 			if (S_ISGITLINK(entry->mode))
 				submodule_update_from_index_entry(sm, entry);
 			else
 				sm->flags |= GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE;
 		} else if (S_ISGITLINK(entry->mode)) {
-			if (!submodule_get(&sm, repo, entry->path, NULL)) {
+			if (!submodule_get(&sm, cache, entry->path, NULL)) {
 				submodule_update_from_index_entry(sm, entry);
 				git_submodule_free(sm);
 			}
-		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0)
-			git_oid_cpy(gitmodules_oid, &entry->id);
+		}
 	}
 
 	if (error == GIT_ITEROVER)
@@ -1423,46 +1541,33 @@ static int load_submodule_config_from_index(
 	return error;
 }
 
-static int load_submodule_config_from_head(
-	git_repository *repo, git_oid *gitmodules_oid)
+static int submodule_cache_refresh_from_head(
+	git_submodule_cache *cache, git_tree *head)
 {
 	int error;
-	git_tree *head;
 	git_iterator *i;
 	const git_index_entry *entry;
 
-	/* if we can't look up current head, then there's no submodule in it */
-	if (git_repository_head_tree(&head, repo) < 0) {
-		giterr_clear();
-		return 0;
-	}
-
-	if ((error = git_iterator_for_tree(&i, head, 0, NULL, NULL)) < 0) {
-		git_tree_free(head);
+	if ((error = git_iterator_for_tree(&i, head, 0, NULL, NULL)) < 0)
 		return error;
-	}
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		khiter_t pos = git_strmap_lookup_index(cache->submodules, entry->path);
 		git_submodule *sm;
 
-		if (git_strmap_valid_index(repo->submodules, pos)) {
-			sm = git_strmap_value_at(repo->submodules, pos);
+		if (git_strmap_valid_index(cache->submodules, pos)) {
+			sm = git_strmap_value_at(cache->submodules, pos);
 
 			if (S_ISGITLINK(entry->mode))
-				submodule_update_from_head_data(
-					sm, entry->mode, &entry->id);
+				submodule_update_from_head_data(sm, entry->mode, &entry->id);
 			else
 				sm->flags |= GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE;
 		} else if (S_ISGITLINK(entry->mode)) {
-			if (!submodule_get(&sm, repo, entry->path, NULL)) {
+			if (!submodule_get(&sm, cache, entry->path, NULL)) {
 				submodule_update_from_head_data(
 					sm, entry->mode, &entry->id);
 				git_submodule_free(sm);
 			}
-		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0 &&
-				   git_oid_iszero(gitmodules_oid)) {
-			git_oid_cpy(gitmodules_oid, &entry->id);
 		}
 	}
 
@@ -1470,17 +1575,15 @@ static int load_submodule_config_from_head(
 		error = 0;
 
 	git_iterator_free(i);
-	git_tree_free(head);
 
 	return error;
 }
 
 static git_config_backend *open_gitmodules(
-	git_repository *repo,
-	bool okay_to_create,
-	const git_oid *gitmodules_oid)
+	git_submodule_cache *cache,
+	int okay_to_create)
 {
-	const char *workdir = git_repository_workdir(repo);
+	const char *workdir = git_repository_workdir(cache->repo);
 	git_buf path = GIT_BUF_INIT;
 	git_config_backend *mods = NULL;
 
@@ -1500,186 +1603,281 @@ static git_config_backend *open_gitmodules(
 		}
 	}
 
-	if (!mods && gitmodules_oid && !git_oid_iszero(gitmodules_oid)) {
-		/* TODO: Retrieve .gitmodules content from ODB */
+	git_buf_free(&path);
 
-		/* Should we actually do this?  Core git does not, but it means you
-		 * can't really get much information about submodules on bare repos.
-		 */
+	return mods;
+}
+
+static void submodule_cache_free(git_submodule_cache *cache)
+{
+	git_submodule *sm;
+
+	if (!cache)
+		return;
+
+	git_strmap_foreach_value(cache->submodules, sm, {
+		sm->repo = NULL; /* disconnect from repo */
+		git_submodule_free(sm);
+	});
+	git_strmap_free(cache->submodules);
+
+	git_buf_free(&cache->gitmodules_path);
+	git_mutex_free(&cache->lock);
+	git__free(cache);
+}
+
+static int submodule_cache_alloc(
+	git_submodule_cache **out, git_repository *repo)
+{
+	git_submodule_cache *cache = git__calloc(1, sizeof(git_submodule_cache));
+	GITERR_CHECK_ALLOC(cache);
+
+	if (git_mutex_init(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to initialize submodule cache lock");
+		git__free(cache);
+		return -1;
 	}
 
-	git_buf_free(&path);
+	cache->submodules = git_strmap_alloc();
+	if (!cache->submodules) {
+		submodule_cache_free(cache);
+		return -1;
+	}
 
-	return mods;
+	cache->repo = repo;
+	git_buf_init(&cache->gitmodules_path, 0);
+
+	*out = cache;
+	return 0;
 }
 
-static int load_submodule_config(git_repository *repo, bool reload)
+static int submodule_cache_refresh(git_submodule_cache *cache, int refresh)
 {
-	int error;
-	git_oid gitmodules_oid;
+	int error = 0, update_index, update_head, update_gitmod;
+	git_index *idx = NULL;
+	git_tree *head = NULL;
+	const char *wd = NULL;
+	git_buf path = GIT_BUF_INIT;
+	git_submodule *sm;
 	git_config_backend *mods = NULL;
+	uint32_t mask;
 
-	if (!reload && repo->submodules)
+	if (!cache || !cache->repo || !refresh)
 		return 0;
 
-	memset(&gitmodules_oid, 0, sizeof(gitmodules_oid));
-
-	/* Submodule data is kept in a hashtable keyed by both name and path.
-	 * These are usually the same, but that is not guaranteed.
-	 */
-	if (!repo->submodules) {
-		repo->submodules = git_strmap_alloc();
-		GITERR_CHECK_ALLOC(repo->submodules);
+	if (git_mutex_lock(&cache->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire lock on submodule cache");
+		return -1;
 	}
 
-	/* TODO: only do the following if the sources appear modified */
+	/* get sources that we will need to check */
 
-	/* add submodule information from index */
+	if (git_repository_index(&idx, cache->repo) < 0)
+		giterr_clear();
+	if (git_repository_head_tree(&head, cache->repo) < 0)
+		giterr_clear();
 
-	if ((error = load_submodule_config_from_index(repo, &gitmodules_oid)) < 0)
+	wd = git_repository_workdir(cache->repo);
+	if (wd && (error = git_buf_joinpath(&path, wd, GIT_MODULES_FILE)) < 0)
 		goto cleanup;
 
+	/* check for invalidation */
+
+	if (refresh == CACHE_FLUSH)
+		update_index = update_head = update_gitmod = true;
+	else {
+		update_index =
+			!idx || git_index__changed_relative_to(idx, &cache->index_stamp);
+		update_head =
+			!head || !git_oid_equal(&cache->head_id, git_tree_id(head));
+
+		update_gitmod = (wd != NULL) ?
+			git_futils_filestamp_check(&cache->gitmodules_stamp, path.ptr) :
+			(cache->gitmodules_stamp.mtime != 0);
+		if (update_gitmod < 0)
+			giterr_clear();
+	}
+
+	/* clear submodule flags that are to be refreshed */
+
+	mask = 0;
+	if (!idx || update_index)
+		mask |= GIT_SUBMODULE_STATUS_IN_INDEX |
+			GIT_SUBMODULE_STATUS__INDEX_FLAGS |
+			GIT_SUBMODULE_STATUS__INDEX_OID_VALID |
+			GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
+	if (!head || update_head)
+		mask |= GIT_SUBMODULE_STATUS_IN_HEAD |
+			GIT_SUBMODULE_STATUS__HEAD_OID_VALID;
+	if (update_gitmod)
+		mask |= GIT_SUBMODULE_STATUS_IN_CONFIG;
+	if (mask != 0)
+		mask |= GIT_SUBMODULE_STATUS_IN_WD |
+			GIT_SUBMODULE_STATUS__WD_SCANNED |
+			GIT_SUBMODULE_STATUS__WD_FLAGS |
+			GIT_SUBMODULE_STATUS__WD_OID_VALID;
+	else
+		goto cleanup; /* nothing to do */
+
+	submodule_cache_clear_flags(cache, mask);
+
+	/* add back submodule information from index */
+
+	if (idx && update_index) {
+		if ((error = submodule_cache_refresh_from_index(cache, idx)) < 0)
+			goto cleanup;
+
+		git_futils_filestamp_set(
+			&cache->index_stamp, git_index__filestamp(idx));
+	}
+
 	/* add submodule information from HEAD */
 
-	if ((error = load_submodule_config_from_head(repo, &gitmodules_oid)) < 0)
-		goto cleanup;
+	if (head && update_head) {
+		if ((error = submodule_cache_refresh_from_head(cache, head)) < 0)
+			goto cleanup;
+
+		git_oid_cpy(&cache->head_id, git_tree_id(head));
+	}
 
 	/* add submodule information from .gitmodules */
 
-	if ((mods = open_gitmodules(repo, false, &gitmodules_oid)) != NULL &&
-		(error = git_config_file_foreach(
-			mods, submodule_load_from_config, repo)) < 0)
-		goto cleanup;
+	if (wd && update_gitmod > 0) {
+		if ((mods = open_gitmodules(cache, false)) != NULL &&
+			(error = git_config_file_foreach(
+				mods, submodule_load_from_config, cache)) < 0)
+			goto cleanup;
+	}
 
-	/* shallow scan submodules in work tree */
+	/* shallow scan submodules in work tree as needed */
 
-	if (!git_repository_is_bare(repo)) {
-		git_submodule *sm;
-
-		git_strmap_foreach_value(repo->submodules, sm, {
-			sm->flags &= ~GIT_SUBMODULE_STATUS__ALL_WD_FLAGS;
-		});
-		git_strmap_foreach_value(repo->submodules, sm, {
+	if (wd && mask != 0) {
+		git_strmap_foreach_value(cache->submodules, sm, {
 			submodule_load_from_wd_lite(sm);
 		});
 	}
 
+	/* remove submodules that no longer exist */
+
+	git_strmap_foreach_value(cache->submodules, sm, {
+		/* purge unless in HEAD, index, or .gitmodules; no sm for wd only */
+		if (sm != NULL &&
+			!(sm->flags &
+			 (GIT_SUBMODULE_STATUS_IN_HEAD |
+			  GIT_SUBMODULE_STATUS_IN_INDEX |
+			  GIT_SUBMODULE_STATUS_IN_CONFIG)))
+			submodule_cache_remove_item(cache, sm, true);
+	});
+
 cleanup:
-	if (mods != NULL)
-		git_config_file_free(mods);
+	git_config_file_free(mods);
+
+	/* TODO: if we got an error, mark submodule config as invalid? */
 
-	if (error)
-		git_submodule_config_free(repo);
+	git_mutex_unlock(&cache->lock);
+
+	git_index_free(idx);
+	git_tree_free(head);
+	git_buf_free(&path);
 
 	return error;
 }
 
-static int lookup_head_remote(git_buf *url, git_repository *repo)
+static int submodule_cache_init(git_repository *repo, int cache_refresh)
 {
-	int error;
-	git_config *cfg;
-	git_reference *head = NULL, *remote = NULL;
-	const char *tgt, *scan;
-	git_buf key = GIT_BUF_INIT;
+	int error = 0;
+	git_submodule_cache *cache = NULL;
 
-	/* 1. resolve HEAD -> refs/heads/BRANCH
-	 * 2. lookup config branch.BRANCH.remote -> ORIGIN
-	 * 3. lookup remote.ORIGIN.url
-	 */
+	/* if submodules already exist, just refresh as requested */
+	if (repo->_submodules)
+		return submodule_cache_refresh(repo->_submodules, cache_refresh);
 
-	if ((error = git_repository_config__weakptr(&cfg, repo)) < 0)
-		return error;
+	/* otherwise create a new cache, load it, and atomically swap it in */
+	if (!(error = submodule_cache_alloc(&cache, repo)) &&
+		!(error = submodule_cache_refresh(cache, CACHE_FLUSH)))
+		cache = git__compare_and_swap(&repo->_submodules, NULL, cache);
 
-	if (git_reference_lookup(&head, repo, GIT_HEAD_FILE) < 0) {
-		giterr_set(GITERR_SUBMODULE,
-			"Cannot resolve relative URL when HEAD cannot be resolved");
-		error = GIT_ENOTFOUND;
-		goto cleanup;
-	}
+	/* might have raced with another thread to set cache, so free if needed */
+	if (cache)
+		submodule_cache_free(cache);
 
-	if (git_reference_type(head) != GIT_REF_SYMBOLIC) {
-		giterr_set(GITERR_SUBMODULE,
-			"Cannot resolve relative URL when HEAD is not symbolic");
-		error = GIT_ENOTFOUND;
-		goto cleanup;
-	}
+	return error;
+}
 
-	if ((error = git_branch_upstream(&remote, head)) < 0)
-		goto cleanup;
+/* Lookup name of remote of the local tracking branch HEAD points to */
+static int lookup_head_remote_key(git_buf *remote_name, git_repository *repo)
+{
+	int error;
+	git_reference *head = NULL;
+	git_buf upstream_name = GIT_BUF_INIT;
 
-	/* remote should refer to something like refs/remotes/ORIGIN/BRANCH */
+	/* lookup and dereference HEAD */
+	if ((error = git_repository_head(&head, repo)) < 0)
+		return error;
 
-	if (git_reference_type(remote) != GIT_REF_SYMBOLIC ||
-		git__prefixcmp(git_reference_symbolic_target(remote), GIT_REFS_REMOTES_DIR) != 0)
+	/* lookup remote tracking branch of HEAD */
+	if (!(error = git_branch_upstream_name(
+			&upstream_name, repo, git_reference_name(head))))
 	{
-		giterr_set(GITERR_SUBMODULE,
-			"Cannot resolve relative URL when HEAD is not symbolic");
-		error = GIT_ENOTFOUND;
-		goto cleanup;
+		/* lookup remote of remote tracking branch */
+		error = git_branch_remote_name(remote_name, repo, upstream_name.ptr);
+
+		git_buf_free(&upstream_name);
 	}
 
-	scan = tgt = git_reference_symbolic_target(remote) + strlen(GIT_REFS_REMOTES_DIR);
-	while (*scan && (*scan != '/' || (scan > tgt && scan[-1] != '\\')))
-		scan++; /* find non-escaped slash to end ORIGIN name */
+	git_reference_free(head);
 
-	error = git_buf_printf(&key, "remote.%.*s.url", (int)(scan - tgt), tgt);
-	if (error < 0)
-		goto cleanup;
+	return error;
+}
 
-	if ((error = git_config_get_string(&tgt, cfg, key.ptr)) < 0)
-		goto cleanup;
+/* Lookup the remote of the local tracking branch HEAD points to */
+static int lookup_head_remote(git_remote **remote, git_repository *repo)
+{
+	int error;
+	git_buf remote_name = GIT_BUF_INIT;
 
-	error = git_buf_sets(url, tgt);
+	/* lookup remote of remote tracking branch name */
+	if (!(error = lookup_head_remote_key(&remote_name, repo)))
+		error = git_remote_load(remote, repo, remote_name.ptr);
 
-cleanup:
-	git_buf_free(&key);
-	git_reference_free(head);
-	git_reference_free(remote);
+	git_buf_free(&remote_name);
 
 	return error;
 }
 
-static int submodule_update_config(
-	git_submodule *submodule,
-	const char *attr,
-	const char *value,
-	bool overwrite,
-	bool only_existing)
+/* Lookup remote, either from HEAD or fall back on origin */
+static int lookup_default_remote(git_remote **remote, git_repository *repo)
 {
-	int error;
-	git_config *config;
-	git_buf key = GIT_BUF_INIT;
-	const git_config_entry *ce = NULL;
+	int error = lookup_head_remote(remote, repo);
 
-	assert(submodule);
+	/* if that failed, use 'origin' instead */
+	if (error == GIT_ENOTFOUND)
+		error = git_remote_load(remote, repo, "origin");
 
-	error = git_repository_config__weakptr(&config, submodule->repo);
-	if (error < 0)
-		return error;
+	if (error == GIT_ENOTFOUND)
+		giterr_set(
+			GITERR_SUBMODULE,
+			"Cannot get default remote for submodule - no local tracking "
+			"branch for HEAD and origin does not exist");
 
-	error = git_buf_printf(&key, "submodule.%s.%s", submodule->name, attr);
-	if (error < 0)
-		goto cleanup;
-
-	if ((error = git_config__lookup_entry(&ce, config, key.ptr, false)) < 0)
-		goto cleanup;
+	return error;
+}
 
-	if (!ce && only_existing)
-		goto cleanup;
-	if (ce && !overwrite)
-		goto cleanup;
-	if (value && ce && ce->value && !strcmp(ce->value, value))
-		goto cleanup;
-	if (!value && (!ce || !ce->value))
-		goto cleanup;
+static int get_url_base(git_buf *url, git_repository *repo)
+{
+	int error;
+	git_remote *remote = NULL;
 
-	if (!value)
-		error = git_config_delete_entry(config, key.ptr);
-	else
-		error = git_config_set_string(config, key.ptr, value);
+	if (!(error = lookup_default_remote(&remote, repo))) {
+		error = git_buf_sets(url, git_remote_url(remote));
+		git_remote_free(remote);
+	}
+	else if (error == GIT_ENOTFOUND) {
+		/* if repository does not have a default remote, use workdir instead */
+		giterr_clear();
+		error = git_buf_sets(url, git_repository_workdir(repo));
+	}
 
-cleanup:
-	git_buf_free(&key);
 	return error;
 }
 
diff --git a/src/submodule.h b/src/submodule.h
index 1c41897e3..a6182beca 100644
--- a/src/submodule.h
+++ b/src/submodule.h
@@ -99,6 +99,29 @@ struct git_submodule {
 	git_oid wd_oid;
 };
 
+/**
+ * The git_submodule_cache stores known submodules along with timestamps,
+ * etc. about when they were loaded
+ */
+typedef struct {
+	git_repository *repo;
+	git_strmap *submodules;
+	git_mutex  lock;
+
+	/* cache invalidation data */
+	git_oid head_id;
+	git_futils_filestamp index_stamp;
+	git_buf gitmodules_path;
+	git_futils_filestamp gitmodules_stamp;
+	git_futils_filestamp config_stamp;
+} git_submodule_cache;
+
+/* Force revalidation of submodule data cache (alloc as needed) */
+extern int git_submodule_cache_refresh(git_repository *repo);
+
+/* Release all submodules */
+extern void git_submodule_cache_free(git_repository *repo);
+
 /* Additional flags on top of public GIT_SUBMODULE_STATUS values */
 enum {
 	GIT_SUBMODULE_STATUS__WD_SCANNED          = (1u << 20),
@@ -111,17 +134,16 @@ enum {
 	GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES = (1u << 27),
 };
 
-#define GIT_SUBMODULE_STATUS__ALL_WD_FLAGS \
-	(GIT_SUBMODULE_STATUS_IN_WD | \
-	 GIT_SUBMODULE_STATUS__WD_OID_VALID | \
-	 GIT_SUBMODULE_STATUS__WD_FLAGS)
-
 #define GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(S) \
 	((S) & ~(0xFFFFFFFFu << 20))
 
 /* Internal submodule check does not attempt to refresh cached data */
 extern bool git_submodule__is_submodule(git_repository *repo, const char *name);
 
+/* Internal lookup does not attempt to refresh cached data */
+extern int git_submodule__lookup(
+	git_submodule **out, git_repository *repo, const char *path);
+
 /* Internal status fn returns status and optionally the various OIDs */
 extern int git_submodule__status(
 	unsigned int *out_status,
@@ -143,5 +165,6 @@ extern int git_submodule_parse_update(
 
 extern const char *git_submodule_ignore_to_str(git_submodule_ignore_t);
 extern const char *git_submodule_update_to_str(git_submodule_update_t);
+extern const char *git_submodule_recurse_to_str(git_submodule_recurse_t);
 
 #endif
diff --git a/src/vector.c b/src/vector.c
index e5d8919d3..c2c67e6b8 100644
--- a/src/vector.c
+++ b/src/vector.c
@@ -54,7 +54,7 @@ int git_vector_dup(git_vector *v, const git_vector *src, git_vector_cmp cmp)
 	bytes = src->length * sizeof(void *);
 
 	v->_alloc_size = src->length;
-	v->_cmp = cmp;
+	v->_cmp = cmp ? cmp : src->_cmp;
 	v->length = src->length;
 	v->flags  = src->flags;
 	if (cmp != src->_cmp)