Merge branch 'jl/submodule-fetch-on-demand'

* jl/submodule-fetch-on-demand:
  fetch/pull: Describe --recurse-submodule restrictions in the BUGS section
  submodule update: Don't fetch when the submodule commit is already present
  fetch/pull: Don't recurse into a submodule when commits are already present
  Submodules: Add 'on-demand' value for the 'fetchRecurseSubmodule' option
  config: teach the fetch.recurseSubmodules option the 'on-demand' value
  fetch/pull: Add the 'on-demand' value to the --recurse-submodules option
  fetch/pull: recurse into submodules when necessary

Conflicts:
	builtin/fetch.c
	submodule.c

13 files changed, 520 insertions, 37 deletions

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 1d0d1b7677..6d4dfa602e 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -897,9 +897,13 @@ diff.wordRegex::
 	characters are *ignorable* whitespace.
 
 fetch.recurseSubmodules::
-	A boolean value which changes the behavior for fetch and pull, the
-	default is to not recursively fetch populated submodules unless
-	configured otherwise.
+	This option can be either set to a boolean value or to 'on-demand'.
+	Setting it to a boolean changes the behavior of fetch and pull to
+	unconditionally recurse into submodules when set to true or to not
+	recurse at all when set to false. When set to 'on-demand' (the default
+	value), fetch and pull will only recurse into a populated submodule
+	when its superproject retrieves a commit that updates the submodule's
+	reference.
 
 fetch.unpackLimit::
 	If the number of objects fetched over the git native
@@ -1823,7 +1827,7 @@ submodule.<name>.update::
 	linkgit:git-submodule[1] and linkgit:gitmodules[5] for details.
 
 submodule.<name>.fetchRecurseSubmodules::
-	This option can be used to enable/disable recursive fetching of this
+	This option can be used to control recursive fetching of this
 	submodule. It can be overridden by using the --[no-]recurse-submodules
 	command line option to "git fetch" and "git pull".
 	This setting will override that from in the linkgit:gitmodules[5]
diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index f37276e5ad..39d326abc6 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -65,14 +65,33 @@ ifndef::git-pull[]
 	specified with the remote.<name>.tagopt setting. See
 	linkgit:git-config[1].
 
---[no-]recurse-submodules::
-	This option controls if new commits of all populated submodules should
-	be fetched too (see linkgit:git-config[1] and linkgit:gitmodules[5]).
+--recurse-submodules[=yes|on-demand|no]::
+	This option controls if and under what conditions new commits of
+	populated submodules should be fetched too. It can be used as a
+	boolean option to completely disable recursion when set to 'no' or to
+	unconditionally recurse into all populated submodules when set to
+	'yes', which is the default when this option is used without any
+	value. Use 'on-demand' to only recurse into a populated submodule
+	when the superproject retrieves a commit that updates the submodule's
+	reference to a commit that isn't already in the local submodule
+	clone.
+
+--no-recurse-submodules::
+	Disable recursive fetching of submodules (this has the same effect as
+	using the '--recurse-submodules=no' option).
 
 --submodule-prefix=<path>::
 	Prepend <path> to paths printed in informative messages
 	such as "Fetching submodule foo".  This option is used
 	internally when recursing over submodules.
+
+--recurse-submodules-default=[yes|on-demand]::
+	This option is used internally to temporarily provide a
+	non-negative default value for the --recurse-submodules
+	option.  All other methods of configuring fetch's submodule
+	recursion (such as settings in linkgit:gitmodules[5] and
+	linkgit:git-config[1]) override this option, as does
+	specifying --[no-]recurse-submodules directly.
 endif::git-pull[]
 
 -u::
diff --git a/Documentation/git-fetch.txt b/Documentation/git-fetch.txt
index 7146f6ba0b..67d221467d 100644
--- a/Documentation/git-fetch.txt
+++ b/Documentation/git-fetch.txt
@@ -76,6 +76,15 @@ The `pu` branch will be updated even if it is does not fast-forward,
 because it is prefixed with a plus sign; `tmp` will not be.
 
 
+BUGS
+----
+Using --recurse-submodules can only fetch new commits in already checked
+out submodules right now. When e.g. upstream added a new submodule in the
+just fetched commits of the superproject the submodule itself can not be
+fetched, making it impossible to check out that submodule later without
+having to do a fetch again. This is expected to be fixed in a future git
+version.
+
 SEE ALSO
 --------
 linkgit:git-pull[1]
diff --git a/Documentation/git-pull.txt b/Documentation/git-pull.txt
index c2a7f103ee..14609cbd4d 100644
--- a/Documentation/git-pull.txt
+++ b/Documentation/git-pull.txt
@@ -84,7 +84,7 @@ must be given before the options meant for 'git fetch'.
 --verbose::
 	Pass --verbose to git-fetch and git-merge.
 
---[no-]recurse-submodules::
+--[no-]recurse-submodules[=yes|on-demand|no]::
 	This option controls if new commits of all populated submodules should
 	be fetched too (see linkgit:git-config[1] and linkgit:gitmodules[5]).
 	That might be necessary to get the data needed for merging submodule
@@ -220,6 +220,15 @@ If you tried a pull which resulted in a complex conflicts and
 would want to start over, you can recover with 'git reset'.
 
 
+BUGS
+----
+Using --recurse-submodules can only fetch new commits in already checked
+out submodules right now. When e.g. upstream added a new submodule in the
+just fetched commits of the superproject the submodule itself can not be
+fetched, making it impossible to check out that submodule later without
+having to do a fetch again. This is expected to be fixed in a future git
+version.
+
 SEE ALSO
 --------
 linkgit:git-fetch[1], linkgit:git-merge[1], linkgit:git-config[1]
diff --git a/Documentation/gitmodules.txt b/Documentation/gitmodules.txt
index 15a2186559..4040941e55 100644
--- a/Documentation/gitmodules.txt
+++ b/Documentation/gitmodules.txt
@@ -45,12 +45,12 @@ submodule.<name>.update::
 	the '--merge' or '--rebase' options.
 
 submodule.<name>.fetchRecurseSubmodules::
-	This option can be used to enable/disable recursive fetching of this
+	This option can be used to control recursive fetching of this
 	submodule. If this option is also present in the submodules entry in
 	.git/config of the superproject, the setting there will override the
 	one found in .gitmodules.
 	Both settings can be overridden on the command line by using the
-	"--[no-]recurse-submodules" option to "git fetch" and "git pull"..
+	"--[no-]recurse-submodules" option to "git fetch" and "git pull".
 
 submodule.<name>.ignore::
 	Defines under what circumstances "git status" and the diff family show
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 6cbb5f69ec..f9c41da475 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -28,12 +28,6 @@ enum {
 	TAGS_SET = 2
 };
 
-enum {
-	RECURSE_SUBMODULES_OFF = 0,
-	RECURSE_SUBMODULES_DEFAULT = 1,
-	RECURSE_SUBMODULES_ON = 2
-};
-
 static int all, append, dry_run, force, keep, multiple, prune, update_head_ok, verbosity;
 static int progress, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT;
@@ -42,6 +36,21 @@ static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
 static struct transport *transport;
 static const char *submodule_prefix = "";
+static const char *recurse_submodules_default;
+
+static int option_parse_recurse_submodules(const struct option *opt,
+				   const char *arg, int unset)
+{
+	if (unset) {
+		recurse_submodules = RECURSE_SUBMODULES_OFF;
+	} else {
+		if (arg)
+			recurse_submodules = parse_fetch_recurse_submodules_arg(opt->long_name, arg);
+		else
+			recurse_submodules = RECURSE_SUBMODULES_ON;
+	}
+	return 0;
+}
 
 static struct option builtin_fetch_options[] = {
 	OPT__VERBOSITY(&verbosity),
@@ -60,9 +69,9 @@ static struct option builtin_fetch_options[] = {
 		    "do not fetch all tags (--no-tags)", TAGS_UNSET),
 	OPT_BOOLEAN('p', "prune", &prune,
 		    "prune remote-tracking branches no longer on remote"),
-	OPT_SET_INT(0, "recurse-submodules", &recurse_submodules,
+	{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, "on-demand",
 		    "control recursive fetching of submodules",
-		    RECURSE_SUBMODULES_ON),
+		    PARSE_OPT_OPTARG, option_parse_recurse_submodules },
 	OPT_BOOLEAN(0, "dry-run", &dry_run,
 		    "dry run"),
 	OPT_BOOLEAN('k', "keep", &keep, "keep downloaded pack"),
@@ -73,6 +82,9 @@ static struct option builtin_fetch_options[] = {
 		   "deepen history of shallow clone"),
 	{ OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, "dir",
 		   "prepend this to submodule path output", PARSE_OPT_HIDDEN },
+	{ OPTION_STRING, 0, "recurse-submodules-default",
+		   &recurse_submodules_default, NULL,
+		   "default mode for recursion", PARSE_OPT_HIDDEN },
 	OPT_END()
 };
 
@@ -284,6 +296,9 @@ static int update_local_ref(struct ref *ref,
 		else {
 			msg = "storing head";
 			what = _("[new branch]");
+			if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
+			    (recurse_submodules != RECURSE_SUBMODULES_ON))
+				check_for_new_submodule_commits(ref->new_sha1);
 		}
 
 		r = s_update_ref(msg, ref, 0);
@@ -299,6 +314,9 @@ static int update_local_ref(struct ref *ref,
 		strcpy(quickref, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV));
 		strcat(quickref, "..");
 		strcat(quickref, find_unique_abbrev(ref->new_sha1, DEFAULT_ABBREV));
+		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
+		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+			check_for_new_submodule_commits(ref->new_sha1);
 		r = s_update_ref("fast-forward", ref, 1);
 		sprintf(display, "%c %-*s %-*s -> %s%s", r ? '!' : ' ',
 			TRANSPORT_SUMMARY_WIDTH, quickref, REFCOL_WIDTH, remote,
@@ -310,6 +328,9 @@ static int update_local_ref(struct ref *ref,
 		strcpy(quickref, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV));
 		strcat(quickref, "...");
 		strcat(quickref, find_unique_abbrev(ref->new_sha1, DEFAULT_ABBREV));
+		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
+		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+			check_for_new_submodule_commits(ref->new_sha1);
 		r = s_update_ref("forced-update", ref, 1);
 		sprintf(display, "%c %-*s %-*s -> %s  (%s)", r ? '!' : '+',
 			TRANSPORT_SUMMARY_WIDTH, quickref, REFCOL_WIDTH, remote,
@@ -810,6 +831,8 @@ static void add_options_to_argv(int *argc, const char **argv)
 		argv[(*argc)++] = "--keep";
 	if (recurse_submodules == RECURSE_SUBMODULES_ON)
 		argv[(*argc)++] = "--recurse-submodules";
+	else if (recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND)
+		argv[(*argc)++] = "--recurse-submodules=on-demand";
 	if (verbosity >= 2)
 		argv[(*argc)++] = "-v";
 	if (verbosity >= 1)
@@ -951,15 +974,16 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 	if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {
 		const char *options[10];
 		int num_options = 0;
-		/* Set recursion as default when we already are recursing */
-		if (submodule_prefix[0])
-			set_config_fetch_recurse_submodules(1);
+		if (recurse_submodules_default) {
+			int arg = parse_fetch_recurse_submodules_arg("--recurse-submodules-default", recurse_submodules_default);
+			set_config_fetch_recurse_submodules(arg);
+		}
 		gitmodules_config();
 		git_config(submodule_config, NULL);
 		add_options_to_argv(&num_options, options);
 		result = fetch_populated_submodules(num_options, options,
 						    submodule_prefix,
-						    recurse_submodules == RECURSE_SUBMODULES_ON,
+						    recurse_submodules,
 						    verbosity < 0);
 	}
 
diff --git a/git-pull.sh b/git-pull.sh
index e31226b62f..4e9e0e49ec 100755
--- a/git-pull.sh
+++ b/git-pull.sh
@@ -110,6 +110,9 @@ do
 	--recurse-submodules)
 		recurse_submodules=--recurse-submodules
 		;;
+	--recurse-submodules=*)
+		recurse_submodules="$1"
+		;;
 	--no-recurse-submodules)
 		recurse_submodules=--no-recurse-submodules
 		;;
diff --git a/git-submodule.sh b/git-submodule.sh
index 7f6b3cf207..b010a67309 100755
--- a/git-submodule.sh
+++ b/git-submodule.sh
@@ -488,8 +488,11 @@ cmd_update()
 
 			if test -z "$nofetch"
 			then
+				# Run fetch only if $sha1 isn't present or it
+				# is not reachable from a ref.
 				(clear_local_git_env; cd "$path" &&
-					git-fetch) ||
+					((rev=$(git rev-list -n 1 $sha1 --not --all 2>/dev/null) &&
+					 test -z "$rev") || git-fetch)) ||
 				die "Unable to fetch in submodule path '$path'"
 			fi
 
diff --git a/submodule.c b/submodule.c
index 0cb6d18299..5294cef641 100644
--- a/submodule.c
+++ b/submodule.c
@@ -12,7 +12,8 @@
 static struct string_list config_name_for_path;
 static struct string_list config_fetch_recurse_submodules_for_name;
 static struct string_list config_ignore_for_name;
-static int config_fetch_recurse_submodules;
+static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
+static struct string_list changed_submodule_paths;
 
 static int add_submodule_odb(const char *path)
 {
@@ -70,7 +71,7 @@ int submodule_config(const char *var, const char *value, void *cb)
 	if (!prefixcmp(var, "submodule."))
 		return parse_submodule_config_option(var, value);
 	else if (!strcmp(var, "fetch.recursesubmodules")) {
-		config_fetch_recurse_submodules = git_config_bool(var, value);
+		config_fetch_recurse_submodules = parse_fetch_recurse_submodules_arg(var, value);
 		return 0;
 	}
 	return 0;
@@ -112,7 +113,7 @@ int parse_submodule_config_option(const char *var, const char *value)
 		if (!config)
 			config = string_list_append(&config_fetch_recurse_submodules_for_name,
 						    strbuf_detach(&submodname, NULL));
-		config->util = git_config_bool(var, value) ? (void *)1 : NULL;
+		config->util = (void *)(intptr_t)parse_fetch_recurse_submodules_arg(var, value);
 		strbuf_release(&submodname);
 	} else if ((len > 7) && !strcmp(var + len - 7, ".ignore")) {
 		if (strcmp(value, "untracked") && strcmp(value, "dirty") &&
@@ -206,6 +207,20 @@ static void print_submodule_summary(struct rev_info *rev, FILE *f,
 	strbuf_release(&sb);
 }
 
+int parse_fetch_recurse_submodules_arg(const char *opt, const char *arg)
+{
+	switch (git_config_maybe_bool(opt, arg)) {
+	case 1:
+		return RECURSE_SUBMODULES_ON;
+	case 0:
+		return RECURSE_SUBMODULES_OFF;
+	default:
+		if (!strcmp(arg, "on-demand"))
+			return RECURSE_SUBMODULES_ON_DEMAND;
+		die("bad %s argument: %s", opt, arg);
+	}
+}
+
 void show_submodule_summary(FILE *f, const char *path,
 		unsigned char one[20], unsigned char two[20],
 		unsigned dirty_submodule,
@@ -267,27 +282,122 @@ void set_config_fetch_recurse_submodules(int value)
 	config_fetch_recurse_submodules = value;
 }
 
+static int is_submodule_commit_present(const char *path, unsigned char sha1[20])
+{
+	int is_present = 0;
+	if (!add_submodule_odb(path) && lookup_commit_reference(sha1)) {
+		/* Even if the submodule is checked out and the commit is
+		 * present, make sure it is reachable from a ref. */
+		struct child_process cp;
+		const char *argv[] = {"rev-list", "-n", "1", NULL, "--not", "--all", NULL};
+		struct strbuf buf = STRBUF_INIT;
+
+		argv[3] = sha1_to_hex(sha1);
+		memset(&cp, 0, sizeof(cp));
+		cp.argv = argv;
+		cp.env = local_repo_env;
+		cp.git_cmd = 1;
+		cp.no_stdin = 1;
+		cp.out = -1;
+		cp.dir = path;
+		if (!run_command(&cp) && !strbuf_read(&buf, cp.out, 1024))
+			is_present = 1;
+
+		close(cp.out);
+		strbuf_release(&buf);
+	}
+	return is_present;
+}
+
+static void submodule_collect_changed_cb(struct diff_queue_struct *q,
+					 struct diff_options *options,
+					 void *data)
+{
+	int i;
+	for (i = 0; i < q->nr; i++) {
+		struct diff_filepair *p = q->queue[i];
+		if (!S_ISGITLINK(p->two->mode))
+			continue;
+
+		if (S_ISGITLINK(p->one->mode)) {
+			/* NEEDSWORK: We should honor the name configured in
+			 * the .gitmodules file of the commit we are examining
+			 * here to be able to correctly follow submodules
+			 * being moved around. */
+			struct string_list_item *path;
+			path = unsorted_string_list_lookup(&changed_submodule_paths, p->two->path);
+			if (!path && !is_submodule_commit_present(p->two->path, p->two->sha1))
+				string_list_append(&changed_submodule_paths, xstrdup(p->two->path));
+		} else {
+			/* Submodule is new or was moved here */
+			/* NEEDSWORK: When the .git directories of submodules
+			 * live inside the superprojects .git directory some
+			 * day we should fetch new submodules directly into
+			 * that location too when config or options request
+			 * that so they can be checked out from there. */
+			continue;
+		}
+	}
+}
+
+void check_for_new_submodule_commits(unsigned char new_sha1[20])
+{
+	struct rev_info rev;
+	struct commit *commit;
+	const char *argv[] = {NULL, NULL, "--not", "--all", NULL};
+	int argc = ARRAY_SIZE(argv) - 1;
+
+	init_revisions(&rev, NULL);
+	argv[1] = xstrdup(sha1_to_hex(new_sha1));
+	setup_revisions(argc, argv, &rev, NULL);
+	if (prepare_revision_walk(&rev))
+		die("revision walk setup failed");
+
+	/*
+	 * Collect all submodules (whether checked out or not) for which new
+	 * commits have been recorded upstream in "changed_submodule_paths".
+	 */
+	while ((commit = get_revision(&rev))) {
+		struct commit_list *parent = commit->parents;
+		while (parent) {
+			struct diff_options diff_opts;
+			diff_setup(&diff_opts);
+			diff_opts.output_format |= DIFF_FORMAT_CALLBACK;
+			diff_opts.format_callback = submodule_collect_changed_cb;
+			if (diff_setup_done(&diff_opts) < 0)
+				die("diff_setup_done failed");
+			diff_tree_sha1(parent->item->object.sha1, commit->object.sha1, "", &diff_opts);
+			diffcore_std(&diff_opts);
+			diff_flush(&diff_opts);
+			parent = parent->next;
+		}
+	}
+	free((char *)argv[1]);
+}
+
 int fetch_populated_submodules(int num_options, const char **options,
-			       const char *prefix, int ignore_config,
+			       const char *prefix, int command_line_option,
 			       int quiet)
 {
-	int i, result = 0, argc = 0;
+	int i, result = 0, argc = 0, default_argc;
 	struct child_process cp;
 	const char **argv;
 	struct string_list_item *name_for_path;
 	const char *work_tree = get_git_work_tree();
 	if (!work_tree)
-		return 0;
+		goto out;
 
 	if (!the_index.initialized)
 		if (read_cache() < 0)
 			die("index file corrupt");
 
-	/* 4: "fetch" (options) "--submodule-prefix" prefix NULL */
-	argv = xcalloc(num_options + 4, sizeof(const char *));
+	/* 6: "fetch" (options) --recurse-submodules-default default "--submodule-prefix" prefix NULL */
+	argv = xcalloc(num_options + 6, sizeof(const char *));
 	argv[argc++] = "fetch";
 	for (i = 0; i < num_options; i++)
 		argv[argc++] = options[i];
+	argv[argc++] = "--recurse-submodules-default";
+	default_argc = argc++;
 	argv[argc++] = "--submodule-prefix";
 
 	memset(&cp, 0, sizeof(cp));
@@ -301,7 +411,7 @@ int fetch_populated_submodules(int num_options, const char **options,
 		struct strbuf submodule_git_dir = STRBUF_INIT;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		struct cache_entry *ce = active_cache[i];
-		const char *git_dir, *name;
+		const char *git_dir, *name, *default_argv;
 
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
@@ -311,16 +421,31 @@ int fetch_populated_submodules(int num_options, const char **options,
 		if (name_for_path)
 			name = name_for_path->util;
 
-		if (!ignore_config) {
+		default_argv = "yes";
+		if (command_line_option == RECURSE_SUBMODULES_DEFAULT) {
 			struct string_list_item *fetch_recurse_submodules_option;
 			fetch_recurse_submodules_option = unsorted_string_list_lookup(&config_fetch_recurse_submodules_for_name, name);
 			if (fetch_recurse_submodules_option) {
-				if (!fetch_recurse_submodules_option->util)
+				if ((intptr_t)fetch_recurse_submodules_option->util == RECURSE_SUBMODULES_OFF)
 					continue;
+				if ((intptr_t)fetch_recurse_submodules_option->util == RECURSE_SUBMODULES_ON_DEMAND) {
+					if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
+						continue;
+					default_argv = "on-demand";
+				}
 			} else {
-				if (!config_fetch_recurse_submodules)
+				if (config_fetch_recurse_submodules == RECURSE_SUBMODULES_OFF)
 					continue;
+				if (config_fetch_recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND) {
+					if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
+						continue;
+					default_argv = "on-demand";
+				}
 			}
+		} else if (command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
+			if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
+				continue;
+			default_argv = "on-demand";
 		}
 
 		strbuf_addf(&submodule_path, "%s/%s", work_tree, ce->name);
@@ -333,6 +458,7 @@ int fetch_populated_submodules(int num_options, const char **options,
 			if (!quiet)
 				printf("Fetching submodule %s%s\n", prefix, ce->name);
 			cp.dir = submodule_path.buf;
+			argv[default_argc] = default_argv;
 			argv[argc] = submodule_prefix.buf;
 			if (run_command(&cp))
 				result = 1;
@@ -342,6 +468,8 @@ int fetch_populated_submodules(int num_options, const char **options,
 		strbuf_release(&submodule_prefix);
 	}
 	free(argv);
+out:
+	string_list_clear(&changed_submodule_paths, 1);
 	return result;
 }
 
diff --git a/submodule.h b/submodule.h
index 4729023aa5..5350b0d5a0 100644
--- a/submodule.h
+++ b/submodule.h
@@ -3,19 +3,28 @@
 
 struct diff_options;
 
+enum {
+	RECURSE_SUBMODULES_ON_DEMAND = -1,
+	RECURSE_SUBMODULES_OFF = 0,
+	RECURSE_SUBMODULES_DEFAULT = 1,
+	RECURSE_SUBMODULES_ON = 2
+};
+
 void set_diffopt_flags_from_submodule_config(struct diff_options *diffopt,
 		const char *path);
 int submodule_config(const char *var, const char *value, void *cb);
 void gitmodules_config();
 int parse_submodule_config_option(const char *var, const char *value);
 void handle_ignore_submodules_arg(struct diff_options *diffopt, const char *);
+int parse_fetch_recurse_submodules_arg(const char *opt, const char *arg);
 void show_submodule_summary(FILE *f, const char *path,
 		unsigned char one[20], unsigned char two[20],
 		unsigned dirty_submodule,
 		const char *del, const char *add, const char *reset);
 void set_config_fetch_recurse_submodules(int value);
+void check_for_new_submodule_commits(unsigned char new_sha1[20]);
 int fetch_populated_submodules(int num_options, const char **options,
-			       const char *prefix, int ignore_config,
+			       const char *prefix, int command_line_option,
 			       int quiet);
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int merge_submodule(unsigned char result[20], const char *path, const unsigned char base[20],
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 8f12379879..af78e21ba9 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -216,4 +216,259 @@ test_expect_success "--no-recurse-submodules overrides config setting" '
 	! test -s actual.err
 '
 
+test_expect_success "Recursion doesn't happen when no new commits are fetched in the superproject" '
+	(
+		cd downstream &&
+		(
+			cd submodule &&
+			git config --unset fetch.recurseSubmodules
+		) &&
+		git config --unset fetch.recurseSubmodules
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	! test -s actual.out &&
+	! test -s actual.err
+'
+
+test_expect_success "Recursion stops when no new submodule commits are fetched" '
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "Fetching submodule submodule" > expect.out.sub &&
+	echo "From $pwd/." > expect.err.sub &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.sub
+	head -2 expect.err >> expect.err.sub &&
+	(
+		cd downstream &&
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	test_cmp expect.err.sub actual.err &&
+	test_cmp expect.out.sub actual.out
+'
+
+test_expect_success "Recursion doesn't happen when new superproject commits don't change any submodules" '
+	add_upstream_commit &&
+	head1=$(git rev-parse --short HEAD) &&
+	echo a > file &&
+	git add file &&
+	git commit -m "new file" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err.file &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.file &&
+	(
+		cd downstream &&
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	! test -s actual.out &&
+	test_cmp expect.err.file actual.err
+'
+
+test_expect_success "Recursion picks up config in submodule" '
+	(
+		cd downstream &&
+		git fetch --recurse-submodules &&
+		(
+			cd submodule &&
+			git config fetch.recurseSubmodules true
+		)
+	) &&
+	add_upstream_commit &&
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err.sub &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.sub &&
+	cat expect.err >> expect.err.sub &&
+	(
+		cd downstream &&
+		git fetch >../actual.out 2>../actual.err &&
+		(
+			cd submodule &&
+			git config --unset fetch.recurseSubmodules
+		)
+	) &&
+	test_cmp expect.err.sub actual.err &&
+	test_cmp expect.out actual.out
+'
+
+test_expect_success "Recursion picks up all submodules when necessary" '
+	add_upstream_commit &&
+	(
+		cd submodule &&
+		(
+			cd deepsubmodule &&
+			git fetch &&
+			git checkout -q FETCH_HEAD
+		) &&
+		head1=$(git rev-parse --short HEAD^) &&
+		git add deepsubmodule &&
+		git commit -m "new deepsubmodule"
+		head2=$(git rev-parse --short HEAD) &&
+		echo "From $pwd/submodule" > ../expect.err.sub &&
+		echo "   $head1..$head2  master     -> origin/master" >> ../expect.err.sub
+	) &&
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err.2 &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.2 &&
+	cat expect.err.sub >> expect.err.2 &&
+	tail -2 expect.err >> expect.err.2 &&
+	(
+		cd downstream &&
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	test_cmp expect.err.2 actual.err &&
+	test_cmp expect.out actual.out
+'
+
+test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no new commits are fetched in the superproject (and ignores config)" '
+	add_upstream_commit &&
+	(
+		cd submodule &&
+		(
+			cd deepsubmodule &&
+			git fetch &&
+			git checkout -q FETCH_HEAD
+		) &&
+		head1=$(git rev-parse --short HEAD^) &&
+		git add deepsubmodule &&
+		git commit -m "new deepsubmodule"
+		head2=$(git rev-parse --short HEAD) &&
+		echo "From $pwd/submodule" > ../expect.err.sub &&
+		echo "   $head1..$head2  master     -> origin/master" >> ../expect.err.sub
+	) &&
+	(
+		cd downstream &&
+		git config fetch.recurseSubmodules true &&
+		git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err &&
+		git config --unset fetch.recurseSubmodules
+	) &&
+	! test -s actual.out &&
+	! test -s actual.err
+'
+
+test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necessary (and ignores config)" '
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	head2=$(git rev-parse --short HEAD) &&
+	tail -2 expect.err > expect.err.deepsub &&
+	echo "From $pwd/." > expect.err &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err
+	cat expect.err.sub >> expect.err &&
+	cat expect.err.deepsub >> expect.err &&
+	(
+		cd downstream &&
+		git config fetch.recurseSubmodules false &&
+		(
+			cd submodule &&
+			git config -f .gitmodules submodule.deepsubmodule.fetchRecursive false
+		) &&
+		git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err &&
+		git config --unset fetch.recurseSubmodules
+		(
+			cd submodule &&
+			git config --unset -f .gitmodules submodule.deepsubmodule.fetchRecursive
+		)
+	) &&
+	test_cmp expect.out actual.out &&
+	test_cmp expect.err actual.err
+'
+
+test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" '
+	add_upstream_commit &&
+	head1=$(git rev-parse --short HEAD) &&
+	echo a >> file &&
+	git add file &&
+	git commit -m "new file" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err.file &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.file &&
+	(
+		cd downstream &&
+		git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err
+	) &&
+	! test -s actual.out &&
+	test_cmp expect.err.file actual.err
+'
+
+test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config" '
+	(
+		cd downstream &&
+		git fetch --recurse-submodules
+	) &&
+	add_upstream_commit &&
+	git config --global fetch.recurseSubmodules false &&
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err.2 &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.2
+	head -2 expect.err >> expect.err.2 &&
+	(
+		cd downstream &&
+		git config fetch.recurseSubmodules on-demand &&
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	git config --global --unset fetch.recurseSubmodules &&
+	(
+		cd downstream &&
+		git config --unset fetch.recurseSubmodules
+	) &&
+	test_cmp expect.out.sub actual.out &&
+	test_cmp expect.err.2 actual.err
+'
+
+test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' overrides fetch.recurseSubmodules" '
+	(
+		cd downstream &&
+		git fetch --recurse-submodules
+	) &&
+	add_upstream_commit &&
+	git config fetch.recurseSubmodules false &&
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err.2 &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err.2
+	head -2 expect.err >> expect.err.2 &&
+	(
+		cd downstream &&
+		git config submodule.submodule.fetchRecurseSubmodules on-demand &&
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	git config --unset fetch.recurseSubmodules &&
+	(
+		cd downstream &&
+		git config --unset submodule.submodule.fetchRecurseSubmodules
+	) &&
+	test_cmp expect.out.sub actual.out &&
+	test_cmp expect.err.2 actual.err
+'
+
+test_expect_success "don't fetch submodule when newly recorded commits are already present" '
+	(
+		cd submodule &&
+		git checkout -q HEAD^^
+	) &&
+	head1=$(git rev-parse --short HEAD) &&
+	git add submodule &&
+	git commit -m "submodule rewound" &&
+	head2=$(git rev-parse --short HEAD) &&
+	echo "From $pwd/." > expect.err &&
+	echo "   $head1..$head2  master     -> origin/master" >> expect.err &&
+	(
+		cd downstream &&
+		git fetch >../actual.out 2>../actual.err
+	) &&
+	! test -s actual.out &&
+	test_cmp expect.err actual.err
+'
+
 test_done
diff --git a/t/t7403-submodule-sync.sh b/t/t7403-submodule-sync.sh
index e5b19538b0..d600583cef 100755
--- a/t/t7403-submodule-sync.sh
+++ b/t/t7403-submodule-sync.sh
@@ -52,7 +52,7 @@ test_expect_success 'change submodule url' '
 
 test_expect_success '"git submodule sync" should update submodule URLs' '
 	(cd super-clone &&
-	 git pull &&
+	 git pull --no-recurse-submodules &&
 	 git submodule sync
 	) &&
 	test -d "$(git config -f super-clone/submodule/.git/config \
diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh
index fa9d23aa31..bf7c788735 100755
--- a/t/t7406-submodule-update.sh
+++ b/t/t7406-submodule-update.sh
@@ -74,6 +74,26 @@ test_expect_success 'submodule update detaching the HEAD ' '
 	)
 '
 
+apos="'";
+test_expect_success 'submodule update does not fetch already present commits' '
+	(cd submodule &&
+	  echo line3 >> file &&
+	  git add file &&
+	  test_tick &&
+	  git commit -m "upstream line3"
+	) &&
+	(cd super/submodule &&
+	  head=$(git rev-parse --verify HEAD) &&
+	  echo "Submodule path ${apos}submodule$apos: checked out $apos$head$apos" > ../../expected &&
+	  git reset --hard HEAD~1
+	) &&
+	(cd super &&
+	  git submodule update > ../actual 2> ../actual.err
+	) &&
+	test_cmp expected actual &&
+	! test -s actual.err
+'
+
 test_expect_success 'submodule update --rebase staying on master' '
 	(cd super/submodule &&
 	  git checkout master