Merge branch 'nd/sparse'

* nd/sparse: (25 commits)
  t7002: test for not using external grep on skip-worktree paths
  t7002: set test prerequisite "external-grep" if supported
  grep: do not do external grep on skip-worktree entries
  commit: correctly respect skip-worktree bit
  ie_match_stat(): do not ignore skip-worktree bit with CE_MATCH_IGNORE_VALID
  tests: rename duplicate t1009
  sparse checkout: inhibit empty worktree
  Add tests for sparse checkout
  read-tree: add --no-sparse-checkout to disable sparse checkout support
  unpack-trees(): ignore worktree check outside checkout area
  unpack_trees(): apply $GIT_DIR/info/sparse-checkout to the final index
  unpack-trees(): "enable" sparse checkout and load $GIT_DIR/info/sparse-checkout
  unpack-trees.c: generalize verify_* functions
  unpack-trees(): add CE_WT_REMOVE to remove on worktree alone
  Introduce "sparse checkout"
  dir.c: export excluded_1() and add_excludes_from_file_1()
  excluded_1(): support exclude files in index
  unpack-trees(): carry skip-worktree bit over in merged_entry()
  Read .gitignore from index if it is skip-worktree
  Avoid writing to buffer in add_excludes_from_file_1()
  ...

Conflicts:
	.gitignore
	Documentation/config.txt
	Documentation/git-update-index.txt
	Makefile
	entry.c
	t/t7002-grep.sh

33 files changed, 1049 insertions, 102 deletions

diff --git a/.gitignore b/.gitignore
index 5d32289664..0746a4d209 100644
--- a/.gitignore
+++ b/.gitignore
@@ -158,6 +158,7 @@
 /test-delta
 /test-dump-cache-tree
 /test-genrandom
+/test-index-version
 /test-match-trees
 /test-parse-options
 /test-path-utils
diff --git a/Documentation/config.txt b/Documentation/config.txt
index 9f40955f8e..8acb613ec3 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -502,6 +502,10 @@ notes should be printed.
 This setting defaults to "refs/notes/commits", and can be overridden by
 the `GIT_NOTES_REF` environment variable.
 
+core.sparseCheckout::
+	Enable "sparse checkout" feature. See section "Sparse checkout" in
+	linkgit:git-read-tree[1] for more information.
+
 add.ignore-errors::
 	Tells 'git-add' to continue adding files when some files cannot be
 	added due to indexing errors. Equivalent to the '--ignore-errors'
diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt
index 625723e41f..98f3b9e758 100644
--- a/Documentation/git-ls-files.txt
+++ b/Documentation/git-ls-files.txt
@@ -109,6 +109,7 @@ OPTIONS
 	Identify the file status with the following tags (followed by
 	a space) at the start of each line:
 	H::	cached
+	S::	skip-worktree
 	M::	unmerged
 	R::	removed/deleted
 	C::	modified/changed
diff --git a/Documentation/git-read-tree.txt b/Documentation/git-read-tree.txt
index a10ce4ba40..d6faa14149 100644
--- a/Documentation/git-read-tree.txt
+++ b/Documentation/git-read-tree.txt
@@ -10,7 +10,7 @@ SYNOPSIS
 --------
 'git read-tree' [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>]
 		[-u [--exclude-per-directory=<gitignore>] | -i]]
-		[--index-output=<file>]
+		[--index-output=<file>] [--no-sparse-checkout]
 		<tree-ish1> [<tree-ish2> [<tree-ish3>]]
 
 
@@ -110,6 +110,10 @@ OPTIONS
 	directories the index file and index output file are
 	located in.
 
+--no-sparse-checkout::
+	Disable sparse checkout support even if `core.sparseCheckout`
+	is true.
+
 <tree-ish#>::
 	The id of the tree object(s) to be read/merged.
 
@@ -360,6 +364,52 @@ middle of doing, and when your working tree is ready (i.e. you
 have finished your work-in-progress), attempt the merge again.
 
 
+Sparse checkout
+---------------
+
+"Sparse checkout" allows to sparsely populate working directory.
+It uses skip-worktree bit (see linkgit:git-update-index[1]) to tell
+Git whether a file on working directory is worth looking at.
+
+"git read-tree" and other merge-based commands ("git merge", "git
+checkout"...) can help maintaining skip-worktree bitmap and working
+directory update. `$GIT_DIR/info/sparse-checkout` is used to
+define the skip-worktree reference bitmap. When "git read-tree" needs
+to update working directory, it will reset skip-worktree bit in index
+based on this file, which uses the same syntax as .gitignore files.
+If an entry matches a pattern in this file, skip-worktree will be
+set on that entry. Otherwise, skip-worktree will be unset.
+
+Then it compares the new skip-worktree value with the previous one. If
+skip-worktree turns from unset to set, it will add the corresponding
+file back. If it turns from set to unset, that file will be removed.
+
+While `$GIT_DIR/info/sparse-checkout` is usually used to specify what
+files are in. You can also specify what files are _not_ in, using
+negate patterns. For example, to remove file "unwanted":
+
+----------------
+*
+!unwanted
+----------------
+
+Another tricky thing is fully repopulating working directory when you
+no longer want sparse checkout. You cannot just disable "sparse
+checkout" because skip-worktree are still in the index and you working
+directory is still sparsely populated. You should re-populate working
+directory with the `$GIT_DIR/info/sparse-checkout` file content as
+follows:
+
+----------------
+*
+----------------
+
+Then you can disable sparse checkout. Sparse checkout support in "git
+read-tree" and similar commands is disabled by default. You need to
+turn `core.sparseCheckout` on in order to have sparse checkout
+support.
+
+
 SEE ALSO
 --------
 linkgit:git-write-tree[1]; linkgit:git-ls-files[1];
diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt
index 6052484ab9..8d88018eed 100644
--- a/Documentation/git-update-index.txt
+++ b/Documentation/git-update-index.txt
@@ -15,6 +15,7 @@ SYNOPSIS
 	     [--cacheinfo <mode> <object> <file>]\*
 	     [--chmod=(+|-)x]
 	     [--assume-unchanged | --no-assume-unchanged]
+	     [--skip-worktree | --no-skip-worktree]
 	     [--ignore-submodules]
 	     [--really-refresh] [--unresolve] [--again | -g]
 	     [--info-only] [--index-info]
@@ -103,6 +104,13 @@ you will need to handle the situation manually.
 	Like '--refresh', but checks stat information unconditionally,
 	without regard to the "assume unchanged" setting.
 
+--skip-worktree::
+--no-skip-worktree::
+	When one of these flags is specified, the object name recorded
+	for the paths are not updated. Instead, these options
+	set and unset the "skip-worktree" bit for the paths. See
+	section "Skip-worktree bit" below for more information.
+
 -g::
 --again::
 	Runs 'git-update-index' itself on the paths whose index
@@ -308,6 +316,27 @@ M foo.c
 <9> now it checks with lstat(2) and finds it has been changed.
 
 
+Skip-worktree bit
+-----------------
+
+Skip-worktree bit can be defined in one (long) sentence: When reading
+an entry, if it is marked as skip-worktree, then Git pretends its
+working directory version is up to date and read the index version
+instead.
+
+To elaborate, "reading" means checking for file existence, reading
+file attributes or file content. The working directory version may be
+present or absent. If present, its content may match against the index
+version or not. Writing is not affected by this bit, content safety
+is still first priority. Note that Git _can_ update working directory
+file, that is marked skip-worktree, if it is safe to do so (i.e.
+working directory version matches index version)
+
+Although this bit looks similar to assume-unchanged bit, its goal is
+different from assume-unchanged bit's. Skip-worktree also takes
+precedence over assume-unchanged bit when both are set.
+
+
 Configuration
 -------------
 
diff --git a/Documentation/technical/api-directory-listing.txt b/Documentation/technical/api-directory-listing.txt
index 5bbd18f020..add6f435b5 100644
--- a/Documentation/technical/api-directory-listing.txt
+++ b/Documentation/technical/api-directory-listing.txt
@@ -58,6 +58,9 @@ The result of the enumeration is left in these fields::
 Calling sequence
 ----------------
 
+Note: index may be looked at for .gitignore files that are CE_SKIP_WORKTREE
+marked. If you to exclude files, make sure you have loaded index first.
+
 * Prepare `struct dir_struct dir` and clear it with `memset(&dir, 0,
   sizeof(dir))`.
 
diff --git a/Makefile b/Makefile
index be18389534..412dda83ac 100644
--- a/Makefile
+++ b/Makefile
@@ -1782,6 +1782,7 @@ TEST_PROGRAMS_NEED_X += test-parse-options
 TEST_PROGRAMS_NEED_X += test-path-utils
 TEST_PROGRAMS_NEED_X += test-sha1
 TEST_PROGRAMS_NEED_X += test-sigchain
+TEST_PROGRAMS_NEED_X += test-index-version
 
 TEST_PROGRAMS = $(patsubst %,%$X,$(TEST_PROGRAMS_NEED_X))
 
diff --git a/builtin-apply.c b/builtin-apply.c
index 36e2f9dda5..541493e1ba 100644
--- a/builtin-apply.c
+++ b/builtin-apply.c
@@ -2666,7 +2666,7 @@ static int verify_index_match(struct cache_entry *ce, struct stat *st)
 			return -1;
 		return 0;
 	}
-	return ce_match_stat(ce, st, CE_MATCH_IGNORE_VALID);
+	return ce_match_stat(ce, st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
 }
 
 static int check_preimage(struct patch *patch, struct cache_entry **ce, struct stat *st)
diff --git a/builtin-clean.c b/builtin-clean.c
index 28cdcd0274..3a70fa81bd 100644
--- a/builtin-clean.c
+++ b/builtin-clean.c
@@ -75,11 +75,13 @@ int cmd_clean(int argc, const char **argv, const char *prefix)
 
 	dir.flags |= DIR_SHOW_OTHER_DIRECTORIES;
 
+	if (read_cache() < 0)
+		die("index file corrupt");
+
 	if (!ignored)
 		setup_standard_excludes(&dir);
 
 	pathspec = get_pathspec(prefix, argv);
-	read_cache();
 
 	fill_directory(&dir, pathspec);
 
diff --git a/builtin-commit.c b/builtin-commit.c
index 073fe90ba1..592b10396d 100644
--- a/builtin-commit.c
+++ b/builtin-commit.c
@@ -183,11 +183,15 @@ static int list_paths(struct string_list *list, const char *with_tree,
 
 	for (i = 0; i < active_nr; i++) {
 		struct cache_entry *ce = active_cache[i];
+		struct string_list_item *item;
+
 		if (ce->ce_flags & CE_UPDATE)
 			continue;
 		if (!match_pathspec(pattern, ce->name, ce_namelen(ce), 0, m))
 			continue;
-		string_list_insert(ce->name, list);
+		item = string_list_insert(ce->name, list);
+		if (ce_skip_worktree(ce))
+			item->util = item; /* better a valid pointer than a fake one */
 	}
 
 	return report_path_error(m, pattern, prefix ? strlen(prefix) : 0);
@@ -200,6 +204,10 @@ static void add_remove_files(struct string_list *list)
 		struct stat st;
 		struct string_list_item *p = &(list->items[i]);
 
+		/* p->util is skip-worktree */
+		if (p->util)
+			continue;
+
 		if (!lstat(p->string, &st)) {
 			if (add_to_cache(p->string, &st, 0))
 				die("updating files failed");
diff --git a/builtin-grep.c b/builtin-grep.c
index af6c6fe8a7..529461fb8f 100644
--- a/builtin-grep.c
+++ b/builtin-grep.c
@@ -220,6 +220,7 @@ static int exec_grep(int argc, const char **argv)
 	int status;
 
 	argv[argc] = NULL;
+	trace_argv_printf(argv, "trace: grep:");
 	pid = fork();
 	if (pid < 0)
 		return pid;
@@ -345,6 +346,21 @@ static void grep_add_color(struct strbuf *sb, const char *escape_seq)
 		strbuf_setlen(sb, sb->len - 1);
 }
 
+static int has_skip_worktree_entry(struct grep_opt *opt, const char **paths)
+{
+	int nr;
+	for (nr = 0; nr < active_nr; nr++) {
+		struct cache_entry *ce = active_cache[nr];
+		if (!S_ISREG(ce->ce_mode))
+			continue;
+		if (!pathspec_matches(paths, ce->name, opt->max_depth))
+			continue;
+		if (ce_skip_worktree(ce))
+			return 1;
+	}
+	return 0;
+}
+
 static int external_grep(struct grep_opt *opt, const char **paths, int cached)
 {
 	int i, nr, argc, hit, len, status;
@@ -353,7 +369,8 @@ static int external_grep(struct grep_opt *opt, const char **paths, int cached)
 	char *argptr = randarg;
 	struct grep_pat *p;
 
-	if (opt->extended || (opt->relative && opt->prefix_length))
+	if (opt->extended || (opt->relative && opt->prefix_length)
+	    || has_skip_worktree_entry(opt, paths))
 		return -1;
 	len = nr = 0;
 	push_arg("grep");
@@ -510,7 +527,7 @@ static int grep_cache(struct grep_opt *opt, const char **paths, int cached,
 		 * are identical, even if worktree file has been modified, so use
 		 * cache version instead
 		 */
-		if (cached || (ce->ce_flags & CE_VALID)) {
+		if (cached || (ce->ce_flags & CE_VALID) || ce_skip_worktree(ce)) {
 			if (ce_stage(ce))
 				continue;
 			hit |= grep_sha1(opt, ce->sha1, ce->name, 0);
diff --git a/builtin-ls-files.c b/builtin-ls-files.c
index c9a03e5427..738215768e 100644
--- a/builtin-ls-files.c
+++ b/builtin-ls-files.c
@@ -37,6 +37,7 @@ static const char *tag_removed = "";
 static const char *tag_other = "";
 static const char *tag_killed = "";
 static const char *tag_modified = "";
+static const char *tag_skip_worktree = "";
 
 static void show_dir_entry(const char *tag, struct dir_entry *ent)
 {
@@ -178,7 +179,8 @@ static void show_files(struct dir_struct *dir, const char *prefix)
 				continue;
 			if (ce->ce_flags & CE_UPDATE)
 				continue;
-			show_ce_entry(ce_stage(ce) ? tag_unmerged : tag_cached, ce);
+			show_ce_entry(ce_stage(ce) ? tag_unmerged :
+				(ce_skip_worktree(ce) ? tag_skip_worktree : tag_cached), ce);
 		}
 	}
 	if (show_deleted | show_modified) {
@@ -192,6 +194,8 @@ static void show_files(struct dir_struct *dir, const char *prefix)
 				continue;
 			if (ce->ce_flags & CE_UPDATE)
 				continue;
+			if (ce_skip_worktree(ce))
+				continue;
 			err = lstat(ce->name, &st);
 			if (show_deleted && err)
 				show_ce_entry(tag_removed, ce);
@@ -481,6 +485,9 @@ int cmd_ls_files(int argc, const char **argv, const char *prefix)
 		prefix_offset = strlen(prefix);
 	git_config(git_default_config, NULL);
 
+	if (read_cache() < 0)
+		die("index file corrupt");
+
 	argc = parse_options(argc, argv, prefix, builtin_ls_files_options,
 			ls_files_usage, 0);
 	if (show_tag || show_valid_bit) {
@@ -490,6 +497,7 @@ int cmd_ls_files(int argc, const char **argv, const char *prefix)
 		tag_modified = "C ";
 		tag_other = "? ";
 		tag_killed = "K ";
+		tag_skip_worktree = "S ";
 	}
 	if (show_modified || show_others || show_deleted || (dir.flags & DIR_SHOW_IGNORED) || show_killed)
 		require_work_tree = 1;
@@ -508,7 +516,6 @@ int cmd_ls_files(int argc, const char **argv, const char *prefix)
 	pathspec = get_pathspec(prefix, argv);
 
 	/* be nice with submodule paths ending in a slash */
-	read_cache();
 	if (pathspec)
 		strip_trailing_slash_from_submodules();
 
diff --git a/builtin-read-tree.c b/builtin-read-tree.c
index 2a3a32cbfe..50413ca17d 100644
--- a/builtin-read-tree.c
+++ b/builtin-read-tree.c
@@ -31,7 +31,7 @@ static int list_tree(unsigned char *sha1)
 }
 
 static const char * const read_tree_usage[] = {
-	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]]  [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
+	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]] [--no-sparse-checkout] [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
 	NULL
 };
 
@@ -98,6 +98,8 @@ int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
 		  PARSE_OPT_NONEG, exclude_per_directory_cb },
 		OPT_SET_INT('i', NULL, &opts.index_only,
 			    "don't check the working tree after merging", 1),
+		OPT_SET_INT(0, "no-sparse-checkout", &opts.skip_sparse_checkout,
+			    "skip applying sparse checkout filter", 1),
 		OPT_END()
 	};
 
diff --git a/builtin-update-index.c b/builtin-update-index.c
index a6b7f2d636..a64a752990 100644
--- a/builtin-update-index.c
+++ b/builtin-update-index.c
@@ -24,8 +24,9 @@ static int info_only;
 static int force_remove;
 static int verbose;
 static int mark_valid_only;
-#define MARK_VALID 1
-#define UNMARK_VALID 2
+static int mark_skip_worktree_only;
+#define MARK_FLAG 1
+#define UNMARK_FLAG 2
 
 __attribute__((format (printf, 1, 2)))
 static void report(const char *fmt, ...)
@@ -41,19 +42,15 @@ static void report(const char *fmt, ...)
 	va_end(vp);
 }
 
-static int mark_valid(const char *path)
+static int mark_ce_flags(const char *path, int flag, int mark)
 {
 	int namelen = strlen(path);
 	int pos = cache_name_pos(path, namelen);
 	if (0 <= pos) {
-		switch (mark_valid_only) {
-		case MARK_VALID:
-			active_cache[pos]->ce_flags |= CE_VALID;
-			break;
-		case UNMARK_VALID:
-			active_cache[pos]->ce_flags &= ~CE_VALID;
-			break;
-		}
+		if (mark)
+			active_cache[pos]->ce_flags |= flag;
+		else
+			active_cache[pos]->ce_flags &= ~flag;
 		cache_tree_invalidate_path(active_cache_tree, path);
 		active_cache_changed = 1;
 		return 0;
@@ -176,29 +173,29 @@ static int process_directory(const char *path, int len, struct stat *st)
 	return error("%s: is a directory - add files inside instead", path);
 }
 
-/*
- * Process a regular file
- */
-static int process_file(const char *path, int len, struct stat *st)
-{
-	int pos = cache_name_pos(path, len);
-	struct cache_entry *ce = pos < 0 ? NULL : active_cache[pos];
-
-	if (ce && S_ISGITLINK(ce->ce_mode))
-		return error("%s is already a gitlink, not replacing", path);
-
-	return add_one_path(ce, path, len, st);
-}
-
 static int process_path(const char *path)
 {
-	int len;
+	int pos, len;
 	struct stat st;
+	struct cache_entry *ce;
 
 	len = strlen(path);
 	if (has_symlink_leading_path(path, len))
 		return error("'%s' is beyond a symbolic link", path);
 
+	pos = cache_name_pos(path, len);
+	ce = pos < 0 ? NULL : active_cache[pos];
+	if (ce && ce_skip_worktree(ce)) {
+		/*
+		 * working directory version is assumed "good"
+		 * so updating it does not make sense.
+		 * On the other hand, removing it from index should work
+		 */
+		if (allow_remove && remove_file_from_cache(path))
+			return error("%s: cannot remove from the index", path);
+		return 0;
+	}
+
 	/*
 	 * First things first: get the stat information, to decide
 	 * what to do about the pathname!
@@ -209,7 +206,13 @@ static int process_path(const char *path)
 	if (S_ISDIR(st.st_mode))
 		return process_directory(path, len, &st);
 
-	return process_file(path, len, &st);
+	/*
+	 * Process a regular file
+	 */
+	if (ce && S_ISGITLINK(ce->ce_mode))
+		return error("%s is already a gitlink, not replacing", path);
+
+	return add_one_path(ce, path, len, &st);
 }
 
 static int add_cacheinfo(unsigned int mode, const unsigned char *sha1,
@@ -277,7 +280,12 @@ static void update_one(const char *path, const char *prefix, int prefix_length)
 		goto free_return;
 	}
 	if (mark_valid_only) {
-		if (mark_valid(p))
+		if (mark_ce_flags(p, CE_VALID, mark_valid_only == MARK_FLAG))
+			die("Unable to mark file %s", path);
+		goto free_return;
+	}
+	if (mark_skip_worktree_only) {
+		if (mark_ce_flags(p, CE_SKIP_WORKTREE, mark_skip_worktree_only == MARK_FLAG))
 			die("Unable to mark file %s", path);
 		goto free_return;
 	}
@@ -389,7 +397,7 @@ static void read_index_info(int line_termination)
 }
 
 static const char update_index_usage[] =
-"git update-index [-q] [--add] [--replace] [--remove] [--unmerged] [--refresh] [--really-refresh] [--cacheinfo] [--chmod=(+|-)x] [--assume-unchanged] [--info-only] [--force-remove] [--stdin] [--index-info] [--unresolve] [--again | -g] [--ignore-missing] [-z] [--verbose] [--] <file>...";
+"git update-index [-q] [--add] [--replace] [--remove] [--unmerged] [--refresh] [--really-refresh] [--cacheinfo] [--chmod=(+|-)x] [--assume-unchanged] [--skip-worktree|--no-skip-worktree] [--info-only] [--force-remove] [--stdin] [--index-info] [--unresolve] [--again | -g] [--ignore-missing] [-z] [--verbose] [--] <file>...";
 
 static unsigned char head_sha1[20];
 static unsigned char merge_head_sha1[20];
@@ -648,11 +656,19 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
 				continue;
 			}
 			if (!strcmp(path, "--assume-unchanged")) {
-				mark_valid_only = MARK_VALID;
+				mark_valid_only = MARK_FLAG;
 				continue;
 			}
 			if (!strcmp(path, "--no-assume-unchanged")) {
-				mark_valid_only = UNMARK_VALID;
+				mark_valid_only = UNMARK_FLAG;
+				continue;
+			}
+			if (!strcmp(path, "--no-skip-worktree")) {
+				mark_skip_worktree_only = UNMARK_FLAG;
+				continue;
+			}
+			if (!strcmp(path, "--skip-worktree")) {
+				mark_skip_worktree_only = MARK_FLAG;
 				continue;
 			}
 			if (!strcmp(path, "--info-only")) {
diff --git a/cache.h b/cache.h
index 3f9ee86147..46606477b6 100644
--- a/cache.h
+++ b/cache.h
@@ -178,14 +178,18 @@ struct cache_entry {
 #define CE_HASHED    (0x100000)
 #define CE_UNHASHED  (0x200000)
 
+/* Only remove in work directory, not index */
+#define CE_WT_REMOVE (0x400000)
+
 /*
  * Extended on-disk flags
  */
 #define CE_INTENT_TO_ADD 0x20000000
+#define CE_SKIP_WORKTREE 0x40000000
 /* CE_EXTENDED2 is for future extension */
 #define CE_EXTENDED2 0x80000000
 
-#define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD)
+#define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE)
 
 /*
  * Safeguard to avoid saving wrong flags:
@@ -234,6 +238,7 @@ static inline size_t ce_namelen(const struct cache_entry *ce)
 			    ondisk_cache_entry_size(ce_namelen(ce)))
 #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT)
 #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE)
+#define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE)
 #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE)
 
 #define ce_permissions(mode) (((mode) & 0100) ? 0755 : 0644)
@@ -464,7 +469,9 @@ extern int index_name_is_other(const struct index_state *, const char *, int);
 /* do stat comparison even if CE_VALID is true */
 #define CE_MATCH_IGNORE_VALID		01
 /* do not check the contents but report dirty on racily-clean entries */
-#define CE_MATCH_RACY_IS_DIRTY	02
+#define CE_MATCH_RACY_IS_DIRTY		02
+/* do stat comparison even if CE_SKIP_WORKTREE is true */
+#define CE_MATCH_IGNORE_SKIP_WORKTREE	04
 extern int ie_match_stat(const struct index_state *, struct cache_entry *, struct stat *, unsigned int);
 extern int ie_modified(const struct index_state *, struct cache_entry *, struct stat *, unsigned int);
 
@@ -529,6 +536,7 @@ extern int auto_crlf;
 extern int read_replace_refs;
 extern int fsync_object_files;
 extern int core_preload_index;
+extern int core_apply_sparse_checkout;
 
 enum safe_crlf {
 	SAFE_CRLF_FALSE = 0,
diff --git a/config.c b/config.c
index 37385ce9d3..f4f2c59a34 100644
--- a/config.c
+++ b/config.c
@@ -518,6 +518,11 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.sparsecheckout")) {
+		core_apply_sparse_checkout = git_config_bool(var, value);
+		return 0;
+	}
+
 	/* Add other config variables here and to Documentation/config.txt. */
 	return 0;
 }
diff --git a/diff-lib.c b/diff-lib.c
index 349f612b61..1c7e652a80 100644
--- a/diff-lib.c
+++ b/diff-lib.c
@@ -159,7 +159,7 @@ int run_diff_files(struct rev_info *revs, unsigned int option)
 				continue;
 		}
 
-		if (ce_uptodate(ce))
+		if (ce_uptodate(ce) || ce_skip_worktree(ce))
 			continue;
 
 		/* If CE_VALID is set, don't look at workdir for file removal */
@@ -323,7 +323,8 @@ static void do_oneway_diff(struct unpack_trees_options *o,
 	int match_missing, cached;
 
 	/* if the entry is not checked out, don't examine work tree */
-	cached = o->index_only || (idx && (idx->ce_flags & CE_VALID));
+	cached = o->index_only ||
+		(idx && ((idx->ce_flags & CE_VALID) || ce_skip_worktree(idx)));
 	/*
 	 * Backward compatibility wart - "diff-index -m" does
 	 * not mean "do not ignore merges", but "match_missing".
diff --git a/diff.c b/diff.c
index aad4b3977a..04beb26a6a 100644
--- a/diff.c
+++ b/diff.c
@@ -1997,7 +1997,7 @@ static int reuse_worktree_file(const char *name, const unsigned char *sha1, int
 	 * If ce is marked as "assume unchanged", there is no
 	 * guarantee that work tree matches what we are looking for.
 	 */
-	if (ce->ce_flags & CE_VALID)
+	if ((ce->ce_flags & CE_VALID) || ce_skip_worktree(ce))
 		return 0;
 
 	/*
diff --git a/dir.c b/dir.c
index d0999ba055..3a8d3e67a5 100644
--- a/dir.c
+++ b/dir.c
@@ -200,11 +200,35 @@ void add_exclude(const char *string, const char *base,
 	which->excludes[which->nr++] = x;
 }
 
-static int add_excludes_from_file_1(const char *fname,
-				    const char *base,
-				    int baselen,
-				    char **buf_p,
-				    struct exclude_list *which)
+static void *read_skip_worktree_file_from_index(const char *path, size_t *size)
+{
+	int pos, len;
+	unsigned long sz;
+	enum object_type type;
+	void *data;
+	struct index_state *istate = &the_index;
+
+	len = strlen(path);
+	pos = index_name_pos(istate, path, len);
+	if (pos < 0)
+		return NULL;
+	if (!ce_skip_worktree(istate->cache[pos]))
+		return NULL;
+	data = read_sha1_file(istate->cache[pos]->sha1, &type, &sz);
+	if (!data || type != OBJ_BLOB) {
+		free(data);
+		return NULL;
+	}
+	*size = xsize_t(sz);
+	return data;
+}
+
+int add_excludes_from_file_to_list(const char *fname,
+				   const char *base,
+				   int baselen,
+				   char **buf_p,
+				   struct exclude_list *which,
+				   int check_index)
 {
 	struct stat st;
 	int fd, i;
@@ -212,27 +236,32 @@ static int add_excludes_from_file_1(const char *fname,
 	char *buf, *entry;
 
 	fd = open(fname, O_RDONLY);
-	if (fd < 0 || fstat(fd, &st) < 0)
-		goto err;
-	size = xsize_t(st.st_size);
-	if (size == 0) {
-		close(fd);
-		return 0;
+	if (fd < 0 || fstat(fd, &st) < 0) {
+		if (0 <= fd)
+			close(fd);
+		if (!check_index ||
+		    (buf = read_skip_worktree_file_from_index(fname, &size)) == NULL)
+			return -1;
 	}
-	buf = xmalloc(size+1);
-	if (read_in_full(fd, buf, size) != size)
-	{
-		free(buf);
-		goto err;
+	else {
+		size = xsize_t(st.st_size);
+		if (size == 0) {
+			close(fd);
+			return 0;
+		}
+		buf = xmalloc(size);
+		if (read_in_full(fd, buf, size) != size) {
+			close(fd);
+			return -1;
+		}
+		close(fd);
 	}
-	close(fd);
 
 	if (buf_p)
 		*buf_p = buf;
-	buf[size++] = '\n';
 	entry = buf;
-	for (i = 0; i < size; i++) {
-		if (buf[i] == '\n') {
+	for (i = 0; i <= size; i++) {
+		if (i == size || buf[i] == '\n') {
 			if (entry != buf + i && entry[0] != '#') {
 				buf[i - (i && buf[i-1] == '\r')] = 0;
 				add_exclude(entry, base, baselen, which);
@@ -241,17 +270,12 @@ static int add_excludes_from_file_1(const char *fname,
 		}
 	}
 	return 0;
-
- err:
-	if (0 <= fd)
-		close(fd);
-	return -1;
 }
 
 void add_excludes_from_file(struct dir_struct *dir, const char *fname)
 {
-	if (add_excludes_from_file_1(fname, "", 0, NULL,
-				     &dir->exclude_list[EXC_FILE]) < 0)
+	if (add_excludes_from_file_to_list(fname, "", 0, NULL,
+					   &dir->exclude_list[EXC_FILE], 0) < 0)
 		die("cannot use %s as an exclude file", fname);
 }
 
@@ -300,9 +324,9 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
 		memcpy(dir->basebuf + current, base + current,
 		       stk->baselen - current);
 		strcpy(dir->basebuf + stk->baselen, dir->exclude_per_dir);
-		add_excludes_from_file_1(dir->basebuf,
-					 dir->basebuf, stk->baselen,
-					 &stk->filebuf, el);
+		add_excludes_from_file_to_list(dir->basebuf,
+					       dir->basebuf, stk->baselen,
+					       &stk->filebuf, el, 1);
 		dir->exclude_stack = stk;
 		current = stk->baselen;
 	}
@@ -312,9 +336,9 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
 /* Scan the list and let the last match determine the fate.
  * Return 1 for exclude, 0 for include and -1 for undecided.
  */
-static int excluded_1(const char *pathname,
-		      int pathlen, const char *basename, int *dtype,
-		      struct exclude_list *el)
+int excluded_from_list(const char *pathname,
+		       int pathlen, const char *basename, int *dtype,
+		       struct exclude_list *el)
 {
 	int i;
 
@@ -325,6 +349,12 @@ static int excluded_1(const char *pathname,
 			int to_exclude = x->to_exclude;
 
 			if (x->flags & EXC_FLAG_MUSTBEDIR) {
+				if (!dtype) {
+					if (!prefixcmp(pathname, exclude))
+						return to_exclude;
+					else
+						continue;
+				}
 				if (*dtype == DT_UNKNOWN)
 					*dtype = get_dtype(NULL, pathname, pathlen);
 				if (*dtype != DT_DIR)
@@ -382,8 +412,8 @@ int excluded(struct dir_struct *dir, const char *pathname, int *dtype_p)
 
 	prep_exclude(dir, pathname, basename-pathname);
 	for (st = EXC_CMDL; st <= EXC_FILE; st++) {
-		switch (excluded_1(pathname, pathlen, basename,
-				   dtype_p, &dir->exclude_list[st])) {
+		switch (excluded_from_list(pathname, pathlen, basename,
+					   dtype_p, &dir->exclude_list[st])) {
 		case 0:
 			return 0;
 		case 1:
diff --git a/dir.h b/dir.h
index 320b6a2f38..3bead5f0e2 100644
--- a/dir.h
+++ b/dir.h
@@ -69,7 +69,11 @@ extern int match_pathspec(const char **pathspec, const char *name, int namelen,
 extern int fill_directory(struct dir_struct *dir, const char **pathspec);
 extern int read_directory(struct dir_struct *, const char *path, int len, const char **pathspec);
 
+extern int excluded_from_list(const char *pathname, int pathlen, const char *basename,
+			      int *dtype, struct exclude_list *el);
 extern int excluded(struct dir_struct *, const char *, int *);
+extern int add_excludes_from_file_to_list(const char *fname, const char *base, int baselen,
+					  char **buf_p, struct exclude_list *which, int check_index);
 extern void add_excludes_from_file(struct dir_struct *, const char *fname);
 extern void add_exclude(const char *string, const char *base,
 			int baselen, struct exclude_list *which);
diff --git a/entry.c b/entry.c
index 06d24f14c6..9d5b232781 100644
--- a/entry.c
+++ b/entry.c
@@ -206,7 +206,7 @@ int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *t
 	len += ce_namelen(ce);
 
 	if (!check_path(path, len, &st, state->base_dir_len)) {
-		unsigned changed = ce_match_stat(ce, &st, CE_MATCH_IGNORE_VALID);
+		unsigned changed = ce_match_stat(ce, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
 		if (!changed)
 			return 0;
 		if (!state->force) {
diff --git a/environment.c b/environment.c
index 5171d9f9a4..739ec27040 100644
--- a/environment.c
+++ b/environment.c
@@ -51,6 +51,7 @@ enum push_default_type push_default = PUSH_DEFAULT_MATCHING;
 enum object_creation_mode object_creation_mode = OBJECT_CREATION_MODE;
 char *notes_ref_name;
 int grafts_replace_parents = 1;
+int core_apply_sparse_checkout;
 
 /* Parallel index stat data preload? */
 int core_preload_index = 0;
diff --git a/read-cache.c b/read-cache.c
index 9033dd3ab9..a0adb272fd 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -259,12 +259,17 @@ int ie_match_stat(const struct index_state *istate,
 {
 	unsigned int changed;
 	int ignore_valid = options & CE_MATCH_IGNORE_VALID;
+	int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
 	int assume_racy_is_modified = options & CE_MATCH_RACY_IS_DIRTY;
 
 	/*
 	 * If it's marked as always valid in the index, it's
 	 * valid whatever the checked-out copy says.
+	 *
+	 * skip-worktree has the same effect with higher precedence
 	 */
+	if (!ignore_skip_worktree && ce_skip_worktree(ce))
+		return 0;
 	if (!ignore_valid && (ce->ce_flags & CE_VALID))
 		return 0;
 
@@ -564,7 +569,7 @@ int add_to_index(struct index_state *istate, const char *path, struct stat *st,
 	int size, namelen, was_same;
 	mode_t st_mode = st->st_mode;
 	struct cache_entry *ce, *alias;
-	unsigned ce_option = CE_MATCH_IGNORE_VALID|CE_MATCH_RACY_IS_DIRTY;
+	unsigned ce_option = CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE|CE_MATCH_RACY_IS_DIRTY;
 	int verbose = flags & (ADD_CACHE_VERBOSE | ADD_CACHE_PRETEND);
 	int pretend = flags & ADD_CACHE_PRETEND;
 	int intent_only = flags & ADD_CACHE_INTENT;
@@ -1000,14 +1005,20 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
 	struct cache_entry *updated;
 	int changed, size;
 	int ignore_valid = options & CE_MATCH_IGNORE_VALID;
+	int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
 
 	if (ce_uptodate(ce))
 		return ce;
 
 	/*
-	 * CE_VALID means the user promised us that the change to
-	 * the work tree does not matter and told us not to worry.
+	 * CE_VALID or CE_SKIP_WORKTREE means the user promised us
+	 * that the change to the work tree does not matter and told
+	 * us not to worry.
 	 */
+	if (!ignore_skip_worktree && ce_skip_worktree(ce)) {
+		ce_mark_uptodate(ce);
+		return ce;
+	}
 	if (!ignore_valid && (ce->ce_flags & CE_VALID)) {
 		ce_mark_uptodate(ce);
 		return ce;
diff --git a/t/t1011-read-tree-sparse-checkout.sh b/t/t1011-read-tree-sparse-checkout.sh
new file mode 100755
index 0000000000..62246dbf95
--- /dev/null
+++ b/t/t1011-read-tree-sparse-checkout.sh
@@ -0,0 +1,150 @@
+#!/bin/sh
+
+test_description='sparse checkout tests'
+
+. ./test-lib.sh
+
+cat >expected <<EOF
+100644 77f0ba1734ed79d12881f81b36ee134de6a3327b 0	init.t
+100644 e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 0	sub/added
+EOF
+test_expect_success 'setup' '
+	test_commit init &&
+	echo modified >> init.t &&
+	mkdir sub &&
+	touch sub/added &&
+	git add init.t sub/added &&
+	git commit -m "modified and added" &&
+	git tag top &&
+	git rm sub/added &&
+	git commit -m removed &&
+	git tag removed &&
+	git checkout top &&
+	git ls-files --stage > result &&
+	test_cmp expected result
+'
+
+cat >expected.swt <<EOF
+H init.t
+H sub/added
+EOF
+test_expect_success 'read-tree without .git/info/sparse-checkout' '
+	git read-tree -m -u HEAD &&
+	git ls-files --stage > result &&
+	test_cmp expected result &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result
+'
+
+test_expect_success 'read-tree with .git/info/sparse-checkout but disabled' '
+	echo > .git/info/sparse-checkout
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test -f sub/added
+'
+
+test_expect_success 'read-tree --no-sparse-checkout with empty .git/info/sparse-checkout and enabled' '
+	git config core.sparsecheckout true &&
+	echo > .git/info/sparse-checkout &&
+	git read-tree --no-sparse-checkout -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test -f sub/added
+'
+
+test_expect_success 'read-tree with empty .git/info/sparse-checkout' '
+	git config core.sparsecheckout true &&
+	echo > .git/info/sparse-checkout &&
+	test_must_fail git read-tree -m -u HEAD &&
+	git ls-files --stage > result &&
+	test_cmp expected result &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test -f sub/added
+'
+
+cat >expected.swt <<EOF
+S init.t
+H sub/added
+EOF
+test_expect_success 'match directories with trailing slash' '
+	echo sub/ > .git/info/sparse-checkout &&
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test ! -f init.t &&
+	test -f sub/added
+'
+
+cat >expected.swt <<EOF
+H init.t
+H sub/added
+EOF
+test_expect_failure 'match directories without trailing slash' '
+	echo init.t > .git/info/sparse-checkout &&
+	echo sub >> .git/info/sparse-checkout &&
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test ! -f init.t &&
+	test -f sub/added
+'
+
+cat >expected.swt <<EOF
+H init.t
+S sub/added
+EOF
+test_expect_success 'checkout area changes' '
+	echo init.t > .git/info/sparse-checkout &&
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test ! -f sub/added
+'
+
+test_expect_success 'read-tree updates worktree, absent case' '
+	echo sub/added > .git/info/sparse-checkout &&
+	git checkout -f top &&
+	git read-tree -m -u HEAD^ &&
+	test ! -f init.t
+'
+
+test_expect_success 'read-tree updates worktree, dirty case' '
+	echo sub/added > .git/info/sparse-checkout &&
+	git checkout -f top &&
+	echo dirty > init.t &&
+	git read-tree -m -u HEAD^ &&
+	grep -q dirty init.t &&
+	rm init.t
+'
+
+test_expect_success 'read-tree removes worktree, dirty case' '
+	echo init.t > .git/info/sparse-checkout &&
+	git checkout -f top &&
+	echo dirty > added &&
+	git read-tree -m -u HEAD^ &&
+	grep -q dirty added
+'
+
+test_expect_success 'read-tree adds to worktree, absent case' '
+	echo init.t > .git/info/sparse-checkout &&
+	git checkout -f removed &&
+	git read-tree -u -m HEAD^ &&
+	test ! -f sub/added
+'
+
+test_expect_success 'read-tree adds to worktree, dirty case' '
+	echo init.t > .git/info/sparse-checkout &&
+	git checkout -f removed &&
+	mkdir sub &&
+	echo dirty > sub/added &&
+	git read-tree -u -m HEAD^ &&
+	grep -q dirty sub/added
+'
+
+test_done
diff --git a/t/t2104-update-index-skip-worktree.sh b/t/t2104-update-index-skip-worktree.sh
new file mode 100755
index 0000000000..1d0879be06
--- /dev/null
+++ b/t/t2104-update-index-skip-worktree.sh
@@ -0,0 +1,57 @@
+#!/bin/sh
+#
+# Copyright (c) 2008 Nguyễn Thái Ngọc Duy
+#
+
+test_description='skip-worktree bit test'
+
+. ./test-lib.sh
+
+cat >expect.full <<EOF
+H 1
+H 2
+H sub/1
+H sub/2
+EOF
+
+cat >expect.skip <<EOF
+S 1
+H 2
+S sub/1
+H sub/2
+EOF
+
+test_expect_success 'setup' '
+	mkdir sub &&
+	touch ./1 ./2 sub/1 sub/2 &&
+	git add 1 2 sub/1 sub/2 &&
+	git ls-files -t | test_cmp expect.full -
+'
+
+test_expect_success 'index is at version 2' '
+	test "$(test-index-version < .git/index)" = 2
+'
+
+test_expect_success 'update-index --skip-worktree' '
+	git update-index --skip-worktree 1 sub/1 &&
+	git ls-files -t | test_cmp expect.skip -
+'
+
+test_expect_success 'index is at version 3 after having some skip-worktree entries' '
+	test "$(test-index-version < .git/index)" = 3
+'
+
+test_expect_success 'ls-files -t' '
+	git ls-files -t | test_cmp expect.skip -
+'
+
+test_expect_success 'update-index --no-skip-worktree' '
+	git update-index --no-skip-worktree 1 sub/1 &&
+	git ls-files -t | test_cmp expect.full -
+'
+
+test_expect_success 'index version is back to 2 when there is no skip-worktree entry' '
+	test "$(test-index-version < .git/index)" = 2
+'
+
+test_done
diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh
index c65bca8388..132c4765cb 100755
--- a/t/t3001-ls-files-others-exclude.sh
+++ b/t/t3001-ls-files-others-exclude.sh
@@ -64,6 +64,8 @@ two/*.4
 echo '!*.2
 !*.8' >one/two/.gitignore
 
+allignores='.gitignore one/.gitignore one/two/.gitignore'
+
 test_expect_success \
     'git ls-files --others with various exclude options.' \
     'git ls-files --others \
@@ -85,6 +87,26 @@ test_expect_success \
        >output &&
      test_cmp expect output'
 
+test_expect_success 'setup skip-worktree gitignore' '
+	git add $allignores &&
+	git update-index --skip-worktree $allignores &&
+	rm $allignores
+'
+
+test_expect_success \
+    'git ls-files --others with various exclude options.' \
+    'git ls-files --others \
+       --exclude=\*.6 \
+       --exclude-per-directory=.gitignore \
+       --exclude-from=.git/ignore \
+       >output &&
+     test_cmp expect output'
+
+test_expect_success 'restore gitignore' '
+	git checkout $allignores &&
+	rm .git/index
+'
+
 cat > excludes-file <<\EOF
 *.[1-8]
 e*
diff --git a/t/t7002-grep.sh b/t/t7002-grep.sh
index abd14bf819..76c5e091b7 100755
--- a/t/t7002-grep.sh
+++ b/t/t7002-grep.sh
@@ -8,6 +8,18 @@ test_description='git grep various.
 
 . ./test-lib.sh
 
+test_expect_success 'Check for external grep support' '
+	case "$(git grep -h 2>&1|grep ext-grep)" in
+	*"(default)"*)
+		test_set_prereq EXTGREP
+		true;;
+	*"(ignored by this build)"*)
+		true;;
+	*)
+		false;;
+	esac
+'
+
 cat >hello.c <<EOF
 #include <stdio.h>
 int main(int argc, const char **argv)
@@ -426,4 +438,16 @@ test_expect_success 'grep -Fi' '
 	test_cmp expected actual
 '
 
+test_expect_success EXTGREP 'external grep is called' '
+	GIT_TRACE=2 git grep foo >/dev/null 2>actual &&
+	grep "trace: grep:.*foo" actual >/dev/null
+'
+
+test_expect_success EXTGREP 'no external grep when skip-worktree entries exist' '
+	git update-index --skip-worktree file &&
+	GIT_TRACE=2 git grep foo >/dev/null 2>actual &&
+	! grep "trace: grep:" actual >/dev/null &&
+	git update-index --no-skip-worktree file
+'
+
 test_done
diff --git a/t/t7011-skip-worktree-reading.sh b/t/t7011-skip-worktree-reading.sh
new file mode 100755
index 0000000000..bb4066f767
--- /dev/null
+++ b/t/t7011-skip-worktree-reading.sh
@@ -0,0 +1,163 @@
+#!/bin/sh
+#
+# Copyright (c) 2008 Nguyễn Thái Ngọc Duy
+#
+
+test_description='skip-worktree bit test'
+
+. ./test-lib.sh
+
+cat >expect.full <<EOF
+H 1
+H 2
+H init.t
+H sub/1
+H sub/2
+EOF
+
+cat >expect.skip <<EOF
+S 1
+H 2
+H init.t
+S sub/1
+H sub/2
+EOF
+
+NULL_SHA1=e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+ZERO_SHA0=0000000000000000000000000000000000000000
+setup_absent() {
+	test -f 1 && rm 1
+	git update-index --remove 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_absent() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	test ! -f 1
+}
+
+setup_dirty() {
+	git update-index --force-remove 1 &&
+	echo dirty > 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_dirty() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	echo dirty > expected
+	test_cmp expected 1
+}
+
+test_expect_success 'setup' '
+	test_commit init &&
+	mkdir sub &&
+	touch ./1 ./2 sub/1 sub/2 &&
+	git add 1 2 sub/1 sub/2 &&
+	git update-index --skip-worktree 1 sub/1 &&
+	git ls-files -t > result &&
+	test_cmp expect.skip result
+'
+
+test_expect_success 'update-index' '
+	setup_absent &&
+	git update-index 1 &&
+	test_absent
+'
+
+test_expect_success 'update-index' '
+	setup_dirty &&
+	git update-index 1 &&
+	test_dirty
+'
+
+test_expect_success 'update-index --remove' '
+	setup_absent &&
+	git update-index --remove 1 &&
+	test -z "$(git ls-files 1)" &&
+	test ! -f 1
+'
+
+test_expect_success 'update-index --remove' '
+	setup_dirty &&
+	git update-index --remove 1 &&
+	test -z "$(git ls-files 1)" &&
+	echo dirty > expected &&
+	test_cmp expected 1
+'
+
+test_expect_success 'ls-files --delete' '
+	setup_absent &&
+	test -z "$(git ls-files -d)"
+'
+
+test_expect_success 'ls-files --delete' '
+	setup_dirty &&
+	test -z "$(git ls-files -d)"
+'
+
+test_expect_success 'ls-files --modified' '
+	setup_absent &&
+	test -z "$(git ls-files -m)"
+'
+
+test_expect_success 'ls-files --modified' '
+	setup_dirty &&
+	test -z "$(git ls-files -m)"
+'
+
+test_expect_success 'grep with skip-worktree file' '
+	git update-index --no-skip-worktree 1 &&
+	echo test > 1 &&
+	git update-index 1 &&
+	git update-index --skip-worktree 1 &&
+	rm 1 &&
+	test "$(git grep --no-ext-grep test)" = "1:test"
+'
+
+echo ":000000 100644 $ZERO_SHA0 $NULL_SHA1 A	1" > expected
+test_expect_success 'diff-index does not examine skip-worktree absent entries' '
+	setup_absent &&
+	git diff-index HEAD -- 1 > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'diff-index does not examine skip-worktree dirty entries' '
+	setup_dirty &&
+	git diff-index HEAD -- 1 > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'diff-files does not examine skip-worktree absent entries' '
+	setup_absent &&
+	test -z "$(git diff-files -- one)"
+'
+
+test_expect_success 'diff-files does not examine skip-worktree dirty entries' '
+	setup_dirty &&
+	test -z "$(git diff-files -- one)"
+'
+
+test_expect_success 'git-rm succeeds on skip-worktree absent entries' '
+	setup_absent &&
+	git rm 1
+'
+
+test_expect_success 'commit on skip-worktree absent entries' '
+	git reset &&
+	setup_absent &&
+	test_must_fail git commit -m null 1
+'
+
+test_expect_success 'commit on skip-worktree dirty entries' '
+	git reset &&
+	setup_dirty &&
+	test_must_fail git commit -m null 1
+'
+
+test_done
diff --git a/t/t7012-skip-worktree-writing.sh b/t/t7012-skip-worktree-writing.sh
new file mode 100755
index 0000000000..8d8b1c0e25
--- /dev/null
+++ b/t/t7012-skip-worktree-writing.sh
@@ -0,0 +1,146 @@
+#!/bin/sh
+#
+# Copyright (c) 2008 Nguyễn Thái Ngọc Duy
+#
+
+test_description='test worktree writing operations when skip-worktree is used'
+
+. ./test-lib.sh
+
+test_expect_success 'setup' '
+	test_commit init &&
+	echo modified >> init.t &&
+	touch added &&
+	git add init.t added &&
+	git commit -m "modified and added" &&
+	git tag top
+'
+
+test_expect_success 'read-tree updates worktree, absent case' '
+	git checkout -f top &&
+	git update-index --skip-worktree init.t &&
+	rm init.t &&
+	git read-tree -m -u HEAD^ &&
+	echo init > expected &&
+	test_cmp expected init.t
+'
+
+test_expect_success 'read-tree updates worktree, dirty case' '
+	git checkout -f top &&
+	git update-index --skip-worktree init.t &&
+	echo dirty >> init.t &&
+	test_must_fail git read-tree -m -u HEAD^ &&
+	grep -q dirty init.t &&
+	test "$(git ls-files -t init.t)" = "S init.t" &&
+	git update-index --no-skip-worktree init.t
+'
+
+test_expect_success 'read-tree removes worktree, absent case' '
+	git checkout -f top &&
+	git update-index --skip-worktree added &&
+	rm added &&
+	git read-tree -m -u HEAD^ &&
+	test ! -f added
+'
+
+test_expect_success 'read-tree removes worktree, dirty case' '
+	git checkout -f top &&
+	git update-index --skip-worktree added &&
+	echo dirty >> added &&
+	test_must_fail git read-tree -m -u HEAD^ &&
+	grep -q dirty added &&
+	test "$(git ls-files -t added)" = "S added" &&
+	git update-index --no-skip-worktree added
+'
+
+NULL_SHA1=e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+ZERO_SHA0=0000000000000000000000000000000000000000
+setup_absent() {
+	test -f 1 && rm 1
+	git update-index --remove 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_absent() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	test ! -f 1
+}
+
+setup_dirty() {
+	git update-index --force-remove 1 &&
+	echo dirty > 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_dirty() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	echo dirty > expected
+	test_cmp expected 1
+}
+
+cat >expected <<EOF
+S 1
+H 2
+H init.t
+S sub/1
+H sub/2
+EOF
+
+test_expect_success 'index setup' '
+	git checkout -f init &&
+	mkdir sub &&
+	touch ./1 ./2 sub/1 sub/2 &&
+	git add 1 2 sub/1 sub/2 &&
+	git update-index --skip-worktree 1 sub/1 &&
+	git ls-files -t > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'git-add ignores worktree content' '
+	setup_absent &&
+	git add 1 &&
+	test_absent
+'
+
+test_expect_success 'git-add ignores worktree content' '
+	setup_dirty &&
+	git add 1 &&
+	test_dirty
+'
+
+test_expect_success 'git-rm fails if worktree is dirty' '
+	setup_dirty &&
+	test_must_fail git rm 1 &&
+	test_dirty
+'
+
+cat >expected <<EOF
+Would remove expected
+Would remove result
+EOF
+test_expect_success 'git-clean, absent case' '
+	setup_absent &&
+	git clean -n > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'git-clean, dirty case' '
+	setup_dirty &&
+	git clean -n > result &&
+	test_cmp expected result
+'
+
+test_expect_failure 'git-apply adds file' false
+test_expect_failure 'git-apply updates file' false
+test_expect_failure 'git-apply removes file' false
+test_expect_failure 'git-mv to skip-worktree' false
+test_expect_failure 'git-mv from skip-worktree' false
+test_expect_failure 'git-checkout' false
+
+test_done
diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh
index 118c6ebb18..7d8ed68bef 100755
--- a/t/t7300-clean.sh
+++ b/t/t7300-clean.sh
@@ -22,6 +22,25 @@ test_expect_success 'setup' '
 
 '
 
+test_expect_success 'git clean with skip-worktree .gitignore' '
+	git update-index --skip-worktree .gitignore &&
+	rm .gitignore &&
+	mkdir -p build docs &&
+	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
+	git clean &&
+	test -f Makefile &&
+	test -f README &&
+	test -f src/part1.c &&
+	test -f src/part2.c &&
+	test ! -f a.out &&
+	test ! -f src/part3.c &&
+	test -f docs/manual.txt &&
+	test -f obj.o &&
+	test -f build/lib.so &&
+	git update-index --no-skip-worktree .gitignore &&
+	git checkout .gitignore
+'
+
 test_expect_success 'git clean' '
 
 	mkdir -p build docs &&
diff --git a/test-index-version.c b/test-index-version.c
new file mode 100644
index 0000000000..bfaad9e09e
--- /dev/null
+++ b/test-index-version.c
@@ -0,0 +1,14 @@
+#include "cache.h"
+
+int main(int argc, const char **argv)
+{
+	struct cache_header hdr;
+	int version;
+
+	memset(&hdr,0,sizeof(hdr));
+	if (read(0, &hdr, sizeof(hdr)) != sizeof(hdr))
+		return 0;
+	version = ntohl(hdr.hdr_version);
+	printf("%d\n", version);
+	return 0;
+}
diff --git a/unpack-trees.c b/unpack-trees.c
index dd5999c356..7570475b45 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -32,6 +32,12 @@ static struct unpack_trees_error_msgs unpack_plumbing_errors = {
 
 	/* bind_overlap */
 	"Entry '%s' overlaps with '%s'.  Cannot bind.",
+
+	/* sparse_not_uptodate_file */
+	"Entry '%s' not uptodate. Cannot update sparse checkout.",
+
+	/* would_lose_orphaned */
+	"Working tree file '%s' would be %s by sparse checkout update.",
 };
 
 #define ERRORMSG(o,fld) \
@@ -78,7 +84,7 @@ static int check_updates(struct unpack_trees_options *o)
 	if (o->update && o->verbose_update) {
 		for (total = cnt = 0; cnt < index->cache_nr; cnt++) {
 			struct cache_entry *ce = index->cache[cnt];
-			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE))
+			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE | CE_WT_REMOVE))
 				total++;
 		}
 
@@ -92,6 +98,13 @@ static int check_updates(struct unpack_trees_options *o)
 	for (i = 0; i < index->cache_nr; i++) {
 		struct cache_entry *ce = index->cache[i];
 
+		if (ce->ce_flags & CE_WT_REMOVE) {
+			display_progress(progress, ++cnt);
+			if (o->update)
+				unlink_entry(ce);
+			continue;
+		}
+
 		if (ce->ce_flags & CE_REMOVE) {
 			display_progress(progress, ++cnt);
 			if (o->update)
@@ -118,6 +131,57 @@ static int check_updates(struct unpack_trees_options *o)
 	return errs != 0;
 }
 
+static int verify_uptodate_sparse(struct cache_entry *ce, struct unpack_trees_options *o);
+static int verify_absent_sparse(struct cache_entry *ce, const char *action, struct unpack_trees_options *o);
+
+static int will_have_skip_worktree(const struct cache_entry *ce, struct unpack_trees_options *o)
+{
+	const char *basename;
+
+	if (ce_stage(ce))
+		return 0;
+
+	basename = strrchr(ce->name, '/');
+	basename = basename ? basename+1 : ce->name;
+	return excluded_from_list(ce->name, ce_namelen(ce), basename, NULL, o->el) <= 0;
+}
+
+static int apply_sparse_checkout(struct cache_entry *ce, struct unpack_trees_options *o)
+{
+	int was_skip_worktree = ce_skip_worktree(ce);
+
+	if (will_have_skip_worktree(ce, o))
+		ce->ce_flags |= CE_SKIP_WORKTREE;
+	else
+		ce->ce_flags &= ~CE_SKIP_WORKTREE;
+
+	/*
+	 * We only care about files getting into the checkout area
+	 * If merge strategies want to remove some, go ahead, this
+	 * flag will be removed eventually in unpack_trees() if it's
+	 * outside checkout area.
+	 */
+	if (ce->ce_flags & CE_REMOVE)
+		return 0;
+
+	if (!was_skip_worktree && ce_skip_worktree(ce)) {
+		/*
+		 * If CE_UPDATE is set, verify_uptodate() must be called already
+		 * also stat info may have lost after merged_entry() so calling
+		 * verify_uptodate() again may fail
+		 */
+		if (!(ce->ce_flags & CE_UPDATE) && verify_uptodate_sparse(ce, o))
+			return -1;
+		ce->ce_flags |= CE_WT_REMOVE;
+	}
+	if (was_skip_worktree && !ce_skip_worktree(ce)) {
+		if (verify_absent_sparse(ce, "overwritten", o))
+			return -1;
+		ce->ce_flags |= CE_UPDATE;
+	}
+	return 0;
+}
+
 static inline int call_unpack_fn(struct cache_entry **src, struct unpack_trees_options *o)
 {
 	int ret = o->fn(src, o);
@@ -369,8 +433,9 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str
  */
 int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options *o)
 {
-	int ret;
+	int i, ret;
 	static struct cache_entry *dfc;
+	struct exclude_list el;
 
 	if (len > MAX_UNPACK_TREES)
 		die("unpack_trees takes at most %d trees", MAX_UNPACK_TREES);
@@ -380,6 +445,16 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 	state.quiet = 1;
 	state.refresh_cache = 1;
 
+	memset(&el, 0, sizeof(el));
+	if (!core_apply_sparse_checkout || !o->update)
+		o->skip_sparse_checkout = 1;
+	if (!o->skip_sparse_checkout) {
+		if (add_excludes_from_file_to_list(git_path("info/sparse-checkout"), "", 0, NULL, &el, 0) < 0)
+			o->skip_sparse_checkout = 1;
+		else
+			o->el = &el;
+	}
+
 	memset(&o->result, 0, sizeof(o->result));
 	o->result.initialized = 1;
 	if (o->src_index) {
@@ -400,26 +475,65 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 		info.fn = unpack_callback;
 		info.data = o;
 
-		if (traverse_trees(len, t, &info) < 0)
-			return unpack_failed(o, NULL);
+		if (traverse_trees(len, t, &info) < 0) {
+			ret = unpack_failed(o, NULL);
+			goto done;
+		}
 	}
 
 	/* Any left-over entries in the index? */
 	if (o->merge) {
 		while (o->pos < o->src_index->cache_nr) {
 			struct cache_entry *ce = o->src_index->cache[o->pos];
-			if (unpack_index_entry(ce, o) < 0)
-				return unpack_failed(o, NULL);
+			if (unpack_index_entry(ce, o) < 0) {
+				ret = unpack_failed(o, NULL);
+				goto done;
+			}
 		}
 	}
 
-	if (o->trivial_merges_only && o->nontrivial_merge)
-		return unpack_failed(o, "Merge requires file-level merging");
+	if (o->trivial_merges_only && o->nontrivial_merge) {
+		ret = unpack_failed(o, "Merge requires file-level merging");
+		goto done;
+	}
+
+	if (!o->skip_sparse_checkout) {
+		int empty_worktree = 1;
+		for (i = 0;i < o->result.cache_nr;i++) {
+			struct cache_entry *ce = o->result.cache[i];
+
+			if (apply_sparse_checkout(ce, o)) {
+				ret = -1;
+				goto done;
+			}
+			/*
+			 * Merge strategies may set CE_UPDATE|CE_REMOVE outside checkout
+			 * area as a result of ce_skip_worktree() shortcuts in
+			 * verify_absent() and verify_uptodate(). Clear them.
+			 */
+			if (ce_skip_worktree(ce))
+				ce->ce_flags &= ~(CE_UPDATE | CE_REMOVE);
+			else
+				empty_worktree = 0;
+
+		}
+		if (o->result.cache_nr && empty_worktree) {
+			ret = unpack_failed(o, "Sparse checkout leaves no entry on working directory");
+			goto done;
+		}
+	}
 
 	o->src_index = NULL;
 	ret = check_updates(o) ? (-2) : 0;
 	if (o->dst_index)
 		*o->dst_index = o->result;
+
+done:
+	for (i = 0;i < el.nr;i++)
+		free(el.excludes[i]);
+	if (el.excludes)
+		free(el.excludes);
+
 	return ret;
 }
 
@@ -445,16 +559,17 @@ static int same(struct cache_entry *a, struct cache_entry *b)
  * When a CE gets turned into an unmerged entry, we
  * want it to be up-to-date
  */
-static int verify_uptodate(struct cache_entry *ce,
-		struct unpack_trees_options *o)
+static int verify_uptodate_1(struct cache_entry *ce,
+				   struct unpack_trees_options *o,
+				   const char *error_msg)
 {
 	struct stat st;
 
-	if (o->index_only || o->reset || ce_uptodate(ce))
+	if (o->index_only || (!ce_skip_worktree(ce) && (o->reset || ce_uptodate(ce))))
 		return 0;
 
 	if (!lstat(ce->name, &st)) {
-		unsigned changed = ie_match_stat(o->src_index, ce, &st, CE_MATCH_IGNORE_VALID);
+		unsigned changed = ie_match_stat(o->src_index, ce, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
 		if (!changed)
 			return 0;
 		/*
@@ -471,7 +586,21 @@ static int verify_uptodate(struct cache_entry *ce,
 	if (errno == ENOENT)
 		return 0;
 	return o->gently ? -1 :
-		error(ERRORMSG(o, not_uptodate_file), ce->name);
+		error(error_msg, ce->name);
+}
+
+static int verify_uptodate(struct cache_entry *ce,
+			   struct unpack_trees_options *o)
+{
+	if (!o->skip_sparse_checkout && will_have_skip_worktree(ce, o))
+		return 0;
+	return verify_uptodate_1(ce, o, ERRORMSG(o, not_uptodate_file));
+}
+
+static int verify_uptodate_sparse(struct cache_entry *ce,
+				  struct unpack_trees_options *o)
+{
+	return verify_uptodate_1(ce, o, ERRORMSG(o, sparse_not_uptodate_file));
 }
 
 static void invalidate_ce_path(struct cache_entry *ce, struct unpack_trees_options *o)
@@ -572,15 +701,16 @@ static int icase_exists(struct unpack_trees_options *o, struct cache_entry *dst,
 	struct cache_entry *src;
 
 	src = index_name_exists(o->src_index, dst->name, ce_namelen(dst), 1);
-	return src && !ie_match_stat(o->src_index, src, st, CE_MATCH_IGNORE_VALID);
+	return src && !ie_match_stat(o->src_index, src, st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
 }
 
 /*
  * We do not want to remove or overwrite a working tree file that
  * is not tracked, unless it is ignored.
  */
-static int verify_absent(struct cache_entry *ce, const char *action,
-			 struct unpack_trees_options *o)
+static int verify_absent_1(struct cache_entry *ce, const char *action,
+				 struct unpack_trees_options *o,
+				 const char *error_msg)
 {
 	struct stat st;
 
@@ -660,6 +790,19 @@ static int verify_absent(struct cache_entry *ce, const char *action,
 	}
 	return 0;
 }
+static int verify_absent(struct cache_entry *ce, const char *action,
+			 struct unpack_trees_options *o)
+{
+	if (!o->skip_sparse_checkout && will_have_skip_worktree(ce, o))
+		return 0;
+	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_untracked));
+}
+
+static int verify_absent_sparse(struct cache_entry *ce, const char *action,
+			 struct unpack_trees_options *o)
+{
+	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_orphaned));
+}
 
 static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		struct unpack_trees_options *o)
@@ -680,6 +823,8 @@ static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		} else {
 			if (verify_uptodate(old, o))
 				return -1;
+			if (ce_skip_worktree(old))
+				update |= CE_SKIP_WORKTREE;
 			invalidate_ce_path(old, o);
 		}
 	}
@@ -1004,10 +1149,10 @@ int oneway_merge(struct cache_entry **src, struct unpack_trees_options *o)
 
 	if (old && same(old, a)) {
 		int update = 0;
-		if (o->reset && !ce_uptodate(old)) {
+		if (o->reset && !ce_uptodate(old) && !ce_skip_worktree(old)) {
 			struct stat st;
 			if (lstat(old->name, &st) ||
-			    ie_match_stat(o->src_index, old, &st, CE_MATCH_IGNORE_VALID))
+			    ie_match_stat(o->src_index, old, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE))
 				update |= CE_UPDATE;
 		}
 		add_entry(o, old, update, 0);
diff --git a/unpack-trees.h b/unpack-trees.h
index d19df44f40..95ff36c824 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -4,6 +4,7 @@
 #define MAX_UNPACK_TREES 8
 
 struct unpack_trees_options;
+struct exclude_list;
 
 typedef int (*merge_fn_t)(struct cache_entry **src,
 		struct unpack_trees_options *options);
@@ -14,6 +15,8 @@ struct unpack_trees_error_msgs {
 	const char *not_uptodate_dir;
 	const char *would_lose_untracked;
 	const char *bind_overlap;
+	const char *sparse_not_uptodate_file;
+	const char *would_lose_orphaned;
 };
 
 struct unpack_trees_options {
@@ -28,6 +31,7 @@ struct unpack_trees_options {
 		     skip_unmerged,
 		     initial_checkout,
 		     diff_index_cached,
+		     skip_sparse_checkout,
 		     gently;
 	const char *prefix;
 	int pos;
@@ -44,6 +48,8 @@ struct unpack_trees_options {
 	struct index_state *dst_index;
 	struct index_state *src_index;
 	struct index_state result;
+
+	struct exclude_list *el; /* for internal use */
 };
 
 extern int unpack_trees(unsigned n, struct tree_desc *t,