Merge branch 'nd/sparse'

* nd/sparse: (25 commits)
  t7002: test for not using external grep on skip-worktree paths
  t7002: set test prerequisite "external-grep" if supported
  grep: do not do external grep on skip-worktree entries
  commit: correctly respect skip-worktree bit
  ie_match_stat(): do not ignore skip-worktree bit with CE_MATCH_IGNORE_VALID
  tests: rename duplicate t1009
  sparse checkout: inhibit empty worktree
  Add tests for sparse checkout
  read-tree: add --no-sparse-checkout to disable sparse checkout support
  unpack-trees(): ignore worktree check outside checkout area
  unpack_trees(): apply $GIT_DIR/info/sparse-checkout to the final index
  unpack-trees(): "enable" sparse checkout and load $GIT_DIR/info/sparse-checkout
  unpack-trees.c: generalize verify_* functions
  unpack-trees(): add CE_WT_REMOVE to remove on worktree alone
  Introduce "sparse checkout"
  dir.c: export excluded_1() and add_excludes_from_file_1()
  excluded_1(): support exclude files in index
  unpack-trees(): carry skip-worktree bit over in merged_entry()
  Read .gitignore from index if it is skip-worktree
  Avoid writing to buffer in add_excludes_from_file_1()
  ...

Conflicts:
	.gitignore
	Documentation/config.txt
	Documentation/git-update-index.txt
	Makefile
	entry.c
	t/t7002-grep.sh

7 files changed, 581 insertions, 0 deletions

diff --git a/t/t1011-read-tree-sparse-checkout.sh b/t/t1011-read-tree-sparse-checkout.sh
new file mode 100755
index 0000000000..62246dbf95
--- /dev/null
+++ b/t/t1011-read-tree-sparse-checkout.sh
@@ -0,0 +1,150 @@
+#!/bin/sh
+
+test_description='sparse checkout tests'
+
+. ./test-lib.sh
+
+cat >expected <<EOF
+100644 77f0ba1734ed79d12881f81b36ee134de6a3327b 0	init.t
+100644 e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 0	sub/added
+EOF
+test_expect_success 'setup' '
+	test_commit init &&
+	echo modified >> init.t &&
+	mkdir sub &&
+	touch sub/added &&
+	git add init.t sub/added &&
+	git commit -m "modified and added" &&
+	git tag top &&
+	git rm sub/added &&
+	git commit -m removed &&
+	git tag removed &&
+	git checkout top &&
+	git ls-files --stage > result &&
+	test_cmp expected result
+'
+
+cat >expected.swt <<EOF
+H init.t
+H sub/added
+EOF
+test_expect_success 'read-tree without .git/info/sparse-checkout' '
+	git read-tree -m -u HEAD &&
+	git ls-files --stage > result &&
+	test_cmp expected result &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result
+'
+
+test_expect_success 'read-tree with .git/info/sparse-checkout but disabled' '
+	echo > .git/info/sparse-checkout
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test -f sub/added
+'
+
+test_expect_success 'read-tree --no-sparse-checkout with empty .git/info/sparse-checkout and enabled' '
+	git config core.sparsecheckout true &&
+	echo > .git/info/sparse-checkout &&
+	git read-tree --no-sparse-checkout -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test -f sub/added
+'
+
+test_expect_success 'read-tree with empty .git/info/sparse-checkout' '
+	git config core.sparsecheckout true &&
+	echo > .git/info/sparse-checkout &&
+	test_must_fail git read-tree -m -u HEAD &&
+	git ls-files --stage > result &&
+	test_cmp expected result &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test -f sub/added
+'
+
+cat >expected.swt <<EOF
+S init.t
+H sub/added
+EOF
+test_expect_success 'match directories with trailing slash' '
+	echo sub/ > .git/info/sparse-checkout &&
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test ! -f init.t &&
+	test -f sub/added
+'
+
+cat >expected.swt <<EOF
+H init.t
+H sub/added
+EOF
+test_expect_failure 'match directories without trailing slash' '
+	echo init.t > .git/info/sparse-checkout &&
+	echo sub >> .git/info/sparse-checkout &&
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test ! -f init.t &&
+	test -f sub/added
+'
+
+cat >expected.swt <<EOF
+H init.t
+S sub/added
+EOF
+test_expect_success 'checkout area changes' '
+	echo init.t > .git/info/sparse-checkout &&
+	git read-tree -m -u HEAD &&
+	git ls-files -t > result &&
+	test_cmp expected.swt result &&
+	test -f init.t &&
+	test ! -f sub/added
+'
+
+test_expect_success 'read-tree updates worktree, absent case' '
+	echo sub/added > .git/info/sparse-checkout &&
+	git checkout -f top &&
+	git read-tree -m -u HEAD^ &&
+	test ! -f init.t
+'
+
+test_expect_success 'read-tree updates worktree, dirty case' '
+	echo sub/added > .git/info/sparse-checkout &&
+	git checkout -f top &&
+	echo dirty > init.t &&
+	git read-tree -m -u HEAD^ &&
+	grep -q dirty init.t &&
+	rm init.t
+'
+
+test_expect_success 'read-tree removes worktree, dirty case' '
+	echo init.t > .git/info/sparse-checkout &&
+	git checkout -f top &&
+	echo dirty > added &&
+	git read-tree -m -u HEAD^ &&
+	grep -q dirty added
+'
+
+test_expect_success 'read-tree adds to worktree, absent case' '
+	echo init.t > .git/info/sparse-checkout &&
+	git checkout -f removed &&
+	git read-tree -u -m HEAD^ &&
+	test ! -f sub/added
+'
+
+test_expect_success 'read-tree adds to worktree, dirty case' '
+	echo init.t > .git/info/sparse-checkout &&
+	git checkout -f removed &&
+	mkdir sub &&
+	echo dirty > sub/added &&
+	git read-tree -u -m HEAD^ &&
+	grep -q dirty sub/added
+'
+
+test_done
diff --git a/t/t2104-update-index-skip-worktree.sh b/t/t2104-update-index-skip-worktree.sh
new file mode 100755
index 0000000000..1d0879be06
--- /dev/null
+++ b/t/t2104-update-index-skip-worktree.sh
@@ -0,0 +1,57 @@
+#!/bin/sh
+#
+# Copyright (c) 2008 Nguyễn Thái Ngọc Duy
+#
+
+test_description='skip-worktree bit test'
+
+. ./test-lib.sh
+
+cat >expect.full <<EOF
+H 1
+H 2
+H sub/1
+H sub/2
+EOF
+
+cat >expect.skip <<EOF
+S 1
+H 2
+S sub/1
+H sub/2
+EOF
+
+test_expect_success 'setup' '
+	mkdir sub &&
+	touch ./1 ./2 sub/1 sub/2 &&
+	git add 1 2 sub/1 sub/2 &&
+	git ls-files -t | test_cmp expect.full -
+'
+
+test_expect_success 'index is at version 2' '
+	test "$(test-index-version < .git/index)" = 2
+'
+
+test_expect_success 'update-index --skip-worktree' '
+	git update-index --skip-worktree 1 sub/1 &&
+	git ls-files -t | test_cmp expect.skip -
+'
+
+test_expect_success 'index is at version 3 after having some skip-worktree entries' '
+	test "$(test-index-version < .git/index)" = 3
+'
+
+test_expect_success 'ls-files -t' '
+	git ls-files -t | test_cmp expect.skip -
+'
+
+test_expect_success 'update-index --no-skip-worktree' '
+	git update-index --no-skip-worktree 1 sub/1 &&
+	git ls-files -t | test_cmp expect.full -
+'
+
+test_expect_success 'index version is back to 2 when there is no skip-worktree entry' '
+	test "$(test-index-version < .git/index)" = 2
+'
+
+test_done
diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh
index c65bca8388..132c4765cb 100755
--- a/t/t3001-ls-files-others-exclude.sh
+++ b/t/t3001-ls-files-others-exclude.sh
@@ -64,6 +64,8 @@ two/*.4
 echo '!*.2
 !*.8' >one/two/.gitignore
 
+allignores='.gitignore one/.gitignore one/two/.gitignore'
+
 test_expect_success \
     'git ls-files --others with various exclude options.' \
     'git ls-files --others \
@@ -85,6 +87,26 @@ test_expect_success \
        >output &&
      test_cmp expect output'
 
+test_expect_success 'setup skip-worktree gitignore' '
+	git add $allignores &&
+	git update-index --skip-worktree $allignores &&
+	rm $allignores
+'
+
+test_expect_success \
+    'git ls-files --others with various exclude options.' \
+    'git ls-files --others \
+       --exclude=\*.6 \
+       --exclude-per-directory=.gitignore \
+       --exclude-from=.git/ignore \
+       >output &&
+     test_cmp expect output'
+
+test_expect_success 'restore gitignore' '
+	git checkout $allignores &&
+	rm .git/index
+'
+
 cat > excludes-file <<\EOF
 *.[1-8]
 e*
diff --git a/t/t7002-grep.sh b/t/t7002-grep.sh
index abd14bf819..76c5e091b7 100755
--- a/t/t7002-grep.sh
+++ b/t/t7002-grep.sh
@@ -8,6 +8,18 @@ test_description='git grep various.
 
 . ./test-lib.sh
 
+test_expect_success 'Check for external grep support' '
+	case "$(git grep -h 2>&1|grep ext-grep)" in
+	*"(default)"*)
+		test_set_prereq EXTGREP
+		true;;
+	*"(ignored by this build)"*)
+		true;;
+	*)
+		false;;
+	esac
+'
+
 cat >hello.c <<EOF
 #include <stdio.h>
 int main(int argc, const char **argv)
@@ -426,4 +438,16 @@ test_expect_success 'grep -Fi' '
 	test_cmp expected actual
 '
 
+test_expect_success EXTGREP 'external grep is called' '
+	GIT_TRACE=2 git grep foo >/dev/null 2>actual &&
+	grep "trace: grep:.*foo" actual >/dev/null
+'
+
+test_expect_success EXTGREP 'no external grep when skip-worktree entries exist' '
+	git update-index --skip-worktree file &&
+	GIT_TRACE=2 git grep foo >/dev/null 2>actual &&
+	! grep "trace: grep:" actual >/dev/null &&
+	git update-index --no-skip-worktree file
+'
+
 test_done
diff --git a/t/t7011-skip-worktree-reading.sh b/t/t7011-skip-worktree-reading.sh
new file mode 100755
index 0000000000..bb4066f767
--- /dev/null
+++ b/t/t7011-skip-worktree-reading.sh
@@ -0,0 +1,163 @@
+#!/bin/sh
+#
+# Copyright (c) 2008 Nguyễn Thái Ngọc Duy
+#
+
+test_description='skip-worktree bit test'
+
+. ./test-lib.sh
+
+cat >expect.full <<EOF
+H 1
+H 2
+H init.t
+H sub/1
+H sub/2
+EOF
+
+cat >expect.skip <<EOF
+S 1
+H 2
+H init.t
+S sub/1
+H sub/2
+EOF
+
+NULL_SHA1=e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+ZERO_SHA0=0000000000000000000000000000000000000000
+setup_absent() {
+	test -f 1 && rm 1
+	git update-index --remove 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_absent() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	test ! -f 1
+}
+
+setup_dirty() {
+	git update-index --force-remove 1 &&
+	echo dirty > 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_dirty() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	echo dirty > expected
+	test_cmp expected 1
+}
+
+test_expect_success 'setup' '
+	test_commit init &&
+	mkdir sub &&
+	touch ./1 ./2 sub/1 sub/2 &&
+	git add 1 2 sub/1 sub/2 &&
+	git update-index --skip-worktree 1 sub/1 &&
+	git ls-files -t > result &&
+	test_cmp expect.skip result
+'
+
+test_expect_success 'update-index' '
+	setup_absent &&
+	git update-index 1 &&
+	test_absent
+'
+
+test_expect_success 'update-index' '
+	setup_dirty &&
+	git update-index 1 &&
+	test_dirty
+'
+
+test_expect_success 'update-index --remove' '
+	setup_absent &&
+	git update-index --remove 1 &&
+	test -z "$(git ls-files 1)" &&
+	test ! -f 1
+'
+
+test_expect_success 'update-index --remove' '
+	setup_dirty &&
+	git update-index --remove 1 &&
+	test -z "$(git ls-files 1)" &&
+	echo dirty > expected &&
+	test_cmp expected 1
+'
+
+test_expect_success 'ls-files --delete' '
+	setup_absent &&
+	test -z "$(git ls-files -d)"
+'
+
+test_expect_success 'ls-files --delete' '
+	setup_dirty &&
+	test -z "$(git ls-files -d)"
+'
+
+test_expect_success 'ls-files --modified' '
+	setup_absent &&
+	test -z "$(git ls-files -m)"
+'
+
+test_expect_success 'ls-files --modified' '
+	setup_dirty &&
+	test -z "$(git ls-files -m)"
+'
+
+test_expect_success 'grep with skip-worktree file' '
+	git update-index --no-skip-worktree 1 &&
+	echo test > 1 &&
+	git update-index 1 &&
+	git update-index --skip-worktree 1 &&
+	rm 1 &&
+	test "$(git grep --no-ext-grep test)" = "1:test"
+'
+
+echo ":000000 100644 $ZERO_SHA0 $NULL_SHA1 A	1" > expected
+test_expect_success 'diff-index does not examine skip-worktree absent entries' '
+	setup_absent &&
+	git diff-index HEAD -- 1 > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'diff-index does not examine skip-worktree dirty entries' '
+	setup_dirty &&
+	git diff-index HEAD -- 1 > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'diff-files does not examine skip-worktree absent entries' '
+	setup_absent &&
+	test -z "$(git diff-files -- one)"
+'
+
+test_expect_success 'diff-files does not examine skip-worktree dirty entries' '
+	setup_dirty &&
+	test -z "$(git diff-files -- one)"
+'
+
+test_expect_success 'git-rm succeeds on skip-worktree absent entries' '
+	setup_absent &&
+	git rm 1
+'
+
+test_expect_success 'commit on skip-worktree absent entries' '
+	git reset &&
+	setup_absent &&
+	test_must_fail git commit -m null 1
+'
+
+test_expect_success 'commit on skip-worktree dirty entries' '
+	git reset &&
+	setup_dirty &&
+	test_must_fail git commit -m null 1
+'
+
+test_done
diff --git a/t/t7012-skip-worktree-writing.sh b/t/t7012-skip-worktree-writing.sh
new file mode 100755
index 0000000000..8d8b1c0e25
--- /dev/null
+++ b/t/t7012-skip-worktree-writing.sh
@@ -0,0 +1,146 @@
+#!/bin/sh
+#
+# Copyright (c) 2008 Nguyễn Thái Ngọc Duy
+#
+
+test_description='test worktree writing operations when skip-worktree is used'
+
+. ./test-lib.sh
+
+test_expect_success 'setup' '
+	test_commit init &&
+	echo modified >> init.t &&
+	touch added &&
+	git add init.t added &&
+	git commit -m "modified and added" &&
+	git tag top
+'
+
+test_expect_success 'read-tree updates worktree, absent case' '
+	git checkout -f top &&
+	git update-index --skip-worktree init.t &&
+	rm init.t &&
+	git read-tree -m -u HEAD^ &&
+	echo init > expected &&
+	test_cmp expected init.t
+'
+
+test_expect_success 'read-tree updates worktree, dirty case' '
+	git checkout -f top &&
+	git update-index --skip-worktree init.t &&
+	echo dirty >> init.t &&
+	test_must_fail git read-tree -m -u HEAD^ &&
+	grep -q dirty init.t &&
+	test "$(git ls-files -t init.t)" = "S init.t" &&
+	git update-index --no-skip-worktree init.t
+'
+
+test_expect_success 'read-tree removes worktree, absent case' '
+	git checkout -f top &&
+	git update-index --skip-worktree added &&
+	rm added &&
+	git read-tree -m -u HEAD^ &&
+	test ! -f added
+'
+
+test_expect_success 'read-tree removes worktree, dirty case' '
+	git checkout -f top &&
+	git update-index --skip-worktree added &&
+	echo dirty >> added &&
+	test_must_fail git read-tree -m -u HEAD^ &&
+	grep -q dirty added &&
+	test "$(git ls-files -t added)" = "S added" &&
+	git update-index --no-skip-worktree added
+'
+
+NULL_SHA1=e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+ZERO_SHA0=0000000000000000000000000000000000000000
+setup_absent() {
+	test -f 1 && rm 1
+	git update-index --remove 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_absent() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	test ! -f 1
+}
+
+setup_dirty() {
+	git update-index --force-remove 1 &&
+	echo dirty > 1 &&
+	git update-index --add --cacheinfo 100644 $NULL_SHA1 1 &&
+	git update-index --skip-worktree 1
+}
+
+test_dirty() {
+	echo "100644 $NULL_SHA1 0	1" > expected &&
+	git ls-files --stage 1 > result &&
+	test_cmp expected result &&
+	echo dirty > expected
+	test_cmp expected 1
+}
+
+cat >expected <<EOF
+S 1
+H 2
+H init.t
+S sub/1
+H sub/2
+EOF
+
+test_expect_success 'index setup' '
+	git checkout -f init &&
+	mkdir sub &&
+	touch ./1 ./2 sub/1 sub/2 &&
+	git add 1 2 sub/1 sub/2 &&
+	git update-index --skip-worktree 1 sub/1 &&
+	git ls-files -t > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'git-add ignores worktree content' '
+	setup_absent &&
+	git add 1 &&
+	test_absent
+'
+
+test_expect_success 'git-add ignores worktree content' '
+	setup_dirty &&
+	git add 1 &&
+	test_dirty
+'
+
+test_expect_success 'git-rm fails if worktree is dirty' '
+	setup_dirty &&
+	test_must_fail git rm 1 &&
+	test_dirty
+'
+
+cat >expected <<EOF
+Would remove expected
+Would remove result
+EOF
+test_expect_success 'git-clean, absent case' '
+	setup_absent &&
+	git clean -n > result &&
+	test_cmp expected result
+'
+
+test_expect_success 'git-clean, dirty case' '
+	setup_dirty &&
+	git clean -n > result &&
+	test_cmp expected result
+'
+
+test_expect_failure 'git-apply adds file' false
+test_expect_failure 'git-apply updates file' false
+test_expect_failure 'git-apply removes file' false
+test_expect_failure 'git-mv to skip-worktree' false
+test_expect_failure 'git-mv from skip-worktree' false
+test_expect_failure 'git-checkout' false
+
+test_done
diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh
index 118c6ebb18..7d8ed68bef 100755
--- a/t/t7300-clean.sh
+++ b/t/t7300-clean.sh
@@ -22,6 +22,25 @@ test_expect_success 'setup' '
 
 '
 
+test_expect_success 'git clean with skip-worktree .gitignore' '
+	git update-index --skip-worktree .gitignore &&
+	rm .gitignore &&
+	mkdir -p build docs &&
+	touch a.out src/part3.c docs/manual.txt obj.o build/lib.so &&
+	git clean &&
+	test -f Makefile &&
+	test -f README &&
+	test -f src/part1.c &&
+	test -f src/part2.c &&
+	test ! -f a.out &&
+	test ! -f src/part3.c &&
+	test -f docs/manual.txt &&
+	test -f obj.o &&
+	test -f build/lib.so &&
+	git update-index --no-skip-worktree .gitignore &&
+	git checkout .gitignore
+'
+
 test_expect_success 'git clean' '
 
 	mkdir -p build docs &&