summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Documentation/config.txt7
-rw-r--r--Documentation/git-ls-files.txt7
-rw-r--r--Documentation/git-update-index.txt45
-rw-r--r--Documentation/githooks.txt28
-rw-r--r--Documentation/technical/index-format.txt19
-rw-r--r--Makefile3
-rw-r--r--builtin/ls-files.c8
-rw-r--r--builtin/update-index.c38
-rw-r--r--cache.h10
-rw-r--r--compat/bswap.h22
-rw-r--r--config.c14
-rw-r--r--config.h1
-rw-r--r--diff-lib.c2
-rw-r--r--dir.c27
-rw-r--r--dir.h2
-rw-r--r--entry.c2
-rw-r--r--environment.c1
-rw-r--r--fsmonitor.c253
-rw-r--r--fsmonitor.h66
-rw-r--r--preload-index.c8
-rw-r--r--read-cache.c45
-rw-r--r--submodule.c2
-rw-r--r--t/helper/.gitignore2
-rw-r--r--t/helper/test-drop-caches.c164
-rw-r--r--t/helper/test-dump-fsmonitor.c21
-rwxr-xr-xt/perf/p7519-fsmonitor.sh184
-rwxr-xr-xt/t1700-split-index.sh1
-rwxr-xr-xt/t7519-status-fsmonitor.sh304
-rwxr-xr-xt/t7519/fsmonitor-all24
-rwxr-xr-xt/t7519/fsmonitor-none22
-rwxr-xr-xt/t7519/fsmonitor-watchman139
-rwxr-xr-xtemplates/hooks--fsmonitor-watchman.sample120
-rw-r--r--unpack-trees.c2
33 files changed, 1570 insertions, 23 deletions
diff --git a/Documentation/config.txt b/Documentation/config.txt
index 671fcbaa0f..5f65fa9234 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -416,6 +416,13 @@ core.protectNTFS::
8.3 "short" names.
Defaults to `true` on Windows, and `false` elsewhere.
+core.fsmonitor::
+ If set, the value of this variable is used as a command which
+ will identify all files that may have changed since the
+ requested date/time. This information is used to speed up git by
+ avoiding unnecessary processing of files that have not changed.
+ See the "fsmonitor-watchman" section of linkgit:githooks[5].
+
core.trustctime::
If false, the ctime differences between the index and the
working tree are ignored; useful when the inode change time
diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt
index d153c17e06..3ac3e3a77d 100644
--- a/Documentation/git-ls-files.txt
+++ b/Documentation/git-ls-files.txt
@@ -9,7 +9,7 @@ git-ls-files - Show information about files in the index and the working tree
SYNOPSIS
--------
[verse]
-'git ls-files' [-z] [-t] [-v]
+'git ls-files' [-z] [-t] [-v] [-f]
(--[cached|deleted|others|ignored|stage|unmerged|killed|modified])*
(-[c|d|o|i|s|u|k|m])*
[--eol]
@@ -133,6 +133,11 @@ a space) at the start of each line:
that are marked as 'assume unchanged' (see
linkgit:git-update-index[1]).
+-f::
+ Similar to `-t`, but use lowercase letters for files
+ that are marked as 'fsmonitor valid' (see
+ linkgit:git-update-index[1]).
+
--full-name::
When run from a subdirectory, the command usually
outputs paths relative to the current directory. This
diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt
index 75c7dd9dea..bdb0342593 100644
--- a/Documentation/git-update-index.txt
+++ b/Documentation/git-update-index.txt
@@ -16,9 +16,11 @@ SYNOPSIS
[--chmod=(+|-)x]
[--[no-]assume-unchanged]
[--[no-]skip-worktree]
+ [--[no-]fsmonitor-valid]
[--ignore-submodules]
[--[no-]split-index]
[--[no-|test-|force-]untracked-cache]
+ [--[no-]fsmonitor]
[--really-refresh] [--unresolve] [--again | -g]
[--info-only] [--index-info]
[-z] [--stdin] [--index-version <n>]
@@ -111,6 +113,12 @@ you will need to handle the situation manually.
set and unset the "skip-worktree" bit for the paths. See
section "Skip-worktree bit" below for more information.
+--[no-]fsmonitor-valid::
+ When one of these flags is specified, the object name recorded
+ for the paths are not updated. Instead, these options
+ set and unset the "fsmonitor valid" bit for the paths. See
+ section "File System Monitor" below for more information.
+
-g::
--again::
Runs 'git update-index' itself on the paths whose index
@@ -201,6 +209,15 @@ will remove the intended effect of the option.
`--untracked-cache` used to imply `--test-untracked-cache` but
this option would enable the extension unconditionally.
+--fsmonitor::
+--no-fsmonitor::
+ Enable or disable files system monitor feature. These options
+ take effect whatever the value of the `core.fsmonitor`
+ configuration variable (see linkgit:git-config[1]). But a warning
+ is emitted when the change goes against the configured value, as
+ the configured value will take effect next time the index is
+ read and this will remove the intended effect of the option.
+
\--::
Do not interpret any more arguments as options.
@@ -447,6 +464,34 @@ command reads the index; while when `--[no-|force-]untracked-cache`
are used, the untracked cache is immediately added to or removed from
the index.
+File System Monitor
+-------------------
+
+This feature is intended to speed up git operations for repos that have
+large working directories.
+
+It enables git to work together with a file system monitor (see the
+"fsmonitor-watchman" section of linkgit:githooks[5]) that can
+inform it as to what files have been modified. This enables git to avoid
+having to lstat() every file to find modified files.
+
+When used in conjunction with the untracked cache, it can further improve
+performance by avoiding the cost of scanning the entire working directory
+looking for new files.
+
+If you want to enable (or disable) this feature, it is easier to use
+the `core.fsmonitor` configuration variable (see
+linkgit:git-config[1]) than using the `--fsmonitor` option to
+`git update-index` in each repository, especially if you want to do so
+across all repositories you use, because you can set the configuration
+variable to `true` (or `false`) in your `$HOME/.gitconfig` just once
+and have it affect all repositories you touch.
+
+When the `core.fsmonitor` configuration variable is changed, the
+file system monitor is added to or removed from the index the next time
+a command reads the index. When `--[no-]fsmonitor` are used, the file
+system monitor is immediately added to or removed from the index.
+
Configuration
-------------
diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt
index 5d3f45560e..0bb0042d8c 100644
--- a/Documentation/githooks.txt
+++ b/Documentation/githooks.txt
@@ -454,6 +454,34 @@ the name of the file that holds the e-mail to be sent. Exiting with a
non-zero status causes 'git send-email' to abort before sending any
e-mails.
+fsmonitor-watchman
+~~~~~~~~~~~~~~~~~~
+
+This hook is invoked when the configuration option core.fsmonitor is
+set to .git/hooks/fsmonitor-watchman. It takes two arguments, a version
+(currently 1) and the time in elapsed nanoseconds since midnight,
+January 1, 1970.
+
+The hook should output to stdout the list of all files in the working
+directory that may have changed since the requested time. The logic
+should be inclusive so that it does not miss any potential changes.
+The paths should be relative to the root of the working directory
+and be separated by a single NUL.
+
+It is OK to include files which have not actually changed. All changes
+including newly-created and deleted files should be included. When
+files are renamed, both the old and the new name should be included.
+
+Git will limit what files it checks for changes as well as which
+directories are checked for untracked files based on the path names
+given.
+
+An optimized way to tell git "all files have changed" is to return
+the filename '/'.
+
+The exit status determines whether git will use the data from the
+hook to limit its search. On error, it will fall back to verifying
+all files and folders.
GIT
---
diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt
index ade0b0c445..db3572626b 100644
--- a/Documentation/technical/index-format.txt
+++ b/Documentation/technical/index-format.txt
@@ -295,3 +295,22 @@ The remaining data of each directory block is grouped by type:
in the previous ewah bitmap.
- One NUL.
+
+== File System Monitor cache
+
+ The file system monitor cache tracks files for which the core.fsmonitor
+ hook has told us about changes. The signature for this extension is
+ { 'F', 'S', 'M', 'N' }.
+
+ The extension starts with
+
+ - 32-bit version number: the current supported version is 1.
+
+ - 64-bit time: the extension data reflects all changes through the given
+ time which is stored as the nanoseconds elapsed since midnight,
+ January 1, 1970.
+
+ - 32-bit bitmap size: the size of the CE_FSMONITOR_VALID bitmap.
+
+ - An ewah bitmap, the n-th bit indicates whether the n-th index entry
+ is not CE_FSMONITOR_VALID.
diff --git a/Makefile b/Makefile
index ee9d5eb11e..e53750ca01 100644
--- a/Makefile
+++ b/Makefile
@@ -646,7 +646,9 @@ TEST_PROGRAMS_NEED_X += test-ctype
TEST_PROGRAMS_NEED_X += test-config
TEST_PROGRAMS_NEED_X += test-date
TEST_PROGRAMS_NEED_X += test-delta
+TEST_PROGRAMS_NEED_X += test-drop-caches
TEST_PROGRAMS_NEED_X += test-dump-cache-tree
+TEST_PROGRAMS_NEED_X += test-dump-fsmonitor
TEST_PROGRAMS_NEED_X += test-dump-split-index
TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
TEST_PROGRAMS_NEED_X += test-fake-ssh
@@ -794,6 +796,7 @@ LIB_OBJS += ewah/ewah_rlw.o
LIB_OBJS += exec_cmd.o
LIB_OBJS += fetch-pack.o
LIB_OBJS += fsck.o
+LIB_OBJS += fsmonitor.o
LIB_OBJS += gettext.o
LIB_OBJS += gpg-interface.o
LIB_OBJS += graph.o
diff --git a/builtin/ls-files.c b/builtin/ls-files.c
index 8c713c47ac..2fc836e330 100644
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@ -31,6 +31,7 @@ static int show_resolve_undo;
static int show_modified;
static int show_killed;
static int show_valid_bit;
+static int show_fsmonitor_bit;
static int line_terminator = '\n';
static int debug_mode;
static int show_eol;
@@ -86,7 +87,8 @@ static const char *get_tag(const struct cache_entry *ce, const char *tag)
{
static char alttag[4];
- if (tag && *tag && show_valid_bit && (ce->ce_flags & CE_VALID)) {
+ if (tag && *tag && ((show_valid_bit && (ce->ce_flags & CE_VALID)) ||
+ (show_fsmonitor_bit && (ce->ce_flags & CE_FSMONITOR_VALID)))) {
memcpy(alttag, tag, 3);
if (isalpha(tag[0])) {
@@ -515,6 +517,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
N_("identify the file status with tags")),
OPT_BOOL('v', NULL, &show_valid_bit,
N_("use lowercase letters for 'assume unchanged' files")),
+ OPT_BOOL('f', NULL, &show_fsmonitor_bit,
+ N_("use lowercase letters for 'fsmonitor clean' files")),
OPT_BOOL('c', "cached", &show_cached,
N_("show cached files in the output (default)")),
OPT_BOOL('d', "deleted", &show_deleted,
@@ -584,7 +588,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
for (i = 0; i < exclude_list.nr; i++) {
add_exclude(exclude_list.items[i].string, "", 0, el, --exclude_args);
}
- if (show_tag || show_valid_bit) {
+ if (show_tag || show_valid_bit || show_fsmonitor_bit) {
tag_cached = "H ";
tag_unmerged = "M ";
tag_removed = "R ";
diff --git a/builtin/update-index.c b/builtin/update-index.c
index fefbe60167..58d1c2d282 100644
--- a/builtin/update-index.c
+++ b/builtin/update-index.c
@@ -16,6 +16,7 @@
#include "pathspec.h"
#include "dir.h"
#include "split-index.h"
+#include "fsmonitor.h"
/*
* Default to not allowing changes to the list of files. The
@@ -32,6 +33,7 @@ static int force_remove;
static int verbose;
static int mark_valid_only;
static int mark_skip_worktree_only;
+static int mark_fsmonitor_only;
#define MARK_FLAG 1
#define UNMARK_FLAG 2
static struct strbuf mtime_dir = STRBUF_INIT;
@@ -228,6 +230,7 @@ static int mark_ce_flags(const char *path, int flag, int mark)
int namelen = strlen(path);
int pos = cache_name_pos(path, namelen);
if (0 <= pos) {
+ mark_fsmonitor_invalid(&the_index, active_cache[pos]);
if (mark)
active_cache[pos]->ce_flags |= flag;
else
@@ -460,6 +463,11 @@ static void update_one(const char *path)
die("Unable to mark file %s", path);
return;
}
+ if (mark_fsmonitor_only) {
+ if (mark_ce_flags(path, CE_FSMONITOR_VALID, mark_fsmonitor_only == MARK_FLAG))
+ die("Unable to mark file %s", path);
+ return;
+ }
if (force_remove) {
if (remove_file_from_cache(path))
@@ -917,6 +925,8 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
struct refresh_params refresh_args = {0, &has_errors};
int lock_error = 0;
int split_index = -1;
+ int force_write = 0;
+ int fsmonitor = -1;
struct lock_file lock_file = LOCK_INIT;
struct parse_opt_ctx_t ctx;
strbuf_getline_fn getline_fn;
@@ -1008,6 +1018,16 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
N_("test if the filesystem supports untracked cache"), UC_TEST),
OPT_SET_INT(0, "force-untracked-cache", &untracked_cache,
N_("enable untracked cache without testing the filesystem"), UC_FORCE),
+ OPT_SET_INT(0, "force-write-index", &force_write,
+ N_("write out the index even if is not flagged as changed"), 1),
+ OPT_BOOL(0, "fsmonitor", &fsmonitor,
+ N_("enable or disable file system monitor")),
+ {OPTION_SET_INT, 0, "fsmonitor-valid", &mark_fsmonitor_only, NULL,
+ N_("mark files as fsmonitor valid"),
+ PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, MARK_FLAG},
+ {OPTION_SET_INT, 0, "no-fsmonitor-valid", &mark_fsmonitor_only, NULL,
+ N_("clear fsmonitor valid bit"),
+ PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, UNMARK_FLAG},
OPT_END()
};
@@ -1146,7 +1166,23 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
die("BUG: bad untracked_cache value: %d", untracked_cache);
}
- if (active_cache_changed) {
+ if (fsmonitor > 0) {
+ if (git_config_get_fsmonitor() == 0)
+ warning(_("core.fsmonitor is unset; "
+ "set it if you really want to "
+ "enable fsmonitor"));
+ add_fsmonitor(&the_index);
+ report(_("fsmonitor enabled"));
+ } else if (!fsmonitor) {
+ if (git_config_get_fsmonitor() == 1)
+ warning(_("core.fsmonitor is set; "
+ "remove it if you really want to "
+ "disable fsmonitor"));
+ remove_fsmonitor(&the_index);
+ report(_("fsmonitor disabled"));
+ }
+
+ if (active_cache_changed || force_write) {
if (newfd < 0) {
if (refresh_args.flags & REFRESH_QUIET)
exit(128);
diff --git a/cache.h b/cache.h
index 89f5d24579..7c71681236 100644
--- a/cache.h
+++ b/cache.h
@@ -204,6 +204,7 @@ struct cache_entry {
#define CE_ADDED (1 << 19)
#define CE_HASHED (1 << 20)
+#define CE_FSMONITOR_VALID (1 << 21)
#define CE_WT_REMOVE (1 << 22) /* remove in work directory */
#define CE_CONFLICTED (1 << 23)
@@ -327,6 +328,7 @@ static inline unsigned int canon_mode(unsigned int mode)
#define CACHE_TREE_CHANGED (1 << 5)
#define SPLIT_INDEX_ORDERED (1 << 6)
#define UNTRACKED_CHANGED (1 << 7)
+#define FSMONITOR_CHANGED (1 << 8)
struct split_index;
struct untracked_cache;
@@ -345,6 +347,7 @@ struct index_state {
struct hashmap dir_hash;
unsigned char sha1[20];
struct untracked_cache *untracked;
+ uint64_t fsmonitor_last_update;
};
extern struct index_state the_index;
@@ -700,8 +703,10 @@ extern void *read_blob_data_from_index(const struct index_state *, const char *,
#define CE_MATCH_IGNORE_MISSING 0x08
/* enable stat refresh */
#define CE_MATCH_REFRESH 0x10
-extern int ie_match_stat(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
-extern int ie_modified(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
+/* don't refresh_fsmonitor state or do stat comparison even if CE_FSMONITOR_VALID is true */
+#define CE_MATCH_IGNORE_FSMONITOR 0X20
+extern int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
+extern int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
#define HASH_WRITE_OBJECT 1
#define HASH_FORMAT_CHECK 2
@@ -799,6 +804,7 @@ extern int core_apply_sparse_checkout;
extern int precomposed_unicode;
extern int protect_hfs;
extern int protect_ntfs;
+extern const char *core_fsmonitor;
/*
* Include broken refs in all ref iterations, which will
diff --git a/compat/bswap.h b/compat/bswap.h
index 7d063e9e40..5078ce5ecc 100644
--- a/compat/bswap.h
+++ b/compat/bswap.h
@@ -158,7 +158,9 @@ static inline uint64_t git_bswap64(uint64_t x)
#define get_be16(p) ntohs(*(unsigned short *)(p))
#define get_be32(p) ntohl(*(unsigned int *)(p))
+#define get_be64(p) ntohll(*(uint64_t *)(p))
#define put_be32(p, v) do { *(unsigned int *)(p) = htonl(v); } while (0)
+#define put_be64(p, v) do { *(uint64_t *)(p) = htonll(v); } while (0)
#else
@@ -178,6 +180,13 @@ static inline uint32_t get_be32(const void *ptr)
(uint32_t)p[3] << 0;
}
+static inline uint64_t get_be64(const void *ptr)
+{
+ const unsigned char *p = ptr;
+ return (uint64_t)get_be32(&p[0]) << 32 |
+ (uint64_t)get_be32(&p[4]) << 0;
+}
+
static inline void put_be32(void *ptr, uint32_t value)
{
unsigned char *p = ptr;
@@ -187,4 +196,17 @@ static inline void put_be32(void *ptr, uint32_t value)
p[3] = value >> 0;
}
+static inline void put_be64(void *ptr, uint64_t value)
+{
+ unsigned char *p = ptr;
+ p[0] = value >> 56;
+ p[1] = value >> 48;
+ p[2] = value >> 40;
+ p[3] = value >> 32;
+ p[4] = value >> 24;
+ p[5] = value >> 16;
+ p[6] = value >> 8;
+ p[7] = value >> 0;
+}
+
#endif
diff --git a/config.c b/config.c
index 903abf9533..03ab56d3d8 100644
--- a/config.c
+++ b/config.c
@@ -2156,6 +2156,20 @@ int git_config_get_max_percent_split_change(void)
return -1; /* default value */
}
+int git_config_get_fsmonitor(void)
+{
+ if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor))
+ core_fsmonitor = getenv("GIT_FSMONITOR_TEST");
+
+ if (core_fsmonitor && !*core_fsmonitor)
+ core_fsmonitor = NULL;
+
+ if (core_fsmonitor)
+ return 1;
+
+ return 0;
+}
+
NORETURN
void git_die_config_linenr(const char *key, const char *filename, int linenr)
{
diff --git a/config.h b/config.h
index a49d264416..524d411823 100644
--- a/config.h
+++ b/config.h
@@ -212,6 +212,7 @@ extern int git_config_get_pathname(const char *key, const char **dest);
extern int git_config_get_untracked_cache(void);
extern int git_config_get_split_index(void);
extern int git_config_get_max_percent_split_change(void);
+extern int git_config_get_fsmonitor(void);
/* This dies if the configured or default date is in the future */
extern int git_config_get_expiry(const char *key, const char **output);
diff --git a/diff-lib.c b/diff-lib.c
index 731f0886d6..5173023cd3 100644
--- a/diff-lib.c
+++ b/diff-lib.c
@@ -12,6 +12,7 @@
#include "refs.h"
#include "submodule.h"
#include "dir.h"
+#include "fsmonitor.h"
/*
* diff-files
@@ -229,6 +230,7 @@ int run_diff_files(struct rev_info *revs, unsigned int option)
if (!changed && !dirty_submodule) {
ce_mark_uptodate(ce);
+ mark_fsmonitor_valid(ce);
if (!revs->diffopt.flags.find_copies_harder)
continue;
}
diff --git a/dir.c b/dir.c
index 047a950f55..3c54366a17 100644
--- a/dir.c
+++ b/dir.c
@@ -18,6 +18,7 @@
#include "utf8.h"
#include "varint.h"
#include "ewah/ewok.h"
+#include "fsmonitor.h"
/*
* Tells read_directory_recursive how a file or directory should be treated.
@@ -1733,17 +1734,23 @@ static int valid_cached_dir(struct dir_struct *dir,
if (!untracked)
return 0;
- if (stat(path->len ? path->buf : ".", &st)) {
- invalidate_directory(dir->untracked, untracked);
- memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
- return 0;
- }
- if (!untracked->valid ||
- match_stat_data_racy(istate, &untracked->stat_data, &st)) {
- if (untracked->valid)
+ /*
+ * With fsmonitor, we can trust the untracked cache's valid field.
+ */
+ refresh_fsmonitor(istate);
+ if (!(dir->untracked->use_fsmonitor && untracked->valid)) {
+ if (stat(path->len ? path->buf : ".", &st)) {
invalidate_directory(dir->untracked, untracked);
- fill_stat_data(&untracked->stat_data, &st);
- return 0;
+ memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
+ return 0;
+ }
+ if (!untracked->valid ||
+ match_stat_data_racy(istate, &untracked->stat_data, &st)) {
+ if (untracked->valid)
+ invalidate_directory(dir->untracked, untracked);
+ fill_stat_data(&untracked->stat_data, &st);
+ return 0;
+ }
}
if (untracked->check_only != !!check_only) {
diff --git a/dir.h b/dir.h
index 57b0943dae..233a2eb36b 100644
--- a/dir.h
+++ b/dir.h
@@ -139,6 +139,8 @@ struct untracked_cache {
int gitignore_invalidated;
int dir_invalidated;
int dir_opened;
+ /* fsmonitor invalidation data */
+ unsigned int use_fsmonitor : 1;
};
struct dir_struct {
diff --git a/entry.c b/entry.c
index 944c183b0f..30211447ac 100644
--- a/entry.c
+++ b/entry.c
@@ -4,6 +4,7 @@
#include "streaming.h"
#include "submodule.h"
#include "progress.h"
+#include "fsmonitor.h"
static void create_directories(const char *path, int path_len,
const struct checkout *state)
@@ -373,6 +374,7 @@ finish:
ce->name);
fill_stat_cache_info(ce, &st);
ce->ce_flags |= CE_UPDATE_IN_BASE;
+ mark_fsmonitor_invalid(state->istate, ce);
state->istate->cache_changed |= CE_ENTRY_CHANGED;
}
delayed:
diff --git a/environment.c b/environment.c
index 8289c25b44..8fa032f307 100644
--- a/environment.c
+++ b/environment.c
@@ -76,6 +76,7 @@ int protect_hfs = PROTECT_HFS_DEFAULT;
#define PROTECT_NTFS_DEFAULT 0
#endif
int protect_ntfs = PROTECT_NTFS_DEFAULT;
+const char *core_fsmonitor;
/*
* The character that begins a commented line in user-editable file
diff --git a/fsmonitor.c b/fsmonitor.c
new file mode 100644
index 0000000000..7c1540c054
--- /dev/null
+++ b/fsmonitor.c
@@ -0,0 +1,253 @@
+#include "cache.h"
+#include "config.h"
+#include "dir.h"
+#include "ewah/ewok.h"
+#include "fsmonitor.h"
+#include "run-command.h"
+#include "strbuf.h"
+
+#define INDEX_EXTENSION_VERSION (1)
+#define HOOK_INTERFACE_VERSION (1)
+
+struct trace_key trace_fsmonitor = TRACE_KEY_INIT(FSMONITOR);
+
+static void fsmonitor_ewah_callback(size_t pos, void *is)
+{
+ struct index_state *istate = (struct index_state *)is;
+ struct cache_entry *ce = istate->cache[pos];
+
+ ce->ce_flags &= ~CE_FSMONITOR_VALID;
+}
+
+int read_fsmonitor_extension(struct index_state *istate, const void *data,
+ unsigned long sz)
+{
+ const char *index = data;
+ uint32_t hdr_version;
+ uint32_t ewah_size;
+ struct ewah_bitmap *fsmonitor_dirty;
+ int i;
+ int ret;
+
+ if (sz < sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t))
+ return error("corrupt fsmonitor extension (too short)");
+
+ hdr_version = get_be32(index);
+ index += sizeof(uint32_t);
+ if (hdr_version != INDEX_EXTENSION_VERSION)
+ return error("bad fsmonitor version %d", hdr_version);
+
+ istate->fsmonitor_last_update = get_be64(index);
+ index += sizeof(uint64_t);
+
+ ewah_size = get_be32(index);
+ index += sizeof(uint32_t);
+
+ fsmonitor_dirty = ewah_new();
+ ret = ewah_read_mmap(fsmonitor_dirty, index, ewah_size);
+ if (ret != ewah_size) {
+ ewah_free(fsmonitor_dirty);
+ return error("failed to parse ewah bitmap reading fsmonitor index extension");
+ }
+
+ if (git_config_get_fsmonitor()) {
+ /* Mark all entries valid */
+ for (i = 0; i < istate->cache_nr; i++)
+ istate->cache[i]->ce_flags |= CE_FSMONITOR_VALID;
+
+ /* Mark all previously saved entries as dirty */
+ ewah_each_bit(fsmonitor_dirty, fsmonitor_ewah_callback, istate);
+
+ /* Now mark the untracked cache for fsmonitor usage */
+ if (istate->untracked)
+ istate->untracked->use_fsmonitor = 1;
+ }
+ ewah_free(fsmonitor_dirty);
+
+ trace_printf_key(&trace_fsmonitor, "read fsmonitor extension successful");
+ return 0;
+}
+
+void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate)
+{
+ uint32_t hdr_version;
+ uint64_t tm;
+ struct ewah_bitmap *bitmap;
+ int i;
+ uint32_t ewah_start;
+ uint32_t ewah_size = 0;
+ int fixup = 0;
+
+ put_be32(&hdr_version, INDEX_EXTENSION_VERSION);
+ strbuf_add(sb, &hdr_version, sizeof(uint32_t));
+
+ put_be64(&tm, istate->fsmonitor_last_update);
+ strbuf_add(sb, &tm, sizeof(uint64_t));
+ fixup = sb->len;
+ strbuf_add(sb, &ewah_size, sizeof(uint32_t)); /* we'll fix this up later */
+
+ ewah_start = sb->len;
+ bitmap = ewah_new();
+ for (i = 0; i < istate->cache_nr; i++)
+ if (!(istate->cache[i]->ce_flags & CE_FSMONITOR_VALID))
+ ewah_set(bitmap, i);
+ ewah_serialize_strbuf(bitmap, sb);
+ ewah_free(bitmap);
+
+ /* fix up size field */
+ put_be32(&ewah_size, sb->len - ewah_start);
+ memcpy(sb->buf + fixup, &ewah_size, sizeof(uint32_t));
+
+ trace_printf_key(&trace_fsmonitor, "write fsmonitor extension successful");
+}
+
+/*
+ * Call the query-fsmonitor hook passing the time of the last saved results.
+ */
+static int query_fsmonitor(int version, uint64_t last_update, struct strbuf *query_result)
+{
+ struct child_process cp = CHILD_PROCESS_INIT;
+ char ver[64];
+ char date[64];
+ const char *argv[4];
+
+ if (!(argv[0] = core_fsmonitor))
+ return -1;
+
+ snprintf(ver, sizeof(version), "%d", version);
+ snprintf(date, sizeof(date), "%" PRIuMAX, (uintmax_t)last_update);
+ argv[1] = ver;
+ argv[2] = date;
+ argv[3] = NULL;
+ cp.argv = argv;
+ cp.use_shell = 1;
+
+ return capture_command(&cp, query_result, 1024);
+}
+
+static void fsmonitor_refresh_callback(struct index_state *istate, const char *name)
+{
+ int pos = index_name_pos(istate, name, strlen(name));
+
+ if (pos >= 0) {
+ struct cache_entry *ce = istate->cache[pos];
+ ce->ce_flags &= ~CE_FSMONITOR_VALID;
+ }
+
+ /*
+ * Mark the untracked cache dirty even if it wasn't found in the index
+ * as it could be a new untracked file.
+ */
+ trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s'", name);
+ untracked_cache_invalidate_path(istate, name);
+}
+
+void refresh_fsmonitor(struct index_state *istate)
+{
+ static int has_run_once = 0;
+ struct strbuf query_result = STRBUF_INIT;
+ int query_success = 0;
+ size_t bol; /* beginning of line */
+ uint64_t last_update;
+ char *buf;
+ int i;
+
+ if (!core_fsmonitor || has_run_once)
+ return;
+ has_run_once = 1;
+
+ trace_printf_key(&trace_fsmonitor, "refresh fsmonitor");
+ /*
+ * This could be racy so save the date/time now and query_fsmonitor
+ * should be inclusive to ensure we don't miss potential changes.
+ */
+ last_update = getnanotime();
+
+ /*
+ * If we have a last update time, call query_fsmonitor for the set of
+ * changes since that time, else assume everything is possibly dirty
+ * and check it all.
+ */
+ if (istate->fsmonitor_last_update) {
+ query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION,
+ istate->fsmonitor_last_update, &query_result);
+ trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor);
+ trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s",
+ core_fsmonitor, query_success ? "success" : "failure");
+ }
+
+ /* a fsmonitor process can return '/' to indicate all entries are invalid */
+ if (query_success && query_result.buf[0] != '/') {
+ /* Mark all entries returned by the monitor as dirty */
+ buf = query_result.buf;
+ bol = 0;
+ for (i = 0; i < query_result.len; i++) {
+ if (buf[i] != '\0')
+ continue;
+ fsmonitor_refresh_callback(istate, buf + bol);
+ bol = i + 1;
+ }
+ if (bol < query_result.len)
+ fsmonitor_refresh_callback(istate, buf + bol);
+ } else {
+ /* Mark all entries invalid */
+ for (i = 0; i < istate->cache_nr; i++)
+ istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
+
+ if (istate->untracked)
+ istate->untracked->use_fsmonitor = 0;
+ }
+ strbuf_release(&query_result);
+
+ /* Now that we've updated istate, save the last_update time */
+ istate->fsmonitor_last_update = last_update;
+}
+
+void add_fsmonitor(struct index_state *istate)
+{
+ int i;
+
+ if (!istate->fsmonitor_last_update) {
+ trace_printf_key(&trace_fsmonitor, "add fsmonitor");
+ istate->cache_changed |= FSMONITOR_CHANGED;
+ istate->fsmonitor_last_update = getnanotime();
+
+ /* reset the fsmonitor state */
+ for (i = 0; i < istate->cache_nr; i++)
+ istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
+
+ /* reset the untracked cache */
+ if (istate->untracked) {
+ add_untracked_cache(istate);
+ istate->untracked->use_fsmonitor = 1;
+ }
+
+ /* Update the fsmonitor state */
+ refresh_fsmonitor(istate);
+ }
+}
+
+void remove_fsmonitor(struct index_state *istate)
+{
+ if (istate->fsmonitor_last_update) {
+ trace_printf_key(&trace_fsmonitor, "remove fsmonitor");
+ istate->cache_changed |= FSMONITOR_CHANGED;
+ istate->fsmonitor_last_update = 0;
+ }
+}
+
+void tweak_fsmonitor(struct index_state *istate)
+{
+ switch (git_config_get_fsmonitor()) {
+ case -1: /* keep: do nothing */
+ break;
+ case 0: /* false */
+ remove_fsmonitor(istate);
+ break;
+ case 1: /* true */
+ add_fsmonitor(istate);
+ break;
+ default: /* unknown value: do nothing */
+ break;
+ }
+}
diff --git a/fsmonitor.h b/fsmonitor.h
new file mode 100644
index 0000000000..0de644e01a
--- /dev/null
+++ b/fsmonitor.h
@@ -0,0 +1,66 @@
+#ifndef FSMONITOR_H
+#define FSMONITOR_H
+
+extern struct trace_key trace_fsmonitor;
+
+/*
+ * Read the fsmonitor index extension and (if configured) restore the
+ * CE_FSMONITOR_VALID state.
+ */
+extern int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz);
+
+/*
+ * Write the CE_FSMONITOR_VALID state into the fsmonitor index extension.
+ */
+extern void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate);
+
+/*
+ * Add/remove the fsmonitor index extension
+ */
+extern void add_fsmonitor(struct index_state *istate);
+extern void remove_fsmonitor(struct index_state *istate);
+
+/*
+ * Add/remove the fsmonitor index extension as necessary based on the current
+ * core.fsmonitor setting.
+ */
+extern void tweak_fsmonitor(struct index_state *istate);
+
+/*
+ * Run the configured fsmonitor integration script and clear the
+ * CE_FSMONITOR_VALID bit for any files returned as dirty. Also invalidate
+ * any corresponding untracked cache directory structures. Optimized to only
+ * run the first time it is called.
+ */
+extern void refresh_fsmonitor(struct index_state *istate);
+
+/*
+ * Set the given cache entries CE_FSMONITOR_VALID bit. This should be
+ * called any time the cache entry has been updated to reflect the
+ * current state of the file on disk.
+ */
+static inline void mark_fsmonitor_valid(struct cache_entry *ce)
+{
+ if (core_fsmonitor) {
+ ce->ce_flags |= CE_FSMONITOR_VALID;
+ trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_clean '%s'", ce->name);
+ }
+}
+
+/*
+ * Clear the given cache entry's CE_FSMONITOR_VALID bit and invalidate
+ * any corresponding untracked cache directory structures. This should
+ * be called any time git creates or modifies a file that should
+ * trigger an lstat() or invalidate the untracked cache for the
+ * corresponding directory
+ */
+static inline void mark_fsmonitor_invalid(struct index_state *istate, struct cache_entry *ce)
+{
+ if (core_fsmonitor) {
+ ce->ce_flags &= ~CE_FSMONITOR_VALID;
+ untracked_cache_invalidate_path(istate, ce->name);
+ trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_invalid '%s'", ce->name);
+ }
+}
+
+#endif
diff --git a/preload-index.c b/preload-index.c
index 70a4c80878..2a83255e4e 100644
--- a/preload-index.c
+++ b/preload-index.c
@@ -4,6 +4,7 @@
#include "cache.h"
#include "pathspec.h"
#include "dir.h"
+#include "fsmonitor.h"
#ifdef NO_PTHREADS
static void preload_index(struct index_state *index,
@@ -55,15 +56,18 @@ static void *preload_thread(void *_data)
continue;
if (ce_skip_worktree(ce))
continue;
+ if (ce->ce_flags & CE_FSMONITOR_VALID)
+ continue;
if (!ce_path_match(ce, &p->pathspec, NULL))
continue;
if (threaded_has_symlink_leading_path(&cache, ce->name, ce_namelen(ce)))
continue;
if (lstat(ce->name, &st))
continue;
- if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY))
+ if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY|CE_MATCH_IGNORE_FSMONITOR))
continue;
ce_mark_uptodate(ce);
+ mark_fsmonitor_valid(ce);
} while (--nr > 0);
cache_def_clear(&cache);
return NULL;
@@ -79,6 +83,8 @@ static void preload_index(struct index_state *index,
return;
threads = index->cache_nr / THREAD_COST;
+ if ((index->cache_nr > 1) && (threads < 2) && getenv("GIT_FORCE_PRELOAD_TEST"))
+ threads = 2;
if (threads < 2)
return;
if (threads > MAX_PARALLEL)
diff --git a/read-cache.c b/read-cache.c
index b13a1cb8f2..87e88b2642 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -19,6 +19,7 @@
#include "varint.h"
#include "split-index.h"
#include "utf8.h"
+#include "fsmonitor.h"
/* Mask for the name length in ce_flags in the on-disk index */
@@ -38,11 +39,12 @@
#define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
#define CACHE_EXT_LINK 0x6c696e6b /* "link" */
#define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */
+#define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */
/* changes that can be kept in $GIT_DIR/index (basically all extensions) */
#define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \
CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \
- SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED)
+ SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED | FSMONITOR_CHANGED)
struct index_state the_index;
static const char *alternate_index_output;
@@ -62,6 +64,7 @@ static void replace_index_entry(struct index_state *istate, int nr, struct cache
free(old);
set_index_entry(istate, nr, ce);
ce->ce_flags |= CE_UPDATE_IN_BASE;
+ mark_fsmonitor_invalid(istate, ce);
istate->cache_changed |= CE_ENTRY_CHANGED;
}
@@ -150,8 +153,10 @@ void fill_stat_cache_info(struct cache_entry *ce, struct stat *st)
if (assume_unchanged)
ce->ce_flags |= CE_VALID;
- if (S_ISREG(st->st_mode))
+ if (S_ISREG(st->st_mode)) {
ce_mark_uptodate(ce);
+ mark_fsmonitor_valid(ce);
+ }
}
static int ce_compare_data(const struct cache_entry *ce, struct stat *st)
@@ -301,7 +306,7 @@ int match_stat_data_racy(const struct index_state *istate,
return match_stat_data(sd, st);
}
-int ie_match_stat(const struct index_state *istate,
+int ie_match_stat(struct index_state *istate,
const struct cache_entry *ce, struct stat *st,
unsigned int options)
{
@@ -309,7 +314,10 @@ int ie_match_stat(const struct index_state *istate,
int ignore_valid = options & CE_MATCH_IGNORE_VALID;
int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
int assume_racy_is_modified = options & CE_MATCH_RACY_IS_DIRTY;
+ int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR;
+ if (!ignore_fsmonitor)
+ refresh_fsmonitor(istate);
/*
* If it's marked as always valid in the index, it's
* valid whatever the checked-out copy says.
@@ -320,6 +328,8 @@ int ie_match_stat(const struct index_state *istate,
return 0;
if (!ignore_valid && (ce->ce_flags & CE_VALID))
return 0;
+ if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID))
+ return 0;
/*
* Intent-to-add entries have not been added, so the index entry
@@ -357,7 +367,7 @@ int ie_match_stat(const struct index_state *istate,
return changed;
}
-int ie_modified(const struct index_state *istate,
+int ie_modified(struct index_state *istate,
const struct cache_entry *ce,
struct stat *st, unsigned int options)
{
@@ -778,6 +788,7 @@ int chmod_index_entry(struct index_state *istate, struct cache_entry *ce,
}
cache_tree_invalidate_path(istate, ce->name);
ce->ce_flags |= CE_UPDATE_IN_BASE;
+ mark_fsmonitor_invalid(istate, ce);
istate->cache_changed |= CE_ENTRY_CHANGED;
return 0;
@@ -1229,10 +1240,13 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
int ignore_valid = options & CE_MATCH_IGNORE_VALID;
int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
int ignore_missing = options & CE_MATCH_IGNORE_MISSING;
+ int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR;
if (!refresh || ce_uptodate(ce))
return ce;
+ if (!ignore_fsmonitor)
+ refresh_fsmonitor(istate);
/*
* CE_VALID or CE_SKIP_WORKTREE means the user promised us
* that the change to the work tree does not matter and told
@@ -1246,6 +1260,10 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
ce_mark_uptodate(ce);
return ce;
}
+ if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID)) {
+ ce_mark_uptodate(ce);
+ return ce;
+ }
if (has_symlink_leading_path(ce->name, ce_namelen(ce))) {
if (ignore_missing)
@@ -1283,8 +1301,10 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
* because CE_UPTODATE flag is in-core only;
* we are not going to write this change out.
*/
- if (!S_ISGITLINK(ce->ce_mode))
+ if (!S_ISGITLINK(ce->ce_mode)) {
ce_mark_uptodate(ce);
+ mark_fsmonitor_valid(ce);
+ }
return ce;
}
}
@@ -1392,6 +1412,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
*/
ce->ce_flags &= ~CE_VALID;
ce->ce_flags |= CE_UPDATE_IN_BASE;
+ mark_fsmonitor_invalid(istate, ce);
istate->cache_changed |= CE_ENTRY_CHANGED;
}
if (quiet)
@@ -1554,6 +1575,9 @@ static int read_index_extension(struct index_state *istate,
case CACHE_EXT_UNTRACKED:
istate->untracked = read_untracked_extension(data, sz);
break;
+ case CACHE_EXT_FSMONITOR:
+ read_fsmonitor_extension(istate, data, sz);
+ break;
default:
if (*ext < 'A' || 'Z' < *ext)
return error("index uses %.4s extension, which we do not understand",
@@ -1729,6 +1753,7 @@ static void post_read_index_from(struct index_state *istate)
check_ce_order(istate);
tweak_untracked_cache(istate);
tweak_split_index(istate);
+ tweak_fsmonitor(istate);
}
/* remember to discard_cache() before reading a different cache! */
@@ -2320,6 +2345,16 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
if (err)
return -1;
}
+ if (!strip_extensions && istate->fsmonitor_last_update) {
+ struct strbuf sb = STRBUF_INIT;
+
+ write_fsmonitor_extension(&sb, istate);
+ err = write_index_ext_header(&c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0
+ || ce_write(&c, newfd, sb.buf, sb.len) < 0;
+ strbuf_release(&sb);
+ if (err)
+ return -1;
+ }
if (ce_flush(&c, newfd, istate->sha1))
return -1;
diff --git a/submodule.c b/submodule.c
index 3ee4a0caa7..bb531e0e59 100644
--- a/submodule.c
+++ b/submodule.c
@@ -62,7 +62,7 @@ int is_staging_gitmodules_ok(const struct index_state *istate)
if ((pos >= 0) && (pos < istate->cache_nr)) {
struct stat st;
if (lstat(GITMODULES_FILE, &st) == 0 &&
- ce_match_stat(istate->cache[pos], &st, 0) & DATA_CHANGED)
+ ce_match_stat(istate->cache[pos], &st, CE_MATCH_IGNORE_FSMONITOR) & DATA_CHANGED)
return 0;
}
diff --git a/t/helper/.gitignore b/t/helper/.gitignore
index 7c9d28a834..d02f9b39ac 100644
--- a/t/helper/.gitignore
+++ b/t/helper/.gitignore
@@ -3,7 +3,9 @@
/test-config
/test-date
/test-delta
+/test-drop-caches
/test-dump-cache-tree
+/test-dump-fsmonitor
/test-dump-split-index
/test-dump-untracked-cache
/test-fake-ssh
diff --git a/t/helper/test-drop-caches.c b/t/helper/test-drop-caches.c
new file mode 100644
index 0000000000..bd1a857d52
--- /dev/null
+++ b/t/helper/test-drop-caches.c
@@ -0,0 +1,164 @@
+#include "git-compat-util.h"
+
+#if defined(GIT_WINDOWS_NATIVE)
+
+static int cmd_sync(void)
+{
+ char Buffer[MAX_PATH];
+ DWORD dwRet;
+ char szVolumeAccessPath[] = "\\\\.\\X:";
+ HANDLE hVolWrite;
+ int success = 0;
+
+ dwRet = GetCurrentDirectory(MAX_PATH, Buffer);
+ if ((0 == dwRet) || (dwRet > MAX_PATH))
+ return error("Error getting current directory");
+
+ if ((Buffer[0] < 'A') || (Buffer[0] > 'Z'))
+ return error("Invalid drive letter '%c'", Buffer[0]);
+
+ szVolumeAccessPath[4] = Buffer[0];
+ hVolWrite = CreateFile(szVolumeAccessPath, GENERIC_READ | GENERIC_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+ if (INVALID_HANDLE_VALUE == hVolWrite)
+ return error("Unable to open volume for writing, need admin access");
+
+ success = FlushFileBuffers(hVolWrite);
+ if (!success)
+ error("Unable to flush volume");
+
+ CloseHandle(hVolWrite);
+
+ return !success;
+}
+
+#define STATUS_SUCCESS (0x00000000L)
+#define STATUS_PRIVILEGE_NOT_HELD (0xC0000061L)
+
+typedef enum _SYSTEM_INFORMATION_CLASS {
+ SystemMemoryListInformation = 80,
+} SYSTEM_INFORMATION_CLASS;
+
+typedef enum _SYSTEM_MEMORY_LIST_COMMAND {
+ MemoryCaptureAccessedBits,
+ MemoryCaptureAndResetAccessedBits,
+ MemoryEmptyWorkingSets,
+ MemoryFlushModifiedList,
+ MemoryPurgeStandbyList,
+ MemoryPurgeLowPriorityStandbyList,
+ MemoryCommandMax
+} SYSTEM_MEMORY_LIST_COMMAND;
+
+static BOOL GetPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
+{
+ BOOL bResult;
+ DWORD dwBufferLength;
+ LUID luid;
+ TOKEN_PRIVILEGES tpPreviousState;
+ TOKEN_PRIVILEGES tpNewState;
+
+ dwBufferLength = 16;
+ bResult = LookupPrivilegeValueA(0, lpName, &luid);
+ if (bResult) {
+ tpNewState.PrivilegeCount = 1;
+ tpNewState.Privileges[0].Luid = luid;
+ tpNewState.Privileges[0].Attributes = 0;
+ bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpNewState,
+ (DWORD)((LPBYTE)&(tpNewState.Privileges[1]) - (LPBYTE)&tpNewState),
+ &tpPreviousState, &dwBufferLength);
+ if (bResult) {
+ tpPreviousState.PrivilegeCount = 1;
+ tpPreviousState.Privileges[0].Luid = luid;
+ tpPreviousState.Privileges[0].Attributes = flags != 0 ? 2 : 0;
+ bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpPreviousState,
+ dwBufferLength, 0, 0);
+ }
+ }
+ return bResult;
+}
+
+static int cmd_dropcaches(void)
+{
+ HANDLE hProcess = GetCurrentProcess();
+ HANDLE hToken;
+ HMODULE ntdll;
+ DWORD(WINAPI *NtSetSystemInformation)(INT, PVOID, ULONG);
+ SYSTEM_MEMORY_LIST_COMMAND command;
+ int status;
+
+ if (!OpenProcessToken(hProcess, TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
+ return error("Can't open current process token");
+
+ if (!GetPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
+ return error("Can't get SeProfileSingleProcessPrivilege");
+
+ CloseHandle(hToken);
+
+ ntdll = LoadLibrary("ntdll.dll");
+ if (!ntdll)
+ return error("Can't load ntdll.dll, wrong Windows version?");
+
+ NtSetSystemInformation =
+ (DWORD(WINAPI *)(INT, PVOID, ULONG))GetProcAddress(ntdll, "NtSetSystemInformation");
+ if (!NtSetSystemInformation)
+ return error("Can't get function addresses, wrong Windows version?");
+
+ command = MemoryPurgeStandbyList;
+ status = NtSetSystemInformation(
+ SystemMemoryListInformation,
+ &command,
+ sizeof(SYSTEM_MEMORY_LIST_COMMAND)
+ );
+ if (status == STATUS_PRIVILEGE_NOT_HELD)
+ error("Insufficient privileges to purge the standby list, need admin access");
+ else if (status != STATUS_SUCCESS)
+ error("Unable to execute the memory list command %d", status);
+
+ FreeLibrary(ntdll);
+
+ return status;
+}
+
+#elif defined(__linux__)
+
+static int cmd_sync(void)
+{
+ return system("sync");
+}
+
+static int cmd_dropcaches(void)
+{
+ return system("echo 3 | sudo tee /proc/sys/vm/drop_caches");
+}
+
+#elif defined(__APPLE__)
+
+static int cmd_sync(void)
+{
+ return system("sync");
+}
+
+static int cmd_dropcaches(void)
+{
+ return system("sudo purge");
+}
+
+#else
+
+static int cmd_sync(void)
+{
+ return 0;
+}
+
+static int cmd_dropcaches(void)
+{
+ return error("drop caches not implemented on this platform");
+}
+
+#endif
+
+int cmd_main(int argc, const char **argv)
+{
+ cmd_sync();
+ return cmd_dropcaches();
+}
diff --git a/t/helper/test-dump-fsmonitor.c b/t/helper/test-dump-fsmonitor.c
new file mode 100644
index 0000000000..ad452707e8
--- /dev/null
+++ b/t/helper/test-dump-fsmonitor.c
@@ -0,0 +1,21 @@
+#include "cache.h"
+
+int cmd_main(int ac, const char **av)
+{
+ struct index_state *istate = &the_index;
+ int i;
+
+ setup_git_directory();
+ if (do_read_index(istate, get_index_file(), 0) < 0)
+ die("unable to read index file");
+ if (!istate->fsmonitor_last_update) {
+ printf("no fsmonitor\n");
+ return 0;
+ }
+ printf("fsmonitor last update %"PRIuMAX"\n", (uintmax_t)istate->fsmonitor_last_update);
+
+ for (i = 0; i < istate->cache_nr; i++)
+ printf((istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) ? "+" : "-");
+
+ return 0;
+}
diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh
new file mode 100755
index 0000000000..16d1bf72e5
--- /dev/null
+++ b/t/perf/p7519-fsmonitor.sh
@@ -0,0 +1,184 @@
+#!/bin/sh
+
+test_description="Test core.fsmonitor"
+
+. ./perf-lib.sh
+
+#
+# Performance test for the fsmonitor feature which enables git to talk to a
+# file system change monitor and avoid having to scan the working directory
+# for new or modified files.
+#
+# By default, the performance test will utilize the Watchman file system
+# monitor if it is installed. If Watchman is not installed, it will use a
+# dummy integration script that does not report any new or modified files.
+# The dummy script has very little overhead which provides optimistic results.
+#
+# The performance test will also use the untracked cache feature if it is
+# available as fsmonitor uses it to speed up scanning for untracked files.
+#
+# There are 3 environment variables that can be used to alter the default
+# behavior of the performance test:
+#
+# GIT_PERF_7519_UNTRACKED_CACHE: used to configure core.untrackedCache
+# GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex
+# GIT_PERF_7519_FSMONITOR: used to configure core.fsMonitor
+#
+# The big win for using fsmonitor is the elimination of the need to scan the
+# working directory looking for changed and untracked files. If the file
+# information is all cached in RAM, the benefits are reduced.
+#
+# GIT_PERF_7519_DROP_CACHE: if set, the OS caches are dropped between tests
+#
+
+test_perf_large_repo
+test_checkout_worktree
+
+test_lazy_prereq UNTRACKED_CACHE '
+ { git update-index --test-untracked-cache; ret=$?; } &&
+ test $ret -ne 1
+'
+
+test_lazy_prereq WATCHMAN '
+ { command -v watchman >/dev/null 2>&1; ret=$?; } &&
+ test $ret -ne 1
+'
+
+if test_have_prereq WATCHMAN
+then
+ # Convert unix style paths to escaped Windows style paths for Watchman
+ case "$(uname -s)" in
+ MSYS_NT*)
+ GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,/,g')"
+ ;;
+ *)
+ GIT_WORK_TREE="$PWD"
+ ;;
+ esac
+fi
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"
+then
+ # When using GIT_PERF_7519_DROP_CACHE, GIT_PERF_REPEAT_COUNT must be 1 to
+ # generate valid results. Otherwise the caching that happens for the nth
+ # run will negate the validity of the comparisons.
+ if test "$GIT_PERF_REPEAT_COUNT" -ne 1
+ then
+ echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
+ GIT_PERF_REPEAT_COUNT=1
+ fi
+fi
+
+test_expect_success "setup for fsmonitor" '
+ # set untrackedCache depending on the environment
+ if test -n "$GIT_PERF_7519_UNTRACKED_CACHE"
+ then
+ git config core.untrackedCache "$GIT_PERF_7519_UNTRACKED_CACHE"
+ else
+ if test_have_prereq UNTRACKED_CACHE
+ then
+ git config core.untrackedCache true
+ else
+ git config core.untrackedCache false
+ fi
+ fi &&
+
+ # set core.splitindex depending on the environment
+ if test -n "$GIT_PERF_7519_SPLIT_INDEX"
+ then
+ git config core.splitIndex "$GIT_PERF_7519_SPLIT_INDEX"
+ fi &&
+
+ # set INTEGRATION_SCRIPT depending on the environment
+ if test -n "$GIT_PERF_7519_FSMONITOR"
+ then
+ INTEGRATION_SCRIPT="$GIT_PERF_7519_FSMONITOR"
+ else
+ #
+ # Choose integration script based on existence of Watchman.
+ # If Watchman exists, watch the work tree and attempt a query.
+ # If everything succeeds, use Watchman integration script,
+ # else fall back to an empty integration script.
+ #
+ mkdir .git/hooks &&
+ if test_have_prereq WATCHMAN
+ then
+ INTEGRATION_SCRIPT=".git/hooks/fsmonitor-watchman" &&
+ cp "$TEST_DIRECTORY/../templates/hooks--fsmonitor-watchman.sample" "$INTEGRATION_SCRIPT" &&
+ watchman watch "$GIT_WORK_TREE" &&
+ watchman watch-list | grep -q -F "$GIT_WORK_TREE"
+ else
+ INTEGRATION_SCRIPT=".git/hooks/fsmonitor-empty" &&
+ write_script "$INTEGRATION_SCRIPT"<<-\EOF
+ EOF
+ fi
+ fi &&
+
+ git config core.fsmonitor "$INTEGRATION_SCRIPT" &&
+ git update-index --fsmonitor
+'
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
+ test-drop-caches
+fi
+
+test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
+ git status
+'
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
+ test-drop-caches
+fi
+
+test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
+ git status -uno
+'
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
+ test-drop-caches
+fi
+
+test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" '
+ git status -uall
+'
+
+test_expect_success "setup without fsmonitor" '
+ unset INTEGRATION_SCRIPT &&
+ git config --unset core.fsmonitor &&
+ git update-index --no-fsmonitor
+'
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
+ test-drop-caches
+fi
+
+test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
+ git status
+'
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
+ test-drop-caches
+fi
+
+test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
+ git status -uno
+'
+
+if test -n "$GIT_PERF_7519_DROP_CACHE"; then
+ test-drop-caches
+fi
+
+test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" '
+ git status -uall
+'
+
+if test_have_prereq WATCHMAN
+then
+ watchman watch-del "$GIT_WORK_TREE" >/dev/null 2>&1 &&
+
+ # Work around Watchman bug on Windows where it holds on to handles
+ # preventing the removal of the trash directory
+ watchman shutdown-server >/dev/null 2>&1
+fi
+
+test_done
diff --git a/t/t1700-split-index.sh b/t/t1700-split-index.sh
index 22f69a410b..af9b847761 100755
--- a/t/t1700-split-index.sh
+++ b/t/t1700-split-index.sh
@@ -6,6 +6,7 @@ test_description='split index mode tests'
# We need total control of index splitting here
sane_unset GIT_TEST_SPLIT_INDEX
+sane_unset GIT_FSMONITOR_TEST
test_expect_success 'enable split index' '
git config splitIndex.maxPercentChange 100 &&
diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh
new file mode 100755
index 0000000000..c6df85af5e
--- /dev/null
+++ b/t/t7519-status-fsmonitor.sh
@@ -0,0 +1,304 @@
+#!/bin/sh
+
+test_description='git status with file system watcher'
+
+. ./test-lib.sh
+
+#
+# To run the entire git test suite using fsmonitor:
+#
+# copy t/t7519/fsmonitor-all to a location in your path and then set
+# GIT_FSMONITOR_TEST=fsmonitor-all and run your tests.
+#
+
+# Note, after "git reset --hard HEAD" no extensions exist other than 'TREE'
+# "git update-index --fsmonitor" can be used to get the extension written
+# before testing the results.
+
+clean_repo () {
+ git reset --hard HEAD &&
+ git clean -fd
+}
+
+dirty_repo () {
+ : >untracked &&
+ : >dir1/untracked &&
+ : >dir2/untracked &&
+ echo 1 >modified &&
+ echo 2 >dir1/modified &&
+ echo 3 >dir2/modified &&
+ echo 4 >new &&
+ echo 5 >dir1/new &&
+ echo 6 >dir2/new
+}
+
+write_integration_script () {
+ write_script .git/hooks/fsmonitor-test<<-\EOF
+ if test "$#" -ne 2
+ then
+ echo "$0: exactly 2 arguments expected"
+ exit 2
+ fi
+ if test "$1" != 1
+ then
+ echo "Unsupported core.fsmonitor hook version." >&2
+ exit 1
+ fi
+ printf "untracked\0"
+ printf "dir1/untracked\0"
+ printf "dir2/untracked\0"
+ printf "modified\0"
+ printf "dir1/modified\0"
+ printf "dir2/modified\0"
+ printf "new\0"
+ printf "dir1/new\0"
+ printf "dir2/new\0"
+ EOF
+}
+
+test_lazy_prereq UNTRACKED_CACHE '
+ { git update-index --test-untracked-cache; ret=$?; } &&
+ test $ret -ne 1
+'
+
+test_expect_success 'setup' '
+ mkdir -p .git/hooks &&
+ : >tracked &&
+ : >modified &&
+ mkdir dir1 &&
+ : >dir1/tracked &&
+ : >dir1/modified &&
+ mkdir dir2 &&
+ : >dir2/tracked &&
+ : >dir2/modified &&
+ git -c core.fsmonitor= add . &&
+ git -c core.fsmonitor= commit -m initial &&
+ git config core.fsmonitor .git/hooks/fsmonitor-test &&
+ cat >.gitignore <<-\EOF
+ .gitignore
+ expect*
+ actual*
+ marker*
+ EOF
+'
+
+# test that the fsmonitor extension is off by default
+test_expect_success 'fsmonitor extension is off by default' '
+ test-dump-fsmonitor >actual &&
+ grep "^no fsmonitor" actual
+'
+
+# test that "update-index --fsmonitor" adds the fsmonitor extension
+test_expect_success 'update-index --fsmonitor" adds the fsmonitor extension' '
+ git update-index --fsmonitor &&
+ test-dump-fsmonitor >actual &&
+ grep "^fsmonitor last update" actual
+'
+
+# test that "update-index --no-fsmonitor" removes the fsmonitor extension
+test_expect_success 'update-index --no-fsmonitor" removes the fsmonitor extension' '
+ git update-index --no-fsmonitor &&
+ test-dump-fsmonitor >actual &&
+ grep "^no fsmonitor" actual
+'
+
+cat >expect <<EOF &&
+h dir1/modified
+H dir1/tracked
+h dir2/modified
+H dir2/tracked
+h modified
+H tracked
+EOF
+
+# test that "update-index --fsmonitor-valid" sets the fsmonitor valid bit
+test_expect_success 'update-index --fsmonitor-valid" sets the fsmonitor valid bit' '
+ git update-index --fsmonitor &&
+ git update-index --fsmonitor-valid dir1/modified &&
+ git update-index --fsmonitor-valid dir2/modified &&
+ git update-index --fsmonitor-valid modified &&
+ git ls-files -f >actual &&
+ test_cmp expect actual
+'
+
+cat >expect <<EOF &&
+H dir1/modified
+H dir1/tracked
+H dir2/modified
+H dir2/tracked
+H modified
+H tracked
+EOF
+
+# test that "update-index --no-fsmonitor-valid" clears the fsmonitor valid bit
+test_expect_success 'update-index --no-fsmonitor-valid" clears the fsmonitor valid bit' '
+ git update-index --no-fsmonitor-valid dir1/modified &&
+ git update-index --no-fsmonitor-valid dir2/modified &&
+ git update-index --no-fsmonitor-valid modified &&
+ git ls-files -f >actual &&
+ test_cmp expect actual
+'
+
+cat >expect <<EOF &&
+H dir1/modified
+H dir1/tracked
+H dir2/modified
+H dir2/tracked
+H modified
+H tracked
+EOF
+
+# test that all files returned by the script get flagged as invalid
+test_expect_success 'all files returned by integration script get flagged as invalid' '
+ write_integration_script &&
+ dirty_repo &&
+ git update-index --fsmonitor &&
+ git ls-files -f >actual &&
+ test_cmp expect actual
+'
+
+cat >expect <<EOF &&
+H dir1/modified
+h dir1/new
+H dir1/tracked
+H dir2/modified
+h dir2/new
+H dir2/tracked
+H modified
+h new
+H tracked
+EOF
+
+# test that newly added files are marked valid
+test_expect_success 'newly added files are marked valid' '
+ git add new &&
+ git add dir1/new &&
+ git add dir2/new &&
+ git ls-files -f >actual &&
+ test_cmp expect actual
+'
+
+cat >expect <<EOF &&
+H dir1/modified
+h dir1/new
+h dir1/tracked
+H dir2/modified
+h dir2/new
+h dir2/tracked
+H modified
+h new
+h tracked
+EOF
+
+# test that all unmodified files get marked valid
+test_expect_success 'all unmodified files get marked valid' '
+ # modified files result in update-index returning 1
+ test_must_fail git update-index --refresh --force-write-index &&
+ git ls-files -f >actual &&
+ test_cmp expect actual
+'
+
+cat >expect <<EOF &&
+H dir1/modified
+h dir1/tracked
+h dir2/modified
+h dir2/tracked
+h modified
+h tracked
+EOF
+
+# test that *only* files returned by the integration script get flagged as invalid
+test_expect_success '*only* files returned by the integration script get flagged as invalid' '
+ write_script .git/hooks/fsmonitor-test<<-\EOF &&
+ printf "dir1/modified\0"
+ EOF
+ clean_repo &&
+ git update-index --refresh --force-write-index &&
+ echo 1 >modified &&
+ echo 2 >dir1/modified &&
+ echo 3 >dir2/modified &&
+ test_must_fail git update-index --refresh --force-write-index &&
+ git ls-files -f >actual &&
+ test_cmp expect actual
+'
+
+# Ensure commands that call refresh_index() to move the index back in time
+# properly invalidate the fsmonitor cache
+test_expect_success 'refresh_index() invalidates fsmonitor cache' '
+ write_script .git/hooks/fsmonitor-test<<-\EOF &&
+ EOF
+ clean_repo &&
+ dirty_repo &&
+ git add . &&
+ git commit -m "to reset" &&
+ git reset HEAD~1 &&
+ git status >actual &&
+ git -c core.fsmonitor= status >expect &&
+ test_i18ncmp expect actual
+'
+
+# test fsmonitor with and without preloadIndex
+preload_values="false true"
+for preload_val in $preload_values
+do
+ test_expect_success "setup preloadIndex to $preload_val" '
+ git config core.preloadIndex $preload_val &&
+ if test $preload_val = true
+ then
+ GIT_FORCE_PRELOAD_TEST=$preload_val; export GIT_FORCE_PRELOAD_TEST
+ else
+ unset GIT_FORCE_PRELOAD_TEST
+ fi
+ '
+
+ # test fsmonitor with and without the untracked cache (if available)
+ uc_values="false"
+ test_have_prereq UNTRACKED_CACHE && uc_values="false true"
+ for uc_val in $uc_values
+ do
+ test_expect_success "setup untracked cache to $uc_val" '
+ git config core.untrackedcache $uc_val
+ '
+
+ # Status is well tested elsewhere so we'll just ensure that the results are
+ # the same when using core.fsmonitor.
+ test_expect_success 'compare status with and without fsmonitor' '
+ write_integration_script &&
+ clean_repo &&
+ dirty_repo &&
+ git add new &&
+ git add dir1/new &&
+ git add dir2/new &&
+ git status >actual &&
+ git -c core.fsmonitor= status >expect &&
+ test_i18ncmp expect actual
+ '
+
+ # Make sure it's actually skipping the check for modified and untracked
+ # (if enabled) files unless it is told about them.
+ test_expect_success "status doesn't detect unreported modifications" '
+ write_script .git/hooks/fsmonitor-test<<-\EOF &&
+ :>marker
+ EOF
+ clean_repo &&
+ git status &&
+ test_path_is_file marker &&
+ dirty_repo &&
+ rm -f marker &&
+ git status >actual &&
+ test_path_is_file marker &&
+ test_i18ngrep ! "Changes not staged for commit:" actual &&
+ if test $uc_val = true
+ then
+ test_i18ngrep ! "Untracked files:" actual
+ fi &&
+ if test $uc_val = false
+ then
+ test_i18ngrep "Untracked files:" actual
+ fi &&
+ rm -f marker
+ '
+ done
+done
+
+test_done
diff --git a/t/t7519/fsmonitor-all b/t/t7519/fsmonitor-all
new file mode 100755
index 0000000000..691bc94dc2
--- /dev/null
+++ b/t/t7519/fsmonitor-all
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# An test hook script to integrate with git to test fsmonitor.
+#
+# The hook is passed a version (currently 1) and a time in nanoseconds
+# formatted as a string and outputs to stdout all files that have been
+# modified since the given time. Paths must be relative to the root of
+# the working tree and separated by a single NUL.
+#
+#echo "$0 $*" >&2
+
+if test "$#" -ne 2
+then
+ echo "$0: exactly 2 arguments expected" >&2
+ exit 2
+fi
+
+if test "$1" != 1
+then
+ echo "Unsupported core.fsmonitor hook version." >&2
+ exit 1
+fi
+
+echo "/"
diff --git a/t/t7519/fsmonitor-none b/t/t7519/fsmonitor-none
new file mode 100755
index 0000000000..ed9cf5a6a9
--- /dev/null
+++ b/t/t7519/fsmonitor-none
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# An test hook script to integrate with git to test fsmonitor.
+#
+# The hook is passed a version (currently 1) and a time in nanoseconds
+# formatted as a string and outputs to stdout all files that have been
+# modified since the given time. Paths must be relative to the root of
+# the working tree and separated by a single NUL.
+#
+#echo "$0 $*" >&2
+
+if test "$#" -ne 2
+then
+ echo "$0: exactly 2 arguments expected" >&2
+ exit 2
+fi
+
+if test "$1" != 1
+then
+ echo "Unsupported core.fsmonitor hook version." >&2
+ exit 1
+fi
diff --git a/t/t7519/fsmonitor-watchman b/t/t7519/fsmonitor-watchman
new file mode 100755
index 0000000000..a3e30bf54f
--- /dev/null
+++ b/t/t7519/fsmonitor-watchman
@@ -0,0 +1,139 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use IPC::Open2;
+
+# An example hook script to integrate Watchman
+# (https://facebook.github.io/watchman/) with git to speed up detecting
+# new and modified files.
+#
+# The hook is passed a version (currently 1) and a time in nanoseconds
+# formatted as a string and outputs to stdout all files that have been
+# modified since the given time. Paths must be relative to the root of
+# the working tree and separated by a single NUL.
+#
+# To enable this hook, rename this file to "query-watchman" and set
+# 'git config core.fsmonitor .git/hooks/query-watchman'
+#
+my ($version, $time) = @ARGV;
+#print STDERR "$0 $version $time\n";
+
+# Check the hook interface version
+
+if ($version == 1) {
+ # convert nanoseconds to seconds
+ $time = int $time / 1000000000;
+} else {
+ die "Unsupported query-fsmonitor hook version '$version'.\n" .
+ "Falling back to scanning...\n";
+}
+
+# Convert unix style paths to escaped Windows style paths when running
+# in Windows command prompt
+
+my $system = `uname -s`;
+$system =~ s/[\r\n]+//g;
+my $git_work_tree;
+
+if ($system =~ m/^MSYS_NT/ || $system =~ m/^MINGW/) {
+ $git_work_tree = `cygpath -aw "\$PWD"`;
+ $git_work_tree =~ s/[\r\n]+//g;
+ $git_work_tree =~ s,\\,/,g;
+} else {
+ $git_work_tree = $ENV{'PWD'};
+}
+
+my $retry = 1;
+
+launch_watchman();
+
+sub launch_watchman {
+
+ my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j')
+ or die "open2() failed: $!\n" .
+ "Falling back to scanning...\n";
+
+ # In the query expression below we're asking for names of files that
+ # changed since $time but were not transient (ie created after
+ # $time but no longer exist).
+ #
+ # To accomplish this, we're using the "since" generator to use the
+ # recency index to select candidate nodes and "fields" to limit the
+ # output to file names only. Then we're using the "expression" term to
+ # further constrain the results.
+ #
+ # The category of transient files that we want to ignore will have a
+ # creation clock (cclock) newer than $time_t value and will also not
+ # currently exist.
+
+ my $query = <<" END";
+ ["query", "$git_work_tree", {
+ "since": $time,
+ "fields": ["name"],
+ "expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]]
+ }]
+ END
+
+ open (my $fh, ">", ".git/watchman-query.json");
+ print $fh $query;
+ close $fh;
+
+ print CHLD_IN $query;
+ close CHLD_IN;
+ my $response = do {local $/; <CHLD_OUT>};
+
+ open ($fh, ">", ".git/watchman-response.json");
+ print $fh $response;
+ close $fh;
+
+ die "Watchman: command returned no output.\n" .
+ "Falling back to scanning...\n" if $response eq "";
+ die "Watchman: command returned invalid output: $response\n" .
+ "Falling back to scanning...\n" unless $response =~ /^\{/;
+
+ my $json_pkg;
+ eval {
+ require JSON::XS;
+ $json_pkg = "JSON::XS";
+ 1;
+ } or do {
+ require JSON::PP;
+ $json_pkg = "JSON::PP";
+ };
+
+ my $o = $json_pkg->new->utf8->decode($response);
+
+ if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
+ print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
+ $retry--;
+ qx/watchman watch "$git_work_tree"/;
+ die "Failed to make watchman watch '$git_work_tree'.\n" .
+ "Falling back to scanning...\n" if $? != 0;
+
+ # Watchman will always return all files on the first query so
+ # return the fast "everything is dirty" flag to git and do the
+ # Watchman query just to get it over with now so we won't pay
+ # the cost in git to look up each individual file.
+
+ open ($fh, ">", ".git/watchman-output.out");
+ print "/\0";
+ close $fh;
+
+ print "/\0";
+ eval { launch_watchman() };
+ exit 0;
+ }
+
+ die "Watchman: $o->{error}.\n" .
+ "Falling back to scanning...\n" if $o->{error};
+
+ open ($fh, ">", ".git/watchman-output.out");
+ binmode $fh, ":utf8";
+ print $fh @{$o->{files}};
+ close $fh;
+
+ binmode STDOUT, ":utf8";
+ local $, = "\0";
+ print @{$o->{files}};
+}
diff --git a/templates/hooks--fsmonitor-watchman.sample b/templates/hooks--fsmonitor-watchman.sample
new file mode 100755
index 0000000000..9eba8a7409
--- /dev/null
+++ b/templates/hooks--fsmonitor-watchman.sample
@@ -0,0 +1,120 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use IPC::Open2;
+
+# An example hook script to integrate Watchman
+# (https://facebook.github.io/watchman/) with git to speed up detecting
+# new and modified files.
+#
+# The hook is passed a version (currently 1) and a time in nanoseconds
+# formatted as a string and outputs to stdout all files that have been
+# modified since the given time. Paths must be relative to the root of
+# the working tree and separated by a single NUL.
+#
+# To enable this hook, rename this file to "query-watchman" and set
+# 'git config core.fsmonitor .git/hooks/query-watchman'
+#
+my ($version, $time) = @ARGV;
+
+# Check the hook interface version
+
+if ($version == 1) {
+ # convert nanoseconds to seconds
+ $time = int $time / 1000000000;
+} else {
+ die "Unsupported query-fsmonitor hook version '$version'.\n" .
+ "Falling back to scanning...\n";
+}
+
+# Convert unix style paths to escaped Windows style paths when running
+# in Windows command prompt
+
+my $system = `uname -s`;
+$system =~ s/[\r\n]+//g;
+my $git_work_tree;
+
+if ($system =~ m/^MSYS_NT/ || $system =~ m/^MINGW/) {
+ $git_work_tree = `cygpath -aw "\$PWD"`;
+ $git_work_tree =~ s/[\r\n]+//g;
+ $git_work_tree =~ s,\\,/,g;
+} else {
+ $git_work_tree = $ENV{'PWD'};
+}
+
+my $retry = 1;
+
+launch_watchman();
+
+sub launch_watchman {
+
+ my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j')
+ or die "open2() failed: $!\n" .
+ "Falling back to scanning...\n";
+
+ # In the query expression below we're asking for names of files that
+ # changed since $time but were not transient (ie created after
+ # $time but no longer exist).
+ #
+ # To accomplish this, we're using the "since" generator to use the
+ # recency index to select candidate nodes and "fields" to limit the
+ # output to file names only. Then we're using the "expression" term to
+ # further constrain the results.
+ #
+ # The category of transient files that we want to ignore will have a
+ # creation clock (cclock) newer than $time_t value and will also not
+ # currently exist.
+
+ my $query = <<" END";
+ ["query", "$git_work_tree", {
+ "since": $time,
+ "fields": ["name"],
+ "expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]]
+ }]
+ END
+
+ print CHLD_IN $query;
+ close CHLD_IN;
+ my $response = do {local $/; <CHLD_OUT>};
+
+ die "Watchman: command returned no output.\n" .
+ "Falling back to scanning...\n" if $response eq "";
+ die "Watchman: command returned invalid output: $response\n" .
+ "Falling back to scanning...\n" unless $response =~ /^\{/;
+
+ my $json_pkg;
+ eval {
+ require JSON::XS;
+ $json_pkg = "JSON::XS";
+ 1;
+ } or do {
+ require JSON::PP;
+ $json_pkg = "JSON::PP";
+ };
+
+ my $o = $json_pkg->new->utf8->decode($response);
+
+ if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
+ print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
+ $retry--;
+ qx/watchman watch "$git_work_tree"/;
+ die "Failed to make watchman watch '$git_work_tree'.\n" .
+ "Falling back to scanning...\n" if $? != 0;
+
+ # Watchman will always return all files on the first query so
+ # return the fast "everything is dirty" flag to git and do the
+ # Watchman query just to get it over with now so we won't pay
+ # the cost in git to look up each individual file.
+ print "/\0";
+ eval { launch_watchman() };
+ exit 0;
+ }
+
+ die "Watchman: $o->{error}.\n" .
+ "Falling back to scanning...\n" if $o->{error};
+
+ binmode STDOUT, ":utf8";
+ local $, = "\0";
+ print @{$o->{files}};
+}
diff --git a/unpack-trees.c b/unpack-trees.c
index 25740cb593..bf8b602901 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -14,6 +14,7 @@
#include "dir.h"
#include "submodule.h"
#include "submodule-config.h"
+#include "fsmonitor.h"
/*
* Error messages expected by scripts out of plumbing commands such as
@@ -408,6 +409,7 @@ static int apply_sparse_checkout(struct index_state *istate,
ce->ce_flags &= ~CE_SKIP_WORKTREE;
if (was_skip_worktree != ce_skip_worktree(ce)) {
ce->ce_flags |= CE_UPDATE_IN_BASE;
+ mark_fsmonitor_invalid(istate, ce);
istate->cache_changed |= CE_ENTRY_CHANGED;
}