diff options
74 files changed, 2255 insertions, 201 deletions
diff --git a/Documentation/RelNotes/2.15.1.txt b/Documentation/RelNotes/2.15.1.txt index 15b24a0b76..a6ae4c1aeb 100644 --- a/Documentation/RelNotes/2.15.1.txt +++ b/Documentation/RelNotes/2.15.1.txt @@ -13,7 +13,7 @@ Fixes since v2.15 latter, which has been fixed. * The experimental "color moved lines differently in diff output" - feature was buggy around "ignore whitespace changes" edges, whihch + feature was buggy around "ignore whitespace changes" edges, which has been corrected. * Instead of using custom line comparison and hashing functions to @@ -24,7 +24,7 @@ Fixes since v2.15 HEAD points at, which have been fixed. * "git commit", after making a commit, did not check for errors when - asking on what branch it made the commit, which has been correted. + asking on what branch it made the commit, which has been corrected. * "git status --ignored -u" did not stop at a working tree of a separate project that is embedded in an ignored directory and @@ -35,7 +35,7 @@ Fixes since v2.15 --recurse-submodules" has been fixed. * A recent regression in "git rebase -i" that broke execution of git - commands from subdirectories via "exec" insn has been fixed. + commands from subdirectories via "exec" instruction has been fixed. * "git check-ref-format --branch @{-1}" bit a "BUG()" when run outside a repository for obvious reasons; clarify the documentation @@ -64,5 +64,22 @@ Fixes since v2.15 * Updates from GfW project. + * "git rebase -i" recently started misbehaving when a submodule that + is configured with 'submodule.<name>.ignore' is dirty; this has + been corrected. + + * Some error messages did not quote filenames shown in it, which have + been fixed. + + * Building with NO_LIBPCRE1_JIT did not disable it, which has been fixed. + + * We used to add an empty alternate object database to the system + that does not help anything; it has been corrected. + + * Error checking in "git imap-send" for empty response has been + improved. + + * An ancient bug in "git apply --ignore-space-change" codepath has + been fixed. Also contains various documentation updates and code clean-ups. diff --git a/Documentation/RelNotes/2.16.0.txt b/Documentation/RelNotes/2.16.0.txt index 12b86ebb4a..3f2909498f 100644 --- a/Documentation/RelNotes/2.16.0.txt +++ b/Documentation/RelNotes/2.16.0.txt @@ -63,6 +63,20 @@ UI, Workflows & Features HTML version via AsciiDoc/Asciidoctor. (merge 049e64aa50 bc/submitting-patches-in-asciidoc later to maint). + * We learned to talk to watchman to speed up "git status" and other + operations that need to see which paths have been modified. + + * The "diff" family of commands learned to ignore differences in + carriage return at the end of line. + + * Places that know about "sendemail.to", like documentation and shell + completion (in contrib/) have been taught about "sendemail.tocmd", + too. + + * "git add --renormalize ." is a new and safer way to record the fact + that you are correcting the end-of-line convention and other + "convert_to_git()" glitches in the in-repository data. + Performance, Internal Implementation, Development Support etc. @@ -90,18 +104,15 @@ Performance, Internal Implementation, Development Support etc. * Conversion from uchar[20] to struct object_id continues. * Code cleanup. - (merge 62a24c8923 rs/hex-to-bytes-cleanup later to maint). * A single-word "unsigned flags" in the diff options is being split into a structure with many bitfields. - (merge 0d1e0e7801 bw/diff-opt-impl-to-bitfields later to maint). * TravisCI build updates. * Parts of a test to drive the long-running content filter interface has been split into its own module, hopefully to eventually become reusable. - (merge 0fe8d516bb cc/git-packet-pm later to maint). * Drop (perhaps overly cautious) sanity check before using the index read from the filesystem at runtime. @@ -119,7 +130,7 @@ Fixes since v2.15 latter, which has been fixed. * The experimental "color moved lines differently in diff output" - feature was buggy around "ignore whitespace changes" edges, whihch + feature was buggy around "ignore whitespace changes" edges, which has been corrected. * Instead of using custom line comparison and hashing functions to @@ -130,7 +141,7 @@ Fixes since v2.15 HEAD points at, which have been fixed. * "git commit", after making a commit, did not check for errors when - asking on what branch it made the commit, which has been correted. + asking on what branch it made the commit, which has been corrected. * "git status --ignored -u" did not stop at a working tree of a separate project that is embedded in an ignored directory and @@ -141,7 +152,7 @@ Fixes since v2.15 --recurse-submodules" has been fixed. * A recent regression in "git rebase -i" that broke execution of git - commands from subdirectories via "exec" insn has been fixed. + commands from subdirectories via "exec" instruction has been fixed. * A (possibly flakey) test fix. @@ -178,7 +189,6 @@ Fixes since v2.15 * Error checking in "git imap-send" for empty response has been improved. - (merge 618ec81abb rs/imap-send-next-arg-fix later to maint). * Recent update to the refs infrastructure implementation started rewriting packed-refs file more often than before; this has been @@ -187,25 +197,69 @@ Fixes since v2.15 * Some error messages did not quote filenames shown in it, which have been fixed. - (merge 0a288d1ee9 sr/wrapper-quote-filenames later to maint). * "git rebase -i" recently started misbehaving when a submodule that is configured with 'submodule.<name>.ignore' is dirty; this has been corrected. - (merge c6d8ccf3a2 bw/rebase-i-ignored-submodule-fix later to maint). * Building with NO_LIBPCRE1_JIT did not disable it, which has been fixed. - (merge 2fff1e196d ab/pcre-v2 later to maint). * We used to add an empty alternate object database to the system that does not help anything; it has been corrected. - (merge f28e36686a jk/info-alternates-fix later to maint). + + * Doc update around use of "format-patch --subject-prefix" etc. + + * A fix for an ancient bug in "git apply --ignore-space-change" codepath. + + * Clarify and enhance documentation for "merge-base --fork-point", as + it was clear what it computed but not why/what for. + (merge 6d1700b8af jc/merge-base-fork-point-doc later to maint). + + * A few scripts (both in production and tests) incorrectly redirected + their error output. These have been corrected. + (merge eadf1c8f45 tz/redirect-fix later to maint). + + * "git notes" sent its error message to its standard output stream, + which was corrected. + (merge 89b9e31dd5 tz/notes-error-to-stderr later to maint). + + * The three-way merge performed by "git cherry-pick" was confused + when a new submodule was added in the meantime, which has been + fixed (or "papered over"). + (merge c641ca6707 sb/test-cherry-pick-submodule-getting-in-a-way later to maint). + + * The sequencer machinery (used by "git cherry-pick A..B", and "git + rebase -i", among other things) would have lost a commit if stopped + due to an unlockable index file, which has been fixed. + (merge bd58886775 pw/sequencer-recover-from-unlockable-index later to maint). + + * "git apply --inaccurate-eof" when used with "--ignore-space-change" + triggered an internal sanity check, which has been fixed. + (merge 4855de1233 rs/apply-inaccurate-eof-with-incomplete-line later to maint). + + * Command line completion (in contrib/) has been taught about the + "--copy" option of "git branch". + (merge 41ca0f773e tz/complete-branch-copy later to maint). + + * When "git rebase" prepared an mailbox of changes and fed it to "git + am" to replay them, it was confused when a stray "From " happened + to be in the log message of one of the replayed changes. This has + been corrected. + (merge ae3b2b04bb ew/rebase-mboxrd later to maint). + + * There was a recent semantic mismerge in the codepath to write out a + section of a configuration section, which has been corrected. + (merge 782c030ea2 rs/config-write-section-fix later to maint). + + * Mentions of "git-rebase" and "git-am" (dashed form) still remained + in end-user visible strings emitted by the "git rebase" command; + they have been corrected. + (merge 82cb775c06 ks/rebase-no-git-foo later to maint). + + * Contrary to the documentation, "git pull -4/-6 other-args" did not + ask the underlying "git fetch" to go over IPv4/IPv6, which has been + corrected. + (merge ffb4568afe sw/pull-ipv46-passthru later to maint). * Other minor doc, test and build updates and code cleanups. - (merge 804862209b ao/merge-verbosity-getenv-just-once later to maint). - (merge 9360ec0002 rs/sequencer-rewrite-file-cleanup later to maint). - (merge f4e45cb3eb ma/bisect-leakfix later to maint). - (merge 4da72644b7 ma/reduce-heads-leakfix later to maint). - (merge 3dc5433fd5 ad/rebase-i-serie-typofix later to maint). - (merge 5313bee032 tz/fsf-address-update later to maint). - (merge 5555a2aa4b cb/t4201-robustify later to maint). + (merge c5e3bc6ec4 sd/branch-copy later to maint). diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index 17419f7901..3ef30922ec 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -203,14 +203,15 @@ lose tabs that way if you are not careful. It is a common convention to prefix your subject line with [PATCH]. This lets people easily distinguish patches from other -e-mail discussions. Use of additional markers after PATCH and -the closing bracket to mark the nature of the patch is also -encouraged. E.g. [PATCH/RFC] is often used when the patch is -not ready to be applied but it is for discussion, [PATCH v2], -[PATCH v3] etc. are often seen when you are sending an update to -what you have previously sent. - -`git format-patch` command follows the best current practice to +e-mail discussions. Use of markers in addition to PATCH within +the brackets to describe the nature of the patch is also +encouraged. E.g. [RFC PATCH] (where RFC stands for "request for +comments") is often used to indicate a patch needs further +discussion before being accepted, [PATCH v2], [PATCH v3] etc. +are often seen when you are sending an update to what you have +previously sent. + +The `git format-patch` command follows the best current practice to format the body of an e-mail message. At the beginning of the patch should come your commit message, ending with the Signed-off-by: lines, and a line that consists of three dashes, @@ -218,6 +219,10 @@ followed by the diffstat information and the patch itself. If you are forwarding a patch from somebody else, optionally, at the beginning of the e-mail message just before the commit message starts, you can put a "From: " line to name that person. +To change the default "[PATCH]" in the subject to "[<text>]", use +`git format-patch --subject-prefix=<text>`. As a shortcut, you +can use `--rfc` instead of `--subject-prefix="RFC PATCH"`, or +`-v <n>` instead of `--subject-prefix="PATCH v<n>"`. You often want to add additional explanation about the patch, other than the commit message itself. Place such "cover letter" diff --git a/Documentation/config.txt b/Documentation/config.txt index 671fcbaa0f..531649cb40 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -416,6 +416,13 @@ core.protectNTFS:: 8.3 "short" names. Defaults to `true` on Windows, and `false` elsewhere. +core.fsmonitor:: + If set, the value of this variable is used as a command which + will identify all files that may have changed since the + requested date/time. This information is used to speed up git by + avoiding unnecessary processing of files that have not changed. + See the "fsmonitor-watchman" section of linkgit:githooks[5]. + core.trustctime:: If false, the ctime differences between the index and the working tree are ignored; useful when the inode change time @@ -3000,6 +3007,7 @@ sendemail.smtpPass:: sendemail.suppresscc:: sendemail.suppressFrom:: sendemail.to:: +sendemail.tocmd:: sendemail.smtpDomain:: sendemail.smtpServer:: sendemail.smtpServerPort:: diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt index dd0dba5b1d..3c93c21683 100644 --- a/Documentation/diff-options.txt +++ b/Documentation/diff-options.txt @@ -557,6 +557,9 @@ endif::git-format-patch[] --text:: Treat all files as text. +--ignore-cr-at-eol:: + Ignore carrige-return at the end of line when doing a comparison. + --ignore-space-at-eol:: Ignore changes in whitespace at EOL. diff --git a/Documentation/git-add.txt b/Documentation/git-add.txt index b700beaff5..d50fa339dc 100644 --- a/Documentation/git-add.txt +++ b/Documentation/git-add.txt @@ -10,7 +10,7 @@ SYNOPSIS [verse] 'git add' [--verbose | -v] [--dry-run | -n] [--force | -f] [--interactive | -i] [--patch | -p] [--edit | -e] [--[no-]all | --[no-]ignore-removal | [--update | -u]] - [--intent-to-add | -N] [--refresh] [--ignore-errors] [--ignore-missing] + [--intent-to-add | -N] [--refresh] [--ignore-errors] [--ignore-missing] [--renormalize] [--chmod=(+|-)x] [--] [<pathspec>...] DESCRIPTION @@ -175,6 +175,13 @@ for "git add --no-all <pathspec>...", i.e. ignored removed files. warning (e.g., if you are manually performing operations on submodules). +--renormalize:: + Apply the "clean" process freshly to all tracked files to + forcibly add them again to the index. This is useful after + changing `core.autocrlf` configuration or the `text` attribute + in order to correct files added with wrong CRLF/LF line endings. + This option implies `-u`. + --chmod=(+|-)x:: Override the executable bit of the added files. The executable bit is only changed in the index, the files on disk are left diff --git a/Documentation/git-branch.txt b/Documentation/git-branch.txt index d6587c5e96..520c53b5e8 100644 --- a/Documentation/git-branch.txt +++ b/Documentation/git-branch.txt @@ -281,6 +281,12 @@ start-point is either a local or remote-tracking branch. and the object it points at. The format is the same as that of linkgit:git-for-each-ref[1]. +CONFIGURATION +------------- +`pager.branch` is only respected when listing branches, i.e., when +`--list` is used or implied. The default is to use a pager. +See linkgit:git-config[1]. + Examples -------- diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt index d153c17e06..3ac3e3a77d 100644 --- a/Documentation/git-ls-files.txt +++ b/Documentation/git-ls-files.txt @@ -9,7 +9,7 @@ git-ls-files - Show information about files in the index and the working tree SYNOPSIS -------- [verse] -'git ls-files' [-z] [-t] [-v] +'git ls-files' [-z] [-t] [-v] [-f] (--[cached|deleted|others|ignored|stage|unmerged|killed|modified])* (-[c|d|o|i|s|u|k|m])* [--eol] @@ -133,6 +133,11 @@ a space) at the start of each line: that are marked as 'assume unchanged' (see linkgit:git-update-index[1]). +-f:: + Similar to `-t`, but use lowercase letters for files + that are marked as 'fsmonitor valid' (see + linkgit:git-update-index[1]). + --full-name:: When run from a subdirectory, the command usually outputs paths relative to the current directory. This diff --git a/Documentation/git-merge-base.txt b/Documentation/git-merge-base.txt index b968b64c38..502e00ec35 100644 --- a/Documentation/git-merge-base.txt +++ b/Documentation/git-merge-base.txt @@ -154,23 +154,71 @@ topic origin/master`, the history of remote-tracking branch `origin/master` may have been rewound and rebuilt, leading to a history of this shape: - o---B1 + o---B2 / - ---o---o---B2--o---o---o---B (origin/master) + ---o---o---B1--o---o---o---B (origin/master) \ - B3 + B0 \ - Derived (topic) + D0---D1---D (topic) -where `origin/master` used to point at commits B3, B2, B1 and now it +where `origin/master` used to point at commits B0, B1, B2 and now it points at B, and your `topic` branch was started on top of it back -when `origin/master` was at B3. This mode uses the reflog of -`origin/master` to find B3 as the fork point, so that the `topic` -can be rebased on top of the updated `origin/master` by: +when `origin/master` was at B0, and you built three commits, D0, D1, +and D, on top of it. Imagine that you now want to rebase the work +you did on the topic on top of the updated origin/master. + +In such a case, `git merge-base origin/master topic` would return the +parent of B0 in the above picture, but B0^..D is *not* the range of +commits you would want to replay on top of B (it includes B0, which +is not what you wrote; it is a commit the other side discarded when +it moved its tip from B0 to B1). + +`git merge-base --fork-point origin/master topic` is designed to +help in such a case. It takes not only B but also B0, B1, and B2 +(i.e. old tips of the remote-tracking branches your repository's +reflog knows about) into account to see on which commit your topic +branch was built and finds B0, allowing you to replay only the +commits on your topic, excluding the commits the other side later +discarded. + +Hence $ fork_point=$(git merge-base --fork-point origin/master topic) + +will find B0, and + $ git rebase --onto origin/master $fork_point topic +will replay D0, D1 and D on top of B to create a new history of this +shape: + + o---B2 + / + ---o---o---B1--o---o---o---B (origin/master) + \ \ + B0 D0'--D1'--D' (topic - updated) + \ + D0---D1---D (topic - old) + +A caveat is that older reflog entries in your repository may be +expired by `git gc`. If B0 no longer appears in the reflog of the +remote-tracking branch `origin/master`, the `--fork-point` mode +obviously cannot find it and fails, avoiding to give a random and +useless result (such as the parent of B0, like the same command +without the `--fork-point` option gives). + +Also, the remote-tracking branch you use the `--fork-point` mode +with must be the one your topic forked from its tip. If you forked +from an older commit than the tip, this mode would not find the fork +point (imagine in the above sample history B0 did not exist, +origin/master started at B1, moved to B2 and then B, and you forked +your topic at origin/master^ when origin/master was B1; the shape of +the history would be the same as above, without B0, and the parent +of B1 is what `git merge-base origin/master topic` correctly finds, +but the `--fork-point` mode will not, because it is not one of the +commits that used to be at the tip of origin/master). + See also -------- diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt index 75c7dd9dea..bdb0342593 100644 --- a/Documentation/git-update-index.txt +++ b/Documentation/git-update-index.txt @@ -16,9 +16,11 @@ SYNOPSIS [--chmod=(+|-)x] [--[no-]assume-unchanged] [--[no-]skip-worktree] + [--[no-]fsmonitor-valid] [--ignore-submodules] [--[no-]split-index] [--[no-|test-|force-]untracked-cache] + [--[no-]fsmonitor] [--really-refresh] [--unresolve] [--again | -g] [--info-only] [--index-info] [-z] [--stdin] [--index-version <n>] @@ -111,6 +113,12 @@ you will need to handle the situation manually. set and unset the "skip-worktree" bit for the paths. See section "Skip-worktree bit" below for more information. +--[no-]fsmonitor-valid:: + When one of these flags is specified, the object name recorded + for the paths are not updated. Instead, these options + set and unset the "fsmonitor valid" bit for the paths. See + section "File System Monitor" below for more information. + -g:: --again:: Runs 'git update-index' itself on the paths whose index @@ -201,6 +209,15 @@ will remove the intended effect of the option. `--untracked-cache` used to imply `--test-untracked-cache` but this option would enable the extension unconditionally. +--fsmonitor:: +--no-fsmonitor:: + Enable or disable files system monitor feature. These options + take effect whatever the value of the `core.fsmonitor` + configuration variable (see linkgit:git-config[1]). But a warning + is emitted when the change goes against the configured value, as + the configured value will take effect next time the index is + read and this will remove the intended effect of the option. + \--:: Do not interpret any more arguments as options. @@ -447,6 +464,34 @@ command reads the index; while when `--[no-|force-]untracked-cache` are used, the untracked cache is immediately added to or removed from the index. +File System Monitor +------------------- + +This feature is intended to speed up git operations for repos that have +large working directories. + +It enables git to work together with a file system monitor (see the +"fsmonitor-watchman" section of linkgit:githooks[5]) that can +inform it as to what files have been modified. This enables git to avoid +having to lstat() every file to find modified files. + +When used in conjunction with the untracked cache, it can further improve +performance by avoiding the cost of scanning the entire working directory +looking for new files. + +If you want to enable (or disable) this feature, it is easier to use +the `core.fsmonitor` configuration variable (see +linkgit:git-config[1]) than using the `--fsmonitor` option to +`git update-index` in each repository, especially if you want to do so +across all repositories you use, because you can set the configuration +variable to `true` (or `false`) in your `$HOME/.gitconfig` just once +and have it affect all repositories you touch. + +When the `core.fsmonitor` configuration variable is changed, the +file system monitor is added to or removed from the index the next time +a command reads the index. When `--[no-]fsmonitor` are used, the file +system monitor is immediately added to or removed from the index. + Configuration ------------- diff --git a/Documentation/git.txt b/Documentation/git.txt index 463b0eb0f5..483a1f3547 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -595,6 +595,10 @@ into it. Unsetting the variable, or setting it to empty, "0" or "false" (case insensitive) disables trace messages. +`GIT_TRACE_FSMONITOR`:: + Enables trace messages for the filesystem monitor extension. + See `GIT_TRACE` for available trace output options. + `GIT_TRACE_PACK_ACCESS`:: Enables trace messages for all accesses to any packs. For each access, the pack file name and an offset in the pack is diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index 4c68bc19d5..30687de81a 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -232,8 +232,7 @@ From a clean working directory: ------------------------------------------------- $ echo "* text=auto" >.gitattributes -$ git read-tree --empty # Clean index, force re-scan of working directory -$ git add . +$ git add --renormalize . $ git status # Show files that will be normalized $ git commit -m "Introduce end-of-line normalization" ------------------------------------------------- @@ -328,6 +327,9 @@ You can declare that a filter turns a content that by itself is unusable into a usable content by setting the filter.<driver>.required configuration variable to `true`. +Note: Whenever the clean filter is changed, the repo should be renormalized: +$ git add --renormalize . + For example, in .gitattributes, you would assign the `filter` attribute for paths. diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index 5d3f45560e..0bb0042d8c 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -454,6 +454,34 @@ the name of the file that holds the e-mail to be sent. Exiting with a non-zero status causes 'git send-email' to abort before sending any e-mails. +fsmonitor-watchman +~~~~~~~~~~~~~~~~~~ + +This hook is invoked when the configuration option core.fsmonitor is +set to .git/hooks/fsmonitor-watchman. It takes two arguments, a version +(currently 1) and the time in elapsed nanoseconds since midnight, +January 1, 1970. + +The hook should output to stdout the list of all files in the working +directory that may have changed since the requested time. The logic +should be inclusive so that it does not miss any potential changes. +The paths should be relative to the root of the working directory +and be separated by a single NUL. + +It is OK to include files which have not actually changed. All changes +including newly-created and deleted files should be included. When +files are renamed, both the old and the new name should be included. + +Git will limit what files it checks for changes as well as which +directories are checked for untracked files based on the path names +given. + +An optimized way to tell git "all files have changed" is to return +the filename '/'. + +The exit status determines whether git will use the data from the +hook to limit its search. On error, it will fall back to verifying +all files and folders. GIT --- diff --git a/Documentation/merge-strategies.txt b/Documentation/merge-strategies.txt index a09d597463..fd5d748d1b 100644 --- a/Documentation/merge-strategies.txt +++ b/Documentation/merge-strategies.txt @@ -58,11 +58,12 @@ diff-algorithm=[patience|minimal|histogram|myers];; ignore-space-change;; ignore-all-space;; ignore-space-at-eol;; +ignore-cr-at-eol;; Treats lines with the indicated type of whitespace change as unchanged for the sake of a three-way merge. Whitespace changes mixed with other changes to a line are not ignored. - See also linkgit:git-diff[1] `-b`, `-w`, and - `--ignore-space-at-eol`. + See also linkgit:git-diff[1] `-b`, `-w`, + `--ignore-space-at-eol`, and `--ignore-cr-at-eol`. + * If 'their' version only introduces whitespace changes to a line, 'our' version is used; diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt index ade0b0c445..db3572626b 100644 --- a/Documentation/technical/index-format.txt +++ b/Documentation/technical/index-format.txt @@ -295,3 +295,22 @@ The remaining data of each directory block is grouped by type: in the previous ewah bitmap. - One NUL. + +== File System Monitor cache + + The file system monitor cache tracks files for which the core.fsmonitor + hook has told us about changes. The signature for this extension is + { 'F', 'S', 'M', 'N' }. + + The extension starts with + + - 32-bit version number: the current supported version is 1. + + - 64-bit time: the extension data reflects all changes through the given + time which is stored as the nanoseconds elapsed since midnight, + January 1, 1970. + + - 32-bit bitmap size: the size of the CE_FSMONITOR_VALID bitmap. + + - An ewah bitmap, the n-th bit indicates whether the n-th index entry + is not CE_FSMONITOR_VALID. @@ -646,7 +646,9 @@ TEST_PROGRAMS_NEED_X += test-ctype TEST_PROGRAMS_NEED_X += test-config TEST_PROGRAMS_NEED_X += test-date TEST_PROGRAMS_NEED_X += test-delta +TEST_PROGRAMS_NEED_X += test-drop-caches TEST_PROGRAMS_NEED_X += test-dump-cache-tree +TEST_PROGRAMS_NEED_X += test-dump-fsmonitor TEST_PROGRAMS_NEED_X += test-dump-split-index TEST_PROGRAMS_NEED_X += test-dump-untracked-cache TEST_PROGRAMS_NEED_X += test-fake-ssh @@ -794,6 +796,7 @@ LIB_OBJS += ewah/ewah_rlw.o LIB_OBJS += exec_cmd.o LIB_OBJS += fetch-pack.o LIB_OBJS += fsck.o +LIB_OBJS += fsmonitor.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o @@ -300,52 +300,33 @@ static uint32_t hash_line(const char *cp, size_t len) static int fuzzy_matchlines(const char *s1, size_t n1, const char *s2, size_t n2) { - const char *last1 = s1 + n1 - 1; - const char *last2 = s2 + n2 - 1; - int result = 0; + const char *end1 = s1 + n1; + const char *end2 = s2 + n2; /* ignore line endings */ - while ((*last1 == '\r') || (*last1 == '\n')) - last1--; - while ((*last2 == '\r') || (*last2 == '\n')) - last2--; - - /* skip leading whitespaces, if both begin with whitespace */ - if (s1 <= last1 && s2 <= last2 && isspace(*s1) && isspace(*s2)) { - while (isspace(*s1) && (s1 <= last1)) - s1++; - while (isspace(*s2) && (s2 <= last2)) - s2++; - } - /* early return if both lines are empty */ - if ((s1 > last1) && (s2 > last2)) - return 1; - while (!result) { - result = *s1++ - *s2++; - /* - * Skip whitespace inside. We check for whitespace on - * both buffers because we don't want "a b" to match - * "ab" - */ - if (isspace(*s1) && isspace(*s2)) { - while (isspace(*s1) && s1 <= last1) + while (s1 < end1 && (end1[-1] == '\r' || end1[-1] == '\n')) + end1--; + while (s2 < end2 && (end2[-1] == '\r' || end2[-1] == '\n')) + end2--; + + while (s1 < end1 && s2 < end2) { + if (isspace(*s1)) { + /* + * Skip whitespace. We check on both buffers + * because we don't want "a b" to match "ab". + */ + if (!isspace(*s2)) + return 0; + while (s1 < end1 && isspace(*s1)) s1++; - while (isspace(*s2) && s2 <= last2) + while (s2 < end2 && isspace(*s2)) s2++; - } - /* - * If we reached the end on one side only, - * lines don't match - */ - if ( - ((s2 > last2) && (s1 <= last1)) || - ((s1 > last1) && (s2 <= last2))) + } else if (*s1++ != *s2++) return 0; - if ((s1 > last1) && (s2 > last2)) - break; } - return !result; + /* If we reached the end on one side only, lines don't match. */ + return s1 == end1 && s2 == end2; } static void add_line_info(struct image *img, const char *bol, size_t len, unsigned flag) @@ -2972,6 +2953,8 @@ static int apply_one_fragment(struct apply_state *state, newlines.len > 0 && newlines.buf[newlines.len - 1] == '\n') { old--; strbuf_setlen(&newlines, newlines.len - 1); + preimage.line_allocated[preimage.nr - 1].len--; + postimage.line_allocated[postimage.nr - 1].len--; } leading = frag->leading; @@ -178,24 +178,40 @@ int read_branch_desc(struct strbuf *buf, const char *branch_name) return 0; } -int validate_new_branchname(const char *name, struct strbuf *ref, - int force, int attr_only) +/* + * Check if 'name' can be a valid name for a branch; die otherwise. + * Return 1 if the named branch already exists; return 0 otherwise. + * Fill ref with the full refname for the branch. + */ +int validate_branchname(const char *name, struct strbuf *ref) { if (strbuf_check_branch_ref(ref, name)) die(_("'%s' is not a valid branch name."), name); - if (!ref_exists(ref->buf)) + return ref_exists(ref->buf); +} + +/* + * Check if a branch 'name' can be created as a new branch; die otherwise. + * 'force' can be used when it is OK for the named branch already exists. + * Return 1 if the named branch already exists; return 0 otherwise. + * Fill ref with the full refname for the branch. + */ +int validate_new_branchname(const char *name, struct strbuf *ref, int force) +{ + const char *head; + + if (!validate_branchname(name, ref)) return 0; - else if (!force && !attr_only) - die(_("A branch named '%s' already exists."), ref->buf + strlen("refs/heads/")); - if (!attr_only) { - const char *head; + if (!force) + die(_("A branch named '%s' already exists."), + ref->buf + strlen("refs/heads/")); + + head = resolve_ref_unsafe("HEAD", 0, NULL, NULL); + if (!is_bare_repository() && head && !strcmp(head, ref->buf)) + die(_("Cannot force update the current branch.")); - head = resolve_ref_unsafe("HEAD", 0, NULL, NULL); - if (!is_bare_repository() && head && !strcmp(head, ref->buf)) - die(_("Cannot force update the current branch.")); - } return 1; } @@ -242,9 +258,9 @@ void create_branch(const char *name, const char *start_name, if (track == BRANCH_TRACK_EXPLICIT || track == BRANCH_TRACK_OVERRIDE) explicit_tracking = 1; - if (validate_new_branchname(name, &ref, force, - track == BRANCH_TRACK_OVERRIDE || - clobber_head)) { + if ((track == BRANCH_TRACK_OVERRIDE || clobber_head) + ? validate_branchname(name, &ref) + : validate_new_branchname(name, &ref, force)) { if (!force) dont_change_ref = 1; else @@ -23,22 +23,19 @@ void create_branch(const char *name, const char *start_name, int clobber_head, int quiet, enum branch_track track); /* - * Validates that the requested branch may be created, returning the - * interpreted ref in ref, force indicates whether (non-head) branches - * may be overwritten. A non-zero return value indicates that the force - * parameter was non-zero and the branch already exists. - * - * Contrary to all of the above, when attr_only is 1, the caller is - * not interested in verifying if it is Ok to update the named - * branch to point at a potentially different commit. It is merely - * asking if it is OK to change some attribute for the named branch - * (e.g. tracking upstream). - * - * NEEDSWORK: This needs to be split into two separate functions in the - * longer run for sanity. - * + * Check if 'name' can be a valid name for a branch; die otherwise. + * Return 1 if the named branch already exists; return 0 otherwise. + * Fill ref with the full refname for the branch. + */ +extern int validate_branchname(const char *name, struct strbuf *ref); + +/* + * Check if a branch 'name' can be created as a new branch; die otherwise. + * 'force' can be used when it is OK for the named branch already exists. + * Return 1 if the named branch already exists; return 0 otherwise. + * Fill ref with the full refname for the branch. */ -int validate_new_branchname(const char *name, struct strbuf *ref, int force, int attr_only); +extern int validate_new_branchname(const char *name, struct strbuf *ref, int force); /* * Remove information about the state of working on the current diff --git a/builtin/add.c b/builtin/add.c index 8d08e99e92..bf01d89e28 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -26,6 +26,7 @@ static const char * const builtin_add_usage[] = { }; static int patch_interactive, add_interactive, edit_interactive; static int take_worktree_changes; +static int add_renormalize; struct update_callback_data { int flags; @@ -123,6 +124,25 @@ int add_files_to_cache(const char *prefix, return !!data.add_errors; } +static int renormalize_tracked_files(const struct pathspec *pathspec, int flags) +{ + int i, retval = 0; + + for (i = 0; i < active_nr; i++) { + struct cache_entry *ce = active_cache[i]; + + if (ce_stage(ce)) + continue; /* do not touch unmerged paths */ + if (!S_ISREG(ce->ce_mode) && !S_ISLNK(ce->ce_mode)) + continue; /* do not touch non blobs */ + if (pathspec && !ce_path_match(ce, pathspec, NULL)) + continue; + retval |= add_file_to_cache(ce->name, flags | HASH_RENORMALIZE); + } + + return retval; +} + static char *prune_directory(struct dir_struct *dir, struct pathspec *pathspec, int prefix) { char *seen; @@ -276,6 +296,7 @@ static struct option builtin_add_options[] = { OPT_BOOL('e', "edit", &edit_interactive, N_("edit current diff and apply")), OPT__FORCE(&ignored_too, N_("allow adding otherwise ignored files")), OPT_BOOL('u', "update", &take_worktree_changes, N_("update tracked files")), + OPT_BOOL(0, "renormalize", &add_renormalize, N_("renormalize EOL of tracked files (implies -u)")), OPT_BOOL('N', "intent-to-add", &intent_to_add, N_("record only the fact that the path will be added later")), OPT_BOOL('A', "all", &addremove_explicit, N_("add changes from all tracked and untracked files")), { OPTION_CALLBACK, 0, "ignore-removal", &addremove_explicit, @@ -406,7 +427,7 @@ int cmd_add(int argc, const char **argv, const char *prefix) chmod_arg[1] != 'x' || chmod_arg[2])) die(_("--chmod param '%s' must be either -x or +x"), chmod_arg); - add_new_files = !take_worktree_changes && !refresh_only; + add_new_files = !take_worktree_changes && !refresh_only && !add_renormalize; require_pathspec = !(take_worktree_changes || (0 < addremove_explicit)); hold_locked_index(&lock_file, LOCK_DIE_ON_ERROR); @@ -500,7 +521,10 @@ int cmd_add(int argc, const char **argv, const char *prefix) plug_bulk_checkin(); - exit_status |= add_files_to_cache(prefix, &pathspec, flags); + if (add_renormalize) + exit_status |= renormalize_tracked_files(&pathspec, flags); + else + exit_status |= add_files_to_cache(prefix, &pathspec, flags); if (add_new_files) exit_status |= add_files(&dir, flags); diff --git a/builtin/branch.c b/builtin/branch.c index 33fd5fcfd1..af95ad2192 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -463,7 +463,6 @@ static void copy_or_rename_branch(const char *oldname, const char *newname, int struct strbuf oldref = STRBUF_INIT, newref = STRBUF_INIT, logmsg = STRBUF_INIT; struct strbuf oldsection = STRBUF_INIT, newsection = STRBUF_INIT; int recovery = 0; - int clobber_head_ok; if (!oldname) { if (copy) @@ -487,9 +486,10 @@ static void copy_or_rename_branch(const char *oldname, const char *newname, int * A command like "git branch -M currentbranch currentbranch" cannot * cause the worktree to become inconsistent with HEAD, so allow it. */ - clobber_head_ok = !strcmp(oldname, newname); - - validate_new_branchname(newname, &newref, force, clobber_head_ok); + if (!strcmp(oldname, newname)) + validate_branchname(newname, &newref); + else + validate_new_branchname(newname, &newref, force); reject_rebase_or_bisect_branch(oldref.buf); @@ -675,6 +675,9 @@ int cmd_branch(int argc, const char **argv, const char *prefix) copy *= 2; } + if (list) + setup_auto_pager("branch", 1); + if (delete) { if (!argc) die(_("branch name required")); @@ -793,9 +796,6 @@ int cmd_branch(int argc, const char **argv, const char *prefix) } else if (argc > 0 && argc <= 2) { struct branch *branch = branch_get(argv[0]); - if (!strcmp(argv[0], "HEAD")) - die(_("it does not make sense to create 'HEAD' manually")); - if (!branch) die(_("no such branch '%s'"), argv[0]); diff --git a/builtin/checkout.c b/builtin/checkout.c index 7d8bcc3833..3faae382de 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -1287,11 +1287,11 @@ int cmd_checkout(int argc, const char **argv, const char *prefix) if (opts.new_branch) { struct strbuf buf = STRBUF_INIT; - opts.branch_exists = - validate_new_branchname(opts.new_branch, &buf, - !!opts.new_branch_force, - !!opts.new_branch_force); - + if (opts.new_branch_force) + opts.branch_exists = validate_branchname(opts.new_branch, &buf); + else + opts.branch_exists = + validate_new_branchname(opts.new_branch, &buf, 0); strbuf_release(&buf); } diff --git a/builtin/ls-files.c b/builtin/ls-files.c index 8c713c47ac..2fc836e330 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -31,6 +31,7 @@ static int show_resolve_undo; static int show_modified; static int show_killed; static int show_valid_bit; +static int show_fsmonitor_bit; static int line_terminator = '\n'; static int debug_mode; static int show_eol; @@ -86,7 +87,8 @@ static const char *get_tag(const struct cache_entry *ce, const char *tag) { static char alttag[4]; - if (tag && *tag && show_valid_bit && (ce->ce_flags & CE_VALID)) { + if (tag && *tag && ((show_valid_bit && (ce->ce_flags & CE_VALID)) || + (show_fsmonitor_bit && (ce->ce_flags & CE_FSMONITOR_VALID)))) { memcpy(alttag, tag, 3); if (isalpha(tag[0])) { @@ -515,6 +517,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) N_("identify the file status with tags")), OPT_BOOL('v', NULL, &show_valid_bit, N_("use lowercase letters for 'assume unchanged' files")), + OPT_BOOL('f', NULL, &show_fsmonitor_bit, + N_("use lowercase letters for 'fsmonitor clean' files")), OPT_BOOL('c', "cached", &show_cached, N_("show cached files in the output (default)")), OPT_BOOL('d', "deleted", &show_deleted, @@ -584,7 +588,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) for (i = 0; i < exclude_list.nr; i++) { add_exclude(exclude_list.items[i].string, "", 0, el, --exclude_args); } - if (show_tag || show_valid_bit) { + if (show_tag || show_valid_bit || show_fsmonitor_bit) { tag_cached = "H "; tag_unmerged = "M "; tag_removed = "R "; diff --git a/builtin/notes.c b/builtin/notes.c index d7754db143..a2a542f82b 100644 --- a/builtin/notes.c +++ b/builtin/notes.c @@ -865,10 +865,10 @@ static int merge(int argc, const char **argv, const char *prefix) if (create_symref("NOTES_MERGE_REF", default_notes_ref(), NULL)) die(_("failed to store link to current notes ref (%s)"), default_notes_ref()); - printf(_("Automatic notes merge failed. Fix conflicts in %s and " - "commit the result with 'git notes merge --commit', or " - "abort the merge with 'git notes merge --abort'.\n"), - git_path(NOTES_MERGE_WORKTREE)); + fprintf(stderr, _("Automatic notes merge failed. Fix conflicts in %s " + "and commit the result with 'git notes merge --commit', " + "or abort the merge with 'git notes merge --abort'.\n"), + git_path(NOTES_MERGE_WORKTREE)); } free_notes(t); diff --git a/builtin/pull.c b/builtin/pull.c index f7e2c4f2ec..166b777ed6 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -113,6 +113,8 @@ static char *opt_depth; static char *opt_unshallow; static char *opt_update_shallow; static char *opt_refmap; +static char *opt_ipv4; +static char *opt_ipv6; static struct option pull_options[] = { /* Shared options */ @@ -218,6 +220,12 @@ static struct option pull_options[] = { OPT_PASSTHRU(0, "refmap", &opt_refmap, N_("refmap"), N_("specify fetch refmap"), PARSE_OPT_NONEG), + OPT_PASSTHRU('4', "ipv4", &opt_ipv4, NULL, + N_("use IPv4 addresses only"), + PARSE_OPT_NOARG), + OPT_PASSTHRU('6', "ipv6", &opt_ipv6, NULL, + N_("use IPv6 addresses only"), + PARSE_OPT_NOARG), OPT_END() }; @@ -522,6 +530,10 @@ static int run_fetch(const char *repo, const char **refspecs) argv_array_push(&args, opt_update_shallow); if (opt_refmap) argv_array_push(&args, opt_refmap); + if (opt_ipv4) + argv_array_push(&args, opt_ipv4); + if (opt_ipv6) + argv_array_push(&args, opt_ipv6); if (repo) { argv_array_push(&args, repo); diff --git a/builtin/update-index.c b/builtin/update-index.c index fefbe60167..58d1c2d282 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -16,6 +16,7 @@ #include "pathspec.h" #include "dir.h" #include "split-index.h" +#include "fsmonitor.h" /* * Default to not allowing changes to the list of files. The @@ -32,6 +33,7 @@ static int force_remove; static int verbose; static int mark_valid_only; static int mark_skip_worktree_only; +static int mark_fsmonitor_only; #define MARK_FLAG 1 #define UNMARK_FLAG 2 static struct strbuf mtime_dir = STRBUF_INIT; @@ -228,6 +230,7 @@ static int mark_ce_flags(const char *path, int flag, int mark) int namelen = strlen(path); int pos = cache_name_pos(path, namelen); if (0 <= pos) { + mark_fsmonitor_invalid(&the_index, active_cache[pos]); if (mark) active_cache[pos]->ce_flags |= flag; else @@ -460,6 +463,11 @@ static void update_one(const char *path) die("Unable to mark file %s", path); return; } + if (mark_fsmonitor_only) { + if (mark_ce_flags(path, CE_FSMONITOR_VALID, mark_fsmonitor_only == MARK_FLAG)) + die("Unable to mark file %s", path); + return; + } if (force_remove) { if (remove_file_from_cache(path)) @@ -917,6 +925,8 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) struct refresh_params refresh_args = {0, &has_errors}; int lock_error = 0; int split_index = -1; + int force_write = 0; + int fsmonitor = -1; struct lock_file lock_file = LOCK_INIT; struct parse_opt_ctx_t ctx; strbuf_getline_fn getline_fn; @@ -1008,6 +1018,16 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) N_("test if the filesystem supports untracked cache"), UC_TEST), OPT_SET_INT(0, "force-untracked-cache", &untracked_cache, N_("enable untracked cache without testing the filesystem"), UC_FORCE), + OPT_SET_INT(0, "force-write-index", &force_write, + N_("write out the index even if is not flagged as changed"), 1), + OPT_BOOL(0, "fsmonitor", &fsmonitor, + N_("enable or disable file system monitor")), + {OPTION_SET_INT, 0, "fsmonitor-valid", &mark_fsmonitor_only, NULL, + N_("mark files as fsmonitor valid"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, MARK_FLAG}, + {OPTION_SET_INT, 0, "no-fsmonitor-valid", &mark_fsmonitor_only, NULL, + N_("clear fsmonitor valid bit"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, UNMARK_FLAG}, OPT_END() }; @@ -1146,7 +1166,23 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) die("BUG: bad untracked_cache value: %d", untracked_cache); } - if (active_cache_changed) { + if (fsmonitor > 0) { + if (git_config_get_fsmonitor() == 0) + warning(_("core.fsmonitor is unset; " + "set it if you really want to " + "enable fsmonitor")); + add_fsmonitor(&the_index); + report(_("fsmonitor enabled")); + } else if (!fsmonitor) { + if (git_config_get_fsmonitor() == 1) + warning(_("core.fsmonitor is set; " + "remove it if you really want to " + "disable fsmonitor")); + remove_fsmonitor(&the_index); + report(_("fsmonitor disabled")); + } + + if (active_cache_changed || force_write) { if (newfd < 0) { if (refresh_args.flags & REFRESH_QUIET) exit(128); @@ -204,6 +204,7 @@ struct cache_entry { #define CE_ADDED (1 << 19) #define CE_HASHED (1 << 20) +#define CE_FSMONITOR_VALID (1 << 21) #define CE_WT_REMOVE (1 << 22) /* remove in work directory */ #define CE_CONFLICTED (1 << 23) @@ -327,6 +328,7 @@ static inline unsigned int canon_mode(unsigned int mode) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) #define UNTRACKED_CHANGED (1 << 7) +#define FSMONITOR_CHANGED (1 << 8) struct split_index; struct untracked_cache; @@ -345,6 +347,8 @@ struct index_state { struct hashmap dir_hash; unsigned char sha1[20]; struct untracked_cache *untracked; + uint64_t fsmonitor_last_update; + struct ewah_bitmap *fsmonitor_dirty; }; extern struct index_state the_index; @@ -700,11 +704,14 @@ extern void *read_blob_data_from_index(const struct index_state *, const char *, #define CE_MATCH_IGNORE_MISSING 0x08 /* enable stat refresh */ #define CE_MATCH_REFRESH 0x10 -extern int ie_match_stat(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int); -extern int ie_modified(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int); +/* don't refresh_fsmonitor state or do stat comparison even if CE_FSMONITOR_VALID is true */ +#define CE_MATCH_IGNORE_FSMONITOR 0X20 +extern int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); +extern int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); #define HASH_WRITE_OBJECT 1 #define HASH_FORMAT_CHECK 2 +#define HASH_RENORMALIZE 4 extern int index_fd(struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); extern int index_path(struct object_id *oid, const char *path, struct stat *st, unsigned flags); @@ -799,6 +806,7 @@ extern int core_apply_sparse_checkout; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; +extern const char *core_fsmonitor; /* * Include broken refs in all ref iterations, which will diff --git a/compat/bswap.h b/compat/bswap.h index 7d063e9e40..5078ce5ecc 100644 --- a/compat/bswap.h +++ b/compat/bswap.h @@ -158,7 +158,9 @@ static inline uint64_t git_bswap64(uint64_t x) #define get_be16(p) ntohs(*(unsigned short *)(p)) #define get_be32(p) ntohl(*(unsigned int *)(p)) +#define get_be64(p) ntohll(*(uint64_t *)(p)) #define put_be32(p, v) do { *(unsigned int *)(p) = htonl(v); } while (0) +#define put_be64(p, v) do { *(uint64_t *)(p) = htonll(v); } while (0) #else @@ -178,6 +180,13 @@ static inline uint32_t get_be32(const void *ptr) (uint32_t)p[3] << 0; } +static inline uint64_t get_be64(const void *ptr) +{ + const unsigned char *p = ptr; + return (uint64_t)get_be32(&p[0]) << 32 | + (uint64_t)get_be32(&p[4]) << 0; +} + static inline void put_be32(void *ptr, uint32_t value) { unsigned char *p = ptr; @@ -187,4 +196,17 @@ static inline void put_be32(void *ptr, uint32_t value) p[3] = value >> 0; } +static inline void put_be64(void *ptr, uint64_t value) +{ + unsigned char *p = ptr; + p[0] = value >> 56; + p[1] = value >> 48; + p[2] = value >> 40; + p[3] = value >> 32; + p[4] = value >> 24; + p[5] = value >> 16; + p[6] = value >> 8; + p[7] = value >> 0; +} + #endif @@ -2156,6 +2156,20 @@ int git_config_get_max_percent_split_change(void) return -1; /* default value */ } +int git_config_get_fsmonitor(void) +{ + if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor)) + core_fsmonitor = getenv("GIT_FSMONITOR_TEST"); + + if (core_fsmonitor && !*core_fsmonitor) + core_fsmonitor = NULL; + + if (core_fsmonitor) + return 1; + + return 0; +} + NORETURN void git_die_config_linenr(const char *key, const char *filename, int linenr) { @@ -2315,7 +2329,7 @@ static ssize_t write_section(int fd, const char *key) struct strbuf sb = store_create_section(key); ssize_t ret; - ret = write_in_full(fd, sb.buf, sb.len) == sb.len; + ret = write_in_full(fd, sb.buf, sb.len); strbuf_release(&sb); return ret; @@ -2810,7 +2824,7 @@ static int git_config_copy_or_rename_section_in_file(const char *config_filename * multiple [branch "$name"] sections. */ if (copystr.len > 0) { - if (write_in_full(out_fd, copystr.buf, copystr.len) != copystr.len) { + if (write_in_full(out_fd, copystr.buf, copystr.len) < 0) { ret = write_error(get_lock_file_path(&lock)); goto out; } @@ -2872,7 +2886,7 @@ static int git_config_copy_or_rename_section_in_file(const char *config_filename * logic in the loop above. */ if (copystr.len > 0) { - if (write_in_full(out_fd, copystr.buf, copystr.len) != copystr.len) { + if (write_in_full(out_fd, copystr.buf, copystr.len) < 0) { ret = write_error(get_lock_file_path(&lock)); goto out; } @@ -212,6 +212,7 @@ extern int git_config_get_pathname(const char *key, const char **dest); extern int git_config_get_untracked_cache(void); extern int git_config_get_split_index(void); extern int git_config_get_max_percent_split_change(void); +extern int git_config_get_fsmonitor(void); /* This dies if the configured or default date is in the future */ extern int git_config_get_expiry(const char *key, const char **output); diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index f07f16b28f..35df6ce9ea 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1204,7 +1204,7 @@ _git_branch () --color --no-color --verbose --abbrev= --no-abbrev --track --no-track --contains --no-contains --merged --no-merged --set-upstream-to= --edit-description --list - --unset-upstream --delete --move --remotes + --unset-upstream --delete --move --copy --remotes --column --no-column --sort= --points-at " ;; @@ -1400,7 +1400,7 @@ __git_diff_common_options="--stat --numstat --shortstat --summary --patch-with-stat --name-only --name-status --color --no-color --color-words --no-renames --check --full-index --binary --abbrev --diff-filter= - --find-copies-harder + --find-copies-harder --ignore-cr-at-eol --text --ignore-space-at-eol --ignore-space-change --ignore-all-space --ignore-blank-lines --exit-code --quiet --ext-diff --no-ext-diff @@ -2641,6 +2641,7 @@ _git_config () sendemail.suppressfrom sendemail.thread sendemail.to + sendemail.tocmd sendemail.validate sendemail.smtpbatchsize sendemail.smtprelogindelay diff --git a/contrib/git-jump/README b/contrib/git-jump/README index 225e3f0954..4484bda410 100644 --- a/contrib/git-jump/README +++ b/contrib/git-jump/README @@ -63,6 +63,9 @@ git jump grep foo_bar # same as above, but case-insensitive; you can give # arbitrary grep options git jump grep -i foo_bar + +# use the silver searcher for git jump grep +git config jump.grepCmd "ag --column" -------------------------------------------------- @@ -92,3 +95,10 @@ how to activate it. The shell snippets to generate the quickfix lines will almost certainly choke on filenames with exotic characters (like newlines). + +Contributing +------------ + +Bug fixes, bug reports, and feature requests should be discussed on the +Git mailing list <git@vger.kernel.org>, and cc'd to the git-jump +maintainer, Jeff King <peff@peff.net>. diff --git a/contrib/git-jump/git-jump b/contrib/git-jump/git-jump index 427f206a45..80ab0590bc 100755 --- a/contrib/git-jump/git-jump +++ b/contrib/git-jump/git-jump @@ -11,7 +11,8 @@ diff: elements are diff hunks. Arguments are given to diff. merge: elements are merge conflicts. Arguments are ignored. -grep: elements are grep hits. Arguments are given to grep. +grep: elements are grep hits. Arguments are given to git grep or, if + configured, to the command in `jump.grepCmd`. ws: elements are whitespace errors. Arguments are given to diff --check. EOF @@ -50,7 +51,9 @@ mode_merge() { # but let's clean up extra whitespace, so they look better if the # editor shows them to us in the status bar. mode_grep() { - git grep -n "$@" | + cmd=$(git config jump.grepCmd) + test -n "$cmd" || cmd="git grep -n" + $cmd "$@" | perl -pe ' s/[ \t]+/ /g; s/^ *//; diff --git a/diff-lib.c b/diff-lib.c index 731f0886d6..5173023cd3 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -12,6 +12,7 @@ #include "refs.h" #include "submodule.h" #include "dir.h" +#include "fsmonitor.h" /* * diff-files @@ -229,6 +230,7 @@ int run_diff_files(struct rev_info *revs, unsigned int option) if (!changed && !dirty_submodule) { ce_mark_uptodate(ce); + mark_fsmonitor_valid(ce); if (!revs->diffopt.flags.find_copies_harder) continue; } @@ -4129,9 +4129,7 @@ void diff_setup_done(struct diff_options *options) * inside contents. */ - if (DIFF_XDL_TST(options, IGNORE_WHITESPACE) || - DIFF_XDL_TST(options, IGNORE_WHITESPACE_CHANGE) || - DIFF_XDL_TST(options, IGNORE_WHITESPACE_AT_EOL)) + if ((options->xdl_opts & XDF_WHITESPACE_FLAGS)) options->flags.diff_from_contents = 1; else options->flags.diff_from_contents = 0; @@ -4588,6 +4586,8 @@ int diff_opt_parse(struct diff_options *options, DIFF_XDL_SET(options, IGNORE_WHITESPACE_CHANGE); else if (!strcmp(arg, "--ignore-space-at-eol")) DIFF_XDL_SET(options, IGNORE_WHITESPACE_AT_EOL); + else if (!strcmp(arg, "--ignore-cr-at-eol")) + DIFF_XDL_SET(options, IGNORE_CR_AT_EOL); else if (!strcmp(arg, "--ignore-blank-lines")) DIFF_XDL_SET(options, IGNORE_BLANK_LINES); else if (!strcmp(arg, "--indent-heuristic")) @@ -18,6 +18,7 @@ #include "utf8.h" #include "varint.h" #include "ewah/ewok.h" +#include "fsmonitor.h" /* * Tells read_directory_recursive how a file or directory should be treated. @@ -1733,17 +1734,23 @@ static int valid_cached_dir(struct dir_struct *dir, if (!untracked) return 0; - if (stat(path->len ? path->buf : ".", &st)) { - invalidate_directory(dir->untracked, untracked); - memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); - return 0; - } - if (!untracked->valid || - match_stat_data_racy(istate, &untracked->stat_data, &st)) { - if (untracked->valid) + /* + * With fsmonitor, we can trust the untracked cache's valid field. + */ + refresh_fsmonitor(istate); + if (!(dir->untracked->use_fsmonitor && untracked->valid)) { + if (stat(path->len ? path->buf : ".", &st)) { invalidate_directory(dir->untracked, untracked); - fill_stat_data(&untracked->stat_data, &st); - return 0; + memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_stat_data_racy(istate, &untracked->stat_data, &st)) { + if (untracked->valid) + invalidate_directory(dir->untracked, untracked); + fill_stat_data(&untracked->stat_data, &st); + return 0; + } } if (untracked->check_only != !!check_only) { @@ -139,6 +139,8 @@ struct untracked_cache { int gitignore_invalidated; int dir_invalidated; int dir_opened; + /* fsmonitor invalidation data */ + unsigned int use_fsmonitor : 1; }; struct dir_struct { @@ -4,6 +4,7 @@ #include "streaming.h" #include "submodule.h" #include "progress.h" +#include "fsmonitor.h" static void create_directories(const char *path, int path_len, const struct checkout *state) @@ -373,6 +374,7 @@ finish: ce->name); fill_stat_cache_info(ce, &st); ce->ce_flags |= CE_UPDATE_IN_BASE; + mark_fsmonitor_invalid(state->istate, ce); state->istate->cache_changed |= CE_ENTRY_CHANGED; } delayed: diff --git a/environment.c b/environment.c index 8289c25b44..8fa032f307 100644 --- a/environment.c +++ b/environment.c @@ -76,6 +76,7 @@ int protect_hfs = PROTECT_HFS_DEFAULT; #define PROTECT_NTFS_DEFAULT 0 #endif int protect_ntfs = PROTECT_NTFS_DEFAULT; +const char *core_fsmonitor; /* * The character that begins a commented line in user-editable file diff --git a/fsmonitor.c b/fsmonitor.c new file mode 100644 index 0000000000..0af7c4edba --- /dev/null +++ b/fsmonitor.c @@ -0,0 +1,266 @@ +#include "cache.h" +#include "config.h" +#include "dir.h" +#include "ewah/ewok.h" +#include "fsmonitor.h" +#include "run-command.h" +#include "strbuf.h" + +#define INDEX_EXTENSION_VERSION (1) +#define HOOK_INTERFACE_VERSION (1) + +struct trace_key trace_fsmonitor = TRACE_KEY_INIT(FSMONITOR); + +static void fsmonitor_ewah_callback(size_t pos, void *is) +{ + struct index_state *istate = (struct index_state *)is; + struct cache_entry *ce = istate->cache[pos]; + + ce->ce_flags &= ~CE_FSMONITOR_VALID; +} + +int read_fsmonitor_extension(struct index_state *istate, const void *data, + unsigned long sz) +{ + const char *index = data; + uint32_t hdr_version; + uint32_t ewah_size; + struct ewah_bitmap *fsmonitor_dirty; + int ret; + + if (sz < sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t)) + return error("corrupt fsmonitor extension (too short)"); + + hdr_version = get_be32(index); + index += sizeof(uint32_t); + if (hdr_version != INDEX_EXTENSION_VERSION) + return error("bad fsmonitor version %d", hdr_version); + + istate->fsmonitor_last_update = get_be64(index); + index += sizeof(uint64_t); + + ewah_size = get_be32(index); + index += sizeof(uint32_t); + + fsmonitor_dirty = ewah_new(); + ret = ewah_read_mmap(fsmonitor_dirty, index, ewah_size); + if (ret != ewah_size) { + ewah_free(fsmonitor_dirty); + return error("failed to parse ewah bitmap reading fsmonitor index extension"); + } + istate->fsmonitor_dirty = fsmonitor_dirty; + + trace_printf_key(&trace_fsmonitor, "read fsmonitor extension successful"); + return 0; +} + +void fill_fsmonitor_bitmap(struct index_state *istate) +{ + int i; + istate->fsmonitor_dirty = ewah_new(); + for (i = 0; i < istate->cache_nr; i++) + if (!(istate->cache[i]->ce_flags & CE_FSMONITOR_VALID)) + ewah_set(istate->fsmonitor_dirty, i); +} + +void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate) +{ + uint32_t hdr_version; + uint64_t tm; + uint32_t ewah_start; + uint32_t ewah_size = 0; + int fixup = 0; + + put_be32(&hdr_version, INDEX_EXTENSION_VERSION); + strbuf_add(sb, &hdr_version, sizeof(uint32_t)); + + put_be64(&tm, istate->fsmonitor_last_update); + strbuf_add(sb, &tm, sizeof(uint64_t)); + fixup = sb->len; + strbuf_add(sb, &ewah_size, sizeof(uint32_t)); /* we'll fix this up later */ + + ewah_start = sb->len; + ewah_serialize_strbuf(istate->fsmonitor_dirty, sb); + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_dirty = NULL; + + /* fix up size field */ + put_be32(&ewah_size, sb->len - ewah_start); + memcpy(sb->buf + fixup, &ewah_size, sizeof(uint32_t)); + + trace_printf_key(&trace_fsmonitor, "write fsmonitor extension successful"); +} + +/* + * Call the query-fsmonitor hook passing the time of the last saved results. + */ +static int query_fsmonitor(int version, uint64_t last_update, struct strbuf *query_result) +{ + struct child_process cp = CHILD_PROCESS_INIT; + char ver[64]; + char date[64]; + const char *argv[4]; + + if (!(argv[0] = core_fsmonitor)) + return -1; + + snprintf(ver, sizeof(version), "%d", version); + snprintf(date, sizeof(date), "%" PRIuMAX, (uintmax_t)last_update); + argv[1] = ver; + argv[2] = date; + argv[3] = NULL; + cp.argv = argv; + cp.use_shell = 1; + cp.dir = get_git_work_tree(); + + return capture_command(&cp, query_result, 1024); +} + +static void fsmonitor_refresh_callback(struct index_state *istate, const char *name) +{ + int pos = index_name_pos(istate, name, strlen(name)); + + if (pos >= 0) { + struct cache_entry *ce = istate->cache[pos]; + ce->ce_flags &= ~CE_FSMONITOR_VALID; + } + + /* + * Mark the untracked cache dirty even if it wasn't found in the index + * as it could be a new untracked file. + */ + trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s'", name); + untracked_cache_invalidate_path(istate, name); +} + +void refresh_fsmonitor(struct index_state *istate) +{ + static int has_run_once = 0; + struct strbuf query_result = STRBUF_INIT; + int query_success = 0; + size_t bol; /* beginning of line */ + uint64_t last_update; + char *buf; + int i; + + if (!core_fsmonitor || has_run_once) + return; + has_run_once = 1; + + trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); + /* + * This could be racy so save the date/time now and query_fsmonitor + * should be inclusive to ensure we don't miss potential changes. + */ + last_update = getnanotime(); + + /* + * If we have a last update time, call query_fsmonitor for the set of + * changes since that time, else assume everything is possibly dirty + * and check it all. + */ + if (istate->fsmonitor_last_update) { + query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION, + istate->fsmonitor_last_update, &query_result); + trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor); + trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s", + core_fsmonitor, query_success ? "success" : "failure"); + } + + /* a fsmonitor process can return '/' to indicate all entries are invalid */ + if (query_success && query_result.buf[0] != '/') { + /* Mark all entries returned by the monitor as dirty */ + buf = query_result.buf; + bol = 0; + for (i = 0; i < query_result.len; i++) { + if (buf[i] != '\0') + continue; + fsmonitor_refresh_callback(istate, buf + bol); + bol = i + 1; + } + if (bol < query_result.len) + fsmonitor_refresh_callback(istate, buf + bol); + } else { + /* Mark all entries invalid */ + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; + + if (istate->untracked) + istate->untracked->use_fsmonitor = 0; + } + strbuf_release(&query_result); + + /* Now that we've updated istate, save the last_update time */ + istate->fsmonitor_last_update = last_update; +} + +void add_fsmonitor(struct index_state *istate) +{ + int i; + + if (!istate->fsmonitor_last_update) { + trace_printf_key(&trace_fsmonitor, "add fsmonitor"); + istate->cache_changed |= FSMONITOR_CHANGED; + istate->fsmonitor_last_update = getnanotime(); + + /* reset the fsmonitor state */ + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; + + /* reset the untracked cache */ + if (istate->untracked) { + add_untracked_cache(istate); + istate->untracked->use_fsmonitor = 1; + } + + /* Update the fsmonitor state */ + refresh_fsmonitor(istate); + } +} + +void remove_fsmonitor(struct index_state *istate) +{ + if (istate->fsmonitor_last_update) { + trace_printf_key(&trace_fsmonitor, "remove fsmonitor"); + istate->cache_changed |= FSMONITOR_CHANGED; + istate->fsmonitor_last_update = 0; + } +} + +void tweak_fsmonitor(struct index_state *istate) +{ + int i; + int fsmonitor_enabled = git_config_get_fsmonitor(); + + if (istate->fsmonitor_dirty) { + if (fsmonitor_enabled) { + /* Mark all entries valid */ + for (i = 0; i < istate->cache_nr; i++) { + istate->cache[i]->ce_flags |= CE_FSMONITOR_VALID; + } + + /* Mark all previously saved entries as dirty */ + ewah_each_bit(istate->fsmonitor_dirty, fsmonitor_ewah_callback, istate); + + /* Now mark the untracked cache for fsmonitor usage */ + if (istate->untracked) + istate->untracked->use_fsmonitor = 1; + } + + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_dirty = NULL; + } + + switch (fsmonitor_enabled) { + case -1: /* keep: do nothing */ + break; + case 0: /* false */ + remove_fsmonitor(istate); + break; + case 1: /* true */ + add_fsmonitor(istate); + break; + default: /* unknown value: do nothing */ + break; + } +} diff --git a/fsmonitor.h b/fsmonitor.h new file mode 100644 index 0000000000..cd3cc0ccf2 --- /dev/null +++ b/fsmonitor.h @@ -0,0 +1,73 @@ +#ifndef FSMONITOR_H +#define FSMONITOR_H + +extern struct trace_key trace_fsmonitor; + +/* + * Read the fsmonitor index extension and (if configured) restore the + * CE_FSMONITOR_VALID state. + */ +extern int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz); + +/* + * Fill the fsmonitor_dirty ewah bits with their state from the index, + * before it is split during writing. + */ +extern void fill_fsmonitor_bitmap(struct index_state *istate); + +/* + * Write the CE_FSMONITOR_VALID state into the fsmonitor index + * extension. Reads from the fsmonitor_dirty ewah in the index. + */ +extern void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate); + +/* + * Add/remove the fsmonitor index extension + */ +extern void add_fsmonitor(struct index_state *istate); +extern void remove_fsmonitor(struct index_state *istate); + +/* + * Add/remove the fsmonitor index extension as necessary based on the current + * core.fsmonitor setting. + */ +extern void tweak_fsmonitor(struct index_state *istate); + +/* + * Run the configured fsmonitor integration script and clear the + * CE_FSMONITOR_VALID bit for any files returned as dirty. Also invalidate + * any corresponding untracked cache directory structures. Optimized to only + * run the first time it is called. + */ +extern void refresh_fsmonitor(struct index_state *istate); + +/* + * Set the given cache entries CE_FSMONITOR_VALID bit. This should be + * called any time the cache entry has been updated to reflect the + * current state of the file on disk. + */ +static inline void mark_fsmonitor_valid(struct cache_entry *ce) +{ + if (core_fsmonitor) { + ce->ce_flags |= CE_FSMONITOR_VALID; + trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_clean '%s'", ce->name); + } +} + +/* + * Clear the given cache entry's CE_FSMONITOR_VALID bit and invalidate + * any corresponding untracked cache directory structures. This should + * be called any time git creates or modifies a file that should + * trigger an lstat() or invalidate the untracked cache for the + * corresponding directory + */ +static inline void mark_fsmonitor_invalid(struct index_state *istate, struct cache_entry *ce) +{ + if (core_fsmonitor) { + ce->ce_flags &= ~CE_FSMONITOR_VALID; + untracked_cache_invalidate_path(istate, ce->name); + trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_invalid '%s'", ce->name); + } +} + +#endif diff --git a/git-rebase--am.sh b/git-rebase--am.sh index 6e64d40d6f..14c50782e0 100644 --- a/git-rebase--am.sh +++ b/git-rebase--am.sh @@ -53,6 +53,7 @@ else git format-patch -k --stdout --full-index --cherry-pick --right-only \ --src-prefix=a/ --dst-prefix=b/ --no-renames --no-cover-letter \ + --pretty=mboxrd \ $git_format_patch_opt \ "$revisions" ${restrict_revision+^$restrict_revision} \ >"$GIT_DIR/rebased-patches" @@ -83,6 +84,7 @@ else fi git am $git_am_opt --rebasing --resolvemsg="$resolvemsg" \ + --patch-format=mboxrd \ $allow_rerere_autoupdate \ ${gpg_sign_opt:+"$gpg_sign_opt"} <"$GIT_DIR/rebased-patches" ret=$? diff --git a/git-rebase.sh b/git-rebase.sh index 6344e8d5e3..60b70f3def 100755 --- a/git-rebase.sh +++ b/git-rebase.sh @@ -9,7 +9,7 @@ OPTIONS_STUCKLONG=t OPTIONS_SPEC="\ git rebase [-i] [options] [--exec <cmd>] [--onto <newbase>] [<upstream>] [<branch>] git rebase [-i] [options] [--exec <cmd>] [--onto <newbase>] --root [<branch>] -git-rebase --continue | --abort | --skip | --edit-todo +git rebase --continue | --abort | --skip | --edit-todo -- Available options are v,verbose! display a diffstat of what changed upstream @@ -166,7 +166,7 @@ apply_autostash () { if test -f "$state_dir/autostash" then stash_sha1=$(cat "$state_dir/autostash") - if git stash apply $stash_sha1 2>&1 >/dev/null + if git stash apply $stash_sha1 >/dev/null 2>&1 then echo "$(gettext 'Applied autostash.')" >&2 else @@ -216,7 +216,7 @@ run_pre_rebase_hook () { } test -f "$apply_dir"/applying && - die "$(gettext "It looks like git-am is in progress. Cannot rebase.")" + die "$(gettext "It looks like 'git am' is in progress. Cannot rebase.")" if test -d "$apply_dir" then @@ -372,7 +372,7 @@ static struct cmd_struct commands[] = { { "archive", cmd_archive, RUN_SETUP_GENTLY }, { "bisect--helper", cmd_bisect__helper, RUN_SETUP }, { "blame", cmd_blame, RUN_SETUP }, - { "branch", cmd_branch, RUN_SETUP }, + { "branch", cmd_branch, RUN_SETUP | DELAY_PAGER_CONFIG }, { "bundle", cmd_bundle, RUN_SETUP_GENTLY }, { "cat-file", cmd_cat_file, RUN_SETUP }, { "check-attr", cmd_check_attr, RUN_SETUP }, diff --git a/merge-recursive.c b/merge-recursive.c index b48b15a6fd..d00b274381 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -1901,8 +1901,9 @@ static int process_entry(struct merge_options *o, oid = b_oid; conf = _("directory/file"); } - if (dir_in_way(path, !o->call_depth, - S_ISGITLINK(a_mode))) { + if (dir_in_way(path, + !o->call_depth && !S_ISGITLINK(a_mode), + 0)) { char *new_path = unique_path(o, path, add_branch); clean_merge = 0; output(o, 1, _("CONFLICT (%s): There is a directory with name %s in %s. " @@ -2252,6 +2253,8 @@ int parse_merge_opt(struct merge_options *o, const char *s) DIFF_XDL_SET(o, IGNORE_WHITESPACE); else if (!strcmp(s, "ignore-space-at-eol")) DIFF_XDL_SET(o, IGNORE_WHITESPACE_AT_EOL); + else if (!strcmp(s, "ignore-cr-at-eol")) + DIFF_XDL_SET(o, IGNORE_CR_AT_EOL); else if (!strcmp(s, "renormalize")) o->renormalize = 1; else if (!strcmp(s, "no-renormalize")) diff --git a/preload-index.c b/preload-index.c index 70a4c80878..2a83255e4e 100644 --- a/preload-index.c +++ b/preload-index.c @@ -4,6 +4,7 @@ #include "cache.h" #include "pathspec.h" #include "dir.h" +#include "fsmonitor.h" #ifdef NO_PTHREADS static void preload_index(struct index_state *index, @@ -55,15 +56,18 @@ static void *preload_thread(void *_data) continue; if (ce_skip_worktree(ce)) continue; + if (ce->ce_flags & CE_FSMONITOR_VALID) + continue; if (!ce_path_match(ce, &p->pathspec, NULL)) continue; if (threaded_has_symlink_leading_path(&cache, ce->name, ce_namelen(ce))) continue; if (lstat(ce->name, &st)) continue; - if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY)) + if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY|CE_MATCH_IGNORE_FSMONITOR)) continue; ce_mark_uptodate(ce); + mark_fsmonitor_valid(ce); } while (--nr > 0); cache_def_clear(&cache); return NULL; @@ -79,6 +83,8 @@ static void preload_index(struct index_state *index, return; threads = index->cache_nr / THREAD_COST; + if ((index->cache_nr > 1) && (threads < 2) && getenv("GIT_FORCE_PRELOAD_TEST")) + threads = 2; if (threads < 2) return; if (threads > MAX_PARALLEL) diff --git a/read-cache.c b/read-cache.c index b13a1cb8f2..2eb81a66b9 100644 --- a/read-cache.c +++ b/read-cache.c @@ -19,6 +19,7 @@ #include "varint.h" #include "split-index.h" #include "utf8.h" +#include "fsmonitor.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -38,11 +39,12 @@ #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ #define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ +#define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */ /* changes that can be kept in $GIT_DIR/index (basically all extensions) */ #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \ CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \ - SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED) + SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED | FSMONITOR_CHANGED) struct index_state the_index; static const char *alternate_index_output; @@ -62,6 +64,7 @@ static void replace_index_entry(struct index_state *istate, int nr, struct cache free(old); set_index_entry(istate, nr, ce); ce->ce_flags |= CE_UPDATE_IN_BASE; + mark_fsmonitor_invalid(istate, ce); istate->cache_changed |= CE_ENTRY_CHANGED; } @@ -150,8 +153,10 @@ void fill_stat_cache_info(struct cache_entry *ce, struct stat *st) if (assume_unchanged) ce->ce_flags |= CE_VALID; - if (S_ISREG(st->st_mode)) + if (S_ISREG(st->st_mode)) { ce_mark_uptodate(ce); + mark_fsmonitor_valid(ce); + } } static int ce_compare_data(const struct cache_entry *ce, struct stat *st) @@ -301,7 +306,7 @@ int match_stat_data_racy(const struct index_state *istate, return match_stat_data(sd, st); } -int ie_match_stat(const struct index_state *istate, +int ie_match_stat(struct index_state *istate, const struct cache_entry *ce, struct stat *st, unsigned int options) { @@ -309,7 +314,10 @@ int ie_match_stat(const struct index_state *istate, int ignore_valid = options & CE_MATCH_IGNORE_VALID; int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE; int assume_racy_is_modified = options & CE_MATCH_RACY_IS_DIRTY; + int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR; + if (!ignore_fsmonitor) + refresh_fsmonitor(istate); /* * If it's marked as always valid in the index, it's * valid whatever the checked-out copy says. @@ -320,6 +328,8 @@ int ie_match_stat(const struct index_state *istate, return 0; if (!ignore_valid && (ce->ce_flags & CE_VALID)) return 0; + if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID)) + return 0; /* * Intent-to-add entries have not been added, so the index entry @@ -357,7 +367,7 @@ int ie_match_stat(const struct index_state *istate, return changed; } -int ie_modified(const struct index_state *istate, +int ie_modified(struct index_state *istate, const struct cache_entry *ce, struct stat *st, unsigned int options) { @@ -631,13 +641,17 @@ int add_to_index(struct index_state *istate, const char *path, struct stat *st, { int size, namelen, was_same; mode_t st_mode = st->st_mode; - struct cache_entry *ce, *alias; + struct cache_entry *ce, *alias = NULL; unsigned ce_option = CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE|CE_MATCH_RACY_IS_DIRTY; int verbose = flags & (ADD_CACHE_VERBOSE | ADD_CACHE_PRETEND); int pretend = flags & ADD_CACHE_PRETEND; int intent_only = flags & ADD_CACHE_INTENT; int add_option = (ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE| (intent_only ? ADD_CACHE_NEW_ONLY : 0)); + int newflags = HASH_WRITE_OBJECT; + + if (flags & HASH_RENORMALIZE) + newflags |= HASH_RENORMALIZE; if (!S_ISREG(st_mode) && !S_ISLNK(st_mode) && !S_ISDIR(st_mode)) return error("%s: can only add regular files, symbolic links or git-directories", path); @@ -678,19 +692,23 @@ int add_to_index(struct index_state *istate, const char *path, struct stat *st, if (ignore_case) { adjust_dirname_case(istate, ce->name); } + if (!(flags & HASH_RENORMALIZE)) { + alias = index_file_exists(istate, ce->name, + ce_namelen(ce), ignore_case); + if (alias && + !ce_stage(alias) && + !ie_match_stat(istate, alias, st, ce_option)) { + /* Nothing changed, really */ + if (!S_ISGITLINK(alias->ce_mode)) + ce_mark_uptodate(alias); + alias->ce_flags |= CE_ADDED; - alias = index_file_exists(istate, ce->name, ce_namelen(ce), ignore_case); - if (alias && !ce_stage(alias) && !ie_match_stat(istate, alias, st, ce_option)) { - /* Nothing changed, really */ - if (!S_ISGITLINK(alias->ce_mode)) - ce_mark_uptodate(alias); - alias->ce_flags |= CE_ADDED; - - free(ce); - return 0; + free(ce); + return 0; + } } if (!intent_only) { - if (index_path(&ce->oid, path, st, HASH_WRITE_OBJECT)) { + if (index_path(&ce->oid, path, st, newflags)) { free(ce); return error("unable to index file %s", path); } @@ -778,6 +796,7 @@ int chmod_index_entry(struct index_state *istate, struct cache_entry *ce, } cache_tree_invalidate_path(istate, ce->name); ce->ce_flags |= CE_UPDATE_IN_BASE; + mark_fsmonitor_invalid(istate, ce); istate->cache_changed |= CE_ENTRY_CHANGED; return 0; @@ -1229,10 +1248,13 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, int ignore_valid = options & CE_MATCH_IGNORE_VALID; int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE; int ignore_missing = options & CE_MATCH_IGNORE_MISSING; + int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR; if (!refresh || ce_uptodate(ce)) return ce; + if (!ignore_fsmonitor) + refresh_fsmonitor(istate); /* * CE_VALID or CE_SKIP_WORKTREE means the user promised us * that the change to the work tree does not matter and told @@ -1246,6 +1268,10 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, ce_mark_uptodate(ce); return ce; } + if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID)) { + ce_mark_uptodate(ce); + return ce; + } if (has_symlink_leading_path(ce->name, ce_namelen(ce))) { if (ignore_missing) @@ -1283,8 +1309,10 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, * because CE_UPTODATE flag is in-core only; * we are not going to write this change out. */ - if (!S_ISGITLINK(ce->ce_mode)) + if (!S_ISGITLINK(ce->ce_mode)) { ce_mark_uptodate(ce); + mark_fsmonitor_valid(ce); + } return ce; } } @@ -1392,6 +1420,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, */ ce->ce_flags &= ~CE_VALID; ce->ce_flags |= CE_UPDATE_IN_BASE; + mark_fsmonitor_invalid(istate, ce); istate->cache_changed |= CE_ENTRY_CHANGED; } if (quiet) @@ -1554,6 +1583,9 @@ static int read_index_extension(struct index_state *istate, case CACHE_EXT_UNTRACKED: istate->untracked = read_untracked_extension(data, sz); break; + case CACHE_EXT_FSMONITOR: + read_fsmonitor_extension(istate, data, sz); + break; default: if (*ext < 'A' || 'Z' < *ext) return error("index uses %.4s extension, which we do not understand", @@ -1729,6 +1761,7 @@ static void post_read_index_from(struct index_state *istate) check_ce_order(istate); tweak_untracked_cache(istate); tweak_split_index(istate); + tweak_fsmonitor(istate); } /* remember to discard_cache() before reading a different cache! */ @@ -2320,6 +2353,16 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, if (err) return -1; } + if (!strip_extensions && istate->fsmonitor_last_update) { + struct strbuf sb = STRBUF_INIT; + + write_fsmonitor_extension(&sb, istate); + err = write_index_ext_header(&c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0 + || ce_write(&c, newfd, sb.buf, sb.len) < 0; + strbuf_release(&sb); + if (err) + return -1; + } if (ce_flush(&c, newfd, istate->sha1)) return -1; @@ -2500,6 +2543,9 @@ int write_locked_index(struct index_state *istate, struct lock_file *lock, int new_shared_index, ret; struct split_index *si = istate->split_index; + if (istate->fsmonitor_last_update) + fill_fsmonitor_bitmap(istate); + if (!si || alternate_index_output || (istate->cache_changed & ~EXTMASK)) { if (si) diff --git a/sequencer.c b/sequencer.c index 19dd575ed9..fa94ed652d 100644 --- a/sequencer.c +++ b/sequencer.c @@ -438,7 +438,8 @@ static int do_recursive_merge(struct commit *base, struct commit *next, char **xopt; static struct lock_file index_lock; - hold_locked_index(&index_lock, LOCK_DIE_ON_ERROR); + if (hold_locked_index(&index_lock, LOCK_REPORT_ON_ERROR) < 0) + return -1; read_cache(); diff --git a/sha1_file.c b/sha1_file.c index 8a7c6b7eba..8ae6cb6285 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -74,6 +74,18 @@ static struct cached_object *find_cached_object(const unsigned char *sha1) return NULL; } + +static enum safe_crlf get_safe_crlf(unsigned flags) +{ + if (flags & HASH_RENORMALIZE) + return SAFE_CRLF_RENORMALIZE; + else if (flags & HASH_WRITE_OBJECT) + return safe_crlf; + else + return SAFE_CRLF_FALSE; +} + + int mkdir_in_gitdir(const char *path) { if (mkdir(path, 0777)) { @@ -1679,7 +1691,7 @@ static int index_mem(struct object_id *oid, void *buf, size_t size, if ((type == OBJ_BLOB) && path) { struct strbuf nbuf = STRBUF_INIT; if (convert_to_git(&the_index, path, buf, size, &nbuf, - write_object ? safe_crlf : SAFE_CRLF_FALSE)) { + get_safe_crlf(flags))) { buf = strbuf_detach(&nbuf, &size); re_allocated = 1; } @@ -1713,7 +1725,7 @@ static int index_stream_convert_blob(struct object_id *oid, int fd, assert(would_convert_to_git_filter_fd(path)); convert_to_git_filter_fd(&the_index, path, fd, &sbuf, - write_object ? safe_crlf : SAFE_CRLF_FALSE); + get_safe_crlf(flags)); if (write_object) ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB), diff --git a/sha1_name.c b/sha1_name.c index 9a2d5caf3b..611c7d24dd 100644 --- a/sha1_name.c +++ b/sha1_name.c @@ -1438,9 +1438,19 @@ int strbuf_check_branch_ref(struct strbuf *sb, const char *name) strbuf_branchname(sb, name, INTERPRET_BRANCH_LOCAL); else strbuf_addstr(sb, name); - if (name[0] == '-') - return -1; + + /* + * This splice must be done even if we end up rejecting the + * name; builtin/branch.c::copy_or_rename_branch() still wants + * to see what the name expanded to so that "branch -m" can be + * used as a tool to correct earlier mistakes. + */ strbuf_splice(sb, 0, 0, "refs/heads/", 11); + + if (*name == '-' || + !strcmp(sb->buf, "refs/heads/HEAD")) + return -1; + return check_refname_format(sb->buf, 0); } diff --git a/submodule.c b/submodule.c index 3ee4a0caa7..bb531e0e59 100644 --- a/submodule.c +++ b/submodule.c @@ -62,7 +62,7 @@ int is_staging_gitmodules_ok(const struct index_state *istate) if ((pos >= 0) && (pos < istate->cache_nr)) { struct stat st; if (lstat(GITMODULES_FILE, &st) == 0 && - ce_match_stat(istate->cache[pos], &st, 0) & DATA_CHANGED) + ce_match_stat(istate->cache[pos], &st, CE_MATCH_IGNORE_FSMONITOR) & DATA_CHANGED) return 0; } diff --git a/t/helper/.gitignore b/t/helper/.gitignore index 7c9d28a834..d02f9b39ac 100644 --- a/t/helper/.gitignore +++ b/t/helper/.gitignore @@ -3,7 +3,9 @@ /test-config /test-date /test-delta +/test-drop-caches /test-dump-cache-tree +/test-dump-fsmonitor /test-dump-split-index /test-dump-untracked-cache /test-fake-ssh diff --git a/t/helper/test-drop-caches.c b/t/helper/test-drop-caches.c new file mode 100644 index 0000000000..bd1a857d52 --- /dev/null +++ b/t/helper/test-drop-caches.c @@ -0,0 +1,164 @@ +#include "git-compat-util.h" + +#if defined(GIT_WINDOWS_NATIVE) + +static int cmd_sync(void) +{ + char Buffer[MAX_PATH]; + DWORD dwRet; + char szVolumeAccessPath[] = "\\\\.\\X:"; + HANDLE hVolWrite; + int success = 0; + + dwRet = GetCurrentDirectory(MAX_PATH, Buffer); + if ((0 == dwRet) || (dwRet > MAX_PATH)) + return error("Error getting current directory"); + + if ((Buffer[0] < 'A') || (Buffer[0] > 'Z')) + return error("Invalid drive letter '%c'", Buffer[0]); + + szVolumeAccessPath[4] = Buffer[0]; + hVolWrite = CreateFile(szVolumeAccessPath, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); + if (INVALID_HANDLE_VALUE == hVolWrite) + return error("Unable to open volume for writing, need admin access"); + + success = FlushFileBuffers(hVolWrite); + if (!success) + error("Unable to flush volume"); + + CloseHandle(hVolWrite); + + return !success; +} + +#define STATUS_SUCCESS (0x00000000L) +#define STATUS_PRIVILEGE_NOT_HELD (0xC0000061L) + +typedef enum _SYSTEM_INFORMATION_CLASS { + SystemMemoryListInformation = 80, +} SYSTEM_INFORMATION_CLASS; + +typedef enum _SYSTEM_MEMORY_LIST_COMMAND { + MemoryCaptureAccessedBits, + MemoryCaptureAndResetAccessedBits, + MemoryEmptyWorkingSets, + MemoryFlushModifiedList, + MemoryPurgeStandbyList, + MemoryPurgeLowPriorityStandbyList, + MemoryCommandMax +} SYSTEM_MEMORY_LIST_COMMAND; + +static BOOL GetPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags) +{ + BOOL bResult; + DWORD dwBufferLength; + LUID luid; + TOKEN_PRIVILEGES tpPreviousState; + TOKEN_PRIVILEGES tpNewState; + + dwBufferLength = 16; + bResult = LookupPrivilegeValueA(0, lpName, &luid); + if (bResult) { + tpNewState.PrivilegeCount = 1; + tpNewState.Privileges[0].Luid = luid; + tpNewState.Privileges[0].Attributes = 0; + bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpNewState, + (DWORD)((LPBYTE)&(tpNewState.Privileges[1]) - (LPBYTE)&tpNewState), + &tpPreviousState, &dwBufferLength); + if (bResult) { + tpPreviousState.PrivilegeCount = 1; + tpPreviousState.Privileges[0].Luid = luid; + tpPreviousState.Privileges[0].Attributes = flags != 0 ? 2 : 0; + bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpPreviousState, + dwBufferLength, 0, 0); + } + } + return bResult; +} + +static int cmd_dropcaches(void) +{ + HANDLE hProcess = GetCurrentProcess(); + HANDLE hToken; + HMODULE ntdll; + DWORD(WINAPI *NtSetSystemInformation)(INT, PVOID, ULONG); + SYSTEM_MEMORY_LIST_COMMAND command; + int status; + + if (!OpenProcessToken(hProcess, TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) + return error("Can't open current process token"); + + if (!GetPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1)) + return error("Can't get SeProfileSingleProcessPrivilege"); + + CloseHandle(hToken); + + ntdll = LoadLibrary("ntdll.dll"); + if (!ntdll) + return error("Can't load ntdll.dll, wrong Windows version?"); + + NtSetSystemInformation = + (DWORD(WINAPI *)(INT, PVOID, ULONG))GetProcAddress(ntdll, "NtSetSystemInformation"); + if (!NtSetSystemInformation) + return error("Can't get function addresses, wrong Windows version?"); + + command = MemoryPurgeStandbyList; + status = NtSetSystemInformation( + SystemMemoryListInformation, + &command, + sizeof(SYSTEM_MEMORY_LIST_COMMAND) + ); + if (status == STATUS_PRIVILEGE_NOT_HELD) + error("Insufficient privileges to purge the standby list, need admin access"); + else if (status != STATUS_SUCCESS) + error("Unable to execute the memory list command %d", status); + + FreeLibrary(ntdll); + + return status; +} + +#elif defined(__linux__) + +static int cmd_sync(void) +{ + return system("sync"); +} + +static int cmd_dropcaches(void) +{ + return system("echo 3 | sudo tee /proc/sys/vm/drop_caches"); +} + +#elif defined(__APPLE__) + +static int cmd_sync(void) +{ + return system("sync"); +} + +static int cmd_dropcaches(void) +{ + return system("sudo purge"); +} + +#else + +static int cmd_sync(void) +{ + return 0; +} + +static int cmd_dropcaches(void) +{ + return error("drop caches not implemented on this platform"); +} + +#endif + +int cmd_main(int argc, const char **argv) +{ + cmd_sync(); + return cmd_dropcaches(); +} diff --git a/t/helper/test-dump-fsmonitor.c b/t/helper/test-dump-fsmonitor.c new file mode 100644 index 0000000000..ad452707e8 --- /dev/null +++ b/t/helper/test-dump-fsmonitor.c @@ -0,0 +1,21 @@ +#include "cache.h" + +int cmd_main(int ac, const char **av) +{ + struct index_state *istate = &the_index; + int i; + + setup_git_directory(); + if (do_read_index(istate, get_index_file(), 0) < 0) + die("unable to read index file"); + if (!istate->fsmonitor_last_update) { + printf("no fsmonitor\n"); + return 0; + } + printf("fsmonitor last update %"PRIuMAX"\n", (uintmax_t)istate->fsmonitor_last_update); + + for (i = 0; i < istate->cache_nr; i++) + printf((istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) ? "+" : "-"); + + return 0; +} diff --git a/t/lib-gpg.sh b/t/lib-gpg.sh index 43679a4c64..a5d3b2cbaa 100755 --- a/t/lib-gpg.sh +++ b/t/lib-gpg.sh @@ -31,7 +31,7 @@ then chmod 0700 ./gpghome && GNUPGHOME="$(pwd)/gpghome" && export GNUPGHOME && - (gpgconf --kill gpg-agent 2>&1 >/dev/null || : ) && + (gpgconf --kill gpg-agent >/dev/null 2>&1 || : ) && gpg --homedir "${GNUPGHOME}" 2>/dev/null --import \ "$TEST_DIRECTORY"/lib-gpg/keyring.gpg && gpg --homedir "${GNUPGHOME}" 2>/dev/null --import-ownertrust \ diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh new file mode 100755 index 0000000000..16d1bf72e5 --- /dev/null +++ b/t/perf/p7519-fsmonitor.sh @@ -0,0 +1,184 @@ +#!/bin/sh + +test_description="Test core.fsmonitor" + +. ./perf-lib.sh + +# +# Performance test for the fsmonitor feature which enables git to talk to a +# file system change monitor and avoid having to scan the working directory +# for new or modified files. +# +# By default, the performance test will utilize the Watchman file system +# monitor if it is installed. If Watchman is not installed, it will use a +# dummy integration script that does not report any new or modified files. +# The dummy script has very little overhead which provides optimistic results. +# +# The performance test will also use the untracked cache feature if it is +# available as fsmonitor uses it to speed up scanning for untracked files. +# +# There are 3 environment variables that can be used to alter the default +# behavior of the performance test: +# +# GIT_PERF_7519_UNTRACKED_CACHE: used to configure core.untrackedCache +# GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex +# GIT_PERF_7519_FSMONITOR: used to configure core.fsMonitor +# +# The big win for using fsmonitor is the elimination of the need to scan the +# working directory looking for changed and untracked files. If the file +# information is all cached in RAM, the benefits are reduced. +# +# GIT_PERF_7519_DROP_CACHE: if set, the OS caches are dropped between tests +# + +test_perf_large_repo +test_checkout_worktree + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +test_lazy_prereq WATCHMAN ' + { command -v watchman >/dev/null 2>&1; ret=$?; } && + test $ret -ne 1 +' + +if test_have_prereq WATCHMAN +then + # Convert unix style paths to escaped Windows style paths for Watchman + case "$(uname -s)" in + MSYS_NT*) + GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,/,g')" + ;; + *) + GIT_WORK_TREE="$PWD" + ;; + esac +fi + +if test -n "$GIT_PERF_7519_DROP_CACHE" +then + # When using GIT_PERF_7519_DROP_CACHE, GIT_PERF_REPEAT_COUNT must be 1 to + # generate valid results. Otherwise the caching that happens for the nth + # run will negate the validity of the comparisons. + if test "$GIT_PERF_REPEAT_COUNT" -ne 1 + then + echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2 + GIT_PERF_REPEAT_COUNT=1 + fi +fi + +test_expect_success "setup for fsmonitor" ' + # set untrackedCache depending on the environment + if test -n "$GIT_PERF_7519_UNTRACKED_CACHE" + then + git config core.untrackedCache "$GIT_PERF_7519_UNTRACKED_CACHE" + else + if test_have_prereq UNTRACKED_CACHE + then + git config core.untrackedCache true + else + git config core.untrackedCache false + fi + fi && + + # set core.splitindex depending on the environment + if test -n "$GIT_PERF_7519_SPLIT_INDEX" + then + git config core.splitIndex "$GIT_PERF_7519_SPLIT_INDEX" + fi && + + # set INTEGRATION_SCRIPT depending on the environment + if test -n "$GIT_PERF_7519_FSMONITOR" + then + INTEGRATION_SCRIPT="$GIT_PERF_7519_FSMONITOR" + else + # + # Choose integration script based on existence of Watchman. + # If Watchman exists, watch the work tree and attempt a query. + # If everything succeeds, use Watchman integration script, + # else fall back to an empty integration script. + # + mkdir .git/hooks && + if test_have_prereq WATCHMAN + then + INTEGRATION_SCRIPT=".git/hooks/fsmonitor-watchman" && + cp "$TEST_DIRECTORY/../templates/hooks--fsmonitor-watchman.sample" "$INTEGRATION_SCRIPT" && + watchman watch "$GIT_WORK_TREE" && + watchman watch-list | grep -q -F "$GIT_WORK_TREE" + else + INTEGRATION_SCRIPT=".git/hooks/fsmonitor-empty" && + write_script "$INTEGRATION_SCRIPT"<<-\EOF + EOF + fi + fi && + + git config core.fsmonitor "$INTEGRATION_SCRIPT" && + git update-index --fsmonitor +' + +if test -n "$GIT_PERF_7519_DROP_CACHE"; then + test-drop-caches +fi + +test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" ' + git status +' + +if test -n "$GIT_PERF_7519_DROP_CACHE"; then + test-drop-caches +fi + +test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" ' + git status -uno +' + +if test -n "$GIT_PERF_7519_DROP_CACHE"; then + test-drop-caches +fi + +test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" ' + git status -uall +' + +test_expect_success "setup without fsmonitor" ' + unset INTEGRATION_SCRIPT && + git config --unset core.fsmonitor && + git update-index --no-fsmonitor +' + +if test -n "$GIT_PERF_7519_DROP_CACHE"; then + test-drop-caches +fi + +test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" ' + git status +' + +if test -n "$GIT_PERF_7519_DROP_CACHE"; then + test-drop-caches +fi + +test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" ' + git status -uno +' + +if test -n "$GIT_PERF_7519_DROP_CACHE"; then + test-drop-caches +fi + +test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" ' + git status -uall +' + +if test_have_prereq WATCHMAN +then + watchman watch-del "$GIT_WORK_TREE" >/dev/null 2>&1 && + + # Work around Watchman bug on Windows where it holds on to handles + # preventing the removal of the trash directory + watchman shutdown-server >/dev/null 2>&1 +fi + +test_done diff --git a/t/t0025-crlf-renormalize.sh b/t/t0025-crlf-renormalize.sh new file mode 100755 index 0000000000..9d9e02a211 --- /dev/null +++ b/t/t0025-crlf-renormalize.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='CRLF renormalization' + +. ./test-lib.sh + +test_expect_success setup ' + git config core.autocrlf false && + printf "LINEONE\nLINETWO\nLINETHREE\n" >LF.txt && + printf "LINEONE\r\nLINETWO\r\nLINETHREE\r\n" >CRLF.txt && + printf "LINEONE\r\nLINETWO\nLINETHREE\n" >CRLF_mix_LF.txt && + git add . && + git commit -m initial +' + +test_expect_success 'renormalize CRLF in repo' ' + echo "*.txt text=auto" >.gitattributes && + git add --renormalize "*.txt" && + cat >expect <<-\EOF && + i/lf w/crlf attr/text=auto CRLF.txt + i/lf w/lf attr/text=auto LF.txt + i/lf w/mixed attr/text=auto CRLF_mix_LF.txt + EOF + git ls-files --eol | + sed -e "s/ / /g" -e "s/ */ /g" | + sort >actual && + test_cmp expect actual +' + +test_done diff --git a/t/t1430-bad-ref-name.sh b/t/t1430-bad-ref-name.sh index e88349c8a0..c7878a60ed 100755 --- a/t/t1430-bad-ref-name.sh +++ b/t/t1430-bad-ref-name.sh @@ -331,4 +331,47 @@ test_expect_success 'update-ref --stdin -z fails delete with bad ref name' ' grep "fatal: invalid ref format: ~a" err ' +test_expect_success 'branch rejects HEAD as a branch name' ' + test_must_fail git branch HEAD HEAD^ && + test_must_fail git show-ref refs/heads/HEAD +' + +test_expect_success 'checkout -b rejects HEAD as a branch name' ' + test_must_fail git checkout -B HEAD HEAD^ && + test_must_fail git show-ref refs/heads/HEAD +' + +test_expect_success 'update-ref can operate on refs/heads/HEAD' ' + git update-ref refs/heads/HEAD HEAD^ && + git show-ref refs/heads/HEAD && + git update-ref -d refs/heads/HEAD && + test_must_fail git show-ref refs/heads/HEAD +' + +test_expect_success 'branch -d can remove refs/heads/HEAD' ' + git update-ref refs/heads/HEAD HEAD^ && + git branch -d HEAD && + test_must_fail git show-ref refs/heads/HEAD +' + +test_expect_success 'branch -m can rename refs/heads/HEAD' ' + git update-ref refs/heads/HEAD HEAD^ && + git branch -m HEAD tail && + test_must_fail git show-ref refs/heads/HEAD && + git show-ref refs/heads/tail +' + +test_expect_success 'branch -d can remove refs/heads/-dash' ' + git update-ref refs/heads/-dash HEAD^ && + git branch -d -- -dash && + test_must_fail git show-ref refs/heads/-dash +' + +test_expect_success 'branch -m can rename refs/heads/-dash' ' + git update-ref refs/heads/-dash HEAD^ && + git branch -m -- -dash dash && + test_must_fail git show-ref refs/heads/-dash && + git show-ref refs/heads/dash +' + test_done diff --git a/t/t1700-split-index.sh b/t/t1700-split-index.sh index 22f69a410b..af9b847761 100755 --- a/t/t1700-split-index.sh +++ b/t/t1700-split-index.sh @@ -6,6 +6,7 @@ test_description='split index mode tests' # We need total control of index splitting here sane_unset GIT_TEST_SPLIT_INDEX +sane_unset GIT_FSMONITOR_TEST test_expect_success 'enable split index' ' git config splitIndex.maxPercentChange 100 && diff --git a/t/t3310-notes-merge-manual-resolve.sh b/t/t3310-notes-merge-manual-resolve.sh index baef2d6924..9c1bf6eb3d 100755 --- a/t/t3310-notes-merge-manual-resolve.sh +++ b/t/t3310-notes-merge-manual-resolve.sh @@ -176,7 +176,7 @@ git rev-parse refs/notes/z > pre_merge_z test_expect_success 'merge z into m (== y) with default ("manual") resolver => Conflicting 3-way merge' ' git update-ref refs/notes/m refs/notes/y && git config core.notesRef refs/notes/m && - test_must_fail git notes merge z >output && + test_must_fail git notes merge z >output 2>&1 && # Output should point to where to resolve conflicts test_i18ngrep "\\.git/NOTES_MERGE_WORKTREE" output && # Inspect merge conflicts @@ -379,7 +379,7 @@ git rev-parse refs/notes/z > pre_merge_z test_expect_success 'redo merge of z into m (== y) with default ("manual") resolver => Conflicting 3-way merge' ' git update-ref refs/notes/m refs/notes/y && git config core.notesRef refs/notes/m && - test_must_fail git notes merge z >output && + test_must_fail git notes merge z >output 2>&1 && # Output should point to where to resolve conflicts test_i18ngrep "\\.git/NOTES_MERGE_WORKTREE" output && # Inspect merge conflicts @@ -413,7 +413,7 @@ git rev-parse refs/notes/y > pre_merge_y git rev-parse refs/notes/z > pre_merge_z test_expect_success 'redo merge of z into m (== y) with default ("manual") resolver => Conflicting 3-way merge' ' - test_must_fail git notes merge z >output && + test_must_fail git notes merge z >output 2>&1 && # Output should point to where to resolve conflicts test_i18ngrep "\\.git/NOTES_MERGE_WORKTREE" output && # Inspect merge conflicts @@ -494,7 +494,7 @@ cp expect_log_y expect_log_m test_expect_success 'redo merge of z into m (== y) with default ("manual") resolver => Conflicting 3-way merge' ' git update-ref refs/notes/m refs/notes/y && - test_must_fail git notes merge z >output && + test_must_fail git notes merge z >output 2>&1 && # Output should point to where to resolve conflicts test_i18ngrep "\\.git/NOTES_MERGE_WORKTREE" output && # Inspect merge conflicts diff --git a/t/t3320-notes-merge-worktrees.sh b/t/t3320-notes-merge-worktrees.sh index b9c3bc2487..10bfc8b947 100755 --- a/t/t3320-notes-merge-worktrees.sh +++ b/t/t3320-notes-merge-worktrees.sh @@ -61,7 +61,7 @@ test_expect_success 'merge z into x while mid-merge on y succeeds' ' ( cd worktree2 && git config core.notesRef refs/notes/x && - test_must_fail git notes merge z 2>&1 >out && + test_must_fail git notes merge z >out 2>&1 && test_i18ngrep "Automatic notes merge failed" out && grep -v "A notes merge into refs/notes/x is already in-progress in" out ) && diff --git a/t/t3400-rebase.sh b/t/t3400-rebase.sh index f5fd15e559..8ac58d5ea5 100755 --- a/t/t3400-rebase.sh +++ b/t/t3400-rebase.sh @@ -255,4 +255,26 @@ test_expect_success 'rebase commit with an ancient timestamp' ' grep "author .* 34567 +0600$" actual ' +test_expect_success 'rebase with "From " line in commit message' ' + git checkout -b preserve-from master~1 && + cat >From_.msg <<EOF && +Somebody embedded an mbox in a commit message + +This is from so-and-so: + +From a@b Mon Sep 17 00:00:00 2001 +From: John Doe <nobody@example.com> +Date: Sat, 11 Nov 2017 00:00:00 +0000 +Subject: not this message + +something +EOF + >From_ && + git add From_ && + git commit -F From_.msg && + git rebase master && + git log -1 --pretty=format:%B >out && + test_cmp From_.msg out +' + test_done diff --git a/t/t3512-cherry-pick-submodule.sh b/t/t3512-cherry-pick-submodule.sh index 6863b7bb6f..ce48c4fcca 100755 --- a/t/t3512-cherry-pick-submodule.sh +++ b/t/t3512-cherry-pick-submodule.sh @@ -10,4 +10,40 @@ KNOWN_FAILURE_NOFF_MERGE_DOESNT_CREATE_EMPTY_SUBMODULE_DIR=1 KNOWN_FAILURE_NOFF_MERGE_ATTEMPTS_TO_MERGE_REMOVED_SUBMODULE_FILES=1 test_submodule_switch "git cherry-pick" +test_expect_success 'unrelated submodule/file conflict is ignored' ' + test_create_repo sub && + + touch sub/file && + git -C sub add file && + git -C sub commit -m "add a file in a submodule" && + + test_create_repo a_repo && + ( + cd a_repo && + >a_file && + git add a_file && + git commit -m "add a file" && + + git branch test && + git checkout test && + + mkdir sub && + >sub/content && + git add sub/content && + git commit -m "add a regular folder with name sub" && + + echo "123" >a_file && + git add a_file && + git commit -m "modify a file" && + + git checkout master && + + git submodule add ../sub sub && + git submodule update sub && + git commit -m "add a submodule info folder with name sub" && + + git cherry-pick test + ) +' + test_done diff --git a/t/t4015-diff-whitespace.sh b/t/t4015-diff-whitespace.sh index 6c9a93b734..559a7541a8 100755 --- a/t/t4015-diff-whitespace.sh +++ b/t/t4015-diff-whitespace.sh @@ -106,6 +106,8 @@ test_expect_success 'another test, without options' ' git diff -w -b --ignore-space-at-eol >out && test_cmp expect out && + git diff -w --ignore-cr-at-eol >out && + test_cmp expect out && tr "Q_" "\015 " <<-\EOF >expect && diff --git a/x b/x @@ -128,6 +130,9 @@ test_expect_success 'another test, without options' ' git diff -b --ignore-space-at-eol >out && test_cmp expect out && + git diff -b --ignore-cr-at-eol >out && + test_cmp expect out && + tr "Q_" "\015 " <<-\EOF >expect && diff --git a/x b/x index d99af23..22d9f73 100644 @@ -145,6 +150,29 @@ test_expect_success 'another test, without options' ' CR at end EOF git diff --ignore-space-at-eol >out && + test_cmp expect out && + + git diff --ignore-space-at-eol --ignore-cr-at-eol >out && + test_cmp expect out && + + tr "Q_" "\015 " <<-\EOF >expect && + diff --git a/x b/x + index_d99af23..22d9f73 100644 + --- a/x + +++ b/x + @@ -1,6 +1,6 @@ + -whitespace at beginning + -whitespace change + -whitespace in the middle + -whitespace at end + +_ whitespace at beginning + +whitespace_ _change + +white space in the middle + +whitespace at end__ + unchanged line + CR at end + EOF + git diff --ignore-cr-at-eol >out && test_cmp expect out ' diff --git a/t/t4107-apply-ignore-whitespace.sh b/t/t4107-apply-ignore-whitespace.sh index 9e29b5262d..ac72eeaf27 100755 --- a/t/t4107-apply-ignore-whitespace.sh +++ b/t/t4107-apply-ignore-whitespace.sh @@ -178,4 +178,18 @@ test_expect_success 'patch5 fails (--no-ignore-whitespace)' ' test_must_fail git apply --no-ignore-whitespace patch5.patch ' +test_expect_success 'apply --ignore-space-change --inaccurate-eof' ' + echo 1 >file && + git apply --ignore-space-change --inaccurate-eof <<-\EOF && + diff --git a/file b/file + --- a/file + +++ b/file + @@ -1 +1 @@ + -1 + +2 + EOF + printf 2 >expect && + test_cmp expect file +' + test_done diff --git a/t/t7006-pager.sh b/t/t7006-pager.sh index 865168ec6a..f5f46a95b4 100755 --- a/t/t7006-pager.sh +++ b/t/t7006-pager.sh @@ -214,6 +214,44 @@ test_expect_success TTY 'git tag as alias respects pager.tag with -l' ' ! test -e paginated.out ' +test_expect_success TTY 'git branch defaults to paging' ' + rm -f paginated.out && + test_terminal git branch && + test -e paginated.out +' + +test_expect_success TTY 'git branch respects pager.branch' ' + rm -f paginated.out && + test_terminal git -c pager.branch=false branch && + ! test -e paginated.out +' + +test_expect_success TTY 'git branch respects --no-pager' ' + rm -f paginated.out && + test_terminal git --no-pager branch && + ! test -e paginated.out +' + +test_expect_success TTY 'git branch --edit-description ignores pager.branch' ' + rm -f paginated.out editor.used && + write_script editor <<-\EOF && + echo "New description" >"$1" + touch editor.used + EOF + EDITOR=./editor test_terminal git -c pager.branch branch --edit-description && + ! test -e paginated.out && + test -e editor.used +' + +test_expect_success TTY 'git branch --set-upstream-to ignores pager.branch' ' + rm -f paginated.out && + git branch other && + test_when_finished "git branch -D other" && + test_terminal git -c pager.branch branch --set-upstream-to=other && + test_when_finished "git branch --unset-upstream" && + ! test -e paginated.out +' + # A colored commit log will begin with an appropriate ANSI escape # for the first color; the text "commit" comes later. colorful() { diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh new file mode 100755 index 0000000000..eb2d13bbcf --- /dev/null +++ b/t/t7519-status-fsmonitor.sh @@ -0,0 +1,317 @@ +#!/bin/sh + +test_description='git status with file system watcher' + +. ./test-lib.sh + +# +# To run the entire git test suite using fsmonitor: +# +# copy t/t7519/fsmonitor-all to a location in your path and then set +# GIT_FSMONITOR_TEST=fsmonitor-all and run your tests. +# + +# Note, after "git reset --hard HEAD" no extensions exist other than 'TREE' +# "git update-index --fsmonitor" can be used to get the extension written +# before testing the results. + +clean_repo () { + git reset --hard HEAD && + git clean -fd +} + +dirty_repo () { + : >untracked && + : >dir1/untracked && + : >dir2/untracked && + echo 1 >modified && + echo 2 >dir1/modified && + echo 3 >dir2/modified && + echo 4 >new && + echo 5 >dir1/new && + echo 6 >dir2/new +} + +write_integration_script () { + write_script .git/hooks/fsmonitor-test<<-\EOF + if test "$#" -ne 2 + then + echo "$0: exactly 2 arguments expected" + exit 2 + fi + if test "$1" != 1 + then + echo "Unsupported core.fsmonitor hook version." >&2 + exit 1 + fi + printf "untracked\0" + printf "dir1/untracked\0" + printf "dir2/untracked\0" + printf "modified\0" + printf "dir1/modified\0" + printf "dir2/modified\0" + printf "new\0" + printf "dir1/new\0" + printf "dir2/new\0" + EOF +} + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +test_expect_success 'setup' ' + mkdir -p .git/hooks && + : >tracked && + : >modified && + mkdir dir1 && + : >dir1/tracked && + : >dir1/modified && + mkdir dir2 && + : >dir2/tracked && + : >dir2/modified && + git -c core.fsmonitor= add . && + git -c core.fsmonitor= commit -m initial && + git config core.fsmonitor .git/hooks/fsmonitor-test && + cat >.gitignore <<-\EOF + .gitignore + expect* + actual* + marker* + EOF +' + +# test that the fsmonitor extension is off by default +test_expect_success 'fsmonitor extension is off by default' ' + test-dump-fsmonitor >actual && + grep "^no fsmonitor" actual +' + +# test that "update-index --fsmonitor" adds the fsmonitor extension +test_expect_success 'update-index --fsmonitor" adds the fsmonitor extension' ' + git update-index --fsmonitor && + test-dump-fsmonitor >actual && + grep "^fsmonitor last update" actual +' + +# test that "update-index --no-fsmonitor" removes the fsmonitor extension +test_expect_success 'update-index --no-fsmonitor" removes the fsmonitor extension' ' + git update-index --no-fsmonitor && + test-dump-fsmonitor >actual && + grep "^no fsmonitor" actual +' + +cat >expect <<EOF && +h dir1/modified +H dir1/tracked +h dir2/modified +H dir2/tracked +h modified +H tracked +EOF + +# test that "update-index --fsmonitor-valid" sets the fsmonitor valid bit +test_expect_success 'update-index --fsmonitor-valid" sets the fsmonitor valid bit' ' + git update-index --fsmonitor && + git update-index --fsmonitor-valid dir1/modified && + git update-index --fsmonitor-valid dir2/modified && + git update-index --fsmonitor-valid modified && + git ls-files -f >actual && + test_cmp expect actual +' + +cat >expect <<EOF && +H dir1/modified +H dir1/tracked +H dir2/modified +H dir2/tracked +H modified +H tracked +EOF + +# test that "update-index --no-fsmonitor-valid" clears the fsmonitor valid bit +test_expect_success 'update-index --no-fsmonitor-valid" clears the fsmonitor valid bit' ' + git update-index --no-fsmonitor-valid dir1/modified && + git update-index --no-fsmonitor-valid dir2/modified && + git update-index --no-fsmonitor-valid modified && + git ls-files -f >actual && + test_cmp expect actual +' + +cat >expect <<EOF && +H dir1/modified +H dir1/tracked +H dir2/modified +H dir2/tracked +H modified +H tracked +EOF + +# test that all files returned by the script get flagged as invalid +test_expect_success 'all files returned by integration script get flagged as invalid' ' + write_integration_script && + dirty_repo && + git update-index --fsmonitor && + git ls-files -f >actual && + test_cmp expect actual +' + +cat >expect <<EOF && +H dir1/modified +h dir1/new +H dir1/tracked +H dir2/modified +h dir2/new +H dir2/tracked +H modified +h new +H tracked +EOF + +# test that newly added files are marked valid +test_expect_success 'newly added files are marked valid' ' + git add new && + git add dir1/new && + git add dir2/new && + git ls-files -f >actual && + test_cmp expect actual +' + +cat >expect <<EOF && +H dir1/modified +h dir1/new +h dir1/tracked +H dir2/modified +h dir2/new +h dir2/tracked +H modified +h new +h tracked +EOF + +# test that all unmodified files get marked valid +test_expect_success 'all unmodified files get marked valid' ' + # modified files result in update-index returning 1 + test_must_fail git update-index --refresh --force-write-index && + git ls-files -f >actual && + test_cmp expect actual +' + +cat >expect <<EOF && +H dir1/modified +h dir1/tracked +h dir2/modified +h dir2/tracked +h modified +h tracked +EOF + +# test that *only* files returned by the integration script get flagged as invalid +test_expect_success '*only* files returned by the integration script get flagged as invalid' ' + write_script .git/hooks/fsmonitor-test<<-\EOF && + printf "dir1/modified\0" + EOF + clean_repo && + git update-index --refresh --force-write-index && + echo 1 >modified && + echo 2 >dir1/modified && + echo 3 >dir2/modified && + test_must_fail git update-index --refresh --force-write-index && + git ls-files -f >actual && + test_cmp expect actual +' + +# Ensure commands that call refresh_index() to move the index back in time +# properly invalidate the fsmonitor cache +test_expect_success 'refresh_index() invalidates fsmonitor cache' ' + write_script .git/hooks/fsmonitor-test<<-\EOF && + EOF + clean_repo && + dirty_repo && + git add . && + git commit -m "to reset" && + git reset HEAD~1 && + git status >actual && + git -c core.fsmonitor= status >expect && + test_i18ncmp expect actual +' + +# test fsmonitor with and without preloadIndex +preload_values="false true" +for preload_val in $preload_values +do + test_expect_success "setup preloadIndex to $preload_val" ' + git config core.preloadIndex $preload_val && + if test $preload_val = true + then + GIT_FORCE_PRELOAD_TEST=$preload_val; export GIT_FORCE_PRELOAD_TEST + else + unset GIT_FORCE_PRELOAD_TEST + fi + ' + + # test fsmonitor with and without the untracked cache (if available) + uc_values="false" + test_have_prereq UNTRACKED_CACHE && uc_values="false true" + for uc_val in $uc_values + do + test_expect_success "setup untracked cache to $uc_val" ' + git config core.untrackedcache $uc_val + ' + + # Status is well tested elsewhere so we'll just ensure that the results are + # the same when using core.fsmonitor. + test_expect_success 'compare status with and without fsmonitor' ' + write_integration_script && + clean_repo && + dirty_repo && + git add new && + git add dir1/new && + git add dir2/new && + git status >actual && + git -c core.fsmonitor= status >expect && + test_i18ncmp expect actual + ' + + # Make sure it's actually skipping the check for modified and untracked + # (if enabled) files unless it is told about them. + test_expect_success "status doesn't detect unreported modifications" ' + write_script .git/hooks/fsmonitor-test<<-\EOF && + :>marker + EOF + clean_repo && + git status && + test_path_is_file marker && + dirty_repo && + rm -f marker && + git status >actual && + test_path_is_file marker && + test_i18ngrep ! "Changes not staged for commit:" actual && + if test $uc_val = true + then + test_i18ngrep ! "Untracked files:" actual + fi && + if test $uc_val = false + then + test_i18ngrep "Untracked files:" actual + fi && + rm -f marker + ' + done +done + +# test that splitting the index dosn't interfere +test_expect_success 'splitting the index results in the same state' ' + write_integration_script && + dirty_repo && + git update-index --fsmonitor && + git ls-files -f >expect && + test-dump-fsmonitor >&2 && echo && + git update-index --fsmonitor --split-index && + test-dump-fsmonitor >&2 && echo && + git ls-files -f >actual && + test_cmp expect actual +' + +test_done diff --git a/t/t7519/fsmonitor-all b/t/t7519/fsmonitor-all new file mode 100755 index 0000000000..691bc94dc2 --- /dev/null +++ b/t/t7519/fsmonitor-all @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An test hook script to integrate with git to test fsmonitor. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +#echo "$0 $*" >&2 + +if test "$#" -ne 2 +then + echo "$0: exactly 2 arguments expected" >&2 + exit 2 +fi + +if test "$1" != 1 +then + echo "Unsupported core.fsmonitor hook version." >&2 + exit 1 +fi + +echo "/" diff --git a/t/t7519/fsmonitor-none b/t/t7519/fsmonitor-none new file mode 100755 index 0000000000..ed9cf5a6a9 --- /dev/null +++ b/t/t7519/fsmonitor-none @@ -0,0 +1,22 @@ +#!/bin/sh +# +# An test hook script to integrate with git to test fsmonitor. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +#echo "$0 $*" >&2 + +if test "$#" -ne 2 +then + echo "$0: exactly 2 arguments expected" >&2 + exit 2 +fi + +if test "$1" != 1 +then + echo "Unsupported core.fsmonitor hook version." >&2 + exit 1 +fi diff --git a/t/t7519/fsmonitor-watchman b/t/t7519/fsmonitor-watchman new file mode 100755 index 0000000000..5514edcf68 --- /dev/null +++ b/t/t7519/fsmonitor-watchman @@ -0,0 +1,133 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $time) = @ARGV; +#print STDERR "$0 $version $time\n"; + +# Check the hook interface version + +if ($version == 1) { + # convert nanoseconds to seconds + $time = int $time / 1000000000; +} else { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree; +if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $git_work_tree = Win32::GetCwd(); + $git_work_tree =~ tr/\\/\//; +} else { + require Cwd; + $git_work_tree = Cwd::cwd(); +} + +my $retry = 1; + +launch_watchman(); + +sub launch_watchman { + + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $time but were not transient (ie created after + # $time but no longer exist). + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + # + # The category of transient files that we want to ignore will have a + # creation clock (cclock) newer than $time_t value and will also not + # currently exist. + + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $time, + "fields": ["name"], + "expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]] + }] + END + + open (my $fh, ">", ".git/watchman-query.json"); + print $fh $query; + close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; <CHLD_OUT>}; + + open ($fh, ">", ".git/watchman-response.json"); + print $fh $response; + close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + my $json_pkg; + eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; + } or do { + require JSON::PP; + $json_pkg = "JSON::PP"; + }; + + my $o = $json_pkg->new->utf8->decode($response); + + if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { + print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; + $retry--; + qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + + open ($fh, ">", ".git/watchman-output.out"); + print "/\0"; + close $fh; + + print "/\0"; + eval { launch_watchman() }; + exit 0; + } + + die "Watchman: $o->{error}.\n" . + "Falling back to scanning...\n" if $o->{error}; + + open ($fh, ">", ".git/watchman-output.out"); + binmode $fh, ":utf8"; + print $fh @{$o->{files}}; + close $fh; + + binmode STDOUT, ":utf8"; + local $, = "\0"; + print @{$o->{files}}; +} diff --git a/templates/hooks--fsmonitor-watchman.sample b/templates/hooks--fsmonitor-watchman.sample new file mode 100755 index 0000000000..e673bb3980 --- /dev/null +++ b/templates/hooks--fsmonitor-watchman.sample @@ -0,0 +1,114 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $time) = @ARGV; + +# Check the hook interface version + +if ($version == 1) { + # convert nanoseconds to seconds + $time = int $time / 1000000000; +} else { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree; +if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $git_work_tree = Win32::GetCwd(); + $git_work_tree =~ tr/\\/\//; +} else { + require Cwd; + $git_work_tree = Cwd::cwd(); +} + +my $retry = 1; + +launch_watchman(); + +sub launch_watchman { + + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $time but were not transient (ie created after + # $time but no longer exist). + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + # + # The category of transient files that we want to ignore will have a + # creation clock (cclock) newer than $time_t value and will also not + # currently exist. + + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $time, + "fields": ["name"], + "expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]] + }] + END + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; <CHLD_OUT>}; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + my $json_pkg; + eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; + } or do { + require JSON::PP; + $json_pkg = "JSON::PP"; + }; + + my $o = $json_pkg->new->utf8->decode($response); + + if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { + print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; + $retry--; + qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + print "/\0"; + eval { launch_watchman() }; + exit 0; + } + + die "Watchman: $o->{error}.\n" . + "Falling back to scanning...\n" if $o->{error}; + + binmode STDOUT, ":utf8"; + local $, = "\0"; + print @{$o->{files}}; +} diff --git a/unpack-trees.c b/unpack-trees.c index 25740cb593..bf8b602901 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -14,6 +14,7 @@ #include "dir.h" #include "submodule.h" #include "submodule-config.h" +#include "fsmonitor.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -408,6 +409,7 @@ static int apply_sparse_checkout(struct index_state *istate, ce->ce_flags &= ~CE_SKIP_WORKTREE; if (was_skip_worktree != ce_skip_worktree(ce)) { ce->ce_flags |= CE_UPDATE_IN_BASE; + mark_fsmonitor_invalid(istate, ce); istate->cache_changed |= CE_ENTRY_CHANGED; } diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index 785ecb0899..915591f7d4 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -27,22 +27,28 @@ extern "C" { #endif /* #ifdef __cplusplus */ +/* xpparm_t.flags */ +#define XDF_NEED_MINIMAL (1 << 0) + +#define XDF_IGNORE_WHITESPACE (1 << 1) +#define XDF_IGNORE_WHITESPACE_CHANGE (1 << 2) +#define XDF_IGNORE_WHITESPACE_AT_EOL (1 << 3) +#define XDF_IGNORE_CR_AT_EOL (1 << 4) +#define XDF_WHITESPACE_FLAGS (XDF_IGNORE_WHITESPACE | \ + XDF_IGNORE_WHITESPACE_CHANGE | \ + XDF_IGNORE_WHITESPACE_AT_EOL | \ + XDF_IGNORE_CR_AT_EOL) -#define XDF_NEED_MINIMAL (1 << 1) -#define XDF_IGNORE_WHITESPACE (1 << 2) -#define XDF_IGNORE_WHITESPACE_CHANGE (1 << 3) -#define XDF_IGNORE_WHITESPACE_AT_EOL (1 << 4) -#define XDF_WHITESPACE_FLAGS (XDF_IGNORE_WHITESPACE | XDF_IGNORE_WHITESPACE_CHANGE | XDF_IGNORE_WHITESPACE_AT_EOL) +#define XDF_IGNORE_BLANK_LINES (1 << 7) -#define XDF_PATIENCE_DIFF (1 << 5) -#define XDF_HISTOGRAM_DIFF (1 << 6) +#define XDF_PATIENCE_DIFF (1 << 14) +#define XDF_HISTOGRAM_DIFF (1 << 15) #define XDF_DIFF_ALGORITHM_MASK (XDF_PATIENCE_DIFF | XDF_HISTOGRAM_DIFF) #define XDF_DIFF_ALG(x) ((x) & XDF_DIFF_ALGORITHM_MASK) -#define XDF_IGNORE_BLANK_LINES (1 << 7) - -#define XDF_INDENT_HEURISTIC (1 << 8) +#define XDF_INDENT_HEURISTIC (1 << 23) +/* xdemitconf_t.flags */ #define XDL_EMIT_FUNCNAMES (1 << 0) #define XDL_EMIT_FUNCCONTEXT (1 << 2) diff --git a/xdiff/xutils.c b/xdiff/xutils.c index 088001db4f..88e5995535 100644 --- a/xdiff/xutils.c +++ b/xdiff/xutils.c @@ -156,6 +156,24 @@ int xdl_blankline(const char *line, long size, long flags) return (i == size); } +/* + * Have we eaten everything on the line, except for an optional + * CR at the very end? + */ +static int ends_with_optional_cr(const char *l, long s, long i) +{ + int complete = s && l[s-1] == '\n'; + + if (complete) + s--; + if (s == i) + return 1; + /* do not ignore CR at the end of an incomplete line */ + if (complete && s == i + 1 && l[i] == '\r') + return 1; + return 0; +} + int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags) { int i1, i2; @@ -170,7 +188,8 @@ int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags) /* * -w matches everything that matches with -b, and -b in turn - * matches everything that matches with --ignore-space-at-eol. + * matches everything that matches with --ignore-space-at-eol, + * which in turn matches everything that matches with --ignore-cr-at-eol. * * Each flavor of ignoring needs different logic to skip whitespaces * while we have both sides to compare. @@ -204,6 +223,14 @@ int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags) i1++; i2++; } + } else if (flags & XDF_IGNORE_CR_AT_EOL) { + /* Find the first difference and see how the line ends */ + while (i1 < s1 && i2 < s2 && l1[i1] == l2[i2]) { + i1++; + i2++; + } + return (ends_with_optional_cr(l1, s1, i1) && + ends_with_optional_cr(l2, s2, i2)); } /* @@ -230,9 +257,16 @@ static unsigned long xdl_hash_record_with_whitespace(char const **data, char const *top, long flags) { unsigned long ha = 5381; char const *ptr = *data; + int cr_at_eol_only = (flags & XDF_WHITESPACE_FLAGS) == XDF_IGNORE_CR_AT_EOL; for (; ptr < top && *ptr != '\n'; ptr++) { - if (XDL_ISSPACE(*ptr)) { + if (cr_at_eol_only) { + /* do not ignore CR at the end of an incomplete line */ + if (*ptr == '\r' && + (ptr + 1 < top && ptr[1] == '\n')) + continue; + } + else if (XDL_ISSPACE(*ptr)) { const char *ptr2 = ptr; int at_eol; while (ptr + 1 < top && XDL_ISSPACE(ptr[1]) |