diff options
80 files changed, 5021 insertions, 1134 deletions
@@ -32,3 +32,7 @@ TAGS # Ninja output .ninja_deps .ninja_log + +# Visual Studio Code project files +/.vscode/ +/.ccls-cache/ diff --git a/.travis.yml b/.travis.yml index 093139b..f76b982 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,16 @@ +matrix: + include: + - os: linux + compiler: gcc + - os: linux + compiler: clang + - os: osx sudo: false language: cpp -compiler: - - gcc - - clang -script: ./configure.py --bootstrap && ./ninja all && ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots && ./misc/ninja_syntax_test.py +script: + - ./misc/ci.py + - ./configure.py --bootstrap + - ./ninja all + - ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots + - ./misc/ninja_syntax_test.py + - ./misc/output_test.py @@ -13,14 +13,50 @@ run `ninja_test` when developing. Ninja is built using itself. To bootstrap the first binary, run the configure script as `./configure.py --bootstrap`. This first compiles all non-test source files together, then re-builds Ninja using itself. -You should end up with a `ninja` binary (or `ninja.exe`) in the source root. +You should end up with a `ninja` binary (or `ninja.exe`) in the project root. #### Windows On Windows, you'll need to install Python to run `configure.py`, and run everything under a Visual Studio Tools Command Prompt (or after -running `vcvarsall` in a normal command prompt). See below if you -want to use mingw or some other compiler instead of Visual Studio. +running `vcvarsall` in a normal command prompt). + +For other combinations such as gcc/clang you will need the compiler +(gcc/cl) in your PATH and you will have to set the appropriate +platform configuration script. + +See below if you want to use mingw or some other compiler instead of +Visual Studio. + +##### Using Visual Studio +Assuming that you now have Python installed, then the steps for building under +Windows using Visual Studio are: + +Clone and checkout the latest release (or whatever branch you want). You +can do this in either a command prompt or by opening a git bash prompt: + +``` + $ git clone git://github.com/ninja-build/ninja.git && cd ninja + $ git checkout release +``` + +Then: + +1. Open a Windows command prompt in the folder where you checked out ninja. +2. Select the Microsoft build environment by running +`vcvarsall.bat` with the appropriate environment. +3. Build ninja and test it. + +The steps for a Visual Studio 2015 64-bit build are outlined here: + +``` + > "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 + > python configure.py --bootstrap + > ninja --help +``` +Copy the ninja executable to another location, if desired, e.g. C:\local\Ninja. + +Finally add the path where ninja.exe is to the PATH variable. ### Adjusting build flags @@ -73,17 +109,9 @@ build "all" before committing to verify the other source still works! ## Testing performance impact of changes -If you have a Chrome build handy, it's a good test case. Otherwise, -[the github downoads page](https://github.com/ninja-build/ninja/releases) -has a copy of the Chrome build files (and depfiles). You can untar -that, then run - - path/to/my/ninja chrome - -and compare that against a baseline Ninja. - -There's a script at `misc/measure.py` that repeatedly runs a command like -the above (to address variance) and summarizes its runtime. E.g. +If you have a Chrome build handy, it's a good test case. There's a +script at `misc/measure.py` that repeatedly runs a command (to address +variance) and summarizes its runtime. E.g. path/to/misc/measure.py path/to/my/ninja chrome @@ -95,7 +123,7 @@ and run that directly on some representative input files. Generally it's the [Google C++ coding style][], but in brief: * Function name are camelcase. -* Member methods are camelcase, expect for trivial getters which are +* Member methods are camelcase, except for trivial getters which are underscore separated. * Local variables are underscore separated. * Member variables are underscore separated and suffixed by an extra @@ -1,19 +1,20 @@ Notes to myself on all the steps to make for a Ninja release. Push new release branch: -1. Consider sending a heads-up to the ninja-build mailing list first -2. Make sure branches 'master' and 'release' are synced up locally -3. update src/version.cc with new version (with ".git"), then +1. Run afl-fuzz for a day or so (see HACKING.md) and run ninja_test +2. Consider sending a heads-up to the ninja-build mailing list first +3. Make sure branches 'master' and 'release' are synced up locally +4. Update src/version.cc with new version (with ".git"), then git commit -am 'mark this 1.5.0.git' -4. git checkout release; git merge master -5. fix version number in src/version.cc (it will likely conflict in the above) -6. fix version in doc/manual.asciidoc (exists only on release branch) -7. commit, tag, push (don't forget to push --tags) +5. git checkout release; git merge master +6. Fix version number in src/version.cc (it will likely conflict in the above) +7. Fix version in doc/manual.asciidoc (exists only on release branch) +8. commit, tag, push (don't forget to push --tags) git commit -am v1.5.0; git push origin release git tag v1.5.0; git push --tags # Push the 1.5.0.git change on master too: git checkout master; git push origin master -8. construct release notes from prior notes +9. Construct release notes from prior notes credits: git shortlog -s --no-merges REV.. Release on github: diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..4c64f29 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,40 @@ +version: 1.0.{build} +image: Visual Studio 2017 + +environment: + CLICOLOR_FORCE: 1 + CHERE_INVOKING: 1 # Tell Bash to inherit the current working directory + matrix: + - MSYSTEM: MINGW64 + - MSYSTEM: MSVC + +for: + - + matrix: + only: + - MSYSTEM: MINGW64 + build_script: + ps: "C:\\msys64\\usr\\bin\\bash -lc @\"\n + pacman -S --quiet --noconfirm --needed re2c 2>&1\n + sed -i 's|cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out|$ar crs $out $in|g' configure.py\n + ./configure.py --bootstrap --platform mingw 2>&1\n + ./ninja all\n + ./ninja_test 2>&1\n + ./misc/ninja_syntax_test.py 2>&1\n\"@" + - + matrix: + only: + - MSYSTEM: MSVC + build_script: + - cmd: >- + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" + + python configure.py --bootstrap + + ninja.bootstrap.exe all + + ninja_test + + python misc/ninja_syntax_test.py + +test: off diff --git a/configure.py b/configure.py index a443748..850bb98 100755 --- a/configure.py +++ b/configure.py @@ -98,7 +98,7 @@ class Platform(object): return self._platform == 'aix' def uses_usr_local(self): - return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly') + return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly', 'netbsd') def supports_ppoll(self): return self._platform in ('freebsd', 'linux', 'openbsd', 'bitrig', @@ -256,7 +256,7 @@ configure_args = sys.argv[1:] if '--bootstrap' in configure_args: configure_args.remove('--bootstrap') n.variable('configure_args', ' '.join(configure_args)) -env_keys = set(['CXX', 'AR', 'CFLAGS', 'LDFLAGS']) +env_keys = set(['CXX', 'AR', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']) configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys) if configure_env: config_str = ' '.join([k + '=' + pipes.quote(configure_env[k]) @@ -356,6 +356,11 @@ else: if platform.uses_usr_local(): cflags.append('-I/usr/local/include') ldflags.append('-L/usr/local/lib') + if platform.is_aix(): + # printf formats for int64_t, uint64_t; large file support + cflags.append('-D__STDC_FORMAT_MACROS') + cflags.append('-D_LARGE_FILES') + libs = [] @@ -397,6 +402,10 @@ def shell_escape(str): if 'CFLAGS' in configure_env: cflags.append(configure_env['CFLAGS']) + ldflags.append(configure_env['CFLAGS']) +if 'CXXFLAGS' in configure_env: + cflags.append(configure_env['CXXFLAGS']) + ldflags.append(configure_env['CXXFLAGS']) n.variable('cflags', ' '.join(shell_escape(flag) for flag in cflags)) if 'LDFLAGS' in configure_env: ldflags.append(configure_env['LDFLAGS']) @@ -405,7 +414,7 @@ n.newline() if platform.is_msvc(): n.rule('cxx', - command='$cxx $cflags -c $in /Fo$out', + command='$cxx $cflags -c $in /Fo$out /Fd' + built('$pdb'), description='CXX $out', deps='msvc' # /showIncludes is included in $cflags. ) @@ -476,6 +485,9 @@ else: n.newline() n.comment('Core source files all build into ninja library.') +cxxvariables = [] +if platform.is_msvc(): + cxxvariables = [('pdb', 'ninja.pdb')] for name in ['build', 'build_log', 'clean', @@ -484,6 +496,8 @@ for name in ['build', 'depfile_parser', 'deps_log', 'disk_interface', + 'dyndep', + 'dyndep_parser', 'edit_distance', 'eval_env', 'graph', @@ -492,19 +506,20 @@ for name in ['build', 'line_printer', 'manifest_parser', 'metrics', + 'parser', 'state', 'string_piece_util', 'util', 'version']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['subprocess-win32', 'includes_normalize-win32', 'msvc_helper-win32', 'msvc_helper_main-win32']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) if platform.is_msvc(): - objs += cxx('minidump-win32') + objs += cxx('minidump-win32', variables=cxxvariables) objs += cc('getopt') else: objs += cxx('subprocess-posix') @@ -527,7 +542,7 @@ if platform.is_aix(): all_targets = [] n.comment('Main executable is library plus main() function.') -objs = cxx('ninja') +objs = cxx('ninja', variables=cxxvariables) ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) n.newline() @@ -542,6 +557,8 @@ if options.bootstrap: n.comment('Tests all build into ninja_test executable.') objs = [] +if platform.is_msvc(): + cxxvariables = [('pdb', 'ninja_test.pdb')] for name in ['build_log_test', 'build_test', @@ -549,6 +566,7 @@ for name in ['build_log_test', 'clparser_test', 'depfile_parser_test', 'deps_log_test', + 'dyndep_parser_test', 'disk_interface_test', 'edit_distance_test', 'graph_test', @@ -560,10 +578,10 @@ for name in ['build_log_test', 'subprocess_test', 'test', 'util_test']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['includes_normalize_test', 'msvc_helper_test']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) @@ -579,7 +597,9 @@ for name in ['build_log_perftest', 'hash_collision_bench', 'manifest_parser_perftest', 'clparser_perftest']: - objs = cxx(name) + if platform.is_msvc(): + cxxvariables = [('pdb', name + '.pdb')] + objs = cxx(name, variables=cxxvariables) all_targets += n.build(binary(name), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) diff --git a/doc/docbook.xsl b/doc/docbook.xsl index 19cc126..2235be2 100644 --- a/doc/docbook.xsl +++ b/doc/docbook.xsl @@ -21,6 +21,9 @@ <!-- Don't put the "Chapter 1." prefix on the "chapters". --> <xsl:param name="chapter.autolabel">0</xsl:param> + <!-- Make builds reproducible by generating the same IDs from the same inputs --> + <xsl:param name="generate.consistent.ids">1</xsl:param> + <!-- Use <ul> for the table of contents. By default DocBook uses a <dl>, which makes no semantic sense. I imagine they just did it because it looks nice? --> diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index df4199a..e49d26d 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -154,18 +154,17 @@ design is quite clever. Ninja's benefit comes from using it in conjunction with a smarter meta-build system. -http://code.google.com/p/gyp/[gyp]:: The meta-build system used to +https://gn.googlesource.com/gn/[gn]:: The meta-build system used to generate build files for Google Chrome and related projects (v8, -node.js). gyp can generate Ninja files for all platforms supported by -Chrome. See the -https://chromium.googlesource.com/chromium/src/+/master/docs/ninja_build.md[Chromium Ninja documentation for more details]. +node.js), as well as Google Fuchsia. gn can generate Ninja files for +all platforms supported by Chrome. https://cmake.org/[CMake]:: A widely used meta-build system that can generate Ninja files on Linux as of CMake version 2.8.8. Newer versions of CMake support generating Ninja files on Windows and Mac OS X too. https://github.com/ninja-build/ninja/wiki/List-of-generators-producing-ninja-build-files[others]:: Ninja ought to fit perfectly into other meta-build software -like http://industriousone.com/premake[premake]. If you do this work, +like https://premake.github.io/[premake]. If you do this work, please let us know! Running Ninja @@ -279,6 +278,14 @@ http://clang.llvm.org/docs/JSONCompilationDatabase.html[JSON format] expected by the Clang tooling interface. _Available since Ninja 1.2._ +`deps`:: show all dependencies stored in the `.ninja_deps` file. When given a +target, show just the target's dependencies. _Available since Ninja 1.4._ + +`recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._ + +`rules`:: output the list of all rules (eventually with their description +if they have one). It can be used to know which rule name to pass to ++ninja -t targets rule _name_+ or +ninja -t compdb+. Writing your own Ninja files ---------------------------- @@ -596,7 +603,7 @@ Ninja supports this processing in two forms. to its stdout. Ninja then filters these lines from the displayed output. No `depfile` attribute is necessary, but the localized string in front of the the header file path. For instance - `msvc_deps_prefix = Note: including file: ` + `msvc_deps_prefix = Note: including file:` for a English Visual Studio (the default). Should be globally defined. + ---- @@ -683,6 +690,7 @@ While a task in the `console` pool is running, Ninja's regular output (such as progress status and output from concurrent tasks) is buffered until it completes. +[[ref_ninja_file]] Ninja file reference -------------------- @@ -714,6 +722,7 @@ the `:` with +| _output1_ _output2_+ and do not appear in `$out`. 6. A pool declaration, which looks like +pool _poolname_+. Pools are explained <<ref_pool, in the section on pools>>. +[[ref_lexer]] Lexical syntax ~~~~~~~~~~~~~~ @@ -818,6 +827,11 @@ keys. the full command or its description; if a command fails, the full command line will always be printed before the command's output. +`dyndep`:: _(Available since Ninja 1.10.)_ Used only on build statements. + If present, must name one of the build statement inputs. Dynamically + discovered dependency information will be loaded from the file. + See the <<ref_dyndep,dynamic dependencies>> section for details. + `generator`:: if present, specifies that this rule is used to re-invoke the generator program. Files built using `generator` rules are treated specially in two ways: firstly, they will not be @@ -883,7 +897,8 @@ quoting rules are deterimined by the called program, which on Windows are usually provided by the C library. If you need shell interpretation of the command (such as the use of `&&` to chain multiple commands), make the command execute the Windows shell by -prefixing the command with `cmd /c`. +prefixing the command with `cmd /c`. Ninja may error with "invalid parameter" +which usually indicates that the command line length has been exceeded. [[ref_outputs]] Build outputs @@ -1006,3 +1021,146 @@ Variable declarations indented in a `build` block are scoped to the 5. Variables from the file that included that file using the `subninja` keyword. + +[[ref_dyndep]] +Dynamic Dependencies +-------------------- + +_Available since Ninja 1.10._ + +Some use cases require implicit dependency information to be dynamically +discovered from source file content _during the build_ in order to build +correctly on the first run (e.g. Fortran module dependencies). This is +unlike <<ref_headers,header dependencies>> which are only needed on the +second run and later to rebuild correctly. A build statement may have a +`dyndep` binding naming one of its inputs to specify that dynamic +dependency information must be loaded from the file. For example: + +---- +build out: ... || foo + dyndep = foo +build foo: ... +---- + +This specifies that file `foo` is a dyndep file. Since it is an input, +the build statement for `out` can never be executed before `foo` is built. +As soon as `foo` is finished Ninja will read it to load dynamically +discovered dependency information for `out`. This may include additional +implicit inputs and/or outputs. Ninja will update the build graph +accordingly and the build will proceed as if the information was known +originally. + +Dyndep file reference +~~~~~~~~~~~~~~~~~~~~~ + +Files specified by `dyndep` bindings use the same <<ref_lexer,lexical syntax>> +as <<ref_ninja_file,ninja build files>> and have the following layout. + +1. A version number in the form `<major>[.<minor>][<suffix>]`: ++ +---- +ninja_dyndep_version = 1 +---- ++ +Currently the version number must always be `1` or `1.0` but may have +an arbitrary suffix. + +2. One or more build statements of the form: ++ +---- +build out | imp-outs... : dyndep | imp-ins... +---- ++ +Every statement must specify exactly one explicit output and must use +the rule name `dyndep`. The `| imp-outs...` and `| imp-ins...` portions +are optional. + +3. An optional `restat` <<ref_rule,variable binding>> on each build statement. + +The build statements in a dyndep file must have a one-to-one correspondence +to build statements in the <<ref_ninja_file,ninja build file>> that name the +dyndep file in a `dyndep` binding. No dyndep build statement may be omitted +and no extra build statements may be specified. + +Dyndep Examples +~~~~~~~~~~~~~~~ + +Fortran Modules +^^^^^^^^^^^^^^^ + +Consider a Fortran source file `foo.f90` that provides a module +`foo.mod` (an implicit output of compilation) and another source file +`bar.f90` that uses the module (an implicit input of compilation). This +implicit dependency must be discovered before we compile either source +in order to ensure that `bar.f90` never compiles before `foo.f90`, and +that `bar.f90` recompiles when `foo.mod` changes. We can achieve this +as follows: + +---- +rule f95 + command = f95 -o $out -c $in +rule fscan + command = fscan -o $out $in + +build foobar.dd: fscan foo.f90 bar.f90 + +build foo.o: f95 foo.f90 || foobar.dd + dyndep = foobar.dd +build bar.o: f95 bar.f90 || foobar.dd + dyndep = foobar.dd +---- + +In this example the order-only dependencies ensure that `foobar.dd` is +generated before either source compiles. The hypothetical `fscan` tool +scans the source files, assumes each will be compiled to a `.o` of the +same name, and writes `foobar.dd` with content such as: + +---- +ninja_dyndep_version = 1 +build foo.o | foo.mod: dyndep +build bar.o: dyndep | foo.mod +---- + +Ninja will load this file to add `foo.mod` as an implicit output of +`foo.o` and implicit input of `bar.o`. This ensures that the Fortran +sources are always compiled in the proper order and recompiled when +needed. + +Tarball Extraction +^^^^^^^^^^^^^^^^^^ + +Consider a tarball `foo.tar` that we want to extract. The extraction time +can be recorded with a `foo.tar.stamp` file so that extraction repeats if +the tarball changes, but we also would like to re-extract if any of the +outputs is missing. However, the list of outputs depends on the content +of the tarball and cannot be spelled out explicitly in the ninja build file. +We can achieve this as follows: + +---- +rule untar + command = tar xf $in && touch $out +rule scantar + command = scantar --stamp=$stamp --dd=$out $in +build foo.tar.dd: scantar foo.tar + stamp = foo.tar.stamp +build foo.tar.stamp: untar foo.tar || foo.tar.dd + dyndep = foo.tar.dd +---- + +In this example the order-only dependency ensures that `foo.tar.dd` is +built before the tarball extracts. The hypothetical `scantar` tool +will read the tarball (e.g. via `tar tf`) and write `foo.tar.dd` with +content such as: + +---- +ninja_dyndep_version = 1 +build foo.tar.stamp | file1.txt file2.txt : dyndep + restat = 1 +---- + +Ninja will load this file to add `file1.txt` and `file2.txt` as implicit +outputs of `foo.tar.stamp`, and to mark the build statement for `restat`. +On future builds, if any implicit output is missing the tarball will be +extracted again. The `restat` binding tells Ninja to tolerate the fact +that the implicit outputs may not have modification times newer than +the tarball itself (avoiding re-extraction on every build). diff --git a/misc/ci.py b/misc/ci.py new file mode 100755 index 0000000..17cbf14 --- /dev/null +++ b/misc/ci.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import os + +ignores = [ + '.git/', + 'misc/afl-fuzz-tokens/', + 'ninja_deps', + 'src/depfile_parser.cc', + 'src/lexer.cc', +] + +error_count = 0 + +def error(path, msg): + global error_count + error_count += 1 + print('\x1b[1;31m{}\x1b[0;31m{}\x1b[0m'.format(path, msg)) + +for root, directory, filenames in os.walk('.'): + for filename in filenames: + path = os.path.join(root, filename)[2:] + if any([path.startswith(x) for x in ignores]): + continue + with open(path, 'rb') as file: + line_nr = 1 + try: + for line in [x.decode() for x in file.readlines()]: + if len(line) == 0 or line[-1] != '\n': + error(path, ' missing newline at end of file.') + if len(line) > 1: + if line[-2] == '\r': + error(path, ' has Windows line endings.') + break + if line[-2] == ' ' or line[-2] == '\t': + error(path, ':{} has trailing whitespace.'.format(line_nr)) + line_nr += 1 + except UnicodeError: + pass # binary file + +exit(error_count) diff --git a/misc/ninja-mode.el b/misc/ninja-mode.el index 639e537..8b975d5 100644 --- a/misc/ninja-mode.el +++ b/misc/ninja-mode.el @@ -56,7 +56,7 @@ (save-excursion (goto-char (line-end-position 0)) (or - ;; If we're continuting the previous line, it's not a + ;; If we're continuing the previous line, it's not a ;; comment. (not (eq ?$ (char-before))) ;; Except if the previous line is a comment as well, as the diff --git a/misc/ninja.vim b/misc/ninja.vim index 190d9ce..c1ffd50 100644 --- a/misc/ninja.vim +++ b/misc/ninja.vim @@ -1,8 +1,8 @@ " ninja build file syntax. " Language: ninja build file as described at " http://ninja-build.org/manual.html -" Version: 1.4 -" Last Change: 2014/05/13 +" Version: 1.5 +" Last Change: 2018/04/05 " Maintainer: Nicolas Weber <nicolasweber@gmx.de> " Version 1.4 of this script is in the upstream vim repository and will be " included in the next vim release. If you change this, please send your change @@ -21,7 +21,10 @@ set cpo&vim syn case match -syn match ninjaComment /#.*/ contains=@Spell +" Comments are only matched when the # is at the beginning of the line (with +" optional whitespace), as long as the prior line didn't end with a $ +" continuation. +syn match ninjaComment /\(\$\n\)\@<!\_^\s*#.*$/ contains=@Spell " Toplevel statements are the ones listed here and " toplevel variable assignments (ident '=' value). @@ -38,12 +41,13 @@ syn match ninjaKeyword "^subninja\>" " limited set of magic variables, 'build' allows general " let assignments. " manifest_parser.cc, ParseRule() -syn region ninjaRule start="^rule" end="^\ze\S" contains=ALL transparent -syn keyword ninjaRuleCommand contained command deps depfile description generator +syn region ninjaRule start="^rule" end="^\ze\S" contains=TOP transparent +syn keyword ninjaRuleCommand contained containedin=ninjaRule command + \ deps depfile description generator \ pool restat rspfile rspfile_content -syn region ninjaPool start="^pool" end="^\ze\S" contains=ALL transparent -syn keyword ninjaPoolCommand contained depth +syn region ninjaPool start="^pool" end="^\ze\S" contains=TOP transparent +syn keyword ninjaPoolCommand contained containedin=ninjaPool depth " Strings are parsed as follows: " lexer.in.cc, ReadEvalString() diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index 5c52ea2..ebe6490 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -21,7 +21,7 @@ class Writer(object): def newline(self): self.output.write('\n') - def comment(self, text, has_path=False): + def comment(self, text): for line in textwrap.wrap(text, self.width - 2, break_long_words=False, break_on_hyphens=False): self.output.write('# ' + line + '\n') @@ -60,7 +60,7 @@ class Writer(object): self.variable('deps', deps, indent=1) def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, - variables=None, implicit_outputs=None): + variables=None, implicit_outputs=None, pool=None): outputs = as_list(outputs) out_outputs = [escape_path(x) for x in outputs] all_inputs = [escape_path(x) for x in as_list(inputs)] @@ -81,6 +81,8 @@ class Writer(object): self._line('build %s: %s' % (' '.join(out_outputs), ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) if variables: if isinstance(variables, dict): diff --git a/misc/ninja_syntax_test.py b/misc/ninja_syntax_test.py index 07e3ed3..90ff9c6 100755 --- a/misc/ninja_syntax_test.py +++ b/misc/ninja_syntax_test.py @@ -46,13 +46,13 @@ class TestLineWordWrap(unittest.TestCase): self.out.getvalue()) def test_comment_wrap(self): - # Filenames shoud not be wrapped + # Filenames should not be wrapped self.n.comment('Hello /usr/local/build-tools/bin') self.assertEqual('# Hello\n# /usr/local/build-tools/bin\n', self.out.getvalue()) def test_short_words_indented(self): - # Test that indent is taking into acount when breaking subsequent lines. + # Test that indent is taking into account when breaking subsequent lines. # The second line should not be ' to tree', as that's longer than the # test layout width of 8. self.n._line('line_one to tree') diff --git a/misc/output_test.py b/misc/output_test.py new file mode 100755 index 0000000..1dcde10 --- /dev/null +++ b/misc/output_test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +"""Runs ./ninja and checks if the output is correct. + +In order to simulate a smart terminal it uses the 'script' command. +""" + +import os +import platform +import subprocess +import sys +import tempfile +import unittest + +default_env = dict(os.environ) +if 'NINJA_STATUS' in default_env: + del default_env['NINJA_STATUS'] +if 'CLICOLOR_FORCE' in default_env: + del default_env['CLICOLOR_FORCE'] +default_env['TERM'] = '' + +def run(build_ninja, flags='', pipe=False, env=default_env): + with tempfile.NamedTemporaryFile('w') as f: + f.write(build_ninja) + f.flush() + ninja_cmd = './ninja {} -f {}'.format(flags, f.name) + try: + if pipe: + output = subprocess.check_output([ninja_cmd], shell=True, env=env) + elif platform.system() == 'Darwin': + output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd], + env=env) + else: + output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'], + env=env) + except subprocess.CalledProcessError as err: + sys.stdout.buffer.write(err.output) + raise err + final_output = '' + for line in output.decode('utf-8').splitlines(True): + if len(line) > 0 and line[-1] == '\r': + continue + final_output += line.replace('\r', '') + return final_output + +class Output(unittest.TestCase): + def test_issue_1418(self): + self.assertEqual(run( +'''rule echo + command = sleep $delay && echo $out + description = echo $out + +build a: echo + delay = 3 +build b: echo + delay = 2 +build c: echo + delay = 1 +'''), +'''[1/3] echo c\x1b[K +c +[2/3] echo b\x1b[K +b +[3/3] echo a\x1b[K +a +''') + + def test_issue_1214(self): + print_red = '''rule echo + command = printf '\x1b[31mred\x1b[0m' + description = echo $out + +build a: echo +''' + # Only strip color when ninja's output is piped. + self.assertEqual(run(print_red), +'''[1/1] echo a\x1b[K +\x1b[31mred\x1b[0m +''') + self.assertEqual(run(print_red, pipe=True), +'''[1/1] echo a +red +''') + # Even in verbose mode, colors should still only be stripped when piped. + self.assertEqual(run(print_red, flags='-v'), +'''[1/1] printf '\x1b[31mred\x1b[0m' +\x1b[31mred\x1b[0m +''') + self.assertEqual(run(print_red, flags='-v', pipe=True), +'''[1/1] printf '\x1b[31mred\x1b[0m' +red +''') + + # CLICOLOR_FORCE=1 can be used to disable escape code stripping. + env = default_env.copy() + env['CLICOLOR_FORCE'] = '1' + self.assertEqual(run(print_red, pipe=True, env=env), +'''[1/1] echo a +\x1b[31mred\x1b[0m +''') + +if __name__ == '__main__': + unittest.main() diff --git a/misc/zsh-completion b/misc/zsh-completion index bf23fac..4cee3b8 100644 --- a/misc/zsh-completion +++ b/misc/zsh-completion @@ -14,7 +14,7 @@ # limitations under the License. # Add the following to your .zshrc to tab-complete ninja targets -# . path/to/ninja/misc/zsh-completion +# fpath=(path/to/ninja/misc/zsh-completion $fpath) __get_targets() { dir="." diff --git a/src/browse.cc b/src/browse.cc index 14900f8..c08c9f4 100644 --- a/src/browse.cc +++ b/src/browse.cc @@ -14,6 +14,7 @@ #include "browse.h" +#include <errno.h> #include <stdio.h> #include <stdlib.h> #include <unistd.h> @@ -57,7 +58,11 @@ void RunBrowsePython(State* state, const char* ninja_command, } command.push_back(NULL); execvp(command[0], (char**)&command[0]); - perror("ninja: execvp"); + if (errno == ENOENT) { + printf("ninja: %s is required for the browse tool\n", NINJA_PYTHON); + } else { + perror("ninja: execvp"); + } } while (false); _exit(1); } else { // Child. diff --git a/src/browse.py b/src/browse.py index 64a16f2..1c9c39b 100755 --- a/src/browse.py +++ b/src/browse.py @@ -24,8 +24,10 @@ from __future__ import print_function try: import http.server as httpserver + import socketserver except ImportError: import BaseHTTPServer as httpserver + import SocketServer as socketserver import argparse import cgi import os @@ -205,10 +207,14 @@ parser.add_argument('-f', default='build.ninja', parser.add_argument('initial_target', default='all', nargs='?', help='Initial target to show (default %(default)s)') +class HTTPServer(socketserver.ThreadingMixIn, httpserver.HTTPServer): + # terminate server immediately when Python exits. + daemon_threads = True + args = parser.parse_args() port = args.port hostname = args.hostname -httpd = httpserver.HTTPServer((hostname,port), RequestHandler) +httpd = HTTPServer((hostname,port), RequestHandler) try: if hostname == "": hostname = socket.gethostname() diff --git a/src/build.cc b/src/build.cc index 61ef0e8..a055738 100644 --- a/src/build.cc +++ b/src/build.cc @@ -97,6 +97,7 @@ void BuildStatus::PlanHasTotalEdges(int total) { } void BuildStatus::BuildEdgeStarted(Edge* edge) { + assert(running_edges_.find(edge) == running_edges_.end()); int start_time = (int)(GetTimeMillis() - start_time_millis_); running_edges_.insert(make_pair(edge, start_time)); ++started_edges_; @@ -154,9 +155,8 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, // (Launching subprocesses in pseudo ttys doesn't work because there are // only a few hundred available on some systems, and ninja can launch // thousands of parallel compile commands.) - // TODO: There should be a flag to disable escape code stripping. string final_output; - if (!printer_.is_smart_terminal()) + if (!printer_.supports_color()) final_output = StripAnsiEscapeCodes(output); else final_output = output; @@ -174,6 +174,20 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, } } +void BuildStatus::BuildLoadDyndeps() { + // The DependencyScan calls EXPLAIN() to print lines explaining why + // it considers a portion of the graph to be out of date. Normally + // this is done before the build starts, but our caller is about to + // load a dyndep file during the build. Doing so may generate more + // exlanation lines (via fprintf directly to stderr), but in an + // interactive console the cursor is currently at the end of a status + // line. Start a new line so that the first explanation does not + // append to the status line. After the explanations are done a + // new build status line will appear. + if (g_explaining) + printer_.PrintOnNewLine(""); +} + void BuildStatus::BuildStarted() { overall_rate_.Restart(); current_rate_.Restart(); @@ -288,7 +302,11 @@ void BuildStatus::PrintStatus(Edge* edge, EdgeStatus status) { force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); } -Plan::Plan() : command_edges_(0), wanted_edges_(0) {} +Plan::Plan(Builder* builder) + : builder_(builder) + , command_edges_(0) + , wanted_edges_(0) +{} void Plan::Reset() { command_edges_ = 0; @@ -298,10 +316,11 @@ void Plan::Reset() { } bool Plan::AddTarget(Node* node, string* err) { - return AddSubTarget(node, NULL, err); + return AddSubTarget(node, NULL, err, NULL); } -bool Plan::AddSubTarget(Node* node, Node* dependent, string* err) { +bool Plan::AddSubTarget(Node* node, Node* dependent, string* err, + set<Edge*>* dyndep_walk) { Edge* edge = node->in_edge(); if (!edge) { // Leaf node. if (node->dirty()) { @@ -318,34 +337,44 @@ bool Plan::AddSubTarget(Node* node, Node* dependent, string* err) { return false; // Don't need to do anything. // If an entry in want_ does not already exist for edge, create an entry which - // maps to false, indicating that we do not want to build this entry itself. - pair<map<Edge*, bool>::iterator, bool> want_ins = - want_.insert(make_pair(edge, false)); - bool& want = want_ins.first->second; + // maps to kWantNothing, indicating that we do not want to build this entry itself. + pair<map<Edge*, Want>::iterator, bool> want_ins = + want_.insert(make_pair(edge, kWantNothing)); + Want& want = want_ins.first->second; + + if (dyndep_walk && want == kWantToFinish) + return false; // Don't need to do anything with already-scheduled edge. // If we do need to build edge and we haven't already marked it as wanted, // mark it now. - if (node->dirty() && !want) { - want = true; - ++wanted_edges_; - if (edge->AllInputsReady()) - ScheduleWork(edge); - if (!edge->is_phony()) - ++command_edges_; + if (node->dirty() && want == kWantNothing) { + want = kWantToStart; + EdgeWanted(edge); + if (!dyndep_walk && edge->AllInputsReady()) + ScheduleWork(want_ins.first); } + if (dyndep_walk) + dyndep_walk->insert(edge); + if (!want_ins.second) return true; // We've already processed the inputs. for (vector<Node*>::iterator i = edge->inputs_.begin(); i != edge->inputs_.end(); ++i) { - if (!AddSubTarget(*i, node, err) && !err->empty()) + if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty()) return false; } return true; } +void Plan::EdgeWanted(Edge* edge) { + ++wanted_edges_; + if (!edge->is_phony()) + ++command_edges_; +} + Edge* Plan::FindWork() { if (ready_.empty()) return NULL; @@ -355,30 +384,32 @@ Edge* Plan::FindWork() { return edge; } -void Plan::ScheduleWork(Edge* edge) { - set<Edge*>::iterator e = ready_.lower_bound(edge); - if (e != ready_.end() && !ready_.key_comp()(edge, *e)) { +void Plan::ScheduleWork(map<Edge*, Want>::iterator want_e) { + if (want_e->second == kWantToFinish) { // This edge has already been scheduled. We can get here again if an edge // and one of its dependencies share an order-only input, or if a node // duplicates an out edge (see https://github.com/ninja-build/ninja/pull/519). // Avoid scheduling the work again. return; } + assert(want_e->second == kWantToStart); + want_e->second = kWantToFinish; + Edge* edge = want_e->first; Pool* pool = edge->pool(); if (pool->ShouldDelayEdge()) { pool->DelayEdge(edge); pool->RetrieveReadyEdges(&ready_); } else { pool->EdgeScheduled(*edge); - ready_.insert(e, edge); + ready_.insert(edge); } } -void Plan::EdgeFinished(Edge* edge, EdgeResult result) { - map<Edge*, bool>::iterator e = want_.find(edge); +bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) { + map<Edge*, Want>::iterator e = want_.find(edge); assert(e != want_.end()); - bool directly_wanted = e->second; + bool directly_wanted = e->second != kWantNothing; // See if this job frees up any delayed jobs. if (directly_wanted) @@ -387,7 +418,7 @@ void Plan::EdgeFinished(Edge* edge, EdgeResult result) { // The rest of this function only applies to successful commands. if (result != kEdgeSucceeded) - return; + return true; if (directly_wanted) --wanted_edges_; @@ -397,29 +428,48 @@ void Plan::EdgeFinished(Edge* edge, EdgeResult result) { // Check off any nodes we were waiting for with this edge. for (vector<Node*>::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { - NodeFinished(*o); + if (!NodeFinished(*o, err)) + return false; } + return true; } -void Plan::NodeFinished(Node* node) { +bool Plan::NodeFinished(Node* node, string* err) { + // If this node provides dyndep info, load it now. + if (node->dyndep_pending()) { + assert(builder_ && "dyndep requires Plan to have a Builder"); + // Load the now-clean dyndep file. This will also update the + // build plan and schedule any new work that is ready. + return builder_->LoadDyndeps(node, err); + } + // See if we we want any edges from this node. for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); oe != node->out_edges().end(); ++oe) { - map<Edge*, bool>::iterator want_e = want_.find(*oe); + map<Edge*, Want>::iterator want_e = want_.find(*oe); if (want_e == want_.end()) continue; // See if the edge is now ready. - if ((*oe)->AllInputsReady()) { - if (want_e->second) { - ScheduleWork(*oe); - } else { - // We do not need to build this edge, but we might need to build one of - // its dependents. - EdgeFinished(*oe, kEdgeSucceeded); - } + if (!EdgeMaybeReady(want_e, err)) + return false; + } + return true; +} + +bool Plan::EdgeMaybeReady(map<Edge*, Want>::iterator want_e, string* err) { + Edge* edge = want_e->first; + if (edge->AllInputsReady()) { + if (want_e->second != kWantNothing) { + ScheduleWork(want_e); + } else { + // We do not need to build this edge, but we might need to build one of + // its dependents. + if (!EdgeFinished(edge, kEdgeSucceeded, err)) + return false; } } + return true; } bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { @@ -428,8 +478,8 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); oe != node->out_edges().end(); ++oe) { // Don't process edges that we don't actually want. - map<Edge*, bool>::iterator want_e = want_.find(*oe); - if (want_e == want_.end() || !want_e->second) + map<Edge*, Want>::iterator want_e = want_.find(*oe); + if (want_e == want_.end() || want_e->second == kWantNothing) continue; // Don't attempt to clean an edge if it failed to load deps. @@ -441,7 +491,12 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { vector<Node*>::iterator begin = (*oe)->inputs_.begin(), end = (*oe)->inputs_.end() - (*oe)->order_only_deps_; - if (find_if(begin, end, mem_fun(&Node::dirty)) == end) { +#if __cplusplus < 201703L +#define MEM_FN mem_fun +#else +#define MEM_FN mem_fn // mem_fun was removed in C++17. +#endif + if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) { // Recompute most_recent_input. Node* most_recent_input = NULL; for (vector<Node*>::iterator i = begin; i != end; ++i) { @@ -464,7 +519,7 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { return false; } - want_e->second = false; + want_e->second = kWantNothing; --wanted_edges_; if (!(*oe)->is_phony()) --command_edges_; @@ -474,10 +529,132 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { return true; } +bool Plan::DyndepsLoaded(DependencyScan* scan, Node* node, + const DyndepFile& ddf, string* err) { + // Recompute the dirty state of all our direct and indirect dependents now + // that our dyndep information has been loaded. + if (!RefreshDyndepDependents(scan, node, err)) + return false; + + // We loaded dyndep information for those out_edges of the dyndep node that + // specify the node in a dyndep binding, but they may not be in the plan. + // Starting with those already in the plan, walk newly-reachable portion + // of the graph through the dyndep-discovered dependencies. + + // Find edges in the the build plan for which we have new dyndep info. + std::vector<DyndepFile::const_iterator> dyndep_roots; + for (DyndepFile::const_iterator oe = ddf.begin(); oe != ddf.end(); ++oe) { + Edge* edge = oe->first; + + // If the edge outputs are ready we do not need to consider it here. + if (edge->outputs_ready()) + continue; + + map<Edge*, Want>::iterator want_e = want_.find(edge); + + // If the edge has not been encountered before then nothing already in the + // plan depends on it so we do not need to consider the edge yet either. + if (want_e == want_.end()) + continue; + + // This edge is already in the plan so queue it for the walk. + dyndep_roots.push_back(oe); + } + + // Walk dyndep-discovered portion of the graph to add it to the build plan. + std::set<Edge*> dyndep_walk; + for (std::vector<DyndepFile::const_iterator>::iterator + oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) { + DyndepFile::const_iterator oe = *oei; + for (vector<Node*>::const_iterator i = oe->second.implicit_inputs_.begin(); + i != oe->second.implicit_inputs_.end(); ++i) { + if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) && + !err->empty()) + return false; + } + } + + // Add out edges from this node that are in the plan (just as + // Plan::NodeFinished would have without taking the dyndep code path). + for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); + oe != node->out_edges().end(); ++oe) { + map<Edge*, Want>::iterator want_e = want_.find(*oe); + if (want_e == want_.end()) + continue; + dyndep_walk.insert(want_e->first); + } + + // See if any encountered edges are now ready. + for (set<Edge*>::iterator wi = dyndep_walk.begin(); + wi != dyndep_walk.end(); ++wi) { + map<Edge*, Want>::iterator want_e = want_.find(*wi); + if (want_e == want_.end()) + continue; + if (!EdgeMaybeReady(want_e, err)) + return false; + } + + return true; +} + +bool Plan::RefreshDyndepDependents(DependencyScan* scan, Node* node, + string* err) { + // Collect the transitive closure of dependents and mark their edges + // as not yet visited by RecomputeDirty. + set<Node*> dependents; + UnmarkDependents(node, &dependents); + + // Update the dirty state of all dependents and check if their edges + // have become wanted. + for (set<Node*>::iterator i = dependents.begin(); + i != dependents.end(); ++i) { + Node* n = *i; + + // Check if this dependent node is now dirty. Also checks for new cycles. + if (!scan->RecomputeDirty(n, err)) + return false; + if (!n->dirty()) + continue; + + // This edge was encountered before. However, we may not have wanted to + // build it if the outputs were not known to be dirty. With dyndep + // information an output is now known to be dirty, so we want the edge. + Edge* edge = n->in_edge(); + assert(edge && !edge->outputs_ready()); + map<Edge*, Want>::iterator want_e = want_.find(edge); + assert(want_e != want_.end()); + if (want_e->second == kWantNothing) { + want_e->second = kWantToStart; + EdgeWanted(edge); + } + } + return true; +} + +void Plan::UnmarkDependents(Node* node, set<Node*>* dependents) { + for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); + oe != node->out_edges().end(); ++oe) { + Edge* edge = *oe; + + map<Edge*, Want>::iterator want_e = want_.find(edge); + if (want_e == want_.end()) + continue; + + if (edge->mark_ != Edge::VisitNone) { + edge->mark_ = Edge::VisitNone; + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + if (dependents->insert(*o).second) + UnmarkDependents(*o, dependents); + } + } + } +} + void Plan::Dump() { printf("pending: %d\n", (int)want_.size()); - for (map<Edge*, bool>::iterator e = want_.begin(); e != want_.end(); ++e) { - if (e->second) + for (map<Edge*, Want>::iterator e = want_.begin(); e != want_.end(); ++e) { + if (e->second != kWantNothing) printf("want "); e->first->Dump(); } @@ -550,8 +727,10 @@ bool RealCommandRunner::WaitForCommand(Result* result) { Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface) - : state_(state), config_(config), disk_interface_(disk_interface), - scan_(state, build_log, deps_log, disk_interface) { + : state_(state), config_(config), + plan_(this), disk_interface_(disk_interface), + scan_(state, build_log, deps_log, disk_interface, + &config_.depfile_parser_options) { status_ = new BuildStatus(config); } @@ -653,7 +832,11 @@ bool Builder::Build(string* err) { } if (edge->is_phony()) { - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) { + Cleanup(); + status_->BuildFinished(); + return false; + } } else { ++pending_commands; } @@ -773,8 +956,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { // The rest of this function only applies to successful commands. if (!result->success()) { - plan_.EdgeFinished(edge, Plan::kEdgeFailed); - return true; + return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err); } // Restat the edge outputs @@ -830,7 +1012,8 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } } - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) + return false; // Delete any left over response file. string rspfile = edge->GetUnescapedRspfile(); @@ -900,7 +1083,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, if (content.empty()) return true; - DepfileParser deps; + DepfileParser deps(config_.depfile_parser_options); if (!deps.Parse(&content, err)) return false; @@ -927,3 +1110,21 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, return true; } + +bool Builder::LoadDyndeps(Node* node, string* err) { + status_->BuildLoadDyndeps(); + + // Load the dyndep information provided by this node. + DyndepFile ddf; + if (!scan_.LoadDyndeps(node, &ddf, err)) + return false; + + // Update the build plan to account for dyndep modifications to the graph. + if (!plan_.DyndepsLoaded(&scan_, node, ddf, err)) + return false; + + // New command edges may have been added to the plan. + status_->PlanHasTotalEdges(plan_.command_edge_count()); + + return true; +} diff --git a/src/build.h b/src/build.h index 43786f1..ab59f0c 100644 --- a/src/build.h +++ b/src/build.h @@ -23,6 +23,7 @@ #include <string> #include <vector> +#include "depfile_parser.h" #include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" #include "line_printer.h" @@ -31,6 +32,7 @@ struct BuildLog; struct BuildStatus; +struct Builder; struct DiskInterface; struct Edge; struct Node; @@ -39,7 +41,7 @@ struct State; /// Plan stores the state of a build plan: what we intend to build, /// which steps we're ready to execute. struct Plan { - Plan(); + Plan(Builder* builder = NULL); /// Add a target to our plan (including all its dependencies). /// Returns false if we don't need to build this target; may @@ -62,7 +64,10 @@ struct Plan { }; /// Mark an edge as done building (whether it succeeded or failed). - void EdgeFinished(Edge* edge, EdgeResult result); + /// If any of the edge's outputs are dyndep bindings of their dependents, + /// this loads dynamic dependencies from the nodes' paths. + /// Returns 'false' if loading dyndep info fails and 'true' otherwise. + bool EdgeFinished(Edge* edge, EdgeResult result, string* err); /// Clean the given node during the build. /// Return false on error. @@ -74,24 +79,53 @@ struct Plan { /// Reset state. Clears want and ready sets. void Reset(); + /// Update the build plan to account for modifications made to the graph + /// by information loaded from a dyndep file. + bool DyndepsLoaded(DependencyScan* scan, Node* node, + const DyndepFile& ddf, string* err); private: - bool AddSubTarget(Node* node, Node* dependent, string* err); - void NodeFinished(Node* node); + bool RefreshDyndepDependents(DependencyScan* scan, Node* node, string* err); + void UnmarkDependents(Node* node, set<Node*>* dependents); + bool AddSubTarget(Node* node, Node* dependent, string* err, + set<Edge*>* dyndep_walk); + + /// Update plan with knowledge that the given node is up to date. + /// If the node is a dyndep binding on any of its dependents, this + /// loads dynamic dependencies from the node's path. + /// Returns 'false' if loading dyndep info fails and 'true' otherwise. + bool NodeFinished(Node* node, string* err); + + /// Enumerate possible steps we want for an edge. + enum Want + { + /// We do not want to build the edge, but we might want to build one of + /// its dependents. + kWantNothing, + /// We want to build the edge, but have not yet scheduled it. + kWantToStart, + /// We want to build the edge, have scheduled it, and are waiting + /// for it to complete. + kWantToFinish + }; + + void EdgeWanted(Edge* edge); + bool EdgeMaybeReady(map<Edge*, Want>::iterator want_e, string* err); /// Submits a ready edge as a candidate for execution. /// The edge may be delayed from running, for example if it's a member of a /// currently-full pool. - void ScheduleWork(Edge* edge); + void ScheduleWork(map<Edge*, Want>::iterator want_e); /// Keep track of which edges we want to build in this plan. If this map does /// not contain an entry for an edge, we do not want to build the entry or its - /// dependents. If an entry maps to false, we do not want to build it, but we - /// might want to build one of its dependents. If the entry maps to true, we - /// want to build it. - map<Edge*, bool> want_; + /// dependents. If it does contain an entry, the enumeration indicates what + /// we want for the edge. + map<Edge*, Want> want_; set<Edge*> ready_; + Builder* builder_; + /// Total number of edges that have commands (not phony). int command_edges_; @@ -139,6 +173,7 @@ struct BuildConfig { /// The maximum load average we must not exceed. A negative value /// means that we do not have any limit. double max_load_average; + DepfileParserOptions depfile_parser_options; }; /// Builder wraps the build process: starting commands, updating status. @@ -175,10 +210,17 @@ struct Builder { scan_.set_build_log(log); } + /// Load the dyndep information provided by the given node. + bool LoadDyndeps(Node* node, string* err); + State* state_; const BuildConfig& config_; Plan plan_; +#if __cplusplus < 201703L auto_ptr<CommandRunner> command_runner_; +#else + unique_ptr<CommandRunner> command_runner_; // auto_ptr was removed in C++17. +#endif BuildStatus* status_; private: @@ -201,6 +243,7 @@ struct BuildStatus { void BuildEdgeStarted(Edge* edge); void BuildEdgeFinished(Edge* edge, bool success, const string& output, int* start_time, int* end_time); + void BuildLoadDyndeps(); void BuildStarted(); void BuildFinished(); diff --git a/src/build_log.cc b/src/build_log.cc index 333915a..774f72f 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -35,6 +35,9 @@ #include "graph.h" #include "metrics.h" #include "util.h" +#if defined(_MSC_VER) && (_MSC_VER < 1800) +#define strtoll _strtoi64 +#endif // Implementation details: // Each run's log appends to the log file. @@ -46,6 +49,7 @@ namespace { const char kFileSignature[] = "# ninja log v%d\n"; +const char kFileColumnLabels[] = "# start_time end_time mtime command hash\n"; const int kOldestSupportedVersion = 4; const int kCurrentVersion = 5; @@ -76,11 +80,17 @@ uint64_t MurmurHash64A(const void* key, size_t len) { switch (len & 7) { case 7: h ^= uint64_t(data[6]) << 48; + NINJA_FALLTHROUGH; case 6: h ^= uint64_t(data[5]) << 40; + NINJA_FALLTHROUGH; case 5: h ^= uint64_t(data[4]) << 32; + NINJA_FALLTHROUGH; case 4: h ^= uint64_t(data[3]) << 24; + NINJA_FALLTHROUGH; case 3: h ^= uint64_t(data[2]) << 16; + NINJA_FALLTHROUGH; case 2: h ^= uint64_t(data[1]) << 8; + NINJA_FALLTHROUGH; case 1: h ^= uint64_t(data[0]); h *= m; }; @@ -135,7 +145,8 @@ bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user, fseek(log_file_, 0, SEEK_END); if (ftell(log_file_) == 0) { - if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) { + if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0 || + fprintf(log_file_, kFileColumnLabels) < 0) { *err = strerror(errno); return false; } @@ -167,6 +178,9 @@ bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, if (log_file_) { if (!WriteEntry(log_file_, *log_entry)) return false; + if (fflush(log_file_) != 0) { + return false; + } } } return true; @@ -290,7 +304,7 @@ bool BuildLog::Load(const string& path, string* err) { if (!end) continue; *end = 0; - restat_mtime = atol(start); + restat_mtime = strtoll(start, NULL, 10); start = end + 1; end = (char*)memchr(start, kFieldSeparator, line_end - start); @@ -353,7 +367,7 @@ BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { } bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { - return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", + return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n", entry.start_time, entry.end_time, entry.mtime, entry.output.c_str(), entry.command_hash) > 0; } diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index b4efb1d..e471d13 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -71,7 +71,7 @@ bool WriteTestData(string* err) { long_rule_command += "$in -o $out\n"; State state; - ManifestParser parser(&state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&state, NULL); if (!parser.ParseTest("rule cxx\n command = " + long_rule_command, err)) return false; diff --git a/src/build_log_test.cc b/src/build_log_test.cc index ad30380..eea818f 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -70,8 +70,9 @@ TEST_F(BuildLogTest, WriteRead) { } TEST_F(BuildLogTest, FirstWriteAddsSignature) { - const char kExpectedVersion[] = "# ninja log vX\n"; - const size_t kVersionPos = strlen(kExpectedVersion) - 2; // Points at 'X'. + const char kExpectedContent[] = "# ninja log vX\n" + "# start_time end_time mtime command hash\n"; + const size_t kVersionPos = 13; // Points at 'X'. BuildLog log; string contents, err; @@ -84,7 +85,7 @@ TEST_F(BuildLogTest, FirstWriteAddsSignature) { ASSERT_EQ("", err); if (contents.size() >= kVersionPos) contents[kVersionPos] = 'X'; - EXPECT_EQ(kExpectedVersion, contents); + EXPECT_EQ(kExpectedContent, contents); // Opening the file anew shouldn't add a second version string. EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err)); @@ -96,7 +97,7 @@ TEST_F(BuildLogTest, FirstWriteAddsSignature) { ASSERT_EQ("", err); if (contents.size() >= kVersionPos) contents[kVersionPos] = 'X'; - EXPECT_EQ(kExpectedVersion, contents); + EXPECT_EQ(kExpectedContent, contents); } TEST_F(BuildLogTest, DoubleEntry) { diff --git a/src/build_test.cc b/src/build_test.cc index a0f898f..b5dbc6c 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -21,6 +21,12 @@ #include "graph.h" #include "test.h" +struct CompareEdgesByOutput { + static bool cmp(const Edge* a, const Edge* b) { + return a->outputs_[0]->path() < b->outputs_[0]->path(); + } +}; + /// Fixture for tests involving Plan. // Though Plan doesn't use State, it's useful to have one around // to create Nodes and Edges. @@ -31,12 +37,6 @@ struct PlanTest : public StateTestWithBuiltinRules { // provide a means to get available Edges in order and in a format which is // easy to write tests around. void FindWorkSorted(deque<Edge*>* ret, int count) { - struct CompareEdgesByOutput { - static bool cmp(const Edge* a, const Edge* b) { - return a->outputs_[0]->path() < b->outputs_[0]->path(); - } - }; - for (int i = 0; i < count; ++i) { ASSERT_TRUE(plan_.more_to_do()); Edge* edge = plan_.FindWork(); @@ -68,14 +68,16 @@ TEST_F(PlanTest, Basic) { ASSERT_FALSE(plan_.FindWork()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); ASSERT_EQ("mid", edge->inputs_[0]->path()); ASSERT_EQ("out", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); ASSERT_FALSE(plan_.more_to_do()); edge = plan_.FindWork(); @@ -99,11 +101,13 @@ TEST_F(PlanTest, DoubleOutputDirect) { Edge* edge; edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat in - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat mid1 mid2 - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_FALSE(edge); // done @@ -129,19 +133,23 @@ TEST_F(PlanTest, DoubleOutputIndirect) { Edge* edge; edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat in - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat a1 - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat a2 - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat b1 b2 - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_FALSE(edge); // done @@ -167,19 +175,23 @@ TEST_F(PlanTest, DoubleDependent) { Edge* edge; edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat in - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat mid - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat mid - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat a1 a2 - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_FALSE(edge); // done @@ -204,7 +216,8 @@ void PlanTest::TestPoolWithDepthOne(const char* test_case) { // This will be false since poolcat is serialized ASSERT_FALSE(plan_.FindWork()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); @@ -213,7 +226,8 @@ void PlanTest::TestPoolWithDepthOne(const char* test_case) { ASSERT_FALSE(plan_.FindWork()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); ASSERT_FALSE(plan_.more_to_do()); edge = plan_.FindWork(); @@ -289,7 +303,8 @@ TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_EQ("outb3", edge->outputs_[0]->path()); // finish out1 - plan_.EdgeFinished(edges.front(), Plan::kEdgeSucceeded); + plan_.EdgeFinished(edges.front(), Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edges.pop_front(); // out3 should be available @@ -300,19 +315,22 @@ TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_FALSE(plan_.FindWork()); - plan_.EdgeFinished(out3, Plan::kEdgeSucceeded); + plan_.EdgeFinished(out3, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); ASSERT_FALSE(plan_.FindWork()); for (deque<Edge*>::iterator it = edges.begin(); it != edges.end(); ++it) { - plan_.EdgeFinished(*it, Plan::kEdgeSucceeded); + plan_.EdgeFinished(*it, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); } Edge* last = plan_.FindWork(); ASSERT_TRUE(last); ASSERT_EQ("allTheThings", last->outputs_[0]->path()); - plan_.EdgeFinished(last, Plan::kEdgeSucceeded); + plan_.EdgeFinished(last, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); ASSERT_FALSE(plan_.more_to_do()); ASSERT_FALSE(plan_.FindWork()); @@ -354,7 +372,8 @@ TEST_F(PlanTest, PoolWithRedundantEdges) { edge = initial_edges[1]; // Foo first ASSERT_EQ("foo.cpp", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); @@ -362,11 +381,13 @@ TEST_F(PlanTest, PoolWithRedundantEdges) { ASSERT_EQ("foo.cpp", edge->inputs_[0]->path()); ASSERT_EQ("foo.cpp", edge->inputs_[1]->path()); ASSERT_EQ("foo.cpp.obj", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = initial_edges[0]; // Now for bar ASSERT_EQ("bar.cpp", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); @@ -374,7 +395,8 @@ TEST_F(PlanTest, PoolWithRedundantEdges) { ASSERT_EQ("bar.cpp", edge->inputs_[0]->path()); ASSERT_EQ("bar.cpp", edge->inputs_[1]->path()); ASSERT_EQ("bar.cpp.obj", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); @@ -382,14 +404,16 @@ TEST_F(PlanTest, PoolWithRedundantEdges) { ASSERT_EQ("foo.cpp.obj", edge->inputs_[0]->path()); ASSERT_EQ("bar.cpp.obj", edge->inputs_[1]->path()); ASSERT_EQ("libfoo.a", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); ASSERT_FALSE(plan_.FindWork()); ASSERT_EQ("libfoo.a", edge->inputs_[0]->path()); ASSERT_EQ("all", edge->outputs_[0]->path()); - plan_.EdgeFinished(edge, Plan::kEdgeSucceeded); + plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_FALSE(edge); @@ -422,7 +446,8 @@ TEST_F(PlanTest, PoolWithFailingEdge) { // This will be false since poolcat is serialized ASSERT_FALSE(plan_.FindWork()); - plan_.EdgeFinished(edge, Plan::kEdgeFailed); + plan_.EdgeFinished(edge, Plan::kEdgeFailed, &err); + ASSERT_EQ("", err); edge = plan_.FindWork(); ASSERT_TRUE(edge); @@ -431,7 +456,8 @@ TEST_F(PlanTest, PoolWithFailingEdge) { ASSERT_FALSE(plan_.FindWork()); - plan_.EdgeFinished(edge, Plan::kEdgeFailed); + plan_.EdgeFinished(edge, Plan::kEdgeFailed, &err); + ASSERT_EQ("", err); ASSERT_TRUE(plan_.more_to_do()); // Jobs have failed edge = plan_.FindWork(); @@ -441,7 +467,7 @@ TEST_F(PlanTest, PoolWithFailingEdge) { /// Fake implementation of CommandRunner, useful for tests. struct FakeCommandRunner : public CommandRunner { explicit FakeCommandRunner(VirtualFileSystem* fs) : - last_command_(NULL), fs_(fs) {} + max_active_edges_(1), fs_(fs) {} // CommandRunner impl virtual bool CanRunMore(); @@ -451,7 +477,8 @@ struct FakeCommandRunner : public CommandRunner { virtual void Abort(); vector<string> commands_ran_; - Edge* last_command_; + vector<Edge*> active_edges_; + size_t max_active_edges_; VirtualFileSystem* fs_; }; @@ -543,12 +570,13 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, } bool FakeCommandRunner::CanRunMore() { - // Only run one at a time. - return last_command_ == NULL; + return active_edges_.size() < max_active_edges_; } bool FakeCommandRunner::StartCommand(Edge* edge) { - assert(!last_command_); + assert(active_edges_.size() < max_active_edges_); + assert(find(active_edges_.begin(), active_edges_.end(), edge) + == active_edges_.end()); commands_ran_.push_back(edge->EvaluateCommand()); if (edge->rule().name() == "cat" || edge->rule().name() == "cat_rsp" || @@ -566,20 +594,38 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { edge->rule().name() == "interrupt" || edge->rule().name() == "console") { // Don't do anything. + } else if (edge->rule().name() == "cp") { + assert(!edge->inputs_.empty()); + assert(edge->outputs_.size() == 1); + string content; + string err; + if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) == + DiskInterface::Okay) + fs_->WriteFile(edge->outputs_[0]->path(), content); } else { printf("unknown command\n"); return false; } - last_command_ = edge; + active_edges_.push_back(edge); + + // Allow tests to control the order by the name of the first output. + sort(active_edges_.begin(), active_edges_.end(), + CompareEdgesByOutput::cmp); + return true; } bool FakeCommandRunner::WaitForCommand(Result* result) { - if (!last_command_) + if (active_edges_.empty()) return false; - Edge* edge = last_command_; + // All active edges were already completed immediately when started, + // so we can pick any edge here. Pick the last edge. Tests can + // control the order of edges by the name of the first output. + vector<Edge*>::iterator edge_iter = active_edges_.end() - 1; + + Edge* edge = *edge_iter; result->edge = edge; if (edge->rule().name() == "interrupt" || @@ -593,7 +639,7 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { result->status = ExitSuccess; else result->status = ExitFailure; - last_command_ = NULL; + active_edges_.erase(edge_iter); return true; } @@ -602,19 +648,33 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { result->status = ExitFailure; else result->status = ExitSuccess; - last_command_ = NULL; + + // Provide a way for test cases to verify when an edge finishes that + // some other edge is still active. This is useful for test cases + // covering behavior involving multiple active edges. + const string& verify_active_edge = edge->GetBinding("verify_active_edge"); + if (!verify_active_edge.empty()) { + bool verify_active_edge_found = false; + for (vector<Edge*>::iterator i = active_edges_.begin(); + i != active_edges_.end(); ++i) { + if ((*i)->outputs_.size() >= 1 && + (*i)->outputs_[0]->path() == verify_active_edge) { + verify_active_edge_found = true; + } + } + EXPECT_TRUE(verify_active_edge_found); + } + + active_edges_.erase(edge_iter); return true; } vector<Edge*> FakeCommandRunner::GetActiveEdges() { - vector<Edge*> edges; - if (last_command_) - edges.push_back(last_command_); - return edges; + return active_edges_; } void FakeCommandRunner::Abort() { - last_command_ = NULL; + active_edges_.clear(); } void BuildTest::Dirty(const string& path) { @@ -1068,6 +1128,19 @@ TEST_F(BuildTest, PhonyNoWork) { EXPECT_TRUE(builder_.AlreadyUpToDate()); } +// Test a self-referencing phony. Ideally this should not work, but +// ninja 1.7 and below tolerated and CMake 2.8.12.x and 3.0.x both +// incorrectly produce it. We tolerate it for compatibility. +TEST_F(BuildTest, PhonySelfReference) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build a: phony a\n")); + + EXPECT_TRUE(builder_.AddTarget("a", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + TEST_F(BuildTest, Fail) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule fail\n" @@ -2295,3 +2368,712 @@ TEST_F(BuildTest, Console) { EXPECT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } + +TEST_F(BuildTest, DyndepMissingAndNoRule) { + // Verify that we can diagnose when a dyndep file is missing and + // has no rule to build it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + + string err; + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("loading 'dd': No such file or directory", err); +} + +TEST_F(BuildTest, DyndepReadyImplicitConnection) { + // Verify that a dyndep file can be loaded immediately to discover + // that one edge has an implicit output that is also an implicit + // input of another edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"build tmp: touch || dd\n" +" dyndep = dd\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out | out.imp: dyndep | tmp.imp\n" +"build tmp | tmp.imp: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(2u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[1]); +} + +TEST_F(BuildTest, DyndepReadySyntaxError) { + // Verify that a dyndep file can be loaded immediately to discover + // and reject a syntax error in it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd", +"build out: dyndep\n" +); + + string err; + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dd:1: expected 'ninja_dyndep_version = ...'\n", err); +} + +TEST_F(BuildTest, DyndepReadyCircular) { + // Verify that a dyndep file can be loaded immediately to discover + // and reject a circular dependency. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" +"build in: r circ\n" + )); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out | circ: dyndep\n" + ); + fs_.Create("out", ""); + + string err; + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dependency cycle: circ -> in -> circ", err); +} + +TEST_F(BuildTest, DyndepBuild) { + // Verify that a dyndep file can be built and loaded to discover nothing. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + size_t files_created = fs_.files_created_.size(); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(2u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[1]); + ASSERT_EQ(2u, fs_.files_read_.size()); + EXPECT_EQ("dd-in", fs_.files_read_[0]); + EXPECT_EQ("dd", fs_.files_read_[1]); + ASSERT_EQ(2u + files_created, fs_.files_created_.size()); + EXPECT_EQ(1u, fs_.files_created_.count("dd")); + EXPECT_EQ(1u, fs_.files_created_.count("out")); +} + +TEST_F(BuildTest, DyndepBuildSyntaxError) { + // Verify that a dyndep file can be built and loaded to discover + // and reject a syntax error in it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd-in", +"build out: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ("dd:1: expected 'ninja_dyndep_version = ...'\n", err); +} + +TEST_F(BuildTest, DyndepBuildUnrelatedOutput) { + // Verify that a dyndep file can have dependents that do not specify + // it as their dyndep binding. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build unrelated: touch || dd\n" +"build out: touch unrelated || dd\n" +" dyndep = dd\n" + )); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch unrelated", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewOutput) { + // Verify that a dyndep file can be built and loaded to discover + // a new output of an edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch in || dd\n" +" dyndep = dd\n" + )); + fs_.Create("in", ""); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out | out.imp: dyndep\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(2u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[1]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewOutputWithMultipleRules1) { + // Verify that a dyndep file can be built and loaded to discover + // a new output of an edge that is already the output of another edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out1 | out-twice.imp: touch in\n" +"build out2: touch in || dd\n" +" dyndep = dd\n" + )); + fs_.Create("in", ""); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out2 | out-twice.imp: dyndep\n" +); + fs_.Tick(); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ("multiple rules generate out-twice.imp", err); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewOutputWithMultipleRules2) { + // Verify that a dyndep file can be built and loaded to discover + // a new output of an edge that is already the output of another + // edge also discovered by dyndep. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd1: cp dd1-in\n" +"build out1: touch || dd1\n" +" dyndep = dd1\n" +"build dd2: cp dd2-in || dd1\n" // make order predictable for test +"build out2: touch || dd2\n" +" dyndep = dd2\n" +)); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + fs_.Create("dd1-in", +"ninja_dyndep_version = 1\n" +"build out1 | out-twice.imp: dyndep\n" +); + fs_.Create("dd2-in", ""); + fs_.Create("dd2", +"ninja_dyndep_version = 1\n" +"build out2 | out-twice.imp: dyndep\n" +); + fs_.Tick(); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ("multiple rules generate out-twice.imp", err); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewInput) { + // Verify that a dyndep file can be built and loaded to discover + // a new input to an edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build in: touch\n" +"build out: touch || dd\n" +" dyndep = dd\n" + )); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep | in\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch in", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { + // Verify that a dyndep file can be built and loaded to discover + // that one edge has an implicit output that is also an implicit + // input of another edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build tmp: touch || dd\n" +" dyndep = dd\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out | out.imp: dyndep | tmp.imp\n" +"build tmp | tmp.imp: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdge) { + // Verify that a dyndep file can be built and loaded to discover + // that an edge is actually wanted due to a missing implicit output. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build tmp: touch || dd\n" +" dyndep = dd\n" +"build out: touch tmp || dd\n" +" dyndep = dd\n" +)); + fs_.Create("tmp", ""); + fs_.Create("out", ""); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +"build tmp | tmp.imp: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdgeAndDependent) { + // Verify that a dyndep file can be built and loaded to discover + // that an edge and a dependent are actually wanted. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build tmp: touch || dd\n" +" dyndep = dd\n" +"build out: touch tmp\n" +)); + fs_.Create("tmp", ""); + fs_.Create("out", ""); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build tmp | tmp.imp: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverCircular) { + // Verify that a dyndep file can be built and loaded to discover + // and reject a circular dependency. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule r\n" +" command = unused\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: r in || dd\n" +" depfile = out.d\n" +" dyndep = dd\n" +"build in: r || dd\n" +" dyndep = dd\n" + )); + fs_.Create("out.d", "out: inimp\n"); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out | circ: dyndep\n" +"build in: dyndep | circ\n" + ); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + // Depending on how the pointers in Plan::ready_ work out, we could have + // discovered the cycle from either starting point. + EXPECT_TRUE(err == "dependency cycle: circ -> in -> circ" || + err == "dependency cycle: in -> circ -> in"); +} + +TEST_F(BuildWithLogTest, DyndepBuildDiscoverRestat) { + // Verify that a dyndep file can be built and loaded to discover + // that an edge has a restat binding. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule true\n" +" command = true\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out1: true in || dd\n" +" dyndep = dd\n" +"build out2: cat out1\n")); + + fs_.Create("out1", ""); + fs_.Create("out2", ""); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out1: dyndep\n" +" restat = 1\n" +); + fs_.Tick(); + fs_.Create("in", ""); + + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause "out2" to + // rebuild regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("true", command_runner_.commands_ran_[1]); + EXPECT_EQ("cat out1 > out2", command_runner_.commands_ran_[2]); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + fs_.Tick(); + fs_.Create("in", ""); + + // We touched "in", so we should build "out1". But because "true" does not + // touch "out1", we should cancel the build of "out2". + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("true", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, DyndepBuildDiscoverScheduledEdge) { + // Verify that a dyndep file can be built and loaded to discover a + // new input that itself is an output from an edge that has already + // been scheduled but not finished. We should not re-schedule it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build out1 | out1.imp: touch\n" +"build zdd: cp zdd-in\n" +" verify_active_edge = out1\n" // verify out1 is active when zdd is finished +"build out2: cp out1 || zdd\n" +" dyndep = zdd\n" +)); + fs_.Create("zdd-in", +"ninja_dyndep_version = 1\n" +"build out2: dyndep | out1.imp\n" +); + + // Enable concurrent builds so that we can load the dyndep file + // while another edge is still active. + command_runner_.max_active_edges_ = 2; + + // During the build "out1" and "zdd" should be built concurrently. + // The fake command runner will finish these in reverse order + // of the names of the first outputs, so "zdd" will finish first + // and we will load the dyndep file while the edge for "out1" is + // still active. This will add a new dependency on "out1.imp", + // also produced by the active edge. The builder should not + // re-schedule the already-active edge. + + string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + // Depending on how the pointers in Plan::ready_ work out, the first + // two commands may have run in either order. + EXPECT_TRUE((command_runner_.commands_ran_[0] == "touch out1 out1.imp" && + command_runner_.commands_ran_[1] == "cp zdd-in zdd") || + (command_runner_.commands_ran_[1] == "touch out1 out1.imp" && + command_runner_.commands_ran_[0] == "cp zdd-in zdd")); + EXPECT_EQ("cp out1 out2", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepTwoLevelDirect) { + // Verify that a clean dyndep file can depend on a dirty dyndep file + // and be loaded properly after the dirty one is built and loaded. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd1: cp dd1-in\n" +"build out1 | out1.imp: touch || dd1\n" +" dyndep = dd1\n" +"build dd2: cp dd2-in || dd1\n" // direct order-only dep on dd1 +"build out2: touch || dd2\n" +" dyndep = dd2\n" +)); + fs_.Create("out1.imp", ""); + fs_.Create("out2", ""); + fs_.Create("out2.imp", ""); + fs_.Create("dd1-in", +"ninja_dyndep_version = 1\n" +"build out1: dyndep\n" +); + fs_.Create("dd2-in", ""); + fs_.Create("dd2", +"ninja_dyndep_version = 1\n" +"build out2 | out2.imp: dyndep | out1.imp\n" +); + + // During the build dd1 should be built and loaded. The RecomputeDirty + // called as a result of loading dd1 should not cause dd2 to be loaded + // because the builder will never get a chance to update the build plan + // to account for dd2. Instead dd2 should only be later loaded once the + // builder recognizes that it is now ready (as its order-only dependency + // on dd1 has been satisfied). This test case verifies that each dyndep + // file is loaded to update the build graph independently. + + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch out1 out1.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out2 out2.imp", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepTwoLevelIndirect) { + // Verify that dyndep files can add to an edge new implicit inputs that + // correspond to implicit outputs added to other edges by other dyndep + // files on which they (order-only) depend. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd1: cp dd1-in\n" +"build out1: touch || dd1\n" +" dyndep = dd1\n" +"build dd2: cp dd2-in || out1\n" // indirect order-only dep on dd1 +"build out2: touch || dd2\n" +" dyndep = dd2\n" +)); + fs_.Create("out1.imp", ""); + fs_.Create("out2", ""); + fs_.Create("out2.imp", ""); + fs_.Create("dd1-in", +"ninja_dyndep_version = 1\n" +"build out1 | out1.imp: dyndep\n" +); + fs_.Create("dd2-in", ""); + fs_.Create("dd2", +"ninja_dyndep_version = 1\n" +"build out2 | out2.imp: dyndep | out1.imp\n" +); + + // During the build dd1 should be built and loaded. Then dd2 should + // be built and loaded. Loading dd2 should cause the builder to + // recognize that out2 needs to be built even though it was originally + // clean without dyndep info. + + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch out1 out1.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out2 out2.imp", command_runner_.commands_ran_[2]); +} + +TEST_F(BuildTest, DyndepTwoLevelDiscoveredReady) { + // Verify that a dyndep file can discover a new input whose + // edge also has a dyndep file that is ready to load immediately. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd0: cp dd0-in\n" +"build dd1: cp dd1-in\n" +"build in: touch\n" +"build tmp: touch || dd0\n" +" dyndep = dd0\n" +"build out: touch || dd1\n" +" dyndep = dd1\n" + )); + fs_.Create("dd1-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep | tmp\n" +); + fs_.Create("dd0-in", ""); + fs_.Create("dd0", +"ninja_dyndep_version = 1\n" +"build tmp: dyndep | in\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(4u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch in", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch tmp", command_runner_.commands_ran_[2]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[3]); +} + +TEST_F(BuildTest, DyndepTwoLevelDiscoveredDirty) { + // Verify that a dyndep file can discover a new input whose + // edge also has a dyndep file that needs to be built. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd0: cp dd0-in\n" +"build dd1: cp dd1-in\n" +"build in: touch\n" +"build tmp: touch || dd0\n" +" dyndep = dd0\n" +"build out: touch || dd1\n" +" dyndep = dd1\n" + )); + fs_.Create("dd1-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep | tmp\n" +); + fs_.Create("dd0-in", +"ninja_dyndep_version = 1\n" +"build tmp: dyndep | in\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(5u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); + EXPECT_EQ("cp dd0-in dd0", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch in", command_runner_.commands_ran_[2]); + EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[4]); +} diff --git a/src/clean.cc b/src/clean.cc index 1d6ba9e..d1f221d 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -22,21 +22,12 @@ #include "state.h" #include "util.h" -Cleaner::Cleaner(State* state, const BuildConfig& config) - : state_(state), - config_(config), - removed_(), - cleaned_(), - cleaned_files_count_(0), - disk_interface_(new RealDiskInterface), - status_(0) { -} - Cleaner::Cleaner(State* state, const BuildConfig& config, DiskInterface* disk_interface) : state_(state), config_(config), + dyndep_loader_(state, disk_interface), removed_(), cleaned_(), cleaned_files_count_(0), @@ -101,6 +92,7 @@ void Cleaner::PrintHeader() { printf("\n"); else printf(" "); + fflush(stdout); } void Cleaner::PrintFooter() { @@ -112,6 +104,7 @@ void Cleaner::PrintFooter() { int Cleaner::CleanAll(bool generator) { Reset(); PrintHeader(); + LoadDyndeps(); for (vector<Edge*>::iterator e = state_->edges_.begin(); e != state_->edges_.end(); ++e) { // Do not try to remove phony targets @@ -157,6 +150,7 @@ int Cleaner::CleanTarget(Node* target) { Reset(); PrintHeader(); + LoadDyndeps(); DoCleanTarget(target); PrintFooter(); return status_; @@ -179,16 +173,24 @@ int Cleaner::CleanTarget(const char* target) { int Cleaner::CleanTargets(int target_count, char* targets[]) { Reset(); PrintHeader(); + LoadDyndeps(); for (int i = 0; i < target_count; ++i) { - const char* target_name = targets[i]; - Node* target = state_->LookupNode(target_name); - if (target) { - if (IsVerbose()) - printf("Target %s\n", target_name); - DoCleanTarget(target); - } else { - Error("unknown target '%s'", target_name); + string target_name = targets[i]; + uint64_t slash_bits; + string err; + if (!CanonicalizePath(&target_name, &slash_bits, &err)) { + Error("failed to canonicalize '%s': %s", target_name.c_str(), err.c_str()); status_ = 1; + } else { + Node* target = state_->LookupNode(target_name); + if (target) { + if (IsVerbose()) + printf("Target %s\n", target_name.c_str()); + DoCleanTarget(target); + } else { + Error("unknown target '%s'", target_name.c_str()); + status_ = 1; + } } } PrintFooter(); @@ -215,6 +217,7 @@ int Cleaner::CleanRule(const Rule* rule) { Reset(); PrintHeader(); + LoadDyndeps(); DoCleanRule(rule); PrintFooter(); return status_; @@ -239,6 +242,7 @@ int Cleaner::CleanRules(int rule_count, char* rules[]) { Reset(); PrintHeader(); + LoadDyndeps(); for (int i = 0; i < rule_count; ++i) { const char* rule_name = rules[i]; const Rule* rule = state_->bindings_.LookupRule(rule_name); @@ -261,3 +265,16 @@ void Cleaner::Reset() { removed_.clear(); cleaned_.clear(); } + +void Cleaner::LoadDyndeps() { + // Load dyndep files that exist, before they are cleaned. + for (vector<Edge*>::iterator e = state_->edges_.begin(); + e != state_->edges_.end(); ++e) { + if (Node* dyndep = (*e)->dyndep_) { + // Capture and ignore errors loading the dyndep file. + // We clean as much of the graph as we know. + std::string err; + dyndep_loader_.LoadDyndeps(dyndep, &err); + } + } +} diff --git a/src/clean.h b/src/clean.h index 19432ab..d044fb1 100644 --- a/src/clean.h +++ b/src/clean.h @@ -19,6 +19,7 @@ #include <string> #include "build.h" +#include "dyndep.h" using namespace std; @@ -28,11 +29,7 @@ struct Rule; struct DiskInterface; struct Cleaner { - /// Build a cleaner object with a real disk interface. - Cleaner(State* state, const BuildConfig& config); - /// Build a cleaner object with the given @a disk_interface - /// (Useful for testing). Cleaner(State* state, const BuildConfig& config, DiskInterface* disk_interface); @@ -95,8 +92,12 @@ struct Cleaner { void DoCleanRule(const Rule* rule); void Reset(); + /// Load dependencies from dyndep bindings. + void LoadDyndeps(); + State* state_; const BuildConfig& config_; + DyndepLoader dyndep_loader_; set<string> removed_; set<Node*> cleaned_; int cleaned_files_count_; diff --git a/src/clean_test.cc b/src/clean_test.cc index 395343b..45187f4 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -285,6 +285,55 @@ TEST_F(CleanTest, CleanDepFileOnCleanRule) { EXPECT_EQ(2u, fs_.files_removed_.size()); } +TEST_F(CleanTest, CleanDyndep) { + // Verify that a dyndep file can be loaded to discover a new output + // to be cleaned. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out: cat in || dd\n" +" dyndep = dd\n" + )); + fs_.Create("in", ""); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out | out.imp: dyndep\n" +); + fs_.Create("out", ""); + fs_.Create("out.imp", ""); + + Cleaner cleaner(&state_, config_, &fs_); + + ASSERT_EQ(0, cleaner.cleaned_files_count()); + EXPECT_EQ(0, cleaner.CleanAll()); + EXPECT_EQ(2, cleaner.cleaned_files_count()); + EXPECT_EQ(2u, fs_.files_removed_.size()); + + string err; + EXPECT_EQ(0, fs_.Stat("out", &err)); + EXPECT_EQ(0, fs_.Stat("out.imp", &err)); +} + +TEST_F(CleanTest, CleanDyndepMissing) { + // Verify that a missing dyndep file is tolerated. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out: cat in || dd\n" +" dyndep = dd\n" + )); + fs_.Create("in", ""); + fs_.Create("out", ""); + fs_.Create("out.imp", ""); + + Cleaner cleaner(&state_, config_, &fs_); + + ASSERT_EQ(0, cleaner.cleaned_files_count()); + EXPECT_EQ(0, cleaner.CleanAll()); + EXPECT_EQ(1, cleaner.cleaned_files_count()); + EXPECT_EQ(1u, fs_.files_removed_.size()); + + string err; + EXPECT_EQ(0, fs_.Stat("out", &err)); + EXPECT_EQ(1, fs_.Stat("out.imp", &err)); +} + TEST_F(CleanTest, CleanRspFile) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n" @@ -325,7 +374,7 @@ TEST_F(CleanTest, CleanRsp) { Cleaner cleaner(&state_, config_, &fs_); ASSERT_EQ(0, cleaner.cleaned_files_count()); ASSERT_EQ(0, cleaner.CleanTarget("out1")); - EXPECT_EQ(2, cleaner.cleaned_files_count()); + EXPECT_EQ(2, cleaner.cleaned_files_count()); ASSERT_EQ(0, cleaner.CleanTarget("in2")); EXPECT_EQ(2, cleaner.cleaned_files_count()); ASSERT_EQ(0, cleaner.CleanRule("cat_rsp")); diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 7cee892..6faeac6 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.13.5 */ +/* Generated by re2c 1.1.1 */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,12 @@ // limitations under the License. #include "depfile_parser.h" +#include "util.h" + +DepfileParser::DepfileParser(DepfileParserOptions options) + : options_(options) +{ +} // A note on backslashes in Makefiles, from reading the docs: // Backslash-newline is the line continuation character. @@ -24,9 +30,15 @@ // How do you end a line with a backslash? The netbsd Make docs suggest // reading the result of a shell command echoing a backslash! // -// Rather than implement all of above, we do a simpler thing here: -// Backslashes escape a set of characters (see "escapes" defined below), -// otherwise they are passed through verbatim. +// Rather than implement all of above, we follow what GCC/Clang produces: +// Backslashes escape a space or hash sign. +// When a space is preceded by 2N+1 backslashes, it is represents N backslashes +// followed by space. +// When a space is preceded by 2N backslashes, it represents 2N backslashes at +// the end of a filename. +// A hash sign is escaped by a single backslash. All other backslashes remain +// unchanged. +// // If anyone actually has depfiles that rely on the more complicated // behavior we can adjust this. bool DepfileParser::Parse(string* content, string* err) { @@ -35,8 +47,13 @@ bool DepfileParser::Parse(string* content, string* err) { // parsing_targets: whether we are parsing targets or dependencies. char* in = &(*content)[0]; char* end = in + content->size(); + bool have_target = false; + bool have_secondary_target_on_this_rule = false; + bool have_newline_since_primary_target = false; + bool warned_distinct_target_lines = false; bool parsing_targets = true; while (in < end) { + bool have_newline = false; // out: current output point (typically same as in, but can fall behind // as we de-escape backslashes). char* out = in; @@ -45,6 +62,7 @@ bool DepfileParser::Parse(string* content, string* err) { for (;;) { // start: beginning of the current parsed span. const char* start = in; + char* yymarker = NULL; { unsigned char yych; @@ -53,14 +71,14 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 128, 0, 0, 0, 0, 0, 0, + 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 0, 0, 128, + 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -82,88 +100,57 @@ bool DepfileParser::Parse(string* content, string* err) { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, }; - yych = *in; - if (yych <= '=') { - if (yych <= '$') { - if (yych <= ' ') { - if (yych <= 0x00) goto yy7; - goto yy9; - } else { - if (yych <= '!') goto yy5; - if (yych <= '#') goto yy9; - goto yy4; - } + if (yybm[0+yych] & 128) { + goto yy9; + } + if (yych <= '\r') { + if (yych <= '\t') { + if (yych >= 0x01) goto yy4; } else { - if (yych <= '*') { - if (yych <= '\'') goto yy9; - if (yych <= ')') goto yy5; - goto yy9; - } else { - if (yych <= ':') goto yy5; - if (yych <= '<') goto yy9; - goto yy5; - } + if (yych <= '\n') goto yy6; + if (yych <= '\f') goto yy4; + goto yy8; } } else { - if (yych <= '_') { - if (yych <= '[') { - if (yych <= '?') goto yy9; - if (yych <= 'Z') goto yy5; - goto yy9; - } else { - if (yych <= '\\') goto yy2; - if (yych <= '^') goto yy9; - goto yy5; - } + if (yych <= '$') { + if (yych <= '#') goto yy4; + goto yy12; } else { - if (yych <= '|') { - if (yych <= '`') goto yy9; - if (yych <= '{') goto yy5; - goto yy9; - } else { - if (yych == 0x7F) goto yy9; - goto yy5; - } + if (yych <= '?') goto yy4; + if (yych <= '\\') goto yy13; + goto yy4; } } -yy2: ++in; - if ((yych = *in) <= '"') { - if (yych <= '\f') { - if (yych <= 0x00) goto yy3; - if (yych != '\n') goto yy14; - } else { - if (yych <= '\r') goto yy3; - if (yych == ' ') goto yy16; - goto yy14; - } - } else { - if (yych <= 'Z') { - if (yych <= '#') goto yy16; - if (yych == '*') goto yy16; - goto yy14; - } else { - if (yych <= '\\') goto yy16; - if (yych == '|') goto yy16; - goto yy14; - } + { + break; } -yy3: +yy4: + ++in; +yy5: { // For any other character (e.g. whitespace), swallow it here, // allowing the outer logic to loop around again. break; } -yy4: - yych = *++in; - if (yych == '$') goto yy12; - goto yy3; -yy5: - ++in; - yych = *in; - goto yy11; yy6: + ++in; + { + // A newline ends the current file name and the current rule. + have_newline = true; + break; + } +yy8: + yych = *++in; + if (yych == '\n') goto yy6; + goto yy5; +yy9: + yych = *++in; + if (yybm[0+yych] & 128) { + goto yy9; + } +yy11: { // Got a span of plain text. int len = (int)(in - start); @@ -173,68 +160,178 @@ yy6: out += len; continue; } -yy7: - ++in; - { - break; - } -yy9: +yy12: yych = *++in; - goto yy3; -yy10: - ++in; - yych = *in; -yy11: - if (yybm[0+yych] & 128) { - goto yy10; + if (yych == '$') goto yy14; + goto yy5; +yy13: + yych = *(yymarker = ++in); + if (yych <= 0x1F) { + if (yych <= '\n') { + if (yych <= 0x00) goto yy5; + if (yych <= '\t') goto yy16; + goto yy17; + } else { + if (yych == '\r') goto yy19; + goto yy16; + } + } else { + if (yych <= '#') { + if (yych <= ' ') goto yy21; + if (yych <= '"') goto yy16; + goto yy23; + } else { + if (yych == '\\') goto yy25; + goto yy16; + } } - goto yy6; -yy12: +yy14: ++in; { // De-escape dollar character. *out++ = '$'; continue; } -yy14: +yy16: + ++in; + goto yy11; +yy17: + ++in; + { + // A line continuation ends the current file name. + break; + } +yy19: + yych = *++in; + if (yych == '\n') goto yy17; + in = yymarker; + goto yy5; +yy21: ++in; { - // Let backslash before other characters through verbatim. - *out++ = '\\'; - *out++ = yych; + // 2N+1 backslashes plus space -> N backslashes plus space. + int len = (int)(in - start); + int n = len / 2 - 1; + if (out < start) + memset(out, '\\', n); + out += n; + *out++ = ' '; continue; } -yy16: +yy23: ++in; { - // De-escape backslashed character. - *out++ = yych; + // De-escape hash sign, but preserve other leading backslashes. + int len = (int)(in - start); + if (len > 2 && out < start) + memset(out, '\\', len - 2); + out += len - 2; + *out++ = '#'; continue; } +yy25: + yych = *++in; + if (yych <= 0x1F) { + if (yych <= '\n') { + if (yych <= 0x00) goto yy11; + if (yych <= '\t') goto yy16; + goto yy11; + } else { + if (yych == '\r') goto yy11; + goto yy16; + } + } else { + if (yych <= '#') { + if (yych <= ' ') goto yy26; + if (yych <= '"') goto yy16; + goto yy23; + } else { + if (yych == '\\') goto yy28; + goto yy16; + } + } +yy26: + ++in; + { + // 2N backslashes plus space -> 2N backslashes, end of filename. + int len = (int)(in - start); + if (out < start) + memset(out, '\\', len - 1); + out += len - 1; + break; + } +yy28: + yych = *++in; + if (yych <= 0x1F) { + if (yych <= '\n') { + if (yych <= 0x00) goto yy11; + if (yych <= '\t') goto yy16; + goto yy11; + } else { + if (yych == '\r') goto yy11; + goto yy16; + } + } else { + if (yych <= '#') { + if (yych <= ' ') goto yy21; + if (yych <= '"') goto yy16; + goto yy23; + } else { + if (yych == '\\') goto yy25; + goto yy16; + } + } } } int len = (int)(out - filename); - const bool is_target = parsing_targets; + const bool is_dependency = !parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. parsing_targets = false; + have_target = true; } - if (len == 0) - continue; + if (len > 0) { + if (is_dependency) { + if (have_secondary_target_on_this_rule) { + if (!have_newline_since_primary_target) { + *err = "depfile has multiple output paths"; + return false; + } else if (options_.depfile_distinct_target_lines_action_ == + kDepfileDistinctTargetLinesActionError) { + *err = + "depfile has multiple output paths (on separate lines)" + " [-w depfilemulti=err]"; + return false; + } else { + if (!warned_distinct_target_lines) { + warned_distinct_target_lines = true; + Warning("depfile has multiple output paths (on separate lines); " + "continuing anyway [-w depfilemulti=warn]"); + } + continue; + } + } + ins_.push_back(StringPiece(filename, len)); + } else if (!out_.str_) { + out_ = StringPiece(filename, len); + } else if (out_ != StringPiece(filename, len)) { + have_secondary_target_on_this_rule = true; + } + } - if (!is_target) { - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths"; - return false; + if (have_newline) { + // A newline ends a rule so the next filename will be a new target. + parsing_targets = true; + have_secondary_target_on_this_rule = false; + if (have_target) { + have_newline_since_primary_target = true; + } } } - if (parsing_targets) { + if (!have_target) { *err = "expected ':' in depfile"; return false; } diff --git a/src/depfile_parser.h b/src/depfile_parser.h index 1e6ebb5..be20374 100644 --- a/src/depfile_parser.h +++ b/src/depfile_parser.h @@ -21,8 +21,24 @@ using namespace std; #include "string_piece.h" +enum DepfileDistinctTargetLinesAction { + kDepfileDistinctTargetLinesActionWarn, + kDepfileDistinctTargetLinesActionError, +}; + +struct DepfileParserOptions { + DepfileParserOptions() + : depfile_distinct_target_lines_action_( + kDepfileDistinctTargetLinesActionWarn) {} + DepfileDistinctTargetLinesAction + depfile_distinct_target_lines_action_; +}; + /// Parser for the dependency information emitted by gcc's -M flags. struct DepfileParser { + explicit DepfileParser(DepfileParserOptions options = + DepfileParserOptions()); + /// Parse an input file. Input must be NUL-terminated. /// Warning: may mutate the content in-place and parsed StringPieces are /// pointers within it. @@ -30,6 +46,7 @@ struct DepfileParser { StringPiece out_; vector<StringPiece> ins_; + DepfileParserOptions options_; }; #endif // NINJA_DEPFILE_PARSER_H_ diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 98c1621..735a0c3 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -13,6 +13,12 @@ // limitations under the License. #include "depfile_parser.h" +#include "util.h" + +DepfileParser::DepfileParser(DepfileParserOptions options) + : options_(options) +{ +} // A note on backslashes in Makefiles, from reading the docs: // Backslash-newline is the line continuation character. @@ -23,9 +29,15 @@ // How do you end a line with a backslash? The netbsd Make docs suggest // reading the result of a shell command echoing a backslash! // -// Rather than implement all of above, we do a simpler thing here: -// Backslashes escape a set of characters (see "escapes" defined below), -// otherwise they are passed through verbatim. +// Rather than implement all of above, we follow what GCC/Clang produces: +// Backslashes escape a space or hash sign. +// When a space is preceded by 2N+1 backslashes, it is represents N backslashes +// followed by space. +// When a space is preceded by 2N backslashes, it represents 2N backslashes at +// the end of a filename. +// A hash sign is escaped by a single backslash. All other backslashes remain +// unchanged. +// // If anyone actually has depfiles that rely on the more complicated // behavior we can adjust this. bool DepfileParser::Parse(string* content, string* err) { @@ -34,8 +46,13 @@ bool DepfileParser::Parse(string* content, string* err) { // parsing_targets: whether we are parsing targets or dependencies. char* in = &(*content)[0]; char* end = in + content->size(); + bool have_target = false; + bool have_secondary_target_on_this_rule = false; + bool have_newline_since_primary_target = false; + bool warned_distinct_target_lines = false; bool parsing_targets = true; while (in < end) { + bool have_newline = false; // out: current output point (typically same as in, but can fall behind // as we de-escape backslashes). char* out = in; @@ -44,10 +61,12 @@ bool DepfileParser::Parse(string* content, string* err) { for (;;) { // start: beginning of the current parsed span. const char* start = in; + char* yymarker = NULL; /*!re2c re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = in; re2c:define:YYLIMIT = end; + re2c:define:YYMARKER = yymarker; re2c:yyfill:enable = 0; @@ -55,11 +74,33 @@ bool DepfileParser::Parse(string* content, string* err) { re2c:indent:string = " "; nul = "\000"; - escape = [ \\#*[|]; + newline = '\r'?'\n'; - '\\' escape { - // De-escape backslashed character. - *out++ = yych; + '\\\\'* '\\ ' { + // 2N+1 backslashes plus space -> N backslashes plus space. + int len = (int)(in - start); + int n = len / 2 - 1; + if (out < start) + memset(out, '\\', n); + out += n; + *out++ = ' '; + continue; + } + '\\\\'+ ' ' { + // 2N backslashes plus space -> 2N backslashes, end of filename. + int len = (int)(in - start); + if (out < start) + memset(out, '\\', len - 1); + out += len - 1; + break; + } + '\\'+ '#' { + // De-escape hash sign, but preserve other leading backslashes. + int len = (int)(in - start); + if (len > 2 && out < start) + memset(out, '\\', len - 2); + out += len - 2; + *out++ = '#'; continue; } '$$' { @@ -67,13 +108,7 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = '$'; continue; } - '\\' [^\000\r\n] { - // Let backslash before other characters through verbatim. - *out++ = '\\'; - *out++ = yych; - continue; - } - [a-zA-Z0-9+,/_:.~()}{@=!\x80-\xFF-]+ { + '\\'+ [^\000\r\n] | [a-zA-Z0-9+,/_:.~()}{%=@\x5B\x5D!\x80-\xFF-]+ { // Got a span of plain text. int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. @@ -85,6 +120,15 @@ bool DepfileParser::Parse(string* content, string* err) { nul { break; } + '\\' newline { + // A line continuation ends the current file name. + break; + } + newline { + // A newline ends the current file name and the current rule. + have_newline = true; + break; + } [^] { // For any other character (e.g. whitespace), swallow it here, // allowing the outer logic to loop around again. @@ -94,25 +138,52 @@ bool DepfileParser::Parse(string* content, string* err) { } int len = (int)(out - filename); - const bool is_target = parsing_targets; + const bool is_dependency = !parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. parsing_targets = false; + have_target = true; } - if (len == 0) - continue; + if (len > 0) { + if (is_dependency) { + if (have_secondary_target_on_this_rule) { + if (!have_newline_since_primary_target) { + *err = "depfile has multiple output paths"; + return false; + } else if (options_.depfile_distinct_target_lines_action_ == + kDepfileDistinctTargetLinesActionError) { + *err = + "depfile has multiple output paths (on separate lines)" + " [-w depfilemulti=err]"; + return false; + } else { + if (!warned_distinct_target_lines) { + warned_distinct_target_lines = true; + Warning("depfile has multiple output paths (on separate lines); " + "continuing anyway [-w depfilemulti=warn]"); + } + continue; + } + } + ins_.push_back(StringPiece(filename, len)); + } else if (!out_.str_) { + out_ = StringPiece(filename, len); + } else if (out_ != StringPiece(filename, len)) { + have_secondary_target_on_this_rule = true; + } + } - if (!is_target) { - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths"; - return false; + if (have_newline) { + // A newline ends a rule so the next filename will be a new target. + parsing_targets = true; + have_secondary_target_on_this_rule = false; + if (have_target) { + have_newline_since_primary_target = true; + } } } - if (parsing_targets) { + if (!have_target) { *err = "expected ':' in depfile"; return false; } diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index ee798f8..19224f3 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -101,15 +101,36 @@ TEST_F(DepfileParserTest, Spaces) { parser_.ins_[2].AsString()); } +TEST_F(DepfileParserTest, MultipleBackslashes) { + // Successive 2N+1 backslashes followed by space (' ') are replaced by N >= 0 + // backslashes and the space. A single backslash before hash sign is removed. + // Other backslashes remain untouched (including 2N backslashes followed by + // space). + string err; + EXPECT_TRUE(Parse( +"a\\ b\\#c.h: \\\\\\\\\\ \\\\\\\\ \\\\share\\info\\\\#1", + &err)); + ASSERT_EQ("", err); + EXPECT_EQ("a b#c.h", + parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("\\\\ ", + parser_.ins_[0].AsString()); + EXPECT_EQ("\\\\\\\\", + parser_.ins_[1].AsString()); + EXPECT_EQ("\\\\share\\info\\#1", + parser_.ins_[2].AsString()); +} + TEST_F(DepfileParserTest, Escapes) { // Put backslashes before a variety of characters, see which ones make // it through. string err; EXPECT_TRUE(Parse( -"\\!\\@\\#$$\\%\\^\\&\\\\:", +"\\!\\@\\#$$\\%\\^\\&\\[\\]\\\\:", &err)); ASSERT_EQ("", err); - EXPECT_EQ("\\!\\@#$\\%\\^\\&\\", + EXPECT_EQ("\\!\\@#$\\%\\^\\&\\[\\]\\\\", parser_.out_.AsString()); ASSERT_EQ(0u, parser_.ins_.size()); } @@ -119,15 +140,16 @@ TEST_F(DepfileParserTest, SpecialChars) { // https://github.com/google/libcxx/tree/master/test/iterators/stream.iterators/istreambuf.iterator/ string err; EXPECT_TRUE(Parse( -"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" -" en@quot.header~ t+t-x!=1 \n" -" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif\n" -" Fu\303\244ball", +"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \\\n" +" en@quot.header~ t+t-x!=1 \\\n" +" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif\\\n" +" Fu\303\244ball\\\n" +" a[1]b@2%c", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", parser_.out_.AsString()); - ASSERT_EQ(4u, parser_.ins_.size()); + ASSERT_EQ(5u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); EXPECT_EQ("t+t-x!=1", @@ -136,6 +158,8 @@ TEST_F(DepfileParserTest, SpecialChars) { parser_.ins_[2].AsString()); EXPECT_EQ("Fu\303\244ball", parser_.ins_[3].AsString()); + EXPECT_EQ("a[1]b@2%c", + parser_.ins_[4].AsString()); } TEST_F(DepfileParserTest, UnifyMultipleOutputs) { @@ -155,3 +179,133 @@ TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { EXPECT_FALSE(Parse("foo bar: x y z", &err)); ASSERT_EQ("depfile has multiple output paths", err); } + +TEST_F(DepfileParserTest, MultipleEmptyRules) { + string err; + EXPECT_TRUE(Parse("foo: x\n" + "foo: \n" + "foo:\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMultipleRulesLF) { + string err; + EXPECT_TRUE(Parse("foo: x\n" + "foo: y\n" + "foo \\\n" + "foo: z\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMultipleRulesCRLF) { + string err; + EXPECT_TRUE(Parse("foo: x\r\n" + "foo: y\r\n" + "foo \\\r\n" + "foo: z\r\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMixedRulesLF) { + string err; + EXPECT_TRUE(Parse("foo: x\\\n" + " y\n" + "foo \\\n" + "foo: z\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMixedRulesCRLF) { + string err; + EXPECT_TRUE(Parse("foo: x\\\r\n" + " y\r\n" + "foo \\\r\n" + "foo: z\r\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, IndentedRulesLF) { + string err; + EXPECT_TRUE(Parse(" foo: x\n" + " foo: y\n" + " foo: z\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, IndentedRulesCRLF) { + string err; + EXPECT_TRUE(Parse(" foo: x\r\n" + " foo: y\r\n" + " foo: z\r\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, TolerateMP) { + string err; + EXPECT_TRUE(Parse("foo: x y z\n" + "x:\n" + "y:\n" + "z:\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, MultipleRulesTolerateMP) { + string err; + EXPECT_TRUE(Parse("foo: x\n" + "x:\n" + "foo: y\n" + "y:\n" + "foo: z\n" + "z:\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, MultipleRulesRejectDifferentOutputs) { + // check that multiple different outputs are rejected by the parser + // when spread across multiple rules + DepfileParserOptions parser_opts; + parser_opts.depfile_distinct_target_lines_action_ = + kDepfileDistinctTargetLinesActionError; + DepfileParser parser(parser_opts); + string err; + string input = + "foo: x y\n" + "bar: y z\n"; + EXPECT_FALSE(parser.Parse(&input, &err)); + ASSERT_EQ("depfile has multiple output paths (on separate lines)" + " [-w depfilemulti=err]", err); +} diff --git a/src/deps_log.cc b/src/deps_log.cc index 89c6023..4aaffeb 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -20,6 +20,9 @@ #include <string.h> #ifndef _WIN32 #include <unistd.h> +#elif defined(_MSC_VER) && (_MSC_VER < 1900) +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; #endif #include "graph.h" @@ -30,7 +33,7 @@ // The version is stored as 4 bytes after the signature and also serves as a // byte order mark. Signature and version combined are 16 bytes long. const char kFileSignature[] = "# ninjadeps\n"; -const int kCurrentVersion = 3; +const int kCurrentVersion = 4; // Record size is currently limited to less than the full 32 bit, due to // internal buffers having to have this size. @@ -45,7 +48,7 @@ bool DepsLog::OpenForWrite(const string& path, string* err) { if (!Recompact(path, err)) return false; } - + file_ = fopen(path.c_str(), "ab"); if (!file_) { *err = strerror(errno); @@ -124,7 +127,7 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, return true; // Update on-disk representation. - unsigned size = 4 * (1 + 1 + node_count); + unsigned size = 4 * (1 + 2 + node_count); if (size > kMaxRecordSize) { errno = ERANGE; return false; @@ -135,8 +138,11 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, int id = node->id(); if (fwrite(&id, 4, 1, file_) < 1) return false; - int timestamp = mtime; - if (fwrite(×tamp, 4, 1, file_) < 1) + uint32_t mtime_part = static_cast<uint32_t>(mtime & 0xffffffff); + if (fwrite(&mtime_part, 4, 1, file_) < 1) + return false; + mtime_part = static_cast<uint32_t>((mtime >> 32) & 0xffffffff); + if (fwrite(&mtime_part, 4, 1, file_) < 1) return false; for (int i = 0; i < node_count; ++i) { id = nodes[i]->id(); @@ -209,7 +215,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { bool is_deps = (size >> 31) != 0; size = size & 0x7FFFFFFF; - if (fread(buf, size, 1, f) < 1 || size > kMaxRecordSize) { + if (size > kMaxRecordSize || fread(buf, size, 1, f) < 1) { read_failed = true; break; } @@ -218,9 +224,11 @@ bool DepsLog::Load(const string& path, State* state, string* err) { assert(size % 4 == 0); int* deps_data = reinterpret_cast<int*>(buf); int out_id = deps_data[0]; - int mtime = deps_data[1]; - deps_data += 2; - int deps_count = (size / 4) - 2; + TimeStamp mtime; + mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) | + (uint64_t)(unsigned int)deps_data[1]); + deps_data += 3; + int deps_count = (size / 4) - 3; Deps* deps = new Deps(mtime, deps_count); for (int i = 0; i < deps_count; ++i) { @@ -323,7 +331,7 @@ bool DepsLog::Recompact(const string& path, string* err) { // will refer to the ordering in new_log, not in the current log. for (vector<Node*>::iterator i = nodes_.begin(); i != nodes_.end(); ++i) (*i)->set_id(-1); - + // Write out all deps again. for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) { Deps* deps = deps_[old_id]; diff --git a/src/deps_log.h b/src/deps_log.h index cec0257..3812a28 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -57,7 +57,9 @@ struct State; /// one's complement of the expected index of the record (to detect /// concurrent writes of multiple ninja processes to the log). /// dependency records are an array of 4-byte integers -/// [output path id, output path mtime, input path id, input path id...] +/// [output path id, +/// output path mtime (lower 4 bytes), output path mtime (upper 4 bytes), +/// input path id, input path id...] /// (The mtime is compared against the on-disk output path mtime /// to verify the stored data is up-to-date.) /// If two records reference the same output the latter one in the file @@ -75,10 +77,10 @@ struct DepsLog { // Reading (startup-time) interface. struct Deps { - Deps(int mtime, int node_count) + Deps(int64_t mtime, int node_count) : mtime(mtime), node_count(node_count), nodes(new Node*[node_count]) {} ~Deps() { delete [] nodes; } - int mtime; + TimeStamp mtime; int node_count; Node** nodes; }; diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 89f7be1..0cdeb45 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -143,7 +143,7 @@ TEST_F(DepsLogTest, DoubleEntry) { ASSERT_GT(file_size, 0); } - // Now reload the file, and readd the same deps. + // Now reload the file, and read the same deps. { State state; DepsLog log; @@ -203,7 +203,7 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_GT(file_size, 0); } - // Now reload the file, and add slighly different deps. + // Now reload the file, and add slightly different deps. int file_size_2; { State state; diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 28530b1..d4c2fb0 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -35,14 +35,15 @@ namespace { string DirName(const string& path) { #ifdef _WIN32 - const char kPathSeparators[] = "\\/"; + static const char kPathSeparators[] = "\\/"; #else - const char kPathSeparators[] = "/"; + static const char kPathSeparators[] = "/"; #endif + static const char* const kEnd = kPathSeparators + sizeof(kPathSeparators) - 1; + string::size_type slash_pos = path.find_last_of(kPathSeparators); if (slash_pos == string::npos) return string(); // Nothing to do. - const char* const kEnd = kPathSeparators + strlen(kPathSeparators); while (slash_pos > 0 && std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd) --slash_pos; @@ -61,17 +62,16 @@ int MakeDir(const string& path) { TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { // FILETIME is in 100-nanosecond increments since the Windows epoch. // We don't much care about epoch correctness but we do want the - // resulting value to fit in an integer. + // resulting value to fit in a 64-bit integer. uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | ((uint64_t)filetime.dwLowDateTime); - mtime /= 1000000000LL / 100; // 100ns -> s. - mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime; + // 1600 epoch -> 2000 epoch (subtract 400 years). + return (TimeStamp)mtime - 12622770400LL * (1000000000LL / 100); } TimeStamp StatSingleFile(const string& path, string* err) { WIN32_FILE_ATTRIBUTE_DATA attrs; - if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { + if (!GetFileAttributesExA(path.c_str(), GetFileExInfoStandard, &attrs)) { DWORD win_err = GetLastError(); if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) return 0; @@ -113,6 +113,11 @@ bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps, } do { string lowername = ffd.cFileName; + if (lowername == "..") { + // Seems to just copy the timestamp for ".." from ".", which is wrong. + // This is the case at least on NTFS under Windows 7. + continue; + } transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower); stamps->insert(make_pair(lowername, TimeStampFromFileTime(ffd.ftLastWriteTime))); @@ -165,6 +170,11 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { string dir = DirName(path); string base(path.substr(dir.size() ? dir.size() + 1 : 0)); + if (base == "..") { + // StatAllFilesInDir does not report any information for base = "..". + base = "."; + dir = path; + } transform(dir.begin(), dir.end(), dir.begin(), ::tolower); transform(base.begin(), base.end(), base.begin(), ::tolower); @@ -192,7 +202,22 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { // that it doesn't exist. if (st.st_mtime == 0) return 1; - return st.st_mtime; +#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE) + return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL + + st.st_mtimespec.tv_nsec); +#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ + defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__)) + // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html + // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above. + // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar + // For bionic, C and POSIX API is always enabled. + // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html. + return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec; +#elif defined(_AIX) + return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n; +#else + return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec; +#endif #endif } diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index d7fb8f8..bac515d 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -87,6 +87,8 @@ TEST_F(DiskInterfaceTest, StatExistingDir) { string err; ASSERT_TRUE(disk_.MakeDir("subdir")); ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir")); + EXPECT_GT(disk_.Stat("..", &err), 1); + EXPECT_EQ("", err); EXPECT_GT(disk_.Stat(".", &err), 1); EXPECT_EQ("", err); EXPECT_GT(disk_.Stat("subdir", &err), 1); @@ -105,7 +107,6 @@ TEST_F(DiskInterfaceTest, StatExistingDir) { #ifdef _WIN32 TEST_F(DiskInterfaceTest, StatCache) { string err; - disk_.AllowStatCache(true); ASSERT_TRUE(Touch("file1")); ASSERT_TRUE(Touch("fiLE2")); @@ -115,6 +116,10 @@ TEST_F(DiskInterfaceTest, StatCache) { ASSERT_TRUE(Touch("subdir\\SUBFILE2")); ASSERT_TRUE(Touch("subdir\\SUBFILE3")); + disk_.AllowStatCache(false); + TimeStamp parent_stat_uncached = disk_.Stat("..", &err); + disk_.AllowStatCache(true); + EXPECT_GT(disk_.Stat("FIle1", &err), 1); EXPECT_EQ("", err); EXPECT_GT(disk_.Stat("file1", &err), 1); @@ -125,6 +130,8 @@ TEST_F(DiskInterfaceTest, StatCache) { EXPECT_GT(disk_.Stat("sUbdir\\suBFile1", &err), 1); EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("..", &err), 1); + EXPECT_EQ("", err); EXPECT_GT(disk_.Stat(".", &err), 1); EXPECT_EQ("", err); EXPECT_GT(disk_.Stat("subdir", &err), 1); @@ -132,11 +139,15 @@ TEST_F(DiskInterfaceTest, StatCache) { EXPECT_GT(disk_.Stat("subdir/subsubdir", &err), 1); EXPECT_EQ("", err); +#ifndef _MSC_VER // TODO: Investigate why. Also see https://github.com/ninja-build/ninja/pull/1423 EXPECT_EQ(disk_.Stat("subdir", &err), disk_.Stat("subdir/.", &err)); EXPECT_EQ("", err); EXPECT_EQ(disk_.Stat("subdir", &err), disk_.Stat("subdir/subsubdir/..", &err)); +#endif + EXPECT_EQ("", err); + EXPECT_EQ(disk_.Stat("..", &err), parent_stat_uncached); EXPECT_EQ("", err); EXPECT_EQ(disk_.Stat("subdir/subsubdir", &err), disk_.Stat("subdir/subsubdir/.", &err)); @@ -202,7 +213,7 @@ TEST_F(DiskInterfaceTest, RemoveFile) { struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { - StatTest() : scan_(&state_, NULL, NULL, this) {} + StatTest() : scan_(&state_, NULL, NULL, this, NULL) {} // DiskInterface implementation. virtual TimeStamp Stat(const string& path, string* err) const; diff --git a/src/dyndep.cc b/src/dyndep.cc new file mode 100644 index 0000000..2aee601 --- /dev/null +++ b/src/dyndep.cc @@ -0,0 +1,124 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dyndep.h" + +#include <assert.h> +#include <stdio.h> + +#include "debug_flags.h" +#include "disk_interface.h" +#include "dyndep_parser.h" +#include "graph.h" +#include "state.h" +#include "util.h" + +bool DyndepLoader::LoadDyndeps(Node* node, std::string* err) const { + DyndepFile ddf; + return LoadDyndeps(node, &ddf, err); +} + +bool DyndepLoader::LoadDyndeps(Node* node, DyndepFile* ddf, + std::string* err) const { + // We are loading the dyndep file now so it is no longer pending. + node->set_dyndep_pending(false); + + // Load the dyndep information from the file. + EXPLAIN("loading dyndep file '%s'", node->path().c_str()); + if (!LoadDyndepFile(node, ddf, err)) + return false; + + // Update each edge that specified this node as its dyndep binding. + std::vector<Edge*> const& out_edges = node->out_edges(); + for (std::vector<Edge*>::const_iterator oe = out_edges.begin(); + oe != out_edges.end(); ++oe) { + Edge* const edge = *oe; + if (edge->dyndep_ != node) + continue; + + DyndepFile::iterator ddi = ddf->find(edge); + if (ddi == ddf->end()) { + *err = ("'" + edge->outputs_[0]->path() + "' " + "not mentioned in its dyndep file " + "'" + node->path() + "'"); + return false; + } + + ddi->second.used_ = true; + Dyndeps const& dyndeps = ddi->second; + if (!UpdateEdge(edge, &dyndeps, err)) { + return false; + } + } + + // Reject extra outputs in dyndep file. + for (DyndepFile::const_iterator oe = ddf->begin(); oe != ddf->end(); + ++oe) { + if (!oe->second.used_) { + Edge* const edge = oe->first; + *err = ("dyndep file '" + node->path() + "' mentions output " + "'" + edge->outputs_[0]->path() + "' whose build statement " + "does not have a dyndep binding for the file"); + return false; + } + } + + return true; +} + +bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps, + std::string* err) const { + // Add dyndep-discovered bindings to the edge. + // We know the edge already has its own binding + // scope because it has a "dyndep" binding. + if (dyndeps->restat_) + edge->env_->AddBinding("restat", "1"); + + // Add the dyndep-discovered outputs to the edge. + edge->outputs_.insert(edge->outputs_.end(), + dyndeps->implicit_outputs_.begin(), + dyndeps->implicit_outputs_.end()); + edge->implicit_outs_ += dyndeps->implicit_outputs_.size(); + + // Add this edge as incoming to each new output. + for (std::vector<Node*>::const_iterator i = + dyndeps->implicit_outputs_.begin(); + i != dyndeps->implicit_outputs_.end(); ++i) { + if ((*i)->in_edge() != NULL) { + *err = "multiple rules generate " + (*i)->path(); + return false; + } + (*i)->set_in_edge(edge); + } + + // Add the dyndep-discovered inputs to the edge. + edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, + dyndeps->implicit_inputs_.begin(), + dyndeps->implicit_inputs_.end()); + edge->implicit_deps_ += dyndeps->implicit_inputs_.size(); + + // Add this edge as outgoing from each new input. + for (std::vector<Node*>::const_iterator i = + dyndeps->implicit_inputs_.begin(); + i != dyndeps->implicit_inputs_.end(); ++i) + (*i)->AddOutEdge(edge); + + return true; +} + +bool DyndepLoader::LoadDyndepFile(Node* file, DyndepFile* ddf, + std::string* err) const { + DyndepParser parser(state_, disk_interface_, ddf); + return parser.Load(file->path(), err); +} diff --git a/src/dyndep.h b/src/dyndep.h new file mode 100644 index 0000000..907f921 --- /dev/null +++ b/src/dyndep.h @@ -0,0 +1,64 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_DYNDEP_LOADER_H_ +#define NINJA_DYNDEP_LOADER_H_ + +#include <map> +#include <string> +#include <vector> + +struct DiskInterface; +struct Edge; +struct Node; +struct State; + +/// Store dynamically-discovered dependency information for one edge. +struct Dyndeps { + Dyndeps() : used_(false), restat_(false) {} + bool used_; + bool restat_; + std::vector<Node*> implicit_inputs_; + std::vector<Node*> implicit_outputs_; +}; + +/// Store data loaded from one dyndep file. Map from an edge +/// to its dynamically-discovered dependency information. +/// This is a struct rather than a typedef so that we can +/// forward-declare it in other headers. +struct DyndepFile: public std::map<Edge*, Dyndeps> {}; + +/// DyndepLoader loads dynamically discovered dependencies, as +/// referenced via the "dyndep" attribute in build files. +struct DyndepLoader { + DyndepLoader(State* state, DiskInterface* disk_interface) + : state_(state), disk_interface_(disk_interface) {} + + /// Load a dyndep file from the given node's path and update the + /// build graph with the new information. One overload accepts + /// a caller-owned 'DyndepFile' object in which to store the + /// information loaded from the dyndep file. + bool LoadDyndeps(Node* node, std::string* err) const; + bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; + + private: + bool LoadDyndepFile(Node* file, DyndepFile* ddf, std::string* err) const; + + bool UpdateEdge(Edge* edge, Dyndeps const* dyndeps, std::string* err) const; + + State* state_; + DiskInterface* disk_interface_; +}; + +#endif // NINJA_DYNDEP_LOADER_H_ diff --git a/src/dyndep_parser.cc b/src/dyndep_parser.cc new file mode 100644 index 0000000..baebbac --- /dev/null +++ b/src/dyndep_parser.cc @@ -0,0 +1,223 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dyndep_parser.h" + +#include <vector> + +#include "dyndep.h" +#include "graph.h" +#include "state.h" +#include "util.h" +#include "version.h" + +DyndepParser::DyndepParser(State* state, FileReader* file_reader, + DyndepFile* dyndep_file) + : Parser(state, file_reader) + , dyndep_file_(dyndep_file) { +} + +bool DyndepParser::Parse(const string& filename, const string& input, + string* err) { + lexer_.Start(filename, input); + + // Require a supported ninja_dyndep_version value immediately so + // we can exit before encountering any syntactic surprises. + bool haveDyndepVersion = false; + + for (;;) { + Lexer::Token token = lexer_.ReadToken(); + switch (token) { + case Lexer::BUILD: { + if (!haveDyndepVersion) + return lexer_.Error("expected 'ninja_dyndep_version = ...'", err); + if (!ParseEdge(err)) + return false; + break; + } + case Lexer::IDENT: { + lexer_.UnreadToken(); + if (haveDyndepVersion) + return lexer_.Error(string("unexpected ") + Lexer::TokenName(token), + err); + if (!ParseDyndepVersion(err)) + return false; + haveDyndepVersion = true; + break; + } + case Lexer::ERROR: + return lexer_.Error(lexer_.DescribeLastError(), err); + case Lexer::TEOF: + if (!haveDyndepVersion) + return lexer_.Error("expected 'ninja_dyndep_version = ...'", err); + return true; + case Lexer::NEWLINE: + break; + default: + return lexer_.Error(string("unexpected ") + Lexer::TokenName(token), + err); + } + } + return false; // not reached +} + +bool DyndepParser::ParseDyndepVersion(string* err) { + string name; + EvalString let_value; + if (!ParseLet(&name, &let_value, err)) + return false; + if (name != "ninja_dyndep_version") { + return lexer_.Error("expected 'ninja_dyndep_version = ...'", err); + } + string version = let_value.Evaluate(&env_); + int major, minor; + ParseVersion(version, &major, &minor); + if (major != 1 || minor != 0) { + return lexer_.Error( + string("unsupported 'ninja_dyndep_version = ") + version + "'", err); + return false; + } + return true; +} + +bool DyndepParser::ParseLet(string* key, EvalString* value, string* err) { + if (!lexer_.ReadIdent(key)) + return lexer_.Error("expected variable name", err); + if (!ExpectToken(Lexer::EQUALS, err)) + return false; + if (!lexer_.ReadVarValue(value, err)) + return false; + return true; +} + +bool DyndepParser::ParseEdge(string* err) { + // Parse one explicit output. We expect it to already have an edge. + // We will record its dynamically-discovered dependency information. + Dyndeps* dyndeps = NULL; + { + EvalString out0; + if (!lexer_.ReadPath(&out0, err)) + return false; + if (out0.empty()) + return lexer_.Error("expected path", err); + + string path = out0.Evaluate(&env_); + string path_err; + uint64_t slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) + return lexer_.Error(path_err, err); + Node* node = state_->LookupNode(path); + if (!node || !node->in_edge()) + return lexer_.Error("no build statement exists for '" + path + "'", err); + Edge* edge = node->in_edge(); + std::pair<DyndepFile::iterator, bool> res = + dyndep_file_->insert(DyndepFile::value_type(edge, Dyndeps())); + if (!res.second) + return lexer_.Error("multiple statements for '" + path + "'", err); + dyndeps = &res.first->second; + } + + // Disallow explicit outputs. + { + EvalString out; + if (!lexer_.ReadPath(&out, err)) + return false; + if (!out.empty()) + return lexer_.Error("explicit outputs not supported", err); + } + + // Parse implicit outputs, if any. + vector<EvalString> outs; + if (lexer_.PeekToken(Lexer::PIPE)) { + for (;;) { + EvalString out; + if (!lexer_.ReadPath(&out, err)) + return err; + if (out.empty()) + break; + outs.push_back(out); + } + } + + if (!ExpectToken(Lexer::COLON, err)) + return false; + + string rule_name; + if (!lexer_.ReadIdent(&rule_name) || rule_name != "dyndep") + return lexer_.Error("expected build command name 'dyndep'", err); + + // Disallow explicit inputs. + { + EvalString in; + if (!lexer_.ReadPath(&in, err)) + return false; + if (!in.empty()) + return lexer_.Error("explicit inputs not supported", err); + } + + // Parse implicit inputs, if any. + vector<EvalString> ins; + if (lexer_.PeekToken(Lexer::PIPE)) { + for (;;) { + EvalString in; + if (!lexer_.ReadPath(&in, err)) + return err; + if (in.empty()) + break; + ins.push_back(in); + } + } + + // Disallow order-only inputs. + if (lexer_.PeekToken(Lexer::PIPE2)) + return lexer_.Error("order-only inputs not supported", err); + + if (!ExpectToken(Lexer::NEWLINE, err)) + return false; + + if (lexer_.PeekToken(Lexer::INDENT)) { + string key; + EvalString val; + if (!ParseLet(&key, &val, err)) + return false; + if (key != "restat") + return lexer_.Error("binding is not 'restat'", err); + string value = val.Evaluate(&env_); + dyndeps->restat_ = !value.empty(); + } + + dyndeps->implicit_inputs_.reserve(ins.size()); + for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) { + string path = i->Evaluate(&env_); + string path_err; + uint64_t slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) + return lexer_.Error(path_err, err); + Node* n = state_->GetNode(path, slash_bits); + dyndeps->implicit_inputs_.push_back(n); + } + + dyndeps->implicit_outputs_.reserve(outs.size()); + for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) { + string path = i->Evaluate(&env_); + string path_err; + uint64_t slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) + return lexer_.Error(path_err, err); + Node* n = state_->GetNode(path, slash_bits); + dyndeps->implicit_outputs_.push_back(n); + } + + return true; +} diff --git a/src/dyndep_parser.h b/src/dyndep_parser.h new file mode 100644 index 0000000..09a3722 --- /dev/null +++ b/src/dyndep_parser.h @@ -0,0 +1,46 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_DYNDEP_PARSER_H_ +#define NINJA_DYNDEP_PARSER_H_ + +#include "eval_env.h" +#include "parser.h" + +struct DyndepFile; +struct EvalString; + +/// Parses dyndep files. +struct DyndepParser: public Parser { + DyndepParser(State* state, FileReader* file_reader, + DyndepFile* dyndep_file); + + /// Parse a text string of input. Used by tests. + bool ParseTest(const string& input, string* err) { + return Parse("input", input, err); + } + +private: + /// Parse a file, given its contents as a string. + bool Parse(const string& filename, const string& input, string* err); + + bool ParseDyndepVersion(string* err); + bool ParseLet(string* key, EvalString* val, string* err); + bool ParseEdge(string* err); + + DyndepFile* dyndep_file_; + BindingEnv env_; +}; + +#endif // NINJA_DYNDEP_PARSER_H_ diff --git a/src/dyndep_parser_test.cc b/src/dyndep_parser_test.cc new file mode 100644 index 0000000..39ec657 --- /dev/null +++ b/src/dyndep_parser_test.cc @@ -0,0 +1,512 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dyndep_parser.h" + +#include <map> +#include <vector> + +#include "dyndep.h" +#include "graph.h" +#include "state.h" +#include "test.h" + +struct DyndepParserTest : public testing::Test { + void AssertParse(const char* input) { + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_TRUE(parser.ParseTest(input, &err)); + ASSERT_EQ("", err); + } + + virtual void SetUp() { + ::AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build out otherout: touch\n"); + } + + State state_; + VirtualFileSystem fs_; + DyndepFile dyndep_file_; +}; + +TEST_F(DyndepParserTest, Empty) { + const char kInput[] = +""; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: expected 'ninja_dyndep_version = ...'\n", err); +} + +TEST_F(DyndepParserTest, Version1) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n")); +} + +TEST_F(DyndepParserTest, Version1Extra) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1-extra\n")); +} + +TEST_F(DyndepParserTest, Version1_0) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1.0\n")); +} + +TEST_F(DyndepParserTest, Version1_0Extra) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1.0-extra\n")); +} + +TEST_F(DyndepParserTest, CommentVersion) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"# comment\n" +"ninja_dyndep_version = 1\n")); +} + +TEST_F(DyndepParserTest, BlankLineVersion) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"\n" +"ninja_dyndep_version = 1\n")); +} + +TEST_F(DyndepParserTest, VersionCRLF) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\r\n")); +} + +TEST_F(DyndepParserTest, CommentVersionCRLF) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"# comment\r\n" +"ninja_dyndep_version = 1\r\n")); +} + +TEST_F(DyndepParserTest, BlankLineVersionCRLF) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"\r\n" +"ninja_dyndep_version = 1\r\n")); +} + +TEST_F(DyndepParserTest, VersionUnexpectedEOF) { + const char kInput[] = +"ninja_dyndep_version = 1.0"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: unexpected EOF\n" + "ninja_dyndep_version = 1.0\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, UnsupportedVersion0) { + const char kInput[] = +"ninja_dyndep_version = 0\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: unsupported 'ninja_dyndep_version = 0'\n" + "ninja_dyndep_version = 0\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, UnsupportedVersion1_1) { + const char kInput[] = +"ninja_dyndep_version = 1.1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: unsupported 'ninja_dyndep_version = 1.1'\n" + "ninja_dyndep_version = 1.1\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, DuplicateVersion) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"ninja_dyndep_version = 1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: unexpected identifier\n", err); +} + +TEST_F(DyndepParserTest, MissingVersionOtherVar) { + const char kInput[] = +"not_ninja_dyndep_version = 1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: expected 'ninja_dyndep_version = ...'\n" + "not_ninja_dyndep_version = 1\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, MissingVersionBuild) { + const char kInput[] = +"build out: dyndep\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: expected 'ninja_dyndep_version = ...'\n", err); +} + +TEST_F(DyndepParserTest, UnexpectedEqual) { + const char kInput[] = +"= 1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: unexpected '='\n", err); +} + +TEST_F(DyndepParserTest, UnexpectedIndent) { + const char kInput[] = +" = 1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:1: unexpected indent\n", err); +} + +TEST_F(DyndepParserTest, OutDuplicate) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +"build out: dyndep\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:3: multiple statements for 'out'\n" + "build out: dyndep\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, OutDuplicateThroughOther) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +"build otherout: dyndep\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:3: multiple statements for 'otherout'\n" + "build otherout: dyndep\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, NoOutEOF) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: unexpected EOF\n" + "build\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, NoOutColon) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build :\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: expected path\n" + "build :\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, OutNoStatement) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build missing: dyndep\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: no build statement exists for 'missing'\n" + "build missing: dyndep\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, OutEOF) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: unexpected EOF\n" + "build out\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, OutNoRule) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out:"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: expected build command name 'dyndep'\n" + "build out:\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, OutBadRule) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: touch"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: expected build command name 'dyndep'\n" + "build out: touch\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, BuildEOF) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: unexpected EOF\n" + "build out: dyndep\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, ExplicitOut) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out exp: dyndep\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: explicit outputs not supported\n" + "build out exp: dyndep\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, ExplicitIn) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep exp\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: explicit inputs not supported\n" + "build out: dyndep exp\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, OrderOnlyIn) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep ||\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:2: order-only inputs not supported\n" + "build out: dyndep ||\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, BadBinding) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +" not_restat = 1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:3: binding is not 'restat'\n" + " not_restat = 1\n" + " ^ near here", err); +} + +TEST_F(DyndepParserTest, RestatTwice) { + const char kInput[] = +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +" restat = 1\n" +" restat = 1\n"; + DyndepParser parser(&state_, &fs_, &dyndep_file_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:4: unexpected indent\n", err); +} + +TEST_F(DyndepParserTest, NoImplicit) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out: dyndep\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); +} + +TEST_F(DyndepParserTest, EmptyImplicit) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out | : dyndep |\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); +} + +TEST_F(DyndepParserTest, ImplicitIn) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out: dyndep | impin\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + ASSERT_EQ(1u, i->second.implicit_inputs_.size()); + EXPECT_EQ("impin", i->second.implicit_inputs_[0]->path()); +} + +TEST_F(DyndepParserTest, ImplicitIns) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out: dyndep | impin1 impin2\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + ASSERT_EQ(2u, i->second.implicit_inputs_.size()); + EXPECT_EQ("impin1", i->second.implicit_inputs_[0]->path()); + EXPECT_EQ("impin2", i->second.implicit_inputs_[1]->path()); +} + +TEST_F(DyndepParserTest, ImplicitOut) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out | impout: dyndep\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + ASSERT_EQ(1u, i->second.implicit_outputs_.size()); + EXPECT_EQ("impout", i->second.implicit_outputs_[0]->path()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); +} + +TEST_F(DyndepParserTest, ImplicitOuts) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out | impout1 impout2 : dyndep\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + ASSERT_EQ(2u, i->second.implicit_outputs_.size()); + EXPECT_EQ("impout1", i->second.implicit_outputs_[0]->path()); + EXPECT_EQ("impout2", i->second.implicit_outputs_[1]->path()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); +} + +TEST_F(DyndepParserTest, ImplicitInsAndOuts) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out | impout1 impout2: dyndep | impin1 impin2\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + ASSERT_EQ(2u, i->second.implicit_outputs_.size()); + EXPECT_EQ("impout1", i->second.implicit_outputs_[0]->path()); + EXPECT_EQ("impout2", i->second.implicit_outputs_[1]->path()); + ASSERT_EQ(2u, i->second.implicit_inputs_.size()); + EXPECT_EQ("impin1", i->second.implicit_inputs_[0]->path()); + EXPECT_EQ("impin2", i->second.implicit_inputs_[1]->path()); +} + +TEST_F(DyndepParserTest, Restat) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +" restat = 1\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(true, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); +} + +TEST_F(DyndepParserTest, OtherOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build otherout: dyndep\n")); + + EXPECT_EQ(1u, dyndep_file_.size()); + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); +} + +TEST_F(DyndepParserTest, MultipleEdges) { + ::AssertParse(&state_, +"build out2: touch\n"); + ASSERT_EQ(2u, state_.edges_.size()); + ASSERT_EQ(1u, state_.edges_[1]->outputs_.size()); + EXPECT_EQ("out2", state_.edges_[1]->outputs_[0]->path()); + EXPECT_EQ(0u, state_.edges_[0]->inputs_.size()); + + ASSERT_NO_FATAL_FAILURE(AssertParse( +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +"build out2: dyndep\n" +" restat = 1\n")); + + EXPECT_EQ(2u, dyndep_file_.size()); + { + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(false, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); + } + { + DyndepFile::iterator i = dyndep_file_.find(state_.edges_[1]); + ASSERT_NE(i, dyndep_file_.end()); + EXPECT_EQ(true, i->second.restat_); + EXPECT_EQ(0u, i->second.implicit_outputs_.size()); + EXPECT_EQ(0u, i->second.implicit_inputs_.size()); + } +} diff --git a/src/eval_env.cc b/src/eval_env.cc index 8817a87..e9b6c43 100644 --- a/src/eval_env.cc +++ b/src/eval_env.cc @@ -65,6 +65,7 @@ const EvalString* Rule::GetBinding(const string& key) const { bool Rule::IsReservedBinding(const string& var) { return var == "command" || var == "depfile" || + var == "dyndep" || var == "description" || var == "deps" || var == "generator" || @@ -130,3 +131,17 @@ string EvalString::Serialize() const { } return result; } + +string EvalString::Unparse() const { + string result; + for (TokenList::const_iterator i = parsed_.begin(); + i != parsed_.end(); ++i) { + bool special = (i->second == SPECIAL); + if (special) + result.append("${"); + result.append(i->first); + if (special) + result.append("}"); + } + return result; +} diff --git a/src/eval_env.h b/src/eval_env.h index 999ce42..8fb9bf4 100644 --- a/src/eval_env.h +++ b/src/eval_env.h @@ -33,8 +33,13 @@ struct Env { /// A tokenized string that contains variable references. /// Can be evaluated relative to an Env. struct EvalString { + /// @return The evaluated string with variable expanded using value found in + /// environment @a env. string Evaluate(Env* env) const; + /// @return The string with variables not expanded. + string Unparse() const; + void Clear() { parsed_.clear(); } bool empty() const { return parsed_.empty(); } diff --git a/src/getopt.c b/src/getopt.c index 0c2ef35..861f07f 100644 --- a/src/getopt.c +++ b/src/getopt.c @@ -75,7 +75,7 @@ COPYRIGHT NOTICE AND DISCLAIMER: Copyright (C) 1997 Gregory Pietsch -This file and the accompanying getopt.h header file are hereby placed in the +This file and the accompanying getopt.h header file are hereby placed in the public domain without restrictions. Just give the author credit, don't claim you wrote it or prevent anyone else from using it. diff --git a/src/graph.cc b/src/graph.cc index 7dd9491..a90c049 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -68,6 +68,31 @@ bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack, edge->outputs_ready_ = true; edge->deps_missing_ = false; + if (!edge->deps_loaded_) { + // This is our first encounter with this edge. + // If there is a pending dyndep file, visit it now: + // * If the dyndep file is ready then load it now to get any + // additional inputs and outputs for this and other edges. + // Once the dyndep file is loaded it will no longer be pending + // if any other edges encounter it, but they will already have + // been updated. + // * If the dyndep file is not ready then since is known to be an + // input to this edge, the edge will not be considered ready below. + // Later during the build the dyndep file will become ready and be + // loaded to update this edge before it can possibly be scheduled. + if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) { + if (!RecomputeDirty(edge->dyndep_, stack, err)) + return false; + + if (!edge->dyndep_->in_edge() || + edge->dyndep_->in_edge()->outputs_ready()) { + // The dyndep file is ready, so load it now. + if (!LoadDyndeps(edge->dyndep_, err)) + return false; + } + } + } + // Load output mtimes so we can compare them to the most recent input below. for (vector<Node*>::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { @@ -75,12 +100,16 @@ bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack, return false; } - if (!dep_loader_.LoadDeps(edge, err)) { - if (!err->empty()) - return false; - // Failed to load dependency info: rebuild to regenerate it. - // LoadDeps() did EXPLAIN() already, no need to do it here. - dirty = edge->deps_missing_ = true; + if (!edge->deps_loaded_) { + // This is our first encounter with this edge. Load discovered deps. + edge->deps_loaded_ = true; + if (!dep_loader_.LoadDeps(edge, err)) { + if (!err->empty()) + return false; + // Failed to load dependency info: rebuild to regenerate it. + // LoadDeps() did EXPLAIN() already, no need to do it here. + dirty = edge->deps_missing_ = true; + } } // Visit all inputs; we're dirty if any of the inputs are dirty. @@ -170,6 +199,13 @@ bool DependencyScan::VerifyDAG(Node* node, vector<Node*>* stack, string* err) { err->append(" -> "); } err->append((*start)->path()); + + if ((start + 1) == stack->end() && edge->maybe_phonycycle_diagnostic()) { + // The manifest parser would have filtered out the self-referencing + // input if it were not configured to allow the error. + err->append(" [-w phonycycle=err]"); + } + return false; } @@ -226,7 +262,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, if (output_mtime < most_recent_input->mtime()) { EXPLAIN("%soutput %s older than most recent input %s " - "(%d vs %d)", + "(%" PRId64 " vs %" PRId64 ")", used_restat ? "restat of " : "", output->path().c_str(), most_recent_input->path().c_str(), output_mtime, most_recent_input->mtime()); @@ -250,7 +286,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, // mtime of the most recent input. This can occur even when the mtime // on disk is newer if a previous run wrote to the output file but // exited with an error or was interrupted. - EXPLAIN("recorded mtime of %s older than most recent input %s (%d vs %d)", + EXPLAIN("recorded mtime of %s older than most recent input %s (%" PRId64 " vs %" PRId64 ")", output->path().c_str(), most_recent_input->path().c_str(), entry->mtime, most_recent_input->mtime()); return true; @@ -265,6 +301,15 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, return false; } +bool DependencyScan::LoadDyndeps(Node* node, string* err) const { + return dyndep_loader_.LoadDyndeps(node, err); +} + +bool DependencyScan::LoadDyndeps(Node* node, DyndepFile* ddf, + string* err) const { + return dyndep_loader_.LoadDyndeps(node, ddf, err); +} + bool Edge::AllInputsReady() const { for (vector<Node*>::const_iterator i = inputs_.begin(); i != inputs_.end(); ++i) { @@ -376,6 +421,11 @@ string Edge::GetUnescapedDepfile() { return env.LookupVariable("depfile"); } +string Edge::GetUnescapedDyndep() { + EdgeEnv env(this, EdgeEnv::kDoNotEscape); + return env.LookupVariable("dyndep"); +} + string Edge::GetUnescapedRspfile() { EdgeEnv env(this, EdgeEnv::kDoNotEscape); return env.LookupVariable("rspfile"); @@ -410,6 +460,14 @@ bool Edge::use_console() const { return pool() == &State::kConsolePool; } +bool Edge::maybe_phonycycle_diagnostic() const { + // CMake 2.8.12.x and 3.0.x produced self-referencing phony rules + // of the form "build a: phony ... a ...". Restrict our + // "phonycycle" diagnostic option to the form it used. + return is_phony() && outputs_.size() == 1 && implicit_outs_ == 0 && + implicit_deps_ == 0; +} + // static string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { string result = path; @@ -426,7 +484,7 @@ string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { } void Node::Dump(const char* prefix) const { - printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", + printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ", prefix, path().c_str(), this, mtime(), mtime() ? "" : " (:missing)", dirty() ? " dirty" : " clean"); @@ -476,7 +534,9 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } - DepfileParser depfile; + DepfileParser depfile(depfile_parser_options_ + ? *depfile_parser_options_ + : DepfileParserOptions()); string depfile_err; if (!depfile.Parse(&content, &depfile_err)) { *err = path + ": " + depfile_err; @@ -524,7 +584,7 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, string* err) { // NOTE: deps are only supported for single-target edges. Node* output = edge->outputs_[0]; - DepsLog::Deps* deps = deps_log_->GetDeps(output); + DepsLog::Deps* deps = deps_log_ ? deps_log_->GetDeps(output) : NULL; if (!deps) { EXPLAIN("deps for '%s' are missing", output->path().c_str()); return false; @@ -532,7 +592,7 @@ bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, string* err) { // Deps are invalid if the output is newer than the deps. if (output->mtime() > deps->mtime) { - EXPLAIN("stored deps info out of date for '%s' (%d vs %d)", + EXPLAIN("stored deps info out of date for '%s' (%" PRId64 " vs %" PRId64 ")", output->path().c_str(), deps->mtime, output->mtime()); return false; } diff --git a/src/graph.h b/src/graph.h index 586c588..75edbc5 100644 --- a/src/graph.h +++ b/src/graph.h @@ -19,11 +19,13 @@ #include <vector> using namespace std; +#include "dyndep.h" #include "eval_env.h" #include "timestamp.h" #include "util.h" struct BuildLog; +struct DepfileParserOptions; struct DiskInterface; struct DepsLog; struct Edge; @@ -39,6 +41,7 @@ struct Node { slash_bits_(slash_bits), mtime_(-1), dirty_(false), + dyndep_pending_(false), in_edge_(NULL), id_(-1) {} @@ -86,6 +89,9 @@ struct Node { void set_dirty(bool dirty) { dirty_ = dirty; } void MarkDirty() { dirty_ = true; } + bool dyndep_pending() const { return dyndep_pending_; } + void set_dyndep_pending(bool pending) { dyndep_pending_ = pending; } + Edge* in_edge() const { return in_edge_; } void set_in_edge(Edge* edge) { in_edge_ = edge; } @@ -115,6 +121,10 @@ private: /// edges to build. bool dirty_; + /// Store whether dyndep information is expected from this node but + /// has not yet been loaded. + bool dyndep_pending_; + /// The Edge that produces this Node, or NULL when there is no /// known edge to produce it. Edge* in_edge_; @@ -134,9 +144,10 @@ struct Edge { VisitDone }; - Edge() : rule_(NULL), pool_(NULL), env_(NULL), mark_(VisitNone), - outputs_ready_(false), deps_missing_(false), - implicit_deps_(0), order_only_deps_(0), implicit_outs_(0) {} + Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), + mark_(VisitNone), outputs_ready_(false), deps_loaded_(false), + deps_missing_(false), implicit_deps_(0), order_only_deps_(0), + implicit_outs_(0) {} /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; @@ -152,6 +163,8 @@ struct Edge { /// Like GetBinding("depfile"), but without shell escaping. string GetUnescapedDepfile(); + /// Like GetBinding("dyndep"), but without shell escaping. + string GetUnescapedDyndep(); /// Like GetBinding("rspfile"), but without shell escaping. string GetUnescapedRspfile(); @@ -161,9 +174,11 @@ struct Edge { Pool* pool_; vector<Node*> inputs_; vector<Node*> outputs_; + Node* dyndep_; BindingEnv* env_; VisitMark mark_; bool outputs_ready_; + bool deps_loaded_; bool deps_missing_; const Rule& rule() const { return *rule_; } @@ -201,6 +216,7 @@ struct Edge { bool is_phony() const; bool use_console() const; + bool maybe_phonycycle_diagnostic() const; }; @@ -208,8 +224,10 @@ struct Edge { /// "depfile" attribute in build files. struct ImplicitDepLoader { ImplicitDepLoader(State* state, DepsLog* deps_log, - DiskInterface* disk_interface) - : state_(state), disk_interface_(disk_interface), deps_log_(deps_log) {} + DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options) + : state_(state), disk_interface_(disk_interface), deps_log_(deps_log), + depfile_parser_options_(depfile_parser_options) {} /// Load implicit dependencies for \a edge. /// @return false on error (without filling \a err if info is just missing @@ -241,6 +259,7 @@ struct ImplicitDepLoader { State* state_; DiskInterface* disk_interface_; DepsLog* deps_log_; + DepfileParserOptions const* depfile_parser_options_; }; @@ -248,10 +267,12 @@ struct ImplicitDepLoader { /// and updating the dirty/outputs_ready state of all the nodes and edges. struct DependencyScan { DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface) + DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options) : build_log_(build_log), disk_interface_(disk_interface), - dep_loader_(state, deps_log, disk_interface) {} + dep_loader_(state, deps_log, disk_interface, depfile_parser_options), + dyndep_loader_(state, disk_interface) {} /// Update the |dirty_| state of the given node by inspecting its input edge. /// Examine inputs, outputs, and command lines to judge whether an edge @@ -276,6 +297,13 @@ struct DependencyScan { return dep_loader_.deps_log(); } + /// Load a dyndep file from the given node's path and update the + /// build graph with the new information. One overload accepts + /// a caller-owned 'DyndepFile' object in which to store the + /// information loaded from the dyndep file. + bool LoadDyndeps(Node* node, string* err) const; + bool LoadDyndeps(Node* node, DyndepFile* ddf, string* err) const; + private: bool RecomputeDirty(Node* node, vector<Node*>* stack, string* err); bool VerifyDAG(Node* node, vector<Node*>* stack, string* err); @@ -288,6 +316,7 @@ struct DependencyScan { BuildLog* build_log_; DiskInterface* disk_interface_; ImplicitDepLoader dep_loader_; + DyndepLoader dyndep_loader_; }; #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index d4d2824..c8cca1c 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -18,7 +18,7 @@ #include "test.h" struct GraphTest : public StateTestWithBuiltinRules { - GraphTest() : scan_(&state_, NULL, NULL, &fs_) {} + GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL) {} VirtualFileSystem fs_; DependencyScan scan_; @@ -323,6 +323,18 @@ TEST_F(GraphTest, NestedPhonyPrintsDone) { ASSERT_FALSE(plan_.more_to_do()); } +TEST_F(GraphTest, PhonySelfReferenceError) { + ManifestParserOptions parser_opts; + parser_opts.phony_cycle_action_ = kPhonyCycleActionError; + AssertParse(&state_, +"build a: phony a\n", + parser_opts); + + string err; + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + ASSERT_EQ("dependency cycle: a -> a [-w phonycycle=err]", err); +} + TEST_F(GraphTest, DependencyCycle) { AssertParse(&state_, "build out: cat mid\n" @@ -467,3 +479,380 @@ TEST_F(GraphTest, Decanonicalize) { EXPECT_EQ(root_nodes[3]->PathDecanonicalized(), "out4\\foo"); } #endif + +TEST_F(GraphTest, DyndepLoadTrivial) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("", err); + EXPECT_FALSE(GetNode("dd")->dyndep_pending()); + + Edge* edge = GetNode("out")->in_edge(); + ASSERT_EQ(1u, edge->outputs_.size()); + EXPECT_EQ("out", edge->outputs_[0]->path()); + ASSERT_EQ(2u, edge->inputs_.size()); + EXPECT_EQ("in", edge->inputs_[0]->path()); + EXPECT_EQ("dd", edge->inputs_[1]->path()); + EXPECT_EQ(0u, edge->implicit_deps_); + EXPECT_EQ(1u, edge->order_only_deps_); + EXPECT_FALSE(edge->GetBindingBool("restat")); +} + +TEST_F(GraphTest, DyndepLoadMissingFile) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("loading 'dd': No such file or directory", err); +} + +TEST_F(GraphTest, DyndepLoadMissingEntry) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("'out' not mentioned in its dyndep file 'dd'", err); +} + +TEST_F(GraphTest, DyndepLoadExtraEntry) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" +"build out2: r in || dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +"build out2: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("dyndep file 'dd' mentions output 'out2' whose build statement " + "does not have a dyndep binding for the file", err); +} + +TEST_F(GraphTest, DyndepLoadOutputWithMultipleRules1) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1 | out-twice.imp: r in1\n" +"build out2: r in2 || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out2 | out-twice.imp: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("multiple rules generate out-twice.imp", err); +} + +TEST_F(GraphTest, DyndepLoadOutputWithMultipleRules2) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in1 || dd1\n" +" dyndep = dd1\n" +"build out2: r in2 || dd2\n" +" dyndep = dd2\n" + ); + fs_.Create("dd1", +"ninja_dyndep_version = 1\n" +"build out1 | out-twice.imp: dyndep\n" + ); + fs_.Create("dd2", +"ninja_dyndep_version = 1\n" +"build out2 | out-twice.imp: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd1")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd1"), &err)); + EXPECT_EQ("", err); + ASSERT_TRUE(GetNode("dd2")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd2"), &err)); + EXPECT_EQ("multiple rules generate out-twice.imp", err); +} + +TEST_F(GraphTest, DyndepLoadMultiple) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in1 || dd\n" +" dyndep = dd\n" +"build out2: r in2 || dd\n" +" dyndep = dd\n" +"build outNot: r in3 || dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out1 | out1imp: dyndep | in1imp\n" +"build out2: dyndep | in2imp\n" +" restat = 1\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("", err); + EXPECT_FALSE(GetNode("dd")->dyndep_pending()); + + Edge* edge1 = GetNode("out1")->in_edge(); + ASSERT_EQ(2u, edge1->outputs_.size()); + EXPECT_EQ("out1", edge1->outputs_[0]->path()); + EXPECT_EQ("out1imp", edge1->outputs_[1]->path()); + EXPECT_EQ(1u, edge1->implicit_outs_); + ASSERT_EQ(3u, edge1->inputs_.size()); + EXPECT_EQ("in1", edge1->inputs_[0]->path()); + EXPECT_EQ("in1imp", edge1->inputs_[1]->path()); + EXPECT_EQ("dd", edge1->inputs_[2]->path()); + EXPECT_EQ(1u, edge1->implicit_deps_); + EXPECT_EQ(1u, edge1->order_only_deps_); + EXPECT_FALSE(edge1->GetBindingBool("restat")); + EXPECT_EQ(edge1, GetNode("out1imp")->in_edge()); + Node* in1imp = GetNode("in1imp"); + ASSERT_EQ(1u, in1imp->out_edges().size()); + EXPECT_EQ(edge1, in1imp->out_edges()[0]); + + Edge* edge2 = GetNode("out2")->in_edge(); + ASSERT_EQ(1u, edge2->outputs_.size()); + EXPECT_EQ("out2", edge2->outputs_[0]->path()); + EXPECT_EQ(0u, edge2->implicit_outs_); + ASSERT_EQ(3u, edge2->inputs_.size()); + EXPECT_EQ("in2", edge2->inputs_[0]->path()); + EXPECT_EQ("in2imp", edge2->inputs_[1]->path()); + EXPECT_EQ("dd", edge2->inputs_[2]->path()); + EXPECT_EQ(1u, edge2->implicit_deps_); + EXPECT_EQ(1u, edge2->order_only_deps_); + EXPECT_TRUE(edge2->GetBindingBool("restat")); + Node* in2imp = GetNode("in2imp"); + ASSERT_EQ(1u, in2imp->out_edges().size()); + EXPECT_EQ(edge2, in2imp->out_edges()[0]); +} + +TEST_F(GraphTest, DyndepFileMissing) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r || dd\n" +" dyndep = dd\n" + ); + + string err; + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("loading 'dd': No such file or directory", err); +} + +TEST_F(GraphTest, DyndepFileError) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" + ); + + string err; + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("'out' not mentioned in its dyndep file 'dd'", err); +} + +TEST_F(GraphTest, DyndepImplicitInputNewer) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out: dyndep | in\n" + ); + fs_.Create("out", ""); + fs_.Tick(); + fs_.Create("in", ""); + + string err; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("", err); + + EXPECT_FALSE(GetNode("in")->dirty()); + EXPECT_FALSE(GetNode("dd")->dirty()); + + // "out" is dirty due to dyndep-specified implicit input + EXPECT_TRUE(GetNode("out")->dirty()); +} + +TEST_F(GraphTest, DyndepFileReady) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build dd: r dd-in\n" +"build out: r || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd-in", ""); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out: dyndep | in\n" + ); + fs_.Create("out", ""); + fs_.Tick(); + fs_.Create("in", ""); + + string err; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("", err); + + EXPECT_FALSE(GetNode("in")->dirty()); + EXPECT_FALSE(GetNode("dd")->dirty()); + EXPECT_TRUE(GetNode("dd")->in_edge()->outputs_ready()); + + // "out" is dirty due to dyndep-specified implicit input + EXPECT_TRUE(GetNode("out")->dirty()); +} + +TEST_F(GraphTest, DyndepFileNotClean) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build dd: r dd-in\n" +"build out: r || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", "this-should-not-be-loaded"); + fs_.Tick(); + fs_.Create("dd-in", ""); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(GetNode("dd")->dirty()); + EXPECT_FALSE(GetNode("dd")->in_edge()->outputs_ready()); + + // "out" is clean but not ready since "dd" is not ready + EXPECT_FALSE(GetNode("out")->dirty()); + EXPECT_FALSE(GetNode("out")->in_edge()->outputs_ready()); +} + +TEST_F(GraphTest, DyndepFileNotReady) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build tmp: r\n" +"build dd: r dd-in || tmp\n" +"build out: r || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", "this-should-not-be-loaded"); + fs_.Create("dd-in", ""); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("", err); + + EXPECT_FALSE(GetNode("dd")->dirty()); + EXPECT_FALSE(GetNode("dd")->in_edge()->outputs_ready()); + EXPECT_FALSE(GetNode("out")->dirty()); + EXPECT_FALSE(GetNode("out")->in_edge()->outputs_ready()); +} + +TEST_F(GraphTest, DyndepFileSecondNotReady) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build dd1: r dd1-in\n" +"build dd2-in: r || dd1\n" +" dyndep = dd1\n" +"build dd2: r dd2-in\n" +"build out: r || dd2\n" +" dyndep = dd2\n" + ); + fs_.Create("dd1", ""); + fs_.Create("dd2", ""); + fs_.Create("dd2-in", ""); + fs_.Tick(); + fs_.Create("dd1-in", ""); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(GetNode("dd1")->dirty()); + EXPECT_FALSE(GetNode("dd1")->in_edge()->outputs_ready()); + EXPECT_FALSE(GetNode("dd2")->dirty()); + EXPECT_FALSE(GetNode("dd2")->in_edge()->outputs_ready()); + EXPECT_FALSE(GetNode("out")->dirty()); + EXPECT_FALSE(GetNode("out")->in_edge()->outputs_ready()); +} + +TEST_F(GraphTest, DyndepFileCircular) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" depfile = out.d\n" +" dyndep = dd\n" +"build in: r circ\n" + ); + fs_.Create("out.d", "out: inimp\n"); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out | circ: dyndep\n" + ); + fs_.Create("out", ""); + + Edge* edge = GetNode("out")->in_edge(); + string err; + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_EQ("dependency cycle: circ -> in -> circ", err); + + // Verify that "out.d" was loaded exactly once despite + // circular reference discovered from dyndep file. + ASSERT_EQ(3u, edge->inputs_.size()); + EXPECT_EQ("in", edge->inputs_[0]->path()); + EXPECT_EQ("inimp", edge->inputs_[1]->path()); + EXPECT_EQ("dd", edge->inputs_[2]->path()); + EXPECT_EQ(1u, edge->implicit_deps_); + EXPECT_EQ(1u, edge->order_only_deps_); +} diff --git a/src/graphviz.cc b/src/graphviz.cc index dce8b32..0d07251 100644 --- a/src/graphviz.cc +++ b/src/graphviz.cc @@ -17,6 +17,7 @@ #include <stdio.h> #include <algorithm> +#include "dyndep.h" #include "graph.h" void GraphViz::AddTarget(Node* node) { @@ -40,6 +41,13 @@ void GraphViz::AddTarget(Node* node) { return; visited_edges_.insert(edge); + if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) { + std::string err; + if (!dyndep_loader_.LoadDyndeps(edge->dyndep_, &err)) { + Warning("%s\n", err.c_str()); + } + } + if (edge->inputs_.size() == 1 && edge->outputs_.size() == 1) { // Can draw simply. // Note extra space before label text -- this is cosmetic and feels diff --git a/src/graphviz.h b/src/graphviz.h index 408496d..601c9b2 100644 --- a/src/graphviz.h +++ b/src/graphviz.h @@ -17,15 +17,22 @@ #include <set> +#include "dyndep.h" + +struct DiskInterface; struct Node; struct Edge; +struct State; /// Runs the process of creating GraphViz .dot file output. struct GraphViz { + GraphViz(State* state, DiskInterface* disk_interface) + : dyndep_loader_(state, disk_interface) {} void Start(); void AddTarget(Node* node); void Finish(); + DyndepLoader dyndep_loader_; std::set<Node*> visited_nodes_; std::set<Edge*> visited_edges_; }; diff --git a/src/hash_map.h b/src/hash_map.h index a91aeb9..55d2c9d 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -18,6 +18,7 @@ #include <algorithm> #include <string.h> #include "string_piece.h" +#include "util.h" // MurmurHash2, by Austin Appleby static inline @@ -40,7 +41,9 @@ unsigned int MurmurHash2(const void* key, size_t len) { } switch (len) { case 3: h ^= data[2] << 16; + NINJA_FALLTHROUGH; case 2: h ^= data[1] << 8; + NINJA_FALLTHROUGH; case 1: h ^= data[0]; h *= m; }; diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 459329b..79bf5b4 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -26,6 +26,21 @@ namespace { +bool InternalGetFullPathName(const StringPiece& file_name, char* buffer, + size_t buffer_length, string *err) { + DWORD result_size = GetFullPathNameA(file_name.AsString().c_str(), + buffer_length, buffer, NULL); + if (result_size == 0) { + *err = "GetFullPathNameA(" + file_name.AsString() + "): " + + GetLastErrorString(); + return false; + } else if (result_size > buffer_length) { + *err = "path too long"; + return false; + } + return true; +} + bool IsPathSeparator(char c) { return c == '/' || c == '\\'; } @@ -54,15 +69,19 @@ bool SameDriveFast(StringPiece a, StringPiece b) { } // Return true if paths a and b are on the same Windows drive. -bool SameDrive(StringPiece a, StringPiece b) { +bool SameDrive(StringPiece a, StringPiece b, string* err) { if (SameDriveFast(a, b)) { return true; } char a_absolute[_MAX_PATH]; char b_absolute[_MAX_PATH]; - GetFullPathName(a.AsString().c_str(), sizeof(a_absolute), a_absolute, NULL); - GetFullPathName(b.AsString().c_str(), sizeof(b_absolute), b_absolute, NULL); + if (!InternalGetFullPathName(a, a_absolute, sizeof(a_absolute), err)) { + return false; + } + if (!InternalGetFullPathName(b, b_absolute, sizeof(b_absolute), err)) { + return false; + } char a_drive[_MAX_DIR]; char b_drive[_MAX_DIR]; _splitpath(a_absolute, a_drive, NULL, NULL, NULL); @@ -106,11 +125,15 @@ bool IsFullPathName(StringPiece s) { } // anonymous namespace IncludesNormalize::IncludesNormalize(const string& relative_to) { - relative_to_ = AbsPath(relative_to); + string err; + relative_to_ = AbsPath(relative_to, &err); + if (!err.empty()) { + Fatal("Initializing IncludesNormalize(): %s", err.c_str()); + } split_relative_to_ = SplitStringPiece(relative_to_, '/'); } -string IncludesNormalize::AbsPath(StringPiece s) { +string IncludesNormalize::AbsPath(StringPiece s, string* err) { if (IsFullPathName(s)) { string result = s.AsString(); for (size_t i = 0; i < result.size(); ++i) { @@ -122,7 +145,9 @@ string IncludesNormalize::AbsPath(StringPiece s) { } char result[_MAX_PATH]; - GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + if (!InternalGetFullPathName(s, result, sizeof(result), err)) { + return ""; + } for (char* c = result; *c; ++c) if (*c == '\\') *c = '/'; @@ -130,8 +155,10 @@ string IncludesNormalize::AbsPath(StringPiece s) { } string IncludesNormalize::Relativize( - StringPiece path, const vector<StringPiece>& start_list) { - string abs_path = AbsPath(path); + StringPiece path, const vector<StringPiece>& start_list, string* err) { + string abs_path = AbsPath(path, err); + if (!err->empty()) + return ""; vector<StringPiece> path_list = SplitStringPiece(abs_path, '/'); int i; for (i = 0; i < static_cast<int>(min(start_list.size(), path_list.size())); @@ -165,12 +192,18 @@ bool IncludesNormalize::Normalize(const string& input, if (!CanonicalizePath(copy, &len, &slash_bits, err)) return false; StringPiece partially_fixed(copy, len); - string abs_input = AbsPath(partially_fixed); + string abs_input = AbsPath(partially_fixed, err); + if (!err->empty()) + return false; - if (!SameDrive(abs_input, relative_to_)) { + if (!SameDrive(abs_input, relative_to_, err)) { + if (!err->empty()) + return false; *result = partially_fixed.AsString(); return true; } - *result = Relativize(abs_input, split_relative_to_); + *result = Relativize(abs_input, split_relative_to_, err); + if (!err->empty()) + return false; return true; } diff --git a/src/includes_normalize.h b/src/includes_normalize.h index 3811e53..0339581 100644 --- a/src/includes_normalize.h +++ b/src/includes_normalize.h @@ -25,9 +25,9 @@ struct IncludesNormalize { IncludesNormalize(const string& relative_to); // Internal utilities made available for testing, maybe useful otherwise. - static string AbsPath(StringPiece s); + static string AbsPath(StringPiece s, string* err); static string Relativize(StringPiece path, - const vector<StringPiece>& start_list); + const vector<StringPiece>& start_list, string* err); /// Normalize by fixing slashes style, fixing redundant .. and . and makes the /// path |input| relative to |this->relative_to_| and store to |result|. diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index 0bb14ec..dbcdbe0 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -58,9 +58,12 @@ TEST(IncludesNormalize, Simple) { } TEST(IncludesNormalize, WithRelative) { + string err; string currentdir = GetCurDir(); EXPECT_EQ("c", NormalizeRelativeAndCheckNoError("a/b/c", "a/b")); - EXPECT_EQ("a", NormalizeAndCheckNoError(IncludesNormalize::AbsPath("a"))); + EXPECT_EQ("a", + NormalizeAndCheckNoError(IncludesNormalize::AbsPath("a", &err))); + EXPECT_EQ("", err); EXPECT_EQ(string("../") + currentdir + string("/a"), NormalizeRelativeAndCheckNoError("a", "../b")); EXPECT_EQ(string("../") + currentdir + string("/a/b"), @@ -108,36 +111,57 @@ TEST(IncludesNormalize, LongInvalidPath) { normalizer.Normalize(kLongInputString, &result, &err)); EXPECT_EQ("path too long", err); - const char kExactlyMaxPath[] = - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "012345678\\" - "0123456789"; + + // Construct max size path having cwd prefix. + // kExactlyMaxPath = "$cwd\\a\\aaaa...aaaa\0"; + char kExactlyMaxPath[_MAX_PATH + 1]; + ASSERT_NE(_getcwd(kExactlyMaxPath, sizeof kExactlyMaxPath), NULL); + + int cwd_len = strlen(kExactlyMaxPath); + ASSERT_LE(cwd_len + 3 + 1, _MAX_PATH) + kExactlyMaxPath[cwd_len] = '\\'; + kExactlyMaxPath[cwd_len + 1] = 'a'; + kExactlyMaxPath[cwd_len + 2] = '\\'; + + kExactlyMaxPath[cwd_len + 3] = 'a'; + + for (int i = cwd_len + 4; i < _MAX_PATH; ++i) { + if (i > cwd_len + 4 && i < _MAX_PATH - 1 && i % 10 == 0) + kExactlyMaxPath[i] = '\\'; + else + kExactlyMaxPath[i] = 'a'; + } + + kExactlyMaxPath[_MAX_PATH] = '\0'; + EXPECT_EQ(strlen(kExactlyMaxPath), _MAX_PATH); + string forward_slashes(kExactlyMaxPath); replace(forward_slashes.begin(), forward_slashes.end(), '\\', '/'); // Make sure a path that's exactly _MAX_PATH long is canonicalized. - EXPECT_EQ(forward_slashes, + EXPECT_EQ(forward_slashes.substr(cwd_len + 1), NormalizeAndCheckNoError(kExactlyMaxPath)); } + +TEST(IncludesNormalize, ShortRelativeButTooLongAbsolutePath) { + string result, err; + IncludesNormalize normalizer("."); + // A short path should work + EXPECT_TRUE(normalizer.Normalize("a", &result, &err)); + EXPECT_EQ("", err); + + // Construct max size path having cwd prefix. + // kExactlyMaxPath = "aaaa\\aaaa...aaaa\0"; + char kExactlyMaxPath[_MAX_PATH + 1]; + for (int i = 0; i < _MAX_PATH; ++i) { + if (i < _MAX_PATH - 1 && i % 10 == 4) + kExactlyMaxPath[i] = '\\'; + else + kExactlyMaxPath[i] = 'a'; + } + kExactlyMaxPath[_MAX_PATH] = '\0'; + EXPECT_EQ(strlen(kExactlyMaxPath), _MAX_PATH); + + // Make sure a path that's exactly _MAX_PATH long fails with a proper error. + EXPECT_FALSE(normalizer.Normalize(kExactlyMaxPath, &result, &err)); + EXPECT_TRUE(err.find("GetFullPathName") != string::npos); +} diff --git a/src/inline.sh b/src/inline.sh index fa282fa..b64e8ca 100755 --- a/src/inline.sh +++ b/src/inline.sh @@ -20,6 +20,6 @@ varname="$1" echo "const char $varname[] =" -od -t x1 -A n -v | sed -e 's|[ \t]||g; s|..|\\x&|g; s|^|"|; s|$|"|' +od -t x1 -A n -v | sed -e 's|^[\t ]\{0,\}$||g; s|[\t ]\{1,\}| |g; s| \{1,\}$||g; s| |\\x|g; s|^|"|; s|$|"|' echo ";" diff --git a/src/lexer.cc b/src/lexer.cc index 37b8678..35ae97b 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.13.5 */ +/* Generated by re2c 0.16 */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,14 +23,14 @@ bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; - const char* context = input_.str_; + const char* line_start = input_.str_; for (const char* p = input_.str_; p < last_token_; ++p) { if (*p == '\n') { ++line; - context = p + 1; + line_start = p + 1; } } - int col = last_token_ ? (int)(last_token_ - context) : 0; + int col = last_token_ ? (int)(last_token_ - line_start) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); @@ -43,12 +43,12 @@ bool Lexer::Error(const string& message, string* err) { int len; bool truncated = true; for (len = 0; len < kTruncateColumn; ++len) { - if (context[len] == 0 || context[len] == '\n') { + if (line_start[len] == 0 || line_start[len] == '\n') { truncated = false; break; } } - *err += string(context, len); + *err += string(line_start, len); if (truncated) *err += "..."; *err += "\n"; @@ -126,305 +126,325 @@ Lexer::Token Lexer::ReadToken() { unsigned char yych; unsigned int yyaccept = 0; static const unsigned char yybm[] = { - 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 192, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 96, 96, 64, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 64, 64, 64, 64, 64, 64, - 64, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 64, 64, 64, 64, 96, - 64, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, + 0, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 160, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 192, 192, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 128, 128, 128, 128, 192, + 128, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, }; - yych = *p; - if (yych <= 'Z') { - if (yych <= '#') { + if (yybm[0+yych] & 32) { + goto yy9; + } + if (yych <= '^') { + if (yych <= ',') { if (yych <= '\f') { - if (yych <= 0x00) goto yy23; - if (yych == '\n') goto yy7; - goto yy25; + if (yych <= 0x00) goto yy2; + if (yych == '\n') goto yy6; + goto yy4; } else { - if (yych <= 0x1F) { - if (yych <= '\r') goto yy6; - goto yy25; - } else { - if (yych <= ' ') goto yy2; - if (yych <= '"') goto yy25; - goto yy4; - } + if (yych <= '\r') goto yy8; + if (yych == '#') goto yy12; + goto yy4; } } else { - if (yych <= '9') { - if (yych <= ',') goto yy25; - if (yych == '/') goto yy25; - goto yy22; + if (yych <= ':') { + if (yych == '/') goto yy4; + if (yych <= '9') goto yy13; + goto yy16; } else { - if (yych <= '<') { - if (yych <= ':') goto yy16; - goto yy25; + if (yych <= '=') { + if (yych <= '<') goto yy4; + goto yy18; } else { - if (yych <= '=') goto yy14; - if (yych <= '@') goto yy25; - goto yy22; + if (yych <= '@') goto yy4; + if (yych <= 'Z') goto yy13; + goto yy4; } } } } else { if (yych <= 'i') { - if (yych <= 'a') { - if (yych == '_') goto yy22; - if (yych <= '`') goto yy25; - goto yy22; + if (yych <= 'b') { + if (yych == '`') goto yy4; + if (yych <= 'a') goto yy13; + goto yy20; } else { - if (yych <= 'c') { - if (yych <= 'b') goto yy9; - goto yy22; - } else { - if (yych <= 'd') goto yy13; - if (yych <= 'h') goto yy22; - goto yy20; - } + if (yych == 'd') goto yy21; + if (yych <= 'h') goto yy13; + goto yy22; } } else { if (yych <= 'r') { - if (yych == 'p') goto yy11; - if (yych <= 'q') goto yy22; - goto yy12; + if (yych == 'p') goto yy23; + if (yych <= 'q') goto yy13; + goto yy24; } else { if (yych <= 'z') { - if (yych <= 's') goto yy21; - goto yy22; + if (yych <= 's') goto yy25; + goto yy13; } else { - if (yych == '|') goto yy18; - goto yy25; + if (yych == '|') goto yy26; + goto yy4; } } } } yy2: - yyaccept = 0; - yych = *(q = ++p); - goto yy73; -yy3: - { token = INDENT; break; } + ++p; + { token = TEOF; break; } yy4: - yyaccept = 1; - yych = *(q = ++p); - if (yych >= 0x01) goto yy68; + ++p; yy5: { token = ERROR; break; } yy6: - yych = *++p; - if (yych == '\n') goto yy65; - goto yy5; -yy7: ++p; -yy8: { token = NEWLINE; break; } +yy8: + yych = *++p; + if (yych == '\n') goto yy28; + goto yy5; yy9: - ++p; - if ((yych = *p) == 'u') goto yy60; - goto yy27; -yy10: - { token = IDENT; break; } + yyaccept = 0; + q = ++p; + yych = *p; + if (yybm[0+yych] & 32) { + goto yy9; + } + if (yych <= '\f') { + if (yych == '\n') goto yy6; + } else { + if (yych <= '\r') goto yy30; + if (yych == '#') goto yy32; + } yy11: - yych = *++p; - if (yych == 'o') goto yy56; - goto yy27; + { token = INDENT; break; } yy12: - yych = *++p; - if (yych == 'u') goto yy52; - goto yy27; + yyaccept = 1; + yych = *(q = ++p); + if (yych <= 0x00) goto yy5; + goto yy33; yy13: - yych = *++p; - if (yych == 'e') goto yy45; - goto yy27; -yy14: ++p; - { token = EQUALS; break; } + yych = *p; +yy14: + if (yybm[0+yych] & 64) { + goto yy13; + } + { token = IDENT; break; } yy16: ++p; { token = COLON; break; } yy18: ++p; - if ((yych = *p) == '|') goto yy43; - { token = PIPE; break; } + { token = EQUALS; break; } yy20: yych = *++p; - if (yych == 'n') goto yy36; - goto yy27; + if (yych == 'u') goto yy36; + goto yy14; yy21: yych = *++p; - if (yych == 'u') goto yy28; - goto yy27; + if (yych == 'e') goto yy37; + goto yy14; yy22: yych = *++p; - goto yy27; + if (yych == 'n') goto yy38; + goto yy14; yy23: - ++p; - { token = TEOF; break; } + yych = *++p; + if (yych == 'o') goto yy39; + goto yy14; +yy24: + yych = *++p; + if (yych == 'u') goto yy40; + goto yy14; yy25: yych = *++p; - goto yy5; + if (yych == 'u') goto yy41; + goto yy14; yy26: ++p; - yych = *p; -yy27: - if (yybm[0+yych] & 32) { - goto yy26; - } - goto yy10; + if ((yych = *p) == '|') goto yy42; + { token = PIPE; break; } yy28: + ++p; + { token = NEWLINE; break; } +yy30: yych = *++p; - if (yych != 'b') goto yy27; - yych = *++p; - if (yych != 'n') goto yy27; - yych = *++p; - if (yych != 'i') goto yy27; - yych = *++p; - if (yych != 'n') goto yy27; - yych = *++p; - if (yych != 'j') goto yy27; - yych = *++p; - if (yych != 'a') goto yy27; + if (yych == '\n') goto yy28; +yy31: + p = q; + if (yyaccept == 0) { + goto yy11; + } else { + goto yy5; + } +yy32: ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; + yych = *p; +yy33: + if (yybm[0+yych] & 128) { + goto yy32; } - { token = SUBNINJA; break; } + if (yych <= 0x00) goto yy31; + ++p; + { continue; } yy36: yych = *++p; - if (yych != 'c') goto yy27; + if (yych == 'i') goto yy44; + goto yy14; +yy37: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'f') goto yy45; + goto yy14; +yy38: yych = *++p; - if (yych != 'u') goto yy27; + if (yych == 'c') goto yy46; + goto yy14; +yy39: yych = *++p; - if (yych != 'd') goto yy27; + if (yych == 'o') goto yy47; + goto yy14; +yy40: yych = *++p; - if (yych != 'e') goto yy27; - ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; - } - { token = INCLUDE; break; } -yy43: + if (yych == 'l') goto yy48; + goto yy14; +yy41: + yych = *++p; + if (yych == 'b') goto yy49; + goto yy14; +yy42: ++p; { token = PIPE2; break; } +yy44: + yych = *++p; + if (yych == 'l') goto yy50; + goto yy14; yy45: yych = *++p; - if (yych != 'f') goto yy27; + if (yych == 'a') goto yy51; + goto yy14; +yy46: yych = *++p; - if (yych != 'a') goto yy27; + if (yych == 'l') goto yy52; + goto yy14; +yy47: yych = *++p; - if (yych != 'u') goto yy27; + if (yych == 'l') goto yy53; + goto yy14; +yy48: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'e') goto yy55; + goto yy14; +yy49: yych = *++p; - if (yych != 't') goto yy27; - ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; - } - { token = DEFAULT; break; } -yy52: + if (yych == 'n') goto yy57; + goto yy14; +yy50: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'd') goto yy58; + goto yy14; +yy51: yych = *++p; - if (yych != 'e') goto yy27; + if (yych == 'u') goto yy60; + goto yy14; +yy52: + yych = *++p; + if (yych == 'u') goto yy61; + goto yy14; +yy53: ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; + } + { token = POOL; break; } +yy55: + ++p; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } { token = RULE; break; } -yy56: +yy57: yych = *++p; - if (yych != 'o') goto yy27; - yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'i') goto yy62; + goto yy14; +yy58: ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } - { token = POOL; break; } + { token = BUILD; break; } yy60: yych = *++p; - if (yych != 'i') goto yy27; + if (yych == 'l') goto yy63; + goto yy14; +yy61: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'd') goto yy64; + goto yy14; +yy62: yych = *++p; - if (yych != 'd') goto yy27; - ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; - } - { token = BUILD; break; } + if (yych == 'n') goto yy65; + goto yy14; +yy63: + yych = *++p; + if (yych == 't') goto yy66; + goto yy14; +yy64: + yych = *++p; + if (yych == 'e') goto yy68; + goto yy14; yy65: + yych = *++p; + if (yych == 'j') goto yy70; + goto yy14; +yy66: ++p; - { token = NEWLINE; break; } -yy67: - ++p; - yych = *p; -yy68: - if (yybm[0+yych] & 64) { - goto yy67; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } - if (yych >= 0x01) goto yy70; -yy69: - p = q; - if (yyaccept <= 0) { - goto yy3; - } else { - goto yy5; - } -yy70: + { token = DEFAULT; break; } +yy68: ++p; - { continue; } -yy72: - yyaccept = 0; - q = ++p; - yych = *p; -yy73: - if (yybm[0+yych] & 128) { - goto yy72; - } - if (yych <= '\f') { - if (yych != '\n') goto yy3; - } else { - if (yych <= '\r') goto yy75; - if (yych == '#') goto yy67; - goto yy3; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } + { token = INCLUDE; break; } +yy70: yych = *++p; - goto yy8; -yy75: + if (yych != 'a') goto yy14; ++p; - if ((yych = *p) == '\n') goto yy65; - goto yy69; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; + } + { token = SUBNINJA; break; } } } @@ -487,49 +507,42 @@ void Lexer::EatWhitespace() { 0, 0, 0, 0, 0, 0, 0, 0, }; yych = *p; - if (yych <= ' ') { - if (yych <= 0x00) goto yy82; - if (yych <= 0x1F) goto yy84; - } else { - if (yych == '$') goto yy80; - goto yy84; + if (yybm[0+yych] & 128) { + goto yy79; } + if (yych <= 0x00) goto yy75; + if (yych == '$') goto yy82; + goto yy77; +yy75: ++p; - yych = *p; - goto yy92; -yy79: - { continue; } -yy80: - yych = *(q = ++p); - if (yych == '\n') goto yy85; - if (yych == '\r') goto yy87; -yy81: { break; } -yy82: +yy77: ++p; +yy78: { break; } -yy84: - yych = *++p; - goto yy81; -yy85: +yy79: ++p; + yych = *p; + if (yybm[0+yych] & 128) { + goto yy79; + } { continue; } -yy87: +yy82: + yych = *(q = ++p); + if (yych == '\n') goto yy83; + if (yych == '\r') goto yy85; + goto yy78; +yy83: + ++p; + { continue; } +yy85: yych = *++p; - if (yych == '\n') goto yy89; + if (yych == '\n') goto yy87; p = q; - goto yy81; -yy89: + goto yy78; +yy87: ++p; { continue; } -yy91: - ++p; - yych = *p; -yy92: - if (yybm[0+yych] & 128) { - goto yy91; - } - goto yy79; } } @@ -537,8 +550,9 @@ yy92: bool Lexer::ReadIdent(string* out) { const char* p = ofs_; + const char* start; for (;;) { - const char* start = p; + start = p; { unsigned char yych; @@ -577,45 +591,28 @@ bool Lexer::ReadIdent(string* out) { 0, 0, 0, 0, 0, 0, 0, 0, }; yych = *p; - if (yych <= '@') { - if (yych <= '.') { - if (yych <= ',') goto yy97; - } else { - if (yych <= '/') goto yy97; - if (yych >= ':') goto yy97; - } - } else { - if (yych <= '_') { - if (yych <= 'Z') goto yy95; - if (yych <= '^') goto yy97; - } else { - if (yych <= '`') goto yy97; - if (yych >= '{') goto yy97; - } + if (yybm[0+yych] & 128) { + goto yy93; } -yy95: ++p; - yych = *p; - goto yy100; -yy96: { - out->assign(start, p - start); - break; + last_token_ = start; + return false; } -yy97: - ++p; - { return false; } -yy99: +yy93: ++p; yych = *p; -yy100: if (yybm[0+yych] & 128) { - goto yy99; + goto yy93; } - goto yy96; + { + out->assign(start, p - start); + break; + } } } + last_token_ = start; ofs_ = p; EatWhitespace(); return true; @@ -631,72 +628,69 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { { unsigned char yych; static const unsigned char yybm[] = { - 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 0, 128, 128, 0, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 16, 128, 128, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 224, 160, 128, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 0, 128, 128, 128, 128, 128, - 128, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 128, 128, 128, 128, 224, - 128, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, + 0, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 0, 16, 16, 0, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 32, 16, 16, 16, 0, 16, 16, 16, + 16, 16, 16, 16, 16, 208, 144, 16, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 0, 16, 16, 16, 16, 16, + 16, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 16, 16, 16, 16, 208, + 16, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 16, 0, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, }; yych = *p; - if (yych <= ' ') { - if (yych <= '\n') { - if (yych <= 0x00) goto yy110; - if (yych >= '\n') goto yy107; - } else { - if (yych == '\r') goto yy105; - if (yych >= ' ') goto yy107; - } + if (yybm[0+yych] & 16) { + goto yy100; + } + if (yych <= '\r') { + if (yych <= 0x00) goto yy98; + if (yych <= '\n') goto yy103; + goto yy105; } else { - if (yych <= '9') { - if (yych == '$') goto yy109; - } else { - if (yych <= ':') goto yy107; - if (yych == '|') goto yy107; - } + if (yych <= ' ') goto yy103; + if (yych <= '$') goto yy107; + goto yy103; } +yy98: ++p; - yych = *p; - goto yy140; -yy104: { - eval->AddText(StringPiece(start, p - start)); - continue; + last_token_ = start; + return Error("unexpected EOF", err); } -yy105: +yy100: ++p; - if ((yych = *p) == '\n') goto yy137; + yych = *p; + if (yybm[0+yych] & 16) { + goto yy100; + } { - last_token_ = start; - return Error(DescribeLastError(), err); + eval->AddText(StringPiece(start, p - start)); + continue; } -yy107: +yy103: ++p; { if (path) { @@ -709,152 +703,121 @@ yy107: continue; } } -yy109: +yy105: + ++p; + if ((yych = *p) == '\n') goto yy108; + { + last_token_ = start; + return Error(DescribeLastError(), err); + } +yy107: yych = *++p; - if (yych <= '-') { - if (yych <= 0x1F) { - if (yych <= '\n') { - if (yych <= '\t') goto yy112; - goto yy124; - } else { - if (yych == '\r') goto yy114; - goto yy112; - } + if (yybm[0+yych] & 64) { + goto yy120; + } + if (yych <= ' ') { + if (yych <= '\f') { + if (yych == '\n') goto yy112; + goto yy110; } else { - if (yych <= '#') { - if (yych <= ' ') goto yy115; - goto yy112; - } else { - if (yych <= '$') goto yy117; - if (yych <= ',') goto yy112; - goto yy119; - } + if (yych <= '\r') goto yy115; + if (yych <= 0x1F) goto yy110; + goto yy116; } } else { - if (yych <= 'Z') { - if (yych <= '9') { - if (yych <= '/') goto yy112; - goto yy119; - } else { - if (yych <= ':') goto yy121; - if (yych <= '@') goto yy112; - goto yy119; - } + if (yych <= '/') { + if (yych == '$') goto yy118; + goto yy110; } else { - if (yych <= '`') { - if (yych == '_') goto yy119; - goto yy112; - } else { - if (yych <= 'z') goto yy119; - if (yych <= '{') goto yy123; - goto yy112; - } + if (yych <= ':') goto yy123; + if (yych <= '`') goto yy110; + if (yych <= '{') goto yy125; + goto yy110; } } +yy108: + ++p; + { + if (path) + p = start; + break; + } yy110: ++p; +yy111: { last_token_ = start; - return Error("unexpected EOF", err); + return Error("bad $-escape (literal $ must be written as $$)", err); } yy112: ++p; -yy113: + yych = *p; + if (yybm[0+yych] & 32) { + goto yy112; + } { - last_token_ = start; - return Error("bad $-escape (literal $ must be written as $$)", err); + continue; } -yy114: - yych = *++p; - if (yych == '\n') goto yy134; - goto yy113; yy115: + yych = *++p; + if (yych == '\n') goto yy126; + goto yy111; +yy116: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy117: +yy118: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy119: +yy120: ++p; yych = *p; - goto yy133; -yy120: + if (yybm[0+yych] & 64) { + goto yy120; + } { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy121: +yy123: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy123: +yy125: yych = *(q = ++p); - if (yybm[0+yych] & 32) { - goto yy127; + if (yybm[0+yych] & 128) { + goto yy129; } - goto yy113; -yy124: + goto yy111; +yy126: ++p; yych = *p; - if (yybm[0+yych] & 16) { - goto yy124; - } + if (yych == ' ') goto yy126; { continue; } -yy127: +yy129: ++p; yych = *p; - if (yybm[0+yych] & 32) { - goto yy127; + if (yybm[0+yych] & 128) { + goto yy129; } - if (yych == '}') goto yy130; + if (yych == '}') goto yy132; p = q; - goto yy113; -yy130: - ++p; - { - eval->AddSpecial(StringPiece(start + 2, p - start - 3)); - continue; - } + goto yy111; yy132: ++p; - yych = *p; -yy133: - if (yybm[0+yych] & 64) { - goto yy132; - } - goto yy120; -yy134: - ++p; - yych = *p; - if (yych == ' ') goto yy134; { + eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy137: - ++p; - { - if (path) - p = start; - break; - } -yy139: - ++p; - yych = *p; -yy140: - if (yybm[0+yych] & 128) { - goto yy139; - } - goto yy104; } } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index f861239..c1fb822 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -22,14 +22,14 @@ bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; - const char* context = input_.str_; + const char* line_start = input_.str_; for (const char* p = input_.str_; p < last_token_; ++p) { if (*p == '\n') { ++line; - context = p + 1; + line_start = p + 1; } } - int col = last_token_ ? (int)(last_token_ - context) : 0; + int col = last_token_ ? (int)(last_token_ - line_start) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); @@ -42,12 +42,12 @@ bool Lexer::Error(const string& message, string* err) { int len; bool truncated = true; for (len = 0; len < kTruncateColumn; ++len) { - if (context[len] == 0 || context[len] == '\n') { + if (line_start[len] == 0 || line_start[len] == '\n') { truncated = false; break; } } - *err += string(context, len); + *err += string(line_start, len); if (truncated) *err += "..."; *err += "\n"; @@ -182,16 +182,21 @@ void Lexer::EatWhitespace() { bool Lexer::ReadIdent(string* out) { const char* p = ofs_; + const char* start; for (;;) { - const char* start = p; + start = p; /*!re2c varname { out->assign(start, p - start); break; } - [^] { return false; } + [^] { + last_token_ = start; + return false; + } */ } + last_token_ = start; ofs_ = p; EatWhitespace(); return true; diff --git a/src/line_printer.cc b/src/line_printer.cc index 2cd3e17..55469d9 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -18,6 +18,9 @@ #include <stdlib.h> #ifdef _WIN32 #include <windows.h> +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x4 +#endif #else #include <unistd.h> #include <sys/ioctl.h> @@ -28,18 +31,36 @@ #include "util.h" LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { -#ifndef _WIN32 const char* term = getenv("TERM"); +#ifndef _WIN32 smart_terminal_ = isatty(1) && term && string(term) != "dumb"; #else // Disable output buffer. It'd be nice to use line buffering but // MSDN says: "For some systems, [_IOLBF] provides line // buffering. However, for Win32, the behavior is the same as _IOFBF // - Full Buffering." - setvbuf(stdout, NULL, _IONBF, 0); - console_ = GetStdHandle(STD_OUTPUT_HANDLE); - CONSOLE_SCREEN_BUFFER_INFO csbi; - smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi); + if (term && string(term) == "dumb") { + smart_terminal_ = false; + } else { + setvbuf(stdout, NULL, _IONBF, 0); + console_ = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO csbi; + smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi); + } +#endif + supports_color_ = smart_terminal_; + if (!supports_color_) { + const char* clicolor_force = getenv("CLICOLOR_FORCE"); + supports_color_ = clicolor_force && string(clicolor_force) != "0"; + } +#ifdef _WIN32 + // Try enabling ANSI escape sequence support on Windows 10 terminals. + if (supports_color_) { + DWORD mode; + if (GetConsoleMode(console_, &mode)) { + SetConsoleMode(console_, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); + } + } #endif } @@ -82,7 +103,7 @@ void LinePrinter::Print(string to_print, LineType type) { // Limit output to width of the terminal if provided so we don't cause // line-wrapping. winsize size; - if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) { + if ((ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0) && size.ws_col) { to_print = ElideMiddle(to_print, size.ws_col); } printf("%s", to_print.c_str()); diff --git a/src/line_printer.h b/src/line_printer.h index 55225e5..92d4dc4 100644 --- a/src/line_printer.h +++ b/src/line_printer.h @@ -27,6 +27,8 @@ struct LinePrinter { bool is_smart_terminal() const { return smart_terminal_; } void set_smart_terminal(bool smart) { smart_terminal_ = smart; } + bool supports_color() const { return supports_color_; } + enum LineType { FULL, ELIDE @@ -46,6 +48,9 @@ struct LinePrinter { /// Whether we can do fancy terminal control codes. bool smart_terminal_; + /// Whether we can use ISO 6429 (ANSI) color sequences. + bool supports_color_; + /// Whether the caret is at the beginning of a blank line. bool have_blank_line_; diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 2164921..2011368 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -18,41 +18,18 @@ #include <stdlib.h> #include <vector> -#include "disk_interface.h" #include "graph.h" -#include "metrics.h" #include "state.h" #include "util.h" #include "version.h" ManifestParser::ManifestParser(State* state, FileReader* file_reader, - DupeEdgeAction dupe_edge_action) - : state_(state), file_reader_(file_reader), - dupe_edge_action_(dupe_edge_action), quiet_(false) { + ManifestParserOptions options) + : Parser(state, file_reader), + options_(options), quiet_(false) { env_ = &state->bindings_; } -bool ManifestParser::Load(const string& filename, string* err, Lexer* parent) { - METRIC_RECORD(".ninja parse"); - string contents; - string read_err; - if (file_reader_->ReadFile(filename, &contents, &read_err) != FileReader::Okay) { - *err = "loading '" + filename + "': " + read_err; - if (parent) - parent->Error(string(*err), err); - return false; - } - - // The lexer needs a nul byte at the end of its input, to know when it's done. - // It takes a StringPiece, and StringPiece's string constructor uses - // string::data(). data()'s return value isn't guaranteed to be - // null-terminated (although in practice - libc++, libstdc++, msvc's stl -- - // it is, and C++11 demands that too), so add an explicit nul byte. - contents.resize(contents.size() + 1); - - return Parse(filename, contents, err); -} - bool ManifestParser::Parse(const string& filename, const string& input, string* err) { lexer_.Start(filename, input); @@ -346,7 +323,7 @@ bool ManifestParser::ParseEdge(string* err) { if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); if (!state_->AddOut(edge, path, slash_bits)) { - if (dupe_edge_action_ == kDupeEdgeActionError) { + if (options_.dupe_edge_action_ == kDupeEdgeActionError) { lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]", err); return false; @@ -383,6 +360,25 @@ bool ManifestParser::ParseEdge(string* err) { edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; + if (options_.phony_cycle_action_ == kPhonyCycleActionWarn && + edge->maybe_phonycycle_diagnostic()) { + // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements + // that reference themselves. Ninja used to tolerate these in the + // build graph but that has since been fixed. Filter them out to + // support users of those old CMake versions. + Node* out = edge->outputs_[0]; + vector<Node*>::iterator new_end = + remove(edge->inputs_.begin(), edge->inputs_.end(), out); + if (new_end != edge->inputs_.end()) { + edge->inputs_.erase(new_end, edge->inputs_.end()); + if (!quiet_) { + Warning("phony target '%s' names itself as an input; " + "ignoring [-w phonycycle=warn]", + out->path().c_str()); + } + } + } + // Multiple outputs aren't (yet?) supported with depslog. string deps_type = edge->GetBinding("deps"); if (!deps_type.empty() && edge->outputs_.size() > 1) { @@ -391,6 +387,23 @@ bool ManifestParser::ParseEdge(string* err) { err); } + // Lookup, validate, and save any dyndep binding. It will be used later + // to load generated dependency information dynamically, but it must + // be one of our manifest-specified inputs. + string dyndep = edge->GetUnescapedDyndep(); + if (!dyndep.empty()) { + uint64_t slash_bits; + if (!CanonicalizePath(&dyndep, &slash_bits, err)) + return false; + edge->dyndep_ = state_->GetNode(dyndep, slash_bits); + edge->dyndep_->set_dyndep_pending(true); + vector<Node*>::iterator dgi = + std::find(edge->inputs_.begin(), edge->inputs_.end(), edge->dyndep_); + if (dgi == edge->inputs_.end()) { + return lexer_.Error("dyndep '" + dyndep + "' is not an input", err); + } + } + return true; } @@ -400,7 +413,7 @@ bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { return false; string path = eval.Evaluate(env_); - ManifestParser subparser(state_, file_reader_, dupe_edge_action_); + ManifestParser subparser(state_, file_reader_, options_); if (new_scope) { subparser.env_ = new BindingEnv(env_); } else { @@ -415,14 +428,3 @@ bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { return true; } - -bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) { - Lexer::Token token = lexer_.ReadToken(); - if (token != expected) { - string message = string("expected ") + Lexer::TokenName(expected); - message += string(", got ") + Lexer::TokenName(token); - message += Lexer::TokenErrorHint(expected); - return lexer_.Error(message, err); - } - return true; -} diff --git a/src/manifest_parser.h b/src/manifest_parser.h index 043e4b2..e14d069 100644 --- a/src/manifest_parser.h +++ b/src/manifest_parser.h @@ -15,29 +15,33 @@ #ifndef NINJA_MANIFEST_PARSER_H_ #define NINJA_MANIFEST_PARSER_H_ -#include <string> - -using namespace std; - -#include "lexer.h" +#include "parser.h" struct BindingEnv; struct EvalString; -struct FileReader; -struct State; enum DupeEdgeAction { kDupeEdgeActionWarn, kDupeEdgeActionError, }; +enum PhonyCycleAction { + kPhonyCycleActionWarn, + kPhonyCycleActionError, +}; + +struct ManifestParserOptions { + ManifestParserOptions() + : dupe_edge_action_(kDupeEdgeActionWarn), + phony_cycle_action_(kPhonyCycleActionWarn) {} + DupeEdgeAction dupe_edge_action_; + PhonyCycleAction phony_cycle_action_; +}; + /// Parses .ninja files. -struct ManifestParser { +struct ManifestParser : public Parser { ManifestParser(State* state, FileReader* file_reader, - DupeEdgeAction dupe_edge_action); - - /// Load and parse a file. - bool Load(const string& filename, string* err, Lexer* parent = NULL); + ManifestParserOptions options = ManifestParserOptions()); /// Parse a text string of input. Used by tests. bool ParseTest(const string& input, string* err) { @@ -59,15 +63,8 @@ private: /// Parse either a 'subninja' or 'include' line. bool ParseFileInclude(bool new_scope, string* err); - /// If the next token is not \a expected, produce an error string - /// saying "expectd foo, got bar". - bool ExpectToken(Lexer::Token expected, string* err); - - State* state_; BindingEnv* env_; - FileReader* file_reader_; - Lexer lexer_; - DupeEdgeAction dupe_edge_action_; + ManifestParserOptions options_; bool quiet_; }; diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc index 60c2054..67d11f9 100644 --- a/src/manifest_parser_perftest.cc +++ b/src/manifest_parser_perftest.cc @@ -56,7 +56,7 @@ int LoadManifests(bool measure_command_evaluation) { string err; RealDiskInterface disk_interface; State state; - ManifestParser parser(&state, &disk_interface, kDupeEdgeActionWarn); + ManifestParser parser(&state, &disk_interface); if (!parser.Load("build.ninja", &err)) { fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); exit(1); diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 3c82dc5..f2b7467 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -23,7 +23,7 @@ struct ParserTest : public testing::Test { void AssertParse(const char* input) { - ManifestParser parser(&state, &fs_, kDupeEdgeActionWarn); + ManifestParser parser(&state, &fs_); string err; EXPECT_TRUE(parser.ParseTest(input, &err)); ASSERT_EQ("", err); @@ -358,7 +358,9 @@ TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputsError) { "build out1 out2: cat in1\n" "build out1: cat in2\n" "build final: cat out1\n"; - ManifestParser parser(&state, &fs_, kDupeEdgeActionError); + ManifestParserOptions parser_opts; + parser_opts.dupe_edge_action_ = kDupeEdgeActionError; + ManifestParser parser(&state, &fs_, parser_opts); string err; EXPECT_FALSE(parser.ParseTest(kInput, &err)); EXPECT_EQ("input:5: multiple rules generate out1 [-w dupbuild=err]\n", err); @@ -373,13 +375,41 @@ TEST_F(ParserTest, DuplicateEdgeInIncludedFile) { "build final: cat out1\n"); const char kInput[] = "subninja sub.ninja\n"; - ManifestParser parser(&state, &fs_, kDupeEdgeActionError); + ManifestParserOptions parser_opts; + parser_opts.dupe_edge_action_ = kDupeEdgeActionError; + ManifestParser parser(&state, &fs_, parser_opts); string err; EXPECT_FALSE(parser.ParseTest(kInput, &err)); EXPECT_EQ("sub.ninja:5: multiple rules generate out1 [-w dupbuild=err]\n", err); } +TEST_F(ParserTest, PhonySelfReferenceIgnored) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"build a: phony a\n" +)); + + Node* node = state.LookupNode("a"); + Edge* edge = node->in_edge(); + ASSERT_TRUE(edge->inputs_.empty()); +} + +TEST_F(ParserTest, PhonySelfReferenceKept) { + const char kInput[] = +"build a: phony a\n"; + ManifestParserOptions parser_opts; + parser_opts.phony_cycle_action_ = kPhonyCycleActionError; + ManifestParser parser(&state, &fs_, parser_opts); + string err; + EXPECT_TRUE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("", err); + + Node* node = state.LookupNode("a"); + Edge* edge = node->in_edge(); + ASSERT_EQ(edge->inputs_.size(), 1); + ASSERT_EQ(edge->inputs_[0], node); +} + TEST_F(ParserTest, ReservedWords) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule build\n" @@ -391,7 +421,7 @@ TEST_F(ParserTest, ReservedWords) { TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest(string("subn", 4), &err)); EXPECT_EQ("input:1: expected '=', got eof\n" @@ -402,7 +432,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("foobar", &err)); EXPECT_EQ("input:1: expected '=', got eof\n" @@ -413,7 +443,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x 3", &err)); EXPECT_EQ("input:1: expected '=', got identifier\n" @@ -424,7 +454,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x = 3", &err)); EXPECT_EQ("input:1: unexpected EOF\n" @@ -435,7 +465,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x = 3\ny 2", &err)); EXPECT_EQ("input:2: expected '=', got identifier\n" @@ -446,7 +476,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x = $", &err)); EXPECT_EQ("input:1: bad $-escape (literal $ must be written as $$)\n" @@ -457,7 +487,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x = $\n $[\n", &err)); EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n" @@ -468,7 +498,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x = a$\n b$\n $\n", &err)); EXPECT_EQ("input:4: unexpected EOF\n" @@ -477,7 +507,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("build\n", &err)); EXPECT_EQ("input:1: expected path\n" @@ -488,29 +518,29 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("build x: y z\n", &err)); EXPECT_EQ("input:1: unknown build rule 'y'\n" "build x: y z\n" - " ^ near here" + " ^ near here" , err); } { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("build x:: y z\n", &err)); EXPECT_EQ("input:1: expected build command name\n" "build x:: y z\n" - " ^ near here" + " ^ near here" , err); } { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n command = cat ok\n" "build x: cat $\n :\n", @@ -523,7 +553,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n", &err)); @@ -532,7 +562,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n" " command = echo\n" @@ -546,7 +576,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n" " command = echo\n" @@ -558,7 +588,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n" " command = ${fafsd\n" @@ -573,7 +603,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n" " command = cat\n" @@ -588,7 +618,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cat\n" " command = cat\n" @@ -602,16 +632,19 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule %foo\n", &err)); - EXPECT_EQ("input:1: expected rule name\n", err); + EXPECT_EQ("input:1: expected rule name\n" + "rule %foo\n" + " ^ near here", + err); } { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cc\n" " command = foo\n" @@ -625,7 +658,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" "build $.: cc bar.cc\n", @@ -638,16 +671,19 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n && bar", &err)); - EXPECT_EQ("input:3: expected variable name\n", err); + EXPECT_EQ("input:3: expected variable name\n" + " && bar\n" + " ^ near here", + err); } { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" "build $: cc bar.cc\n", @@ -660,7 +696,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("default\n", &err)); @@ -672,7 +708,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("default nonexistent\n", &err)); @@ -684,7 +720,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule r\n command = r\n" "build b: r\n" @@ -698,7 +734,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("default $a\n", &err)); EXPECT_EQ("input:1: empty path\n" @@ -709,7 +745,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule r\n" " command = r\n" @@ -721,7 +757,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; // the indented blank line must terminate the rule // this also verifies that "unexpected (token)" errors are correct @@ -734,15 +770,17 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("pool\n", &err)); - EXPECT_EQ("input:1: expected pool name\n", err); + EXPECT_EQ("input:1: expected pool name\n" + "pool\n" + " ^ near here", err); } { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("pool foo\n", &err)); EXPECT_EQ("input:2: expected 'depth =' line\n", err); @@ -750,7 +788,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("pool foo\n" " depth = 4\n" @@ -763,7 +801,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("pool foo\n" " depth = -1\n", &err)); @@ -775,7 +813,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("pool foo\n" " bar = 1\n", &err)); @@ -787,7 +825,7 @@ TEST_F(ParserTest, Errors) { { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; // Pool names are dereferenced at edge parsing time. EXPECT_FALSE(parser.ParseTest("rule run\n" @@ -800,7 +838,7 @@ TEST_F(ParserTest, Errors) { TEST_F(ParserTest, MissingInput) { State local_state; - ManifestParser parser(&local_state, &fs_, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, &fs_); string err; EXPECT_FALSE(parser.Load("build.ninja", &err)); EXPECT_EQ("loading 'build.ninja': No such file or directory", err); @@ -808,7 +846,7 @@ TEST_F(ParserTest, MissingInput) { TEST_F(ParserTest, MultipleOutputs) { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n depfile = bar\n" "build a.o b.o: cc c.cc\n", @@ -818,7 +856,7 @@ TEST_F(ParserTest, MultipleOutputs) { TEST_F(ParserTest, MultipleOutputsWithDeps) { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n" "build a.o b.o: cc c.cc\n", @@ -853,7 +891,7 @@ TEST_F(ParserTest, SubNinja) { } TEST_F(ParserTest, MissingSubNinja) { - ManifestParser parser(&state, &fs_, kDupeEdgeActionWarn); + ManifestParser parser(&state, &fs_); string err; EXPECT_FALSE(parser.ParseTest("subninja foo.ninja\n", &err)); EXPECT_EQ("input:1: loading 'foo.ninja': No such file or directory\n" @@ -866,7 +904,7 @@ TEST_F(ParserTest, DuplicateRuleInDifferentSubninjas) { // Test that rules are scoped to subninjas. fs_.Create("test.ninja", "rule cat\n" " command = cat\n"); - ManifestParser parser(&state, &fs_, kDupeEdgeActionWarn); + ManifestParser parser(&state, &fs_); string err; EXPECT_TRUE(parser.ParseTest("rule cat\n" " command = cat\n" @@ -879,7 +917,7 @@ TEST_F(ParserTest, DuplicateRuleInDifferentSubninjasWithInclude) { " command = cat\n"); fs_.Create("test.ninja", "include rules.ninja\n" "build x : cat\n"); - ManifestParser parser(&state, &fs_, kDupeEdgeActionWarn); + ManifestParser parser(&state, &fs_); string err; EXPECT_TRUE(parser.ParseTest("include rules.ninja\n" "subninja test.ninja\n" @@ -899,7 +937,7 @@ TEST_F(ParserTest, Include) { TEST_F(ParserTest, BrokenInclude) { fs_.Create("include.ninja", "build\n"); - ManifestParser parser(&state, &fs_, kDupeEdgeActionWarn); + ManifestParser parser(&state, &fs_); string err; EXPECT_FALSE(parser.ParseTest("include include.ninja\n", &err)); EXPECT_EQ("include.ninja:1: expected path\n" @@ -974,7 +1012,7 @@ TEST_F(ParserTest, ImplicitOutputDupes) { } TEST_F(ParserTest, NoExplicitOutput) { - ManifestParser parser(&state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&state, NULL); string err; EXPECT_TRUE(parser.ParseTest( "rule cat\n" @@ -1034,7 +1072,7 @@ TEST_F(ParserTest, UTF8) { TEST_F(ParserTest, CRLF) { State local_state; - ManifestParser parser(&local_state, NULL, kDupeEdgeActionWarn); + ManifestParser parser(&local_state, NULL); string err; EXPECT_TRUE(parser.ParseTest("# comment with crlf\r\n", &err)); @@ -1047,3 +1085,73 @@ TEST_F(ParserTest, CRLF) { " description = YAY!\r\n", &err)); } + +TEST_F(ParserTest, DyndepNotSpecified) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build result: cat in\n")); + Edge* edge = state.GetNode("result", 0)->in_edge(); + ASSERT_FALSE(edge->dyndep_); +} + +TEST_F(ParserTest, DyndepNotInput) { + State lstate; + ManifestParser parser(&lstate, NULL); + string err; + EXPECT_FALSE(parser.ParseTest( +"rule touch\n" +" command = touch $out\n" +"build result: touch\n" +" dyndep = notin\n", + &err)); + EXPECT_EQ("input:5: dyndep 'notin' is not an input\n", err); +} + +TEST_F(ParserTest, DyndepExplicitInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build result: cat in\n" +" dyndep = in\n")); + Edge* edge = state.GetNode("result", 0)->in_edge(); + ASSERT_TRUE(edge->dyndep_); + EXPECT_TRUE(edge->dyndep_->dyndep_pending()); + EXPECT_EQ(edge->dyndep_->path(), "in"); +} + +TEST_F(ParserTest, DyndepImplicitInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build result: cat in | dd\n" +" dyndep = dd\n")); + Edge* edge = state.GetNode("result", 0)->in_edge(); + ASSERT_TRUE(edge->dyndep_); + EXPECT_TRUE(edge->dyndep_->dyndep_pending()); + EXPECT_EQ(edge->dyndep_->path(), "dd"); +} + +TEST_F(ParserTest, DyndepOrderOnlyInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build result: cat in || dd\n" +" dyndep = dd\n")); + Edge* edge = state.GetNode("result", 0)->in_edge(); + ASSERT_TRUE(edge->dyndep_); + EXPECT_TRUE(edge->dyndep_->dyndep_pending()); + EXPECT_EQ(edge->dyndep_->path(), "dd"); +} + +TEST_F(ParserTest, DyndepRuleInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +" dyndep = $in\n" +"build result: cat in\n")); + Edge* edge = state.GetNode("result", 0)->in_edge(); + ASSERT_TRUE(edge->dyndep_); + EXPECT_TRUE(edge->dyndep_->dyndep_pending()); + EXPECT_EQ(edge->dyndep_->path(), "in"); +} diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc index 1efb085..ca93638 100644 --- a/src/minidump-win32.cc +++ b/src/minidump-win32.cc @@ -32,17 +32,17 @@ typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( /// Creates a windows minidump in temp folder. void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { char temp_path[MAX_PATH]; - GetTempPath(sizeof(temp_path), temp_path); + GetTempPathA(sizeof(temp_path), temp_path); char temp_file[MAX_PATH]; sprintf(temp_file, "%s\\ninja_crash_dump_%lu.dmp", temp_path, GetCurrentProcessId()); // Delete any previous minidump of the same name. - DeleteFile(temp_file); + DeleteFileA(temp_file); // Load DbgHelp.dll dynamically, as library is not present on all // Windows versions. - HMODULE dbghelp = LoadLibrary("dbghelp.dll"); + HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); if (dbghelp == NULL) { Error("failed to create minidump: LoadLibrary('dbghelp.dll'): %s", GetLastErrorString().c_str()); diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index e37a26e..de6147a 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -43,10 +43,10 @@ int CLWrapper::Run(const string& command, string* output) { security_attributes.bInheritHandle = TRUE; // Must be inheritable so subprocesses can dup to children. - HANDLE nul = CreateFile("NUL", GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | - FILE_SHARE_DELETE, - &security_attributes, OPEN_EXISTING, 0, NULL); + HANDLE nul = + CreateFileA("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); if (nul == INVALID_HANDLE_VALUE) Fatal("couldn't open nul"); @@ -58,8 +58,8 @@ int CLWrapper::Run(const string& command, string* output) { Win32Fatal("SetHandleInformation"); PROCESS_INFORMATION process_info = {}; - STARTUPINFO startup_info = {}; - startup_info.cb = sizeof(STARTUPINFO); + STARTUPINFOA startup_info = {}; + startup_info.cb = sizeof(STARTUPINFOA); startup_info.hStdInput = nul; startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); startup_info.hStdOutput = stdout_write; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index e419cd7..644b2a2 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -113,7 +113,7 @@ int MSVCHelperMain(int argc, char** argv) { PushPathIntoEnvironment(env); } - char* command = GetCommandLine(); + char* command = GetCommandLineA(); command = strstr(command, " -- "); if (!command) { Fatal("expected command line to end with \" -- command args\""); diff --git a/src/ninja.cc b/src/ninja.cc index 54de7b9..a093cd1 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -70,6 +70,13 @@ struct Options { /// Whether duplicate rules for one target should warn or print an error. bool dupe_edges_should_err; + + /// Whether phony cycles should warn or print an error. + bool phony_cycle_should_err; + + /// Whether a depfile with multiple targets on separate lines should + /// warn or print an error. + bool depfile_distinct_target_lines_should_err; }; /// The Ninja main() loads up a series of data structures; various tools need @@ -119,6 +126,7 @@ struct NinjaMain : public BuildLogUser { int ToolCompilationDatabase(const Options* options, int argc, char* argv[]); int ToolRecompact(const Options* options, int argc, char* argv[]); int ToolUrtle(const Options* options, int argc, char** argv); + int ToolRules(const Options* options, int argc, char* argv[]); /// Open the build log. /// @return false on error. @@ -151,7 +159,7 @@ struct NinjaMain : public BuildLogUser { // Just checking n isn't enough: If an old output is both in the build log // and in the deps log, it will have a Node object in state_. (It will also // have an in edge if one of its inputs is another output that's in the deps - // log, but having a deps edge product an output thats input to another deps + // log, but having a deps edge product an output that's input to another deps // edge is rare, and the first recompaction will delete all old outputs from // the deps log, and then a second recompaction will clear the build log, // which seems good enough for this corner case.) @@ -198,21 +206,21 @@ void Usage(const BuildConfig& config) { "if targets are unspecified, builds the 'default' target (see manual).\n" "\n" "options:\n" -" --version print ninja version (\"%s\")\n" +" --version print ninja version (\"%s\")\n" +" -v, --verbose show all command lines while building\n" "\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" "\n" -" -j N run N jobs in parallel [default=%d, derived from CPUs available]\n" -" -k N keep going until N jobs fail [default=1]\n" +" -j N run N jobs in parallel (0 means infinity) [default=%d on this system]\n" +" -k N keep going until N jobs fail (0 means infinity) [default=1]\n" " -l N do not start new jobs if the load average is greater than N\n" " -n dry run (don't run commands but act like they succeeded)\n" -" -v show all command lines while building\n" "\n" -" -d MODE enable debugging (use -d list to list modes)\n" -" -t TOOL run a subtool (use -t list to list subtools)\n" +" -d MODE enable debugging (use '-d list' to list modes)\n" +" -t TOOL run a subtool (use '-t list' to list subtools)\n" " terminates toplevel options; further flags are passed to the tool\n" -" -w FLAG adjust warnings (use -w list to list warnings)\n", +" -w FLAG adjust warnings (use '-w list' to list warnings)\n", kNinjaVersion, config.parallelism); } @@ -331,7 +339,7 @@ int NinjaMain::ToolGraph(const Options* options, int argc, char* argv[]) { return 1; } - GraphViz graph; + GraphViz graph(&state_, &disk_interface_); graph.Start(); for (vector<Node*>::const_iterator n = nodes.begin(); n != nodes.end(); ++n) graph.AddTarget(*n); @@ -346,6 +354,8 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { return 1; } + DyndepLoader dyndep_loader(&state_, &disk_interface_); + for (int i = 0; i < argc; ++i) { string err; Node* node = CollectTarget(argv[i], &err); @@ -356,6 +366,11 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { printf("%s:\n", node->path().c_str()); if (Edge* edge = node->in_edge()) { + if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) { + if (!dyndep_loader.LoadDyndeps(edge->dyndep_, &err)) { + Warning("%s\n", err.c_str()); + } + } printf(" input: %s\n", edge->rule_->name().c_str()); for (int in = 0; in < (int)edge->inputs_.size(); in++) { const char* label = ""; @@ -384,7 +399,12 @@ int NinjaMain::ToolBrowse(const Options* options, int argc, char* argv[]) { // If we get here, the browse failed. return 1; } -#endif // _WIN32 +#else +int NinjaMain::ToolBrowse(const Options*, int, char**) { + Fatal("browse tool not supported on this platform"); + return 1; +} +#endif #if defined(_MSC_VER) int NinjaMain::ToolMSVC(const Options* options, int argc, char* argv[]) { @@ -491,7 +511,7 @@ int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) { TimeStamp mtime = disk_interface.Stat((*it)->path(), &err); if (mtime == -1) Error("%s", err.c_str()); // Log and ignore Stat() errors; - printf("%s: #deps %d, deps mtime %d (%s)\n", + printf("%s: #deps %d, deps mtime %" PRId64 " (%s)\n", (*it)->path().c_str(), deps->node_count, deps->mtime, (!mtime || mtime > deps->mtime ? "STALE":"VALID")); for (int i = 0; i < deps->node_count; ++i) @@ -542,6 +562,55 @@ int NinjaMain::ToolTargets(const Options* options, int argc, char* argv[]) { } } +int NinjaMain::ToolRules(const Options* options, int argc, char* argv[]) { + // Parse options. + + // The rules tool uses getopt, and expects argv[0] to contain the name of + // the tool, i.e. "rules". + argc++; + argv--; + + bool print_description = false; + + optind = 1; + int opt; + while ((opt = getopt(argc, argv, const_cast<char*>("hd"))) != -1) { + switch (opt) { + case 'd': + print_description = true; + break; + case 'h': + default: + printf("usage: ninja -t rules [options]\n" + "\n" + "options:\n" + " -d also print the description of the rule\n" + " -h print this message\n" + ); + return 1; + } + } + argv += optind; + argc -= optind; + + // Print rules + + typedef map<string, const Rule*> Rules; + const Rules& rules = state_.bindings_.GetRules(); + for (Rules::const_iterator i = rules.begin(); i != rules.end(); ++i) { + printf("%s", i->first.c_str()); + if (print_description) { + const Rule* rule = i->second; + const EvalString* description = rule->GetBinding("description"); + if (description != NULL) { + printf(": %s", description->Unparse().c_str()); + } + } + printf("\n"); + } + return 0; +} + enum PrintCommandMode { PCM_Single, PCM_All }; void PrintCommands(Edge* edge, set<Edge*>* seen, PrintCommandMode mode) { if (!edge) @@ -639,7 +708,7 @@ int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) { return 1; } - Cleaner cleaner(&state_, config_); + Cleaner cleaner(&state_, config_, &disk_interface_); if (argc >= 1) { if (clean_rules) return cleaner.CleanRules(argc, argv); @@ -659,7 +728,65 @@ void EncodeJSONString(const char *str) { } } -int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, char* argv[]) { +enum EvaluateCommandMode { + ECM_NORMAL, + ECM_EXPAND_RSPFILE +}; +string EvaluateCommandWithRspfile(Edge* edge, EvaluateCommandMode mode) { + string command = edge->EvaluateCommand(); + if (mode == ECM_NORMAL) + return command; + + string rspfile = edge->GetUnescapedRspfile(); + if (rspfile.empty()) + return command; + + size_t index = command.find(rspfile); + if (index == 0 || index == string::npos || command[index - 1] != '@') + return command; + + string rspfile_content = edge->GetBinding("rspfile_content"); + size_t newline_index = 0; + while ((newline_index = rspfile_content.find('\n', newline_index)) != + string::npos) { + rspfile_content.replace(newline_index, 1, 1, ' '); + ++newline_index; + } + command.replace(index - 1, rspfile.length() + 1, rspfile_content); + return command; +} + +int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, + char* argv[]) { + // The compdb tool uses getopt, and expects argv[0] to contain the name of + // the tool, i.e. "compdb". + argc++; + argv--; + + EvaluateCommandMode eval_mode = ECM_NORMAL; + + optind = 1; + int opt; + while ((opt = getopt(argc, argv, const_cast<char*>("hx"))) != -1) { + switch(opt) { + case 'x': + eval_mode = ECM_EXPAND_RSPFILE; + break; + + case 'h': + default: + printf( + "usage: ninja -t compdb [options] [rules]\n" + "\n" + "options:\n" + " -x expand @rspfile style response file invocations\n" + ); + return 1; + } + } + argv += optind; + argc -= optind; + bool first = true; vector<char> cwd; @@ -685,9 +812,11 @@ int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, char* a printf("\n {\n \"directory\": \""); EncodeJSONString(&cwd[0]); printf("\",\n \"command\": \""); - EncodeJSONString((*e)->EvaluateCommand().c_str()); + EncodeJSONString(EvaluateCommandWithRspfile(*e, eval_mode).c_str()); printf("\",\n \"file\": \""); EncodeJSONString((*e)->inputs_[0]->path().c_str()); + printf("\",\n \"output\": \""); + EncodeJSONString((*e)->outputs_[0]->path().c_str()); printf("\"\n }"); first = false; @@ -740,10 +869,8 @@ int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) { /// Returns a Tool, or NULL if Ninja should exit. const Tool* ChooseTool(const string& tool_name) { static const Tool kTools[] = { -#if defined(NINJA_HAVE_BROWSE) { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse }, -#endif #if defined(_MSC_VER) { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolMSVC }, @@ -764,6 +891,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase }, { "recompact", "recompacts ninja-internal data structures", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact }, + { "rules", "list all rules", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRules }, { "urtle", NULL, Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle }, { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } @@ -845,7 +974,10 @@ bool DebugEnable(const string& name) { bool WarningEnable(const string& name, Options* options) { if (name == "list") { printf("warning flags:\n" -" dupbuild={err,warn} multiple build lines for one target\n"); +" dupbuild={err,warn} multiple build lines for one target\n" +" phonycycle={err,warn} phony build statement references itself\n" +" depfilemulti={err,warn} depfile has multiple output paths on separate lines\n" + ); return false; } else if (name == "dupbuild=err") { options->dupe_edges_should_err = true; @@ -853,9 +985,22 @@ bool WarningEnable(const string& name, Options* options) { } else if (name == "dupbuild=warn") { options->dupe_edges_should_err = false; return true; + } else if (name == "phonycycle=err") { + options->phony_cycle_should_err = true; + return true; + } else if (name == "phonycycle=warn") { + options->phony_cycle_should_err = false; + return true; + } else if (name == "depfilemulti=err") { + options->depfile_distinct_target_lines_should_err = true; + return true; + } else if (name == "depfilemulti=warn") { + options->depfile_distinct_target_lines_should_err = false; + return true; } else { const char* suggestion = - SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn", NULL); + SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn", + "phonycycle=err", "phonycycle=warn", NULL); if (suggestion) { Error("unknown warning flag '%s', did you mean '%s'?", name.c_str(), suggestion); @@ -1031,6 +1176,7 @@ int ReadFlags(int* argc, char*** argv, const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, OPT_VERSION }, + { "verbose", no_argument, NULL, 'v' }, { NULL, 0, NULL, 0 } }; @@ -1049,9 +1195,12 @@ int ReadFlags(int* argc, char*** argv, case 'j': { char* end; int value = strtol(optarg, &end, 10); - if (*end != 0 || value <= 0) + if (*end != 0 || value < 0) Fatal("invalid -j parameter"); - config->parallelism = value; + + // We want to run N jobs in parallel. For N = 0, INT_MAX + // is close enough to infinite for most sane builds. + config->parallelism = value > 0 ? value : INT_MAX; break; } case 'k': { @@ -1107,17 +1256,25 @@ int ReadFlags(int* argc, char*** argv, return -1; } -int real_main(int argc, char** argv) { +NORETURN void real_main(int argc, char** argv) { + // Use exit() instead of return in this function to avoid potentially + // expensive cleanup when destructing NinjaMain. BuildConfig config; Options options = {}; options.input_file = "build.ninja"; + options.dupe_edges_should_err = true; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); const char* ninja_command = argv[0]; int exit_code = ReadFlags(&argc, &argv, &options, &config); if (exit_code >= 0) - return exit_code; + exit(exit_code); + + if (options.depfile_distinct_target_lines_should_err) { + config.depfile_parser_options.depfile_distinct_target_lines_action_ = + kDepfileDistinctTargetLinesActionError; + } if (options.working_dir) { // The formatting of this string, complete with funny quotes, is @@ -1136,7 +1293,7 @@ int real_main(int argc, char** argv) { // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed // by other tools. NinjaMain ninja(ninja_command, config); - return (ninja.*options.tool->func)(&options, argc, argv); + exit((ninja.*options.tool->func)(&options, argc, argv)); } // Limit number of rebuilds, to prevent infinite loops. @@ -1144,50 +1301,54 @@ int real_main(int argc, char** argv) { for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { NinjaMain ninja(ninja_command, config); - ManifestParser parser(&ninja.state_, &ninja.disk_interface_, - options.dupe_edges_should_err - ? kDupeEdgeActionError - : kDupeEdgeActionWarn); + ManifestParserOptions parser_opts; + if (options.dupe_edges_should_err) { + parser_opts.dupe_edge_action_ = kDupeEdgeActionError; + } + if (options.phony_cycle_should_err) { + parser_opts.phony_cycle_action_ = kPhonyCycleActionError; + } + ManifestParser parser(&ninja.state_, &ninja.disk_interface_, parser_opts); string err; if (!parser.Load(options.input_file, &err)) { Error("%s", err.c_str()); - return 1; + exit(1); } if (options.tool && options.tool->when == Tool::RUN_AFTER_LOAD) - return (ninja.*options.tool->func)(&options, argc, argv); + exit((ninja.*options.tool->func)(&options, argc, argv)); if (!ninja.EnsureBuildDirExists()) - return 1; + exit(1); if (!ninja.OpenBuildLog() || !ninja.OpenDepsLog()) - return 1; + exit(1); if (options.tool && options.tool->when == Tool::RUN_AFTER_LOGS) - return (ninja.*options.tool->func)(&options, argc, argv); + exit((ninja.*options.tool->func)(&options, argc, argv)); // Attempt to rebuild the manifest before building anything else if (ninja.RebuildManifest(options.input_file, &err)) { // In dry_run mode the regeneration will succeed without changing the // manifest forever. Better to return immediately. if (config.dry_run) - return 0; + exit(0); // Start the build over with the new manifest. continue; } else if (!err.empty()) { Error("rebuilding '%s': %s", options.input_file, err.c_str()); - return 1; + exit(1); } int result = ninja.RunBuild(argc, argv); if (g_metrics) ninja.DumpMetrics(); - return result; + exit(result); } Error("manifest '%s' still dirty after %d tries\n", options.input_file, kCycleLimit); - return 1; + exit(1); } } // anonymous namespace @@ -1200,7 +1361,7 @@ int main(int argc, char** argv) { __try { // Running inside __try ... __except suppresses any Windows error // dialogs for errors such as bad_alloc. - return real_main(argc, argv); + real_main(argc, argv); } __except(ExceptionFilter(GetExceptionCode(), GetExceptionInformation())) { // Common error situations return exitCode=1. 2 was chosen to @@ -1208,6 +1369,6 @@ int main(int argc, char** argv) { return 2; } #else - return real_main(argc, argv); + real_main(argc, argv); #endif } diff --git a/src/parser.cc b/src/parser.cc new file mode 100644 index 0000000..745c532 --- /dev/null +++ b/src/parser.cc @@ -0,0 +1,51 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser.h" + +#include "disk_interface.h" +#include "metrics.h" + +bool Parser::Load(const string& filename, string* err, Lexer* parent) { + METRIC_RECORD(".ninja parse"); + string contents; + string read_err; + if (file_reader_->ReadFile(filename, &contents, &read_err) != + FileReader::Okay) { + *err = "loading '" + filename + "': " + read_err; + if (parent) + parent->Error(string(*err), err); + return false; + } + + // The lexer needs a nul byte at the end of its input, to know when it's done. + // It takes a StringPiece, and StringPiece's string constructor uses + // string::data(). data()'s return value isn't guaranteed to be + // null-terminated (although in practice - libc++, libstdc++, msvc's stl -- + // it is, and C++11 demands that too), so add an explicit nul byte. + contents.resize(contents.size() + 1); + + return Parse(filename, contents, err); +} + +bool Parser::ExpectToken(Lexer::Token expected, string* err) { + Lexer::Token token = lexer_.ReadToken(); + if (token != expected) { + string message = string("expected ") + Lexer::TokenName(expected); + message += string(", got ") + Lexer::TokenName(token); + message += Lexer::TokenErrorHint(expected); + return lexer_.Error(message, err); + } + return true; +} diff --git a/src/parser.h b/src/parser.h new file mode 100644 index 0000000..e2d2b97 --- /dev/null +++ b/src/parser.h @@ -0,0 +1,50 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_PARSER_H_ +#define NINJA_PARSER_H_ + +#include <string> + +using namespace std; + +#include "lexer.h" + +struct FileReader; +struct State; + +/// Base class for parsers. +struct Parser { + Parser(State* state, FileReader* file_reader) + : state_(state), file_reader_(file_reader) {} + + /// Load and parse a file. + bool Load(const string& filename, string* err, Lexer* parent = NULL); + +protected: + /// If the next token is not \a expected, produce an error string + /// saying "expected foo, got bar". + bool ExpectToken(Lexer::Token expected, string* err); + + State* state_; + FileReader* file_reader_; + Lexer lexer_; + +private: + /// Parse a file, given its contents as a string. + virtual bool Parse(const string& filename, const string& input, + string* err) = 0; +}; + +#endif // NINJA_PARSER_H_ diff --git a/src/state.cc b/src/state.cc index 9b3c7e1..74cf4c1 100644 --- a/src/state.cc +++ b/src/state.cc @@ -186,6 +186,7 @@ void State::Reset() { i->second->ResetState(); for (vector<Edge*>::iterator e = edges_.begin(); e != edges_.end(); ++e) { (*e)->outputs_ready_ = false; + (*e)->deps_loaded_ = false; (*e)->mark_ = Edge::VisitNone; } } diff --git a/src/state.h b/src/state.h index 54e9dc5..6fe886c 100644 --- a/src/state.h +++ b/src/state.h @@ -33,7 +33,7 @@ struct Rule; /// Pools are scoped to a State. Edges within a State will share Pools. A Pool /// will keep a count of the total 'weight' of the currently scheduled edges. If /// a Plan attempts to schedule an Edge which would cause the total weight to -/// exceed the depth of the Pool, the Pool will enque the Edge instead of +/// exceed the depth of the Pool, the Pool will enqueue the Edge instead of /// allowing the Plan to schedule it. The Pool will relinquish queued Edges when /// the total scheduled weight diminishes enough (i.e. when a scheduled edge /// completes). diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index 1de22c3..fc5543e 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -14,6 +14,7 @@ #include "subprocess.h" +#include <sys/select.h> #include <assert.h> #include <errno.h> #include <fcntl.h> @@ -54,21 +55,25 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { SetCloseOnExec(fd_); posix_spawn_file_actions_t action; - if (posix_spawn_file_actions_init(&action) != 0) - Fatal("posix_spawn_file_actions_init: %s", strerror(errno)); + int err = posix_spawn_file_actions_init(&action); + if (err != 0) + Fatal("posix_spawn_file_actions_init: %s", strerror(err)); - if (posix_spawn_file_actions_addclose(&action, output_pipe[0]) != 0) - Fatal("posix_spawn_file_actions_addclose: %s", strerror(errno)); + err = posix_spawn_file_actions_addclose(&action, output_pipe[0]); + if (err != 0) + Fatal("posix_spawn_file_actions_addclose: %s", strerror(err)); posix_spawnattr_t attr; - if (posix_spawnattr_init(&attr) != 0) - Fatal("posix_spawnattr_init: %s", strerror(errno)); + err = posix_spawnattr_init(&attr); + if (err != 0) + Fatal("posix_spawnattr_init: %s", strerror(err)); short flags = 0; flags |= POSIX_SPAWN_SETSIGMASK; - if (posix_spawnattr_setsigmask(&attr, &set->old_mask_) != 0) - Fatal("posix_spawnattr_setsigmask: %s", strerror(errno)); + err = posix_spawnattr_setsigmask(&attr, &set->old_mask_); + if (err != 0) + Fatal("posix_spawnattr_setsigmask: %s", strerror(err)); // Signals which are set to be caught in the calling process image are set to // default action in the new process image, so no explicit // POSIX_SPAWN_SETSIGDEF parameter is needed. @@ -79,17 +84,21 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // No need to posix_spawnattr_setpgroup(&attr, 0), it's the default. // Open /dev/null over stdin. - if (posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY, - 0) != 0) { - Fatal("posix_spawn_file_actions_addopen: %s", strerror(errno)); + err = posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY, + 0); + if (err != 0) { + Fatal("posix_spawn_file_actions_addopen: %s", strerror(err)); } - if (posix_spawn_file_actions_adddup2(&action, output_pipe[1], 1) != 0) - Fatal("posix_spawn_file_actions_adddup2: %s", strerror(errno)); - if (posix_spawn_file_actions_adddup2(&action, output_pipe[1], 2) != 0) - Fatal("posix_spawn_file_actions_adddup2: %s", strerror(errno)); - if (posix_spawn_file_actions_addclose(&action, output_pipe[1]) != 0) - Fatal("posix_spawn_file_actions_addclose: %s", strerror(errno)); + err = posix_spawn_file_actions_adddup2(&action, output_pipe[1], 1); + if (err != 0) + Fatal("posix_spawn_file_actions_adddup2: %s", strerror(err)); + err = posix_spawn_file_actions_adddup2(&action, output_pipe[1], 2); + if (err != 0) + Fatal("posix_spawn_file_actions_adddup2: %s", strerror(err)); + err = posix_spawn_file_actions_addclose(&action, output_pipe[1]); + if (err != 0) + Fatal("posix_spawn_file_actions_addclose: %s", strerror(err)); // In the console case, output_pipe is still inherited by the child and // closed when the subprocess finishes, which then notifies ninja. } @@ -97,18 +106,22 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { flags |= POSIX_SPAWN_USEVFORK; #endif - if (posix_spawnattr_setflags(&attr, flags) != 0) - Fatal("posix_spawnattr_setflags: %s", strerror(errno)); + err = posix_spawnattr_setflags(&attr, flags); + if (err != 0) + Fatal("posix_spawnattr_setflags: %s", strerror(err)); const char* spawned_args[] = { "/bin/sh", "-c", command.c_str(), NULL }; - if (posix_spawn(&pid_, "/bin/sh", &action, &attr, - const_cast<char**>(spawned_args), environ) != 0) - Fatal("posix_spawn: %s", strerror(errno)); - - if (posix_spawnattr_destroy(&attr) != 0) - Fatal("posix_spawnattr_destroy: %s", strerror(errno)); - if (posix_spawn_file_actions_destroy(&action) != 0) - Fatal("posix_spawn_file_actions_destroy: %s", strerror(errno)); + err = posix_spawn(&pid_, "/bin/sh", &action, &attr, + const_cast<char**>(spawned_args), environ); + if (err != 0) + Fatal("posix_spawn: %s", strerror(err)); + + err = posix_spawnattr_destroy(&attr); + if (err != 0) + Fatal("posix_spawnattr_destroy: %s", strerror(err)); + err = posix_spawn_file_actions_destroy(&action); + if (err != 0) + Fatal("posix_spawn_file_actions_destroy: %s", strerror(err)); close(output_pipe[1]); return true; diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 4bab719..a4a7669 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -59,8 +59,8 @@ HANDLE Subprocess::SetupPipe(HANDLE ioport) { } // Get the write end of the pipe as a handle inheritable across processes. - HANDLE output_write_handle = CreateFile(pipe_name, GENERIC_WRITE, 0, - NULL, OPEN_EXISTING, 0, NULL); + HANDLE output_write_handle = + CreateFileA(pipe_name, GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL); HANDLE output_write_child; if (!DuplicateHandle(GetCurrentProcess(), output_write_handle, GetCurrentProcess(), &output_write_child, @@ -80,9 +80,10 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); security_attributes.bInheritHandle = TRUE; // Must be inheritable so subprocesses can dup to children. - HANDLE nul = CreateFile("NUL", GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - &security_attributes, OPEN_EXISTING, 0, NULL); + HANDLE nul = + CreateFileA("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); if (nul == INVALID_HANDLE_VALUE) Fatal("couldn't open nul"); @@ -123,6 +124,10 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { buf_ = "CreateProcess failed: The system cannot find the file " "specified.\n"; return true; + } else if (error == ERROR_INVALID_PARAMETER) { + // This generally means that the command line was too long. Give extra + // context for this case. + Win32Fatal("CreateProcess", "is the command line too long?"); } else { Win32Fatal("CreateProcess"); // pass all other errors to Win32Fatal } diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 0a8c206..6e487db 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -182,7 +182,7 @@ TEST_F(SubprocessTest, SetWithMulti) { "cmd /c echo hi", "cmd /c time /t", #else - "whoami", + "id -u", "pwd", #endif }; diff --git a/src/test.cc b/src/test.cc index 51882f0..a9816bc 100644 --- a/src/test.cc +++ b/src/test.cc @@ -95,8 +95,9 @@ Node* StateTestWithBuiltinRules::GetNode(const string& path) { return state_.GetNode(path, 0); } -void AssertParse(State* state, const char* input) { - ManifestParser parser(state, NULL, kDupeEdgeActionWarn); +void AssertParse(State* state, const char* input, + ManifestParserOptions opts) { + ManifestParser parser(state, NULL, opts); string err; EXPECT_TRUE(parser.ParseTest(input, &err)); ASSERT_EQ("", err); @@ -16,6 +16,7 @@ #define NINJA_TEST_H_ #include "disk_interface.h" +#include "manifest_parser.h" #include "state.h" #include "util.h" @@ -103,7 +104,7 @@ extern testing::Test* g_current_test; } \ } -// Support utilites for tests. +// Support utilities for tests. struct Node; @@ -122,7 +123,8 @@ struct StateTestWithBuiltinRules : public testing::Test { State state_; }; -void AssertParse(State* state, const char* input); +void AssertParse(State* state, const char* input, + ManifestParserOptions = ManifestParserOptions()); void AssertHash(const char* expected, uint64_t actual); void VerifyGraph(const State& state); diff --git a/src/timestamp.h b/src/timestamp.h index cee7ba8..6a7ccd0 100644 --- a/src/timestamp.h +++ b/src/timestamp.h @@ -15,10 +15,19 @@ #ifndef NINJA_TIMESTAMP_H_ #define NINJA_TIMESTAMP_H_ +#ifdef _WIN32 +#include "win32port.h" +#else +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include <inttypes.h> +#endif + // When considering file modification times we only care to compare // them against one another -- we never convert them to an absolute -// real time. On POSIX we use time_t (seconds since epoch) and on -// Windows we use a different value. Both fit in an int. -typedef int TimeStamp; +// real time. On POSIX we use timespec (seconds&nanoseconds since epoch) +// and on Windows we use a different value. Both fit in an int64. +typedef int64_t TimeStamp; #endif // NINJA_TIMESTAMP_H_ diff --git a/src/util.cc b/src/util.cc index 84de879..ee810d6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -102,18 +102,13 @@ bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err) { return true; } +static bool IsPathSeparator(char c) { #ifdef _WIN32 -static uint64_t ShiftOverBit(int offset, uint64_t bits) { - // e.g. for |offset| == 2: - // | ... 9 8 7 6 5 4 3 2 1 0 | - // \_________________/ \_/ - // above below - // So we drop the bit at offset and move above "down" into its place. - uint64_t above = bits & ~((1 << (offset + 1)) - 1); - uint64_t below = bits & ((1 << offset) - 1); - return (above >> 1) | below; -} + return c == '/' || c == '\\'; +#else + return c == '/'; #endif +} bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, string* err) { @@ -134,37 +129,13 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, const char* src = start; const char* end = start + *len; + if (IsPathSeparator(*src)) { #ifdef _WIN32 - uint64_t bits = 0; - uint64_t bits_mask = 1; - int bits_offset = 0; - // Convert \ to /, setting a bit in |bits| for each \ encountered. - for (char* c = path; c < end; ++c) { - switch (*c) { - case '\\': - bits |= bits_mask; - *c = '/'; - // Intentional fallthrough. - case '/': - bits_mask <<= 1; - bits_offset++; - } - } - if (bits_offset > 64) { - *err = "too many path components"; - return false; - } - bits_offset = 0; -#endif - if (*src == '/') { -#ifdef _WIN32 - bits_offset++; // network path starts with // - if (*len > 1 && *(src + 1) == '/') { + if (*len > 1 && IsPathSeparator(*(src + 1))) { src += 2; dst += 2; - bits_offset++; } else { ++src; ++dst; @@ -177,24 +148,16 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, while (src < end) { if (*src == '.') { - if (src + 1 == end || src[1] == '/') { + if (src + 1 == end || IsPathSeparator(src[1])) { // '.' component; eliminate. src += 2; -#ifdef _WIN32 - bits = ShiftOverBit(bits_offset, bits); -#endif continue; - } else if (src[1] == '.' && (src + 2 == end || src[2] == '/')) { + } else if (src[1] == '.' && (src + 2 == end || IsPathSeparator(src[2]))) { // '..' component. Back up if possible. if (component_count > 0) { dst = components[component_count - 1]; src += 3; --component_count; -#ifdef _WIN32 - bits = ShiftOverBit(bits_offset, bits); - bits_offset--; - bits = ShiftOverBit(bits_offset, bits); -#endif } else { *dst++ = *src++; *dst++ = *src++; @@ -204,11 +167,8 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, } } - if (*src == '/') { + if (IsPathSeparator(*src)) { src++; -#ifdef _WIN32 - bits = ShiftOverBit(bits_offset, bits); -#endif continue; } @@ -217,11 +177,8 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, components[component_count] = dst; ++component_count; - while (*src != '/' && src != end) + while (src != end && !IsPathSeparator(*src)) *dst++ = *src++; -#ifdef _WIN32 - bits_offset++; -#endif *dst++ = *src++; // Copy '/' or final \0 character as well. } @@ -232,6 +189,20 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, *len = dst - start - 1; #ifdef _WIN32 + uint64_t bits = 0; + uint64_t bits_mask = 1; + + for (char* c = start; c < start + *len; ++c) { + switch (*c) { + case '\\': + bits |= bits_mask; + *c = '/'; + NINJA_FALLTHROUGH; + case '/': + bits_mask <<= 1; + } + } + *slash_bits = bits; #else *slash_bits = 0; @@ -347,13 +318,8 @@ int ReadFile(const string& path, string* contents, string* err) { // This makes a ninja run on a set of 1500 manifest files about 4% faster // than using the generic fopen code below. err->clear(); - HANDLE f = ::CreateFile(path.c_str(), - GENERIC_READ, - FILE_SHARE_READ, - NULL, - OPEN_EXISTING, - FILE_FLAG_SEQUENTIAL_SCAN, - NULL); + HANDLE f = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); if (f == INVALID_HANDLE_VALUE) { err->assign(GetLastErrorString()); return -ENOENT; @@ -380,9 +346,19 @@ int ReadFile(const string& path, string* contents, string* err) { return -errno; } + struct stat st; + if (fstat(fileno(f), &st) < 0) { + err->assign(strerror(errno)); + fclose(f); + return -errno; + } + + // +1 is for the resize in ManifestParser::Load + contents->reserve(st.st_size + 1); + char buf[64 << 10]; size_t len; - while ((len = fread(buf, 1, sizeof(buf), f)) > 0) { + while (!feof(f) && (len = fread(buf, 1, sizeof(buf), f)) > 0) { contents->append(buf, len); } if (ferror(f)) { @@ -466,8 +442,12 @@ string GetLastErrorString() { return msg; } -void Win32Fatal(const char* function) { - Fatal("%s: %s", function, GetLastErrorString().c_str()); +void Win32Fatal(const char* function, const char* hint) { + if (hint) { + Fatal("%s: %s (%s)", function, GetLastErrorString().c_str(), hint); + } else { + Fatal("%s: %s", function, GetLastErrorString().c_str()); + } } #endif @@ -505,6 +485,15 @@ int GetProcessorCount() { GetNativeSystemInfo(&info); return info.dwNumberOfProcessors; #else +#ifdef CPU_COUNT + // The number of exposed processors might not represent the actual number of + // processors threads can run on. This happens when a CPU set limitation is + // active, see https://github.com/ninja-build/ninja/issues/1278 + cpu_set_t set; + if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) { + return CPU_COUNT(&set); + } +#endif return sysconf(_SC_NPROCESSORS_ONLN); #endif } @@ -607,7 +596,7 @@ double GetLoadAverage() { string ElideMiddle(const string& str, size_t width) { const int kMargin = 3; // Space for "...". string result = str; - if (result.size() + kMargin > width) { + if (result.size() > width) { size_t elide_size = (width - kMargin) / 2; result = result.substr(0, elide_size) + "..." @@ -34,6 +34,20 @@ using namespace std; /// Log a fatal message and exit. NORETURN void Fatal(const char* msg, ...); +// Have a generic fall-through for different versions of C/C++. +#if defined(__cplusplus) && __cplusplus >= 201703L +#define NINJA_FALLTHROUGH [[fallthrough]] +#elif defined(__cplusplus) && __cplusplus >= 201103L && defined(__clang__) +#define NINJA_FALLTHROUGH [[clang::fallthrough]] +#elif defined(__cplusplus) && __cplusplus >= 201103L && defined(__GNUC__) && \ + __GNUC__ >= 7 +#define NINJA_FALLTHROUGH [[gnu::fallthrough]] +#elif defined(__GNUC__) && __GNUC__ >= 7 // gcc 7 +#define NINJA_FALLTHROUGH __attribute__ ((fallthrough)) +#else // C++11 on gcc 6, and all other cases +#define NINJA_FALLTHROUGH +#endif + /// Log a warning message. void Warning(const char* msg, ...); @@ -105,7 +119,7 @@ bool Truncate(const string& path, size_t size, string* err); string GetLastErrorString(); /// Calls Fatal() with a function name and GetLastErrorString. -NORETURN void Win32Fatal(const char* function); +NORETURN void Win32Fatal(const char* function, const char* hint = NULL); #endif #endif // NINJA_UTIL_H_ diff --git a/src/util_test.cc b/src/util_test.cc index 45d0727..d97b48c 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -292,12 +292,12 @@ TEST(CanonicalizePath, TooManyComponents) { EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); EXPECT_EQ(slash_bits, 0xffffffff); - // 65 is not. + // 65 is OK if #component is less than 60 after path canonicalization. err = ""; path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./" "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x/y.h"; - EXPECT_FALSE(CanonicalizePath(&path, &slash_bits, &err)); - EXPECT_EQ(err, "too many path components"); + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(slash_bits, 0x0); // Backslashes version. err = ""; @@ -306,8 +306,28 @@ TEST(CanonicalizePath, TooManyComponents) { "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\x\\y.h"; - EXPECT_FALSE(CanonicalizePath(&path, &slash_bits, &err)); - EXPECT_EQ(err, "too many path components"); + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(slash_bits, 0x1ffffffff); + + + // 59 after canonicalization is OK. + err = ""; + path = "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/x/y.h"; + EXPECT_EQ(58, std::count(path.begin(), path.end(), '/')); + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(slash_bits, 0x0); + + // Backslashes version. + err = ""; + path = + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\x\\y.h"; + EXPECT_EQ(58, std::count(path.begin(), path.end(), '\\')); + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(slash_bits, 0x3ffffffffffffff); } #endif @@ -399,10 +419,12 @@ TEST(StripAnsiEscapeCodes, StripColors) { TEST(ElideMiddle, NothingToElide) { string input = "Nothing to elide in this short string."; EXPECT_EQ(input, ElideMiddle(input, 80)); + EXPECT_EQ(input, ElideMiddle(input, 38)); } TEST(ElideMiddle, ElideInTheMiddle) { string input = "01234567890123456789"; string elided = ElideMiddle(input, 10); EXPECT_EQ("012...789", elided); + EXPECT_EQ("01234567...23456789", ElideMiddle(input, 19)); } diff --git a/src/version.cc b/src/version.cc index e2a83bb..1c906ae 100644 --- a/src/version.cc +++ b/src/version.cc @@ -18,7 +18,7 @@ #include "util.h" -const char* kNinjaVersion = "1.7.2.git"; +const char* kNinjaVersion = "1.9.0.git"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.'); diff --git a/src/win32port.h b/src/win32port.h index ce3c949..e542536 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -15,6 +15,13 @@ #ifndef NINJA_WIN32PORT_H_ #define NINJA_WIN32PORT_H_ +#if defined(__MINGW32__) || defined(__MINGW64__) +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include <inttypes.h> +#endif + typedef signed short int16_t; typedef unsigned short uint16_t; /// A 64-bit integer type @@ -23,6 +30,7 @@ typedef unsigned long long uint64_t; // printf format specifier for uint64_t, from C99. #ifndef PRIu64 +#define PRId64 "I64d" #define PRIu64 "I64u" #define PRIx64 "I64x" #endif |