diff options
author | Jan Niklas Hasse <jhasse@bixense.com> | 2020-01-17 09:35:30 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-01-17 09:35:30 +0100 |
commit | 069ac4a8444fcb80bc6dc120a0664cfd51ff24cd (patch) | |
tree | d3dea7d3ca42aaeb09bcd270a470adf36ea6ae39 | |
parent | 02bea7088a2fc95f4a9965985a8e19dcbd7192db (diff) | |
parent | d986e4db5630cf1c5547e69b5556f006f7d3444a (diff) | |
download | ninja-069ac4a8444fcb80bc6dc120a0664cfd51ff24cd.tar.gz |
Merge branch 'master' into github-actions-static-msvc-cmake
-rw-r--r-- | .editorconfig | 11 | ||||
-rw-r--r-- | .github/workflows/linux.yml | 55 | ||||
-rw-r--r-- | .github/workflows/release-ninja-binaries.yml | 12 | ||||
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | CMakeLists.txt | 12 | ||||
-rw-r--r-- | CONTRIBUTING.md | 34 | ||||
-rw-r--r-- | HACKING.md | 252 | ||||
-rw-r--r-- | README | 21 | ||||
-rw-r--r-- | README.md | 50 | ||||
-rw-r--r-- | RELEASING | 2 | ||||
-rw-r--r-- | doc/manual.asciidoc | 3 | ||||
-rwxr-xr-x | misc/output_test.py | 16 | ||||
-rw-r--r-- | src/build.cc | 18 | ||||
-rw-r--r-- | src/build_log.cc | 70 | ||||
-rw-r--r-- | src/build_log.h | 8 | ||||
-rw-r--r-- | src/build_log_test.cc | 51 | ||||
-rw-r--r-- | src/build_test.cc | 223 | ||||
-rw-r--r-- | src/clean.cc | 13 | ||||
-rw-r--r-- | src/clean.h | 5 | ||||
-rw-r--r-- | src/clean_test.cc | 82 | ||||
-rw-r--r-- | src/depfile_parser.cc | 49 | ||||
-rw-r--r-- | src/depfile_parser.h | 13 | ||||
-rw-r--r-- | src/depfile_parser.in.cc | 49 | ||||
-rw-r--r-- | src/depfile_parser_test.cc | 100 | ||||
-rw-r--r-- | src/deps_log.cc | 14 | ||||
-rw-r--r-- | src/deps_log.h | 3 | ||||
-rw-r--r-- | src/graph.cc | 36 | ||||
-rw-r--r-- | src/load_status.h | 24 | ||||
-rw-r--r-- | src/manifest_parser.cc | 8 | ||||
-rw-r--r-- | src/manifest_parser_test.cc | 5 | ||||
-rw-r--r-- | src/ninja.cc | 120 |
31 files changed, 903 insertions, 459 deletions
diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0cc68d6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +end_of_line = lf + +[CMakeLists.txt] +indent_style = tab diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 0000000..2febee2 --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,55 @@ +name: Linux + +on: + pull_request: + push: + release: + types: published + +jobs: + build: + runs-on: [ubuntu-latest] + container: + image: centos:7 + steps: + - uses: actions/checkout@v1 + - name: Install dependencies + run: | + curl -L -O https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2-Linux-x86_64.sh + chmod +x cmake-3.16.2-Linux-x86_64.sh + ./cmake-3.16.2-Linux-x86_64.sh --skip-license --prefix=/usr/local + curl -L -O https://www.mirrorservice.org/sites/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/p7zip-16.02-10.el7.x86_64.rpm + curl -L -O https://www.mirrorservice.org/sites/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/p7zip-plugins-16.02-10.el7.x86_64.rpm + rpm -U --quiet p7zip-16.02-10.el7.x86_64.rpm + rpm -U --quiet p7zip-plugins-16.02-10.el7.x86_64.rpm + yum install -y make gcc-c++ + - name: Build ninja + shell: bash + run: | + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --parallel --config Release + ctest -vv + strip ninja + - name: Create ninja archive + run: | + mkdir artifact + 7z a artifact/ninja-linux.zip ./build/ninja + + # Upload ninja binary archive as an artifact + - name: Upload artifact + uses: actions/upload-artifact@v1 + with: + name: ninja-binary-archives + path: artifact + + - name: Upload release asset + if: github.event.action == 'published' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./artifact/ninja-linux.zip + asset_name: ninja-linux.zip + asset_content_type: application/zip diff --git a/.github/workflows/release-ninja-binaries.yml b/.github/workflows/release-ninja-binaries.yml index deffe13..a45b6e9 100644 --- a/.github/workflows/release-ninja-binaries.yml +++ b/.github/workflows/release-ninja-binaries.yml @@ -11,11 +11,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] + os: [macOS-latest, windows-latest] include: - - os: ubuntu-latest - zip_name: ninja-linux - extra_cmake_flags: '' - os: macOS-latest zip_name: ninja-mac extra_cmake_flags: '' @@ -27,9 +24,6 @@ jobs: - uses: actions/checkout@v1 # Install OS specific dependencies - - name: Install Linux dependencies - if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install re2c - name: Install macOS dependencies if: matrix.os == 'macOS-latest' run: brew install re2c p7zip cmake @@ -45,10 +39,6 @@ jobs: cmake --build . --parallel --config Release ctest -vv - - name: Strip Linux binary - if: matrix.os == 'ubuntu-latest' - run: cd build && strip ninja - - name: Create ninja archive shell: bash env: @@ -3,7 +3,6 @@ *.exe *.pdb *.ilk -TAGS /build*/ /build.ninja /ninja @@ -18,8 +17,8 @@ TAGS /graph.png /doc/manual.html /doc/doxygen -/gtest-1.6.0 *.patch +.DS_Store # Eclipse project files .project diff --git a/CMakeLists.txt b/CMakeLists.txt index c71a38f..206435e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,5 +114,17 @@ if(WIN32) endif() target_link_libraries(ninja_test PRIVATE libninja libninja-re2c) +foreach(perftest + build_log_perftest + canon_perftest + clparser_perftest + depfile_parser_perftest + hash_collision_bench + manifest_parser_perftest +) + add_executable(${perftest} src/${perftest}.cc) + target_link_libraries(${perftest} PRIVATE libninja libninja-re2c) +endforeach() + enable_testing() add_test(NinjaTest ninja_test) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..be1fc02 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# How to successfully make changes to Ninja + +We're very wary of changes that increase the complexity of Ninja (in particular, +new build file syntax or command-line flags) or increase the maintenance burden +of Ninja. Ninja is already successfully used by hundreds of developers for large +projects and it already achieves (most of) the goals we set out for it to do. +It's probably best to discuss new feature ideas on the +[mailing list](https://groups.google.com/forum/#!forum/ninja-build) or in an +issue before creating a PR. + +## Coding guidelines + +Generally it's the +[Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html) with +a few additions: + +* Any code merged into the Ninja codebase which will be part of the main + executable must compile as C++03. You may use C++11 features in a test or an + unimportant tool if you guard your code with `#if __cplusplus >= 201103L`. +* We have used `using namespace std;` a lot in the past. For new contributions, + please try to avoid relying on it and instead whenever possible use `std::`. + However, please do not change existing code simply to add `std::` unless your + contribution already needs to change that line of code anyway. +* All source files should have the Google Inc. license header. +* Use `///` for [Doxygen](http://www.doxygen.nl/) (use `\a` to refer to + arguments). +* It's not necessary to document each argument, especially when they're + relatively self-evident (e.g. in + `CanonicalizePath(string* path, string* err)`, the arguments are hopefully + obvious). + +If you're unsure about code formatting, please use +[clang-format](https://clang.llvm.org/docs/ClangFormat.html). However, please do +not format code that is not otherwise part of your contribution. diff --git a/HACKING.md b/HACKING.md deleted file mode 100644 index bd6fec7..0000000 --- a/HACKING.md +++ /dev/null @@ -1,252 +0,0 @@ -## Basic overview - -`./configure.py` generates the `build.ninja` files used to build -ninja. It accepts various flags to adjust build parameters. -Run './configure.py --help' for more configuration options. - -The primary build target of interest is `ninja`, but when hacking on -Ninja your changes should be testable so it's more useful to build and -run `ninja_test` when developing. - -### Bootstrapping - -Ninja is built using itself. To bootstrap the first binary, run the -configure script as `./configure.py --bootstrap`. This first compiles -all non-test source files together, then re-builds Ninja using itself. -You should end up with a `ninja` binary (or `ninja.exe`) in the project root. - -#### Windows - -On Windows, you'll need to install Python to run `configure.py`, and -run everything under a Visual Studio Tools Command Prompt (or after -running `vcvarsall` in a normal command prompt). - -For other combinations such as gcc/clang you will need the compiler -(gcc/cl) in your PATH and you will have to set the appropriate -platform configuration script. - -See below if you want to use mingw or some other compiler instead of -Visual Studio. - -##### Using Visual Studio -Assuming that you now have Python installed, then the steps for building under -Windows using Visual Studio are: - -Clone and checkout the latest release (or whatever branch you want). You -can do this in either a command prompt or by opening a git bash prompt: - -``` - $ git clone git://github.com/ninja-build/ninja.git && cd ninja - $ git checkout release -``` - -Then: - -1. Open a Windows command prompt in the folder where you checked out ninja. -2. Select the Microsoft build environment by running -`vcvarsall.bat` with the appropriate environment. -3. Build ninja and test it. - -The steps for a Visual Studio 2015 64-bit build are outlined here: - -``` - > "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 - > python configure.py --bootstrap - > ninja --help -``` -Copy the ninja executable to another location, if desired, e.g. C:\local\Ninja. - -Finally add the path where ninja.exe is to the PATH variable. - -### Adjusting build flags - -Build in "debug" mode while developing (disables optimizations and builds -way faster on Windows): - - ./configure.py --debug - -To use clang, set `CXX`: - - CXX=clang++ ./configure.py - -## How to successfully make changes to Ninja - -Github pull requests are convenient for me to merge (I can just click -a button and it's all handled server-side), but I'm also comfortable -accepting pre-github git patches (via `send-email` etc.). - -Good pull requests have all of these attributes: - -* Are scoped to one specific issue -* Include a test to demonstrate their correctness -* Update the docs where relevant -* Match the Ninja coding style (see below) -* Don't include a mess of "oops, fix typo" commits - -These are typically merged without hesitation. If a change is lacking -any of the above I usually will ask you to fix it, though there are -obvious exceptions (fixing typos in comments don't need tests). - -I am very wary of changes that increase the complexity of Ninja (in -particular, new build file syntax or command-line flags) or increase -the maintenance burden of Ninja. Ninja is already successfully used -by hundreds of developers for large projects and it already achieves -(most of) the goals I set out for it to do. It's probably best to -discuss new feature ideas on the [mailing list](https://groups.google.com/forum/#!forum/ninja-build) -before I shoot down your patch. - -## Testing - -### Test-driven development - -Set your build command to - - ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name - -now you can repeatedly run that while developing until the tests pass -(I frequently set it as my compilation command in Emacs). Remember to -build "all" before committing to verify the other source still works! - -## Testing performance impact of changes - -If you have a Chrome build handy, it's a good test case. There's a -script at `misc/measure.py` that repeatedly runs a command (to address -variance) and summarizes its runtime. E.g. - - path/to/misc/measure.py path/to/my/ninja chrome - -For changing the depfile parser, you can also build `parser_perftest` -and run that directly on some representative input files. - -## Coding guidelines - -Generally it's the [Google C++ coding style][], but in brief: - -* Function name are camelcase. -* Member methods are camelcase, except for trivial getters which are - underscore separated. -* Local variables are underscore separated. -* Member variables are underscore separated and suffixed by an extra - underscore. -* Two spaces indentation. -* Opening braces is at the end of line. -* Lines are 80 columns maximum. -* All source files should have the Google Inc. license header. - -[Google C++ coding style]: https://google.github.io/styleguide/cppguide.html - -## Documentation - -### Style guidelines - -* Use `///` for doxygen. -* Use `\a` to refer to arguments. -* It's not necessary to document each argument, especially when they're - relatively self-evident (e.g. in `CanonicalizePath(string* path, string* err)`, - the arguments are hopefully obvious) - -### Building the manual - - sudo apt-get install asciidoc --no-install-recommends - ./ninja manual - -### Building the code documentation - - sudo apt-get install doxygen - ./ninja doxygen - -## Building for Windows - -While developing, it's helpful to copy `ninja.exe` to another name like -`n.exe`; otherwise, rebuilds will be unable to write `ninja.exe` because -it's locked while in use. - -### Via Visual Studio - -* Install Visual Studio (Express is fine), [Python for Windows][], - and (if making changes) googletest (see above instructions) -* In a Visual Studio command prompt: `python configure.py --bootstrap` - -[Python for Windows]: http://www.python.org/getit/windows/ - -### Via mingw on Windows (not well supported) - -* Install mingw, msys, and python -* In the mingw shell, put Python in your path, and - `python configure.py --bootstrap` -* To reconfigure, run `python configure.py` -* Remember to strip the resulting executable if size matters to you - -### Via mingw on Linux (not well supported) - -Setup on Ubuntu Lucid: -* `sudo apt-get install gcc-mingw32 wine` -* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar` - -Setup on Ubuntu Precise: -* `sudo apt-get install gcc-mingw-w64-i686 g++-mingw-w64-i686 wine` -* `export CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ AR=i686-w64-mingw32-ar` - -Setup on Arch: -* Uncomment the `[multilib]` section of `/etc/pacman.conf` and `sudo pacman -Sy`. -* `sudo pacman -S mingw-w64-gcc wine` -* `export CC=x86_64-w64-mingw32-cc CXX=x86_64-w64-mingw32-c++ AR=x86_64-w64-mingw32-ar` -* `export CFLAGS=-I/usr/x86_64-w64-mingw32/include` - -Then run: -* `./configure.py --platform=mingw --host=linux` -* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` -* Run: `./ninja.exe` (implicitly runs through wine(!)) - -### Using Microsoft compilers on Linux (extremely flaky) - -The trick is to install just the compilers, and not all of Visual Studio, -by following [these instructions][win7sdk]. - -[win7sdk]: http://www.kegel.com/wine/cl-howto-win7sdk.html - -### Using gcov - -Do a clean debug build with the right flags: - - CFLAGS=-coverage LDFLAGS=-coverage ./configure.py --debug - ninja -t clean ninja_test && ninja ninja_test - -Run the test binary to generate `.gcda` and `.gcno` files in the build -directory, then run gcov on the .o files to generate `.gcov` files in the -root directory: - - ./ninja_test - gcov build/*.o - -Look at the generated `.gcov` files directly, or use your favorite gcov viewer. - -### Using afl-fuzz - -Build with afl-clang++: - - CXX=path/to/afl-1.20b/afl-clang++ ./configure.py - ninja - -Then run afl-fuzz like so: - - afl-fuzz -i misc/afl-fuzz -o /tmp/afl-fuzz-out ./ninja -n -f @@ - -You can pass `-x misc/afl-fuzz-tokens` to use the token dictionary. In my -testing, that did not seem more effective though. - -#### Using afl-fuzz with asan - -If you want to use asan (the `isysroot` bit is only needed on OS X; if clang -can't find C++ standard headers make sure your LLVM checkout includes a libc++ -checkout and has libc++ installed in the build directory): - - CFLAGS="-fsanitize=address -isysroot $(xcrun -show-sdk-path)" \ - LDFLAGS=-fsanitize=address CXX=path/to/afl-1.20b/afl-clang++ \ - ./configure.py - AFL_CXX=path/to/clang++ ninja - -Make sure ninja can find the asan runtime: - - DYLD_LIBRARY_PATH=path/to//lib/clang/3.7.0/lib/darwin/ \ - afl-fuzz -i misc/afl-fuzz -o /tmp/afl-fuzz-out ./ninja -n -f @@ @@ -1,21 +0,0 @@ -Ninja is a small build system with a focus on speed. -https://ninja-build.org/ - -See the manual -- https://ninja-build.org/manual.html or -doc/manual.asciidoc included in the distribution -- for background -and more details. - -Binaries for Linux, Mac, and Windows are available at - https://github.com/ninja-build/ninja/releases -Run './ninja -h' for Ninja help. - -To build your own binary, on many platforms it should be sufficient to -just run `./configure.py --bootstrap`; for more details see HACKING.md. -(Also read that before making changes to Ninja, as it has advice.) - -Installation is not necessary because the only required file is the -resulting ninja binary. However, to enable features like Bash -completion and Emacs and Vim editing modes, some files in misc/ must be -copied to appropriate locations. - -If you're interested in making changes to Ninja, read HACKING.md first. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3326f81 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +# Ninja + +Ninja is a small build system with a focus on speed. +https://ninja-build.org/ + +See [the manual](https://ninja-build.org/manual.html) or +`doc/manual.asciidoc` included in the distribution for background +and more details. + +Binaries for Linux, Mac, and Windows are available at + [GitHub](https://github.com/ninja-build/ninja/releases). +Run `./ninja -h` for Ninja help. + +Installation is not necessary because the only required file is the +resulting ninja binary. However, to enable features like Bash +completion and Emacs and Vim editing modes, some files in misc/ must be +copied to appropriate locations. + +If you're interested in making changes to Ninja, read CONTRIBUTING.md first. + +## Building Ninja itself + +You can either build Ninja via the custom generator script written in Python or +via CMake. For more details see +[the wiki](https://github.com/ninja-build/ninja/wiki). + +### Python + +``` +./configure.py --bootstrap +``` + +This will generate the `ninja` binary and a `build.ninja` file you can now use +to built Ninja with itself. + +### CMake + +``` +cmake -Bbuild-cmake -H. +cmake --build build-cmake +``` + +The `ninja` binary will now be inside the `build-cmake` directory (you can +choose any other name you like). + +To run the unit tests: + +``` +./build-cmake/ninja_test +``` @@ -1,7 +1,7 @@ Notes to myself on all the steps to make for a Ninja release. Push new release branch: -1. Run afl-fuzz for a day or so (see HACKING.md) and run ninja_test +1. Run afl-fuzz for a day or so and run ninja_test 2. Consider sending a heads-up to the ninja-build mailing list first 3. Make sure branches 'master' and 'release' are synced up locally 4. Update src/version.cc with new version (with ".git"), then diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e49d26d..8f42efb 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -283,6 +283,9 @@ target, show just the target's dependencies. _Available since Ninja 1.4._ `recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._ +`restat`:: updates all recorded file modification timestamps in the `.ninja_log` +file. _Available since Ninja 1.10._ + `rules`:: output the list of all rules (eventually with their description if they have one). It can be used to know which rule name to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. diff --git a/misc/output_test.py b/misc/output_test.py index fb73d72..3fd9c32 100755 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -18,12 +18,15 @@ if 'NINJA_STATUS' in default_env: if 'CLICOLOR_FORCE' in default_env: del default_env['CLICOLOR_FORCE'] default_env['TERM'] = '' +NINJA_PATH = os.path.abspath('./ninja') def run(build_ninja, flags='', pipe=False, env=default_env): - with tempfile.NamedTemporaryFile('w') as f: - f.write(build_ninja) - f.flush() - ninja_cmd = './ninja {} -f {}'.format(flags, f.name) + with tempfile.TemporaryDirectory() as d: + os.chdir(d) + with open('build.ninja', 'w') as f: + f.write(build_ninja) + f.flush() + ninja_cmd = '{} {}'.format(NINJA_PATH, flags) try: if pipe: output = subprocess.check_output([ninja_cmd], shell=True, env=env) @@ -99,5 +102,10 @@ red \x1b[31mred\x1b[0m ''') + def test_pr_1685(self): + # Running those tools without .ninja_deps and .ninja_log shouldn't fail. + self.assertEqual(run('', flags='-t recompact'), '') + self.assertEqual(run('', flags='-t restat'), '') + if __name__ == '__main__': unittest.main() diff --git a/src/build.cc b/src/build.cc index 75671a6..cd8df4e 100644 --- a/src/build.cc +++ b/src/build.cc @@ -1033,14 +1033,16 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } if (!deps_type.empty() && !config_.dry_run) { - assert(edge->outputs_.size() == 1 && "should have been rejected by parser"); - Node* out = edge->outputs_[0]; - TimeStamp deps_mtime = disk_interface_->Stat(out->path(), err); - if (deps_mtime == -1) - return false; - if (!scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes)) { - *err = string("Error writing to deps log: ") + strerror(errno); - return false; + assert(edge->outputs_.size() >= 1 && "should have been rejected by parser"); + for (std::vector<Node*>::const_iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err); + if (deps_mtime == -1) + return false; + if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes)) { + *err = std::string("Error writing to deps log: ") + strerror(errno); + return false; + } } } return true; diff --git a/src/build_log.cc b/src/build_log.cc index c4a08a0..98543b6 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -21,6 +21,7 @@ #endif #include "build_log.h" +#include "disk_interface.h" #include <errno.h> #include <stdlib.h> @@ -241,14 +242,14 @@ struct LineReader { char* line_end_; }; -bool BuildLog::Load(const string& path, string* err) { +LoadStatus BuildLog::Load(const string& path, string* err) { METRIC_RECORD(".ninja_log load"); FILE* file = fopen(path.c_str(), "r"); if (!file) { if (errno == ENOENT) - return true; + return LOAD_NOT_FOUND; *err = strerror(errno); - return false; + return LOAD_ERROR; } int log_version = 0; @@ -269,7 +270,7 @@ bool BuildLog::Load(const string& path, string* err) { unlink(path.c_str()); // Don't report this as a failure. An empty build log will cause // us to rebuild the outputs anyway. - return true; + return LOAD_SUCCESS; } } @@ -339,7 +340,7 @@ bool BuildLog::Load(const string& path, string* err) { fclose(file); if (!line_start) { - return true; // file was empty + return LOAD_SUCCESS; // file was empty } // Decide whether it's time to rebuild the log: @@ -354,7 +355,7 @@ bool BuildLog::Load(const string& path, string* err) { needs_recompaction_ = true; } - return true; + return LOAD_SUCCESS; } BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { @@ -418,3 +419,60 @@ bool BuildLog::Recompact(const string& path, const BuildLogUser& user, return true; } + +bool BuildLog::Restat(const StringPiece path, + const DiskInterface& disk_interface, + const int output_count, char** outputs, + std::string* const err) { + METRIC_RECORD(".ninja_log restat"); + + Close(); + std::string temp_path = path.AsString() + ".restat"; + FILE* f = fopen(temp_path.c_str(), "wb"); + if (!f) { + *err = strerror(errno); + return false; + } + + if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { + *err = strerror(errno); + fclose(f); + return false; + } + for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { + bool skip = output_count > 0; + for (int j = 0; j < output_count; ++j) { + if (i->second->output == outputs[j]) { + skip = false; + break; + } + } + if (!skip) { + const TimeStamp mtime = disk_interface.Stat(i->second->output, err); + if (mtime == -1) { + fclose(f); + return false; + } + i->second->mtime = mtime; + } + + if (!WriteEntry(f, *i->second)) { + *err = strerror(errno); + fclose(f); + return false; + } + } + + fclose(f); + if (unlink(path.str_) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.str_) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} diff --git a/src/build_log.h b/src/build_log.h index 5268fab..ebe0530 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -20,9 +20,11 @@ using namespace std; #include "hash_map.h" +#include "load_status.h" #include "timestamp.h" #include "util.h" // uint64_t +struct DiskInterface; struct Edge; /// Can answer questions about the manifest for the BuildLog. @@ -49,7 +51,7 @@ struct BuildLog { void Close(); /// Load the on-disk log. - bool Load(const string& path, string* err); + LoadStatus Load(const string& path, string* err); struct LogEntry { string output; @@ -81,6 +83,10 @@ struct BuildLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, const BuildLogUser& user, string* err); + /// Restat all outputs in the log + bool Restat(StringPiece path, const DiskInterface& disk_interface, + int output_count, char** outputs, std::string* err); + typedef ExternalStringHashMap<LogEntry*>::Type Entries; const Entries& entries() const { return entries_; } diff --git a/src/build_log_test.cc b/src/build_log_test.cc index ad30380..a8b1733 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -25,6 +25,7 @@ #include <sys/types.h> #include <unistd.h> #endif +#include <cassert> namespace { @@ -150,7 +151,7 @@ TEST_F(BuildLogTest, Truncate) { BuildLog log3; err.clear(); - ASSERT_TRUE(log3.Load(kTestFilename, &err) || !err.empty()); + ASSERT_TRUE(log3.Load(kTestFilename, &err) == LOAD_SUCCESS || !err.empty()); } } @@ -216,6 +217,54 @@ TEST_F(BuildLogTest, DuplicateVersionHeader) { ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash)); } +struct TestDiskInterface : public DiskInterface { + virtual TimeStamp Stat(const string& path, string* err) const { + return 4; + } + virtual bool WriteFile(const string& path, const string& contents) { + assert(false); + return true; + } + virtual bool MakeDir(const string& path) { + assert(false); + return false; + } + virtual Status ReadFile(const string& path, string* contents, string* err) { + assert(false); + return NotFound; + } + virtual int RemoveFile(const string& path) { + assert(false); + return 0; + } +}; + +TEST_F(BuildLogTest, Restat) { + FILE* f = fopen(kTestFilename, "wb"); + fprintf(f, "# ninja log v4\n" + "1\t2\t3\tout\tcommand\n"); + fclose(f); + std::string err; + BuildLog log; + EXPECT_TRUE(log.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + BuildLog::LogEntry* e = log.LookupByOutput("out"); + ASSERT_EQ(3, e->mtime); + + TestDiskInterface testDiskInterface; + char out2[] = { 'o', 'u', 't', '2' }; + char* filter2[] = { out2 }; + EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, 1, filter2, &err)); + ASSERT_EQ("", err); + e = log.LookupByOutput("out"); + ASSERT_EQ(3, e->mtime); // unchanged, since the filter doesn't match + + EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, 0, NULL, &err)); + ASSERT_EQ("", err); + e = log.LookupByOutput("out"); + ASSERT_EQ(4, e->mtime); +} + TEST_F(BuildLogTest, VeryLongInputLine) { // Ninja's build log buffer is currently 256kB. Lines longer than that are // silently ignored, but don't affect parsing of other lines. diff --git a/src/build_test.cc b/src/build_test.cc index ddf8574..426e825 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -488,6 +488,11 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { status_(config_) { } + BuildTest(DepsLog* log) : config_(MakeConfig()), command_runner_(&fs_), + builder_(&state_, config_, NULL, log, &fs_), + status_(config_) { + } + virtual void SetUp() { StateTestWithBuiltinRules::SetUp(); @@ -582,6 +587,8 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { edge->rule().name() == "cat_rsp" || edge->rule().name() == "cat_rsp_out" || edge->rule().name() == "cc" || + edge->rule().name() == "cp_multi_msvc" || + edge->rule().name() == "cp_multi_gcc" || edge->rule().name() == "touch" || edge->rule().name() == "touch-interrupt" || edge->rule().name() == "touch-fail-tick2") { @@ -643,6 +650,14 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { return true; } + if (edge->rule().name() == "cp_multi_msvc") { + const std::string prefix = edge->GetBinding("msvc_deps_prefix"); + for (std::vector<Node*>::iterator in = edge->inputs_.begin(); + in != edge->inputs_.end(); ++in) { + result->output += prefix + (*in)->path() + '\n'; + } + } + if (edge->rule().name() == "fail" || (edge->rule().name() == "touch-fail-tick2" && fs_->now_ == 2)) result->status = ExitFailure; @@ -1855,6 +1870,214 @@ TEST_F(BuildTest, FailedDepsParse) { EXPECT_EQ("subcommand failed", err); } +struct BuildWithQueryDepsLogTest : public BuildTest { + BuildWithQueryDepsLogTest() : BuildTest(&log_) { + } + + ~BuildWithQueryDepsLogTest() { + log_.Close(); + } + + virtual void SetUp() { + BuildTest::SetUp(); + + temp_dir_.CreateAndEnter("BuildWithQueryDepsLogTest"); + + std::string err; + ASSERT_TRUE(log_.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + } + + ScopedTempDir temp_dir_; + + DepsLog log_; +}; + +/// Test a MSVC-style deps log with multiple outputs. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileMSVC) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_msvc\n" +" command = echo 'using $in' && for file in $out; do cp $in $$file; done\n" +" deps = msvc\n" +" msvc_deps_prefix = using \n" +"build out1 out2: cp_multi_msvc in1\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'using in1' && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(1, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(1, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); +} + +/// Test a GCC-style deps log with multiple outputs. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOneLine) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo '$out: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1 out2: in1 in2"); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1 out2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs using a line per input. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCMultiLineInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo '$out: in1\\n$out: in2' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1 out2: in1\nout1 out2: in2"); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1 out2: in1\\nout1 out2: in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs using a line per output. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCMultiLineOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo 'out1: $in\\nout2: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1: in1 in2\nout2: in1 in2"); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1: in1 in2\\nout2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs mentioning only the main output. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOnlyMainOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo 'out1: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1: in1 in2"); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs mentioning only the secondary output. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOnlySecondaryOutput) { + // Note: This ends up short-circuiting the node creation due to the primary + // output not being present, but it should still work. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo 'out2: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out2: in1 in2"); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + /// Tests of builds involving deps logs necessarily must span /// multiple builds. We reuse methods on BuildTest but not the /// builder_ it sets up, because we want pristine objects for diff --git a/src/clean.cc b/src/clean.cc index d1f221d..ec6e7d7 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -124,6 +124,19 @@ int Cleaner::CleanAll(bool generator) { return status_; } +int Cleaner::CleanDead(const BuildLog::Entries& entries) { + Reset(); + PrintHeader(); + for (BuildLog::Entries::const_iterator i = entries.begin(); i != entries.end(); ++i) { + Node* n = state_->LookupNode(i->first); + if (!n || !n->in_edge()) { + Remove(i->first.AsString()); + } + } + PrintFooter(); + return status_; +} + void Cleaner::DoCleanTarget(Node* target) { if (Edge* e = target->in_edge()) { // Do not try to remove phony targets diff --git a/src/clean.h b/src/clean.h index d044fb1..4c02ff6 100644 --- a/src/clean.h +++ b/src/clean.h @@ -20,6 +20,7 @@ #include "build.h" #include "dyndep.h" +#include "build_log.h" using namespace std; @@ -58,6 +59,10 @@ struct Cleaner { /// Clean the file produced by the given @a rules. /// @return non-zero if an error occurs. int CleanRules(int rule_count, char* rules[]); + /// Clean the files produced by previous builds that are no longer in the + /// manifest. + /// @return non-zero if an error occurs. + int CleanDead(const BuildLog::Entries& entries); /// @return the number of file cleaned. int cleaned_files_count() const { diff --git a/src/clean_test.cc b/src/clean_test.cc index 45187f4..d068f3c 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -15,8 +15,17 @@ #include "clean.h" #include "build.h" +#include "util.h" #include "test.h" +#ifndef _WIN32 +#include <unistd.h> +#endif + +namespace { + +const char kTestFilename[] = "CleanTest-tempfile"; + struct CleanTest : public StateTestWithBuiltinRules { VirtualFileSystem fs_; BuildConfig config_; @@ -454,3 +463,76 @@ TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) { EXPECT_EQ(0, fs_.Stat("out 1.d", &err)); EXPECT_EQ(0, fs_.Stat("out 2.rsp", &err)); } + +struct CleanDeadTest : public CleanTest, public BuildLogUser{ + virtual void SetUp() { + // In case a crashing test left a stale file behind. + unlink(kTestFilename); + CleanTest::SetUp(); + } + virtual void TearDown() { + unlink(kTestFilename); + } + virtual bool IsPathDead(StringPiece) const { return false; } +}; + +TEST_F(CleanDeadTest, CleanDead) { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, +"rule cat\n" +" command = cat $in > $out\n" +"build out1: cat in\n" +"build out2: cat in\n" +)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out2: cat in\n" +)); + fs_.Create("in", ""); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + BuildLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); + ASSERT_EQ("", err); + log1.RecordCommand(state.edges_[0], 15, 18); + log1.RecordCommand(state.edges_[1], 20, 25); + log1.Close(); + + BuildLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(2u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out1")); + ASSERT_TRUE(log2.LookupByOutput("out2")); + + // First use the manifest that describe how to build out1. + Cleaner cleaner1(&state, config_, &fs_); + EXPECT_EQ(0, cleaner1.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner1.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Then use the manifest that does not build out1 anymore. + Cleaner cleaner2(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(1, cleaner2.cleaned_files_count()); + EXPECT_EQ(1u, fs_.files_removed_.size()); + EXPECT_EQ("out1", *(fs_.files_removed_.begin())); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Nothing to do now. + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(1u, fs_.files_removed_.size()); + EXPECT_EQ("out1", *(fs_.files_removed_.begin())); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + log2.Close(); +} +} // anonymous namespace diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 6faeac6..90d4a8a 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -16,6 +16,8 @@ #include "depfile_parser.h" #include "util.h" +#include <algorithm> + DepfileParser::DepfileParser(DepfileParserOptions options) : options_(options) { @@ -48,10 +50,8 @@ bool DepfileParser::Parse(string* content, string* err) { char* in = &(*content)[0]; char* end = in + content->size(); bool have_target = false; - bool have_secondary_target_on_this_rule = false; - bool have_newline_since_primary_target = false; - bool warned_distinct_target_lines = false; bool parsing_targets = true; + bool poisoned_input = false; while (in < end) { bool have_newline = false; // out: current output point (typically same as in, but can fall behind @@ -294,41 +294,32 @@ yy28: } if (len > 0) { - if (is_dependency) { - if (have_secondary_target_on_this_rule) { - if (!have_newline_since_primary_target) { - *err = "depfile has multiple output paths"; - return false; - } else if (options_.depfile_distinct_target_lines_action_ == - kDepfileDistinctTargetLinesActionError) { - *err = - "depfile has multiple output paths (on separate lines)" - " [-w depfilemulti=err]"; + StringPiece piece = StringPiece(filename, len); + // If we've seen this as an input before, skip it. + std::vector<StringPiece>::iterator pos = std::find(ins_.begin(), ins_.end(), piece); + if (pos == ins_.end()) { + if (is_dependency) { + if (poisoned_input) { + *err = "inputs may not also have inputs"; return false; - } else { - if (!warned_distinct_target_lines) { - warned_distinct_target_lines = true; - Warning("depfile has multiple output paths (on separate lines); " - "continuing anyway [-w depfilemulti=warn]"); - } - continue; } + // New input. + ins_.push_back(piece); + } else { + // Check for a new output. + if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end()) + outs_.push_back(piece); } - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - have_secondary_target_on_this_rule = true; + } else if (!is_dependency) { + // We've passed an input on the left side; reject new inputs. + poisoned_input = true; } } if (have_newline) { // A newline ends a rule so the next filename will be a new target. parsing_targets = true; - have_secondary_target_on_this_rule = false; - if (have_target) { - have_newline_since_primary_target = true; - } + poisoned_input = false; } } if (!have_target) { diff --git a/src/depfile_parser.h b/src/depfile_parser.h index be20374..11b1228 100644 --- a/src/depfile_parser.h +++ b/src/depfile_parser.h @@ -21,17 +21,8 @@ using namespace std; #include "string_piece.h" -enum DepfileDistinctTargetLinesAction { - kDepfileDistinctTargetLinesActionWarn, - kDepfileDistinctTargetLinesActionError, -}; - struct DepfileParserOptions { - DepfileParserOptions() - : depfile_distinct_target_lines_action_( - kDepfileDistinctTargetLinesActionWarn) {} - DepfileDistinctTargetLinesAction - depfile_distinct_target_lines_action_; + DepfileParserOptions() {} }; /// Parser for the dependency information emitted by gcc's -M flags. @@ -44,7 +35,7 @@ struct DepfileParser { /// pointers within it. bool Parse(string* content, string* err); - StringPiece out_; + std::vector<StringPiece> outs_; vector<StringPiece> ins_; DepfileParserOptions options_; }; diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 735a0c3..b32b942 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -15,6 +15,8 @@ #include "depfile_parser.h" #include "util.h" +#include <algorithm> + DepfileParser::DepfileParser(DepfileParserOptions options) : options_(options) { @@ -47,10 +49,8 @@ bool DepfileParser::Parse(string* content, string* err) { char* in = &(*content)[0]; char* end = in + content->size(); bool have_target = false; - bool have_secondary_target_on_this_rule = false; - bool have_newline_since_primary_target = false; - bool warned_distinct_target_lines = false; bool parsing_targets = true; + bool poisoned_input = false; while (in < end) { bool have_newline = false; // out: current output point (typically same as in, but can fall behind @@ -146,41 +146,32 @@ bool DepfileParser::Parse(string* content, string* err) { } if (len > 0) { - if (is_dependency) { - if (have_secondary_target_on_this_rule) { - if (!have_newline_since_primary_target) { - *err = "depfile has multiple output paths"; - return false; - } else if (options_.depfile_distinct_target_lines_action_ == - kDepfileDistinctTargetLinesActionError) { - *err = - "depfile has multiple output paths (on separate lines)" - " [-w depfilemulti=err]"; + StringPiece piece = StringPiece(filename, len); + // If we've seen this as an input before, skip it. + std::vector<StringPiece>::iterator pos = std::find(ins_.begin(), ins_.end(), piece); + if (pos == ins_.end()) { + if (is_dependency) { + if (poisoned_input) { + *err = "inputs may not also have inputs"; return false; - } else { - if (!warned_distinct_target_lines) { - warned_distinct_target_lines = true; - Warning("depfile has multiple output paths (on separate lines); " - "continuing anyway [-w depfilemulti=warn]"); - } - continue; } + // New input. + ins_.push_back(piece); + } else { + // Check for a new output. + if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end()) + outs_.push_back(piece); } - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - have_secondary_target_on_this_rule = true; + } else if (!is_dependency) { + // We've passed an input on the left side; reject new inputs. + poisoned_input = true; } } if (have_newline) { // A newline ends a rule so the next filename will be a new target. parsing_targets = true; - have_secondary_target_on_this_rule = false; - if (have_target) { - have_newline_since_primary_target = true; - } + poisoned_input = false; } } if (!have_target) { diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index 19224f3..bf1a0bc 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -34,7 +34,8 @@ TEST_F(DepfileParserTest, Basic) { "build/ninja.o: ninja.cc ninja.h eval_env.h manifest_parser.h\n", &err)); ASSERT_EQ("", err); - EXPECT_EQ("build/ninja.o", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("build/ninja.o", parser_.outs_[0].AsString()); EXPECT_EQ(4u, parser_.ins_.size()); } @@ -54,7 +55,8 @@ TEST_F(DepfileParserTest, Continuation) { " bar.h baz.h\n", &err)); ASSERT_EQ("", err); - EXPECT_EQ("foo.o", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("foo.o", parser_.outs_[0].AsString()); EXPECT_EQ(2u, parser_.ins_.size()); } @@ -65,7 +67,8 @@ TEST_F(DepfileParserTest, CarriageReturnContinuation) { " bar.h baz.h\r\n", &err)); ASSERT_EQ("", err); - EXPECT_EQ("foo.o", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("foo.o", parser_.outs_[0].AsString()); EXPECT_EQ(2u, parser_.ins_.size()); } @@ -79,8 +82,9 @@ TEST_F(DepfileParserTest, BackSlashes) { " Project\\Thing\\Bar.tlb \\\n", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("Project\\Dir\\Build\\Release8\\Foo\\Foo.res", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); EXPECT_EQ(4u, parser_.ins_.size()); } @@ -90,8 +94,9 @@ TEST_F(DepfileParserTest, Spaces) { "a\\ bc\\ def: a\\ b c d", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("a bc def", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("a b", parser_.ins_[0].AsString()); @@ -111,8 +116,9 @@ TEST_F(DepfileParserTest, MultipleBackslashes) { "a\\ b\\#c.h: \\\\\\\\\\ \\\\\\\\ \\\\share\\info\\\\#1", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("a b#c.h", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("\\\\ ", parser_.ins_[0].AsString()); @@ -130,8 +136,9 @@ TEST_F(DepfileParserTest, Escapes) { "\\!\\@\\#$$\\%\\^\\&\\[\\]\\\\:", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("\\!\\@#$\\%\\^\\&\\[\\]\\\\", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(0u, parser_.ins_.size()); } @@ -147,8 +154,9 @@ TEST_F(DepfileParserTest, SpecialChars) { " a[1]b@2%c", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(5u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); @@ -166,18 +174,25 @@ TEST_F(DepfileParserTest, UnifyMultipleOutputs) { // check that multiple duplicate targets are properly unified string err; EXPECT_TRUE(Parse("foo foo: x y z", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); EXPECT_EQ("z", parser_.ins_[2].AsString()); } -TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { - // check that multiple different outputs are rejected by the parser +TEST_F(DepfileParserTest, MultipleDifferentOutputs) { + // check that multiple different outputs are accepted by the parser string err; - EXPECT_FALSE(Parse("foo bar: x y z", &err)); - ASSERT_EQ("depfile has multiple output paths", err); + EXPECT_TRUE(Parse("foo bar: x y z", &err)); + ASSERT_EQ(2u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); + ASSERT_EQ("bar", parser_.outs_[1].AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); } TEST_F(DepfileParserTest, MultipleEmptyRules) { @@ -185,7 +200,8 @@ TEST_F(DepfileParserTest, MultipleEmptyRules) { EXPECT_TRUE(Parse("foo: x\n" "foo: \n" "foo:\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(1u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); } @@ -196,7 +212,8 @@ TEST_F(DepfileParserTest, UnifyMultipleRulesLF) { "foo: y\n" "foo \\\n" "foo: z\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -209,7 +226,8 @@ TEST_F(DepfileParserTest, UnifyMultipleRulesCRLF) { "foo: y\r\n" "foo \\\r\n" "foo: z\r\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -222,7 +240,8 @@ TEST_F(DepfileParserTest, UnifyMixedRulesLF) { " y\n" "foo \\\n" "foo: z\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -235,7 +254,8 @@ TEST_F(DepfileParserTest, UnifyMixedRulesCRLF) { " y\r\n" "foo \\\r\n" "foo: z\r\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -247,7 +267,8 @@ TEST_F(DepfileParserTest, IndentedRulesLF) { EXPECT_TRUE(Parse(" foo: x\n" " foo: y\n" " foo: z\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -259,7 +280,8 @@ TEST_F(DepfileParserTest, IndentedRulesCRLF) { EXPECT_TRUE(Parse(" foo: x\r\n" " foo: y\r\n" " foo: z\r\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -272,7 +294,8 @@ TEST_F(DepfileParserTest, TolerateMP) { "x:\n" "y:\n" "z:\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -287,25 +310,34 @@ TEST_F(DepfileParserTest, MultipleRulesTolerateMP) { "y:\n" "foo: z\n" "z:\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); EXPECT_EQ("z", parser_.ins_[2].AsString()); } -TEST_F(DepfileParserTest, MultipleRulesRejectDifferentOutputs) { - // check that multiple different outputs are rejected by the parser +TEST_F(DepfileParserTest, MultipleRulesDifferentOutputs) { + // check that multiple different outputs are accepted by the parser // when spread across multiple rules - DepfileParserOptions parser_opts; - parser_opts.depfile_distinct_target_lines_action_ = - kDepfileDistinctTargetLinesActionError; - DepfileParser parser(parser_opts); string err; - string input = - "foo: x y\n" - "bar: y z\n"; - EXPECT_FALSE(parser.Parse(&input, &err)); - ASSERT_EQ("depfile has multiple output paths (on separate lines)" - " [-w depfilemulti=err]", err); + EXPECT_TRUE(Parse("foo: x y\n" + "bar: y z\n", &err)); + ASSERT_EQ(2u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); + ASSERT_EQ("bar", parser_.outs_[1].AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, BuggyMP) { + std::string err; + EXPECT_FALSE(Parse("foo: x y z\n" + "x: alsoin\n" + "y:\n" + "z:\n", &err)); + ASSERT_EQ("inputs may not also have inputs", err); } diff --git a/src/deps_log.cc b/src/deps_log.cc index 4aaffeb..cf55194 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -167,15 +167,15 @@ void DepsLog::Close() { file_ = NULL; } -bool DepsLog::Load(const string& path, State* state, string* err) { +LoadStatus DepsLog::Load(const string& path, State* state, string* err) { METRIC_RECORD(".ninja_deps load"); char buf[kMaxRecordSize + 1]; FILE* f = fopen(path.c_str(), "rb"); if (!f) { if (errno == ENOENT) - return true; + return LOAD_NOT_FOUND; *err = strerror(errno); - return false; + return LOAD_ERROR; } bool valid_header = true; @@ -196,7 +196,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { unlink(path.c_str()); // Don't report this as a failure. An empty deps log will cause // us to rebuild the outputs anyway. - return true; + return LOAD_SUCCESS; } long offset; @@ -284,12 +284,12 @@ bool DepsLog::Load(const string& path, State* state, string* err) { fclose(f); if (!Truncate(path, offset, err)) - return false; + return LOAD_ERROR; // The truncate succeeded; we'll just report the load error as a // warning because the build can proceed. *err += "; recovering"; - return true; + return LOAD_SUCCESS; } fclose(f); @@ -302,7 +302,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { needs_recompaction_ = true; } - return true; + return LOAD_SUCCESS; } DepsLog::Deps* DepsLog::GetDeps(Node* node) { diff --git a/src/deps_log.h b/src/deps_log.h index 3812a28..e7974a1 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -21,6 +21,7 @@ using namespace std; #include <stdio.h> +#include "load_status.h" #include "timestamp.h" struct Node; @@ -84,7 +85,7 @@ struct DepsLog { int node_count; Node** nodes; }; - bool Load(const string& path, State* state, string* err); + LoadStatus Load(const string& path, State* state, string* err); Deps* GetDeps(Node* node); /// Rewrite the known log entries, throwing away old data. diff --git a/src/graph.cc b/src/graph.cc index e24a954..28a9653 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -14,6 +14,7 @@ #include "graph.h" +#include <algorithm> #include <assert.h> #include <stdio.h> @@ -511,6 +512,17 @@ bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { return true; } +struct matches { + matches(std::vector<StringPiece>::iterator i) : i_(i) {} + + bool operator()(const Node* node) const { + StringPiece opath = StringPiece(node->path()); + return *i_ == opath; + } + + std::vector<StringPiece>::iterator i_; +}; + bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, string* err) { METRIC_RECORD("depfile load"); @@ -541,9 +553,15 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } + if (depfile.outs_.empty()) { + *err = path + ": no outputs declared"; + return false; + } + uint64_t unused; - if (!CanonicalizePath(const_cast<char*>(depfile.out_.str_), - &depfile.out_.len_, &unused, err)) { + std::vector<StringPiece>::iterator primary_out = depfile.outs_.begin(); + if (!CanonicalizePath(const_cast<char*>(primary_out->str_), + &primary_out->len_, &unused, err)) { *err = path + ": " + *err; return false; } @@ -552,12 +570,22 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, // mark the edge as dirty. Node* first_output = edge->outputs_[0]; StringPiece opath = StringPiece(first_output->path()); - if (opath != depfile.out_) { + if (opath != *primary_out) { EXPLAIN("expected depfile '%s' to mention '%s', got '%s'", path.c_str(), - first_output->path().c_str(), depfile.out_.AsString().c_str()); + first_output->path().c_str(), primary_out->AsString().c_str()); return false; } + // Ensure that all mentioned outputs are outputs of the edge. + for (std::vector<StringPiece>::iterator o = depfile.outs_.begin(); + o != depfile.outs_.end(); ++o) { + matches m(o); + if (std::find_if(edge->outputs_.begin(), edge->outputs_.end(), m) == edge->outputs_.end()) { + *err = path + ": depfile mentions '" + o->AsString() + "' as an output, but no such output was declared"; + return false; + } + } + // Preallocate space in edge->inputs_ to be filled in below. vector<Node*>::iterator implicit_dep = PreallocateSpace(edge, depfile.ins_.size()); diff --git a/src/load_status.h b/src/load_status.h new file mode 100644 index 0000000..0b16b1a --- /dev/null +++ b/src/load_status.h @@ -0,0 +1,24 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_LOAD_STATUS_H_ +#define NINJA_LOAD_STATUS_H_ + +enum LoadStatus { + LOAD_ERROR, + LOAD_SUCCESS, + LOAD_NOT_FOUND, +}; + +#endif // NINJA_LOAD_STATUS_H_ diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index e28be2f..bb53dc2 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -379,14 +379,6 @@ bool ManifestParser::ParseEdge(string* err) { } } - // Multiple outputs aren't (yet?) supported with depslog. - string deps_type = edge->GetBinding("deps"); - if (!deps_type.empty() && edge->outputs_.size() > 1) { - return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; " - "bring this up on the mailing list if it affects you", - err); - } - // Lookup, validate, and save any dyndep binding. It will be used later // to load generated dependency information dynamically, but it must // be one of our manifest-specified inputs. diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index f2b7467..f4aee2d 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -858,11 +858,10 @@ TEST_F(ParserTest, MultipleOutputsWithDeps) { State local_state; ManifestParser parser(&local_state, NULL); string err; - EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n" + EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n" "build a.o b.o: cc c.cc\n", &err)); - EXPECT_EQ("input:5: multiple outputs aren't (yet?) supported by depslog; " - "bring this up on the mailing list if it affects you\n", err); + EXPECT_EQ("", err); } TEST_F(ParserTest, SubNinja) { diff --git a/src/ninja.cc b/src/ninja.cc index c24f09d..1429639 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -17,6 +17,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <cstdlib> #ifdef _WIN32 #include "getopt.h" @@ -73,10 +74,6 @@ struct Options { /// Whether phony cycles should warn or print an error. bool phony_cycle_should_err; - - /// Whether a depfile with multiple targets on separate lines should - /// warn or print an error. - bool depfile_distinct_target_lines_should_err; }; /// The Ninja main() loads up a series of data structures; various tools need @@ -123,17 +120,19 @@ struct NinjaMain : public BuildLogUser { int ToolTargets(const Options* options, int argc, char* argv[]); int ToolCommands(const Options* options, int argc, char* argv[]); int ToolClean(const Options* options, int argc, char* argv[]); + int ToolCleanDead(const Options* options, int argc, char* argv[]); int ToolCompilationDatabase(const Options* options, int argc, char* argv[]); int ToolRecompact(const Options* options, int argc, char* argv[]); + int ToolRestat(const Options* options, int argc, char* argv[]); int ToolUrtle(const Options* options, int argc, char** argv); int ToolRules(const Options* options, int argc, char* argv[]); /// Open the build log. - /// @return false on error. + /// @return LOAD_ERROR on error. bool OpenBuildLog(bool recompact_only = false); /// Open the deps log: load it, then open for writing. - /// @return false on error. + /// @return LOAD_ERROR on error. bool OpenDepsLog(bool recompact_only = false); /// Ensure the build directory exists, creating it if necessary. @@ -154,7 +153,7 @@ struct NinjaMain : public BuildLogUser { virtual bool IsPathDead(StringPiece s) const { Node* n = state_.LookupNode(s); - if (!n || !n->in_edge()) + if (n && n->in_edge()) return false; // Just checking n isn't enough: If an old output is both in the build log // and in the deps log, it will have a Node object in state_. (It will also @@ -719,6 +718,11 @@ int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) { } } +int NinjaMain::ToolCleanDead(const Options* options, int argc, char* argv[]) { + Cleaner cleaner(&state_, config_, &disk_interface_); + return cleaner.CleanDead(build_log_.entries()); +} + void EncodeJSONString(const char *str) { while (*str) { if (*str == '"' || *str == '\\') @@ -803,12 +807,14 @@ int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, bool first = true; vector<char> cwd; + char* success = NULL; do { cwd.resize(cwd.size() + 1024); errno = 0; - } while (!getcwd(&cwd[0], cwd.size()) && errno == ERANGE); - if (errno != 0 && errno != ERANGE) { + success = getcwd(&cwd[0], cwd.size()); + } while (!success && errno == ERANGE); + if (!success) { Error("cannot determine working directory: %s", strerror(errno)); return 1; } @@ -845,13 +851,71 @@ int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) { if (!EnsureBuildDirExists()) return 1; - if (!OpenBuildLog(/*recompact_only=*/true) || - !OpenDepsLog(/*recompact_only=*/true)) + if (OpenBuildLog(/*recompact_only=*/true) == LOAD_ERROR || + OpenDepsLog(/*recompact_only=*/true) == LOAD_ERROR) return 1; return 0; } +int NinjaMain::ToolRestat(const Options* options, int argc, char* argv[]) { + // The restat tool uses getopt, and expects argv[0] to contain the name of the + // tool, i.e. "restat" + argc++; + argv--; + + optind = 1; + int opt; + while ((opt = getopt(argc, argv, const_cast<char*>("h"))) != -1) { + switch (opt) { + case 'h': + default: + printf("usage: ninja -t restat [outputs]\n"); + return 1; + } + } + argv += optind; + argc -= optind; + + if (!EnsureBuildDirExists()) + return 1; + + string log_path = ".ninja_log"; + if (!build_dir_.empty()) + log_path = build_dir_ + "/" + log_path; + + string err; + const LoadStatus status = build_log_.Load(log_path, &err); + if (status == LOAD_ERROR) { + Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + return EXIT_FAILURE; + } + if (status == LOAD_NOT_FOUND) { + // Nothing to restat, ignore this + return EXIT_SUCCESS; + } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. + Warning("%s", err.c_str()); + err.clear(); + } + + bool success = build_log_.Restat(log_path, disk_interface_, argc, argv, &err); + if (!success) { + Error("failed recompaction: %s", err.c_str()); + return EXIT_FAILURE; + } + + if (!config_.dry_run) { + if (!build_log_.OpenForWrite(log_path, *this, &err)) { + Error("opening build log: %s", err.c_str()); + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; +} + int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) { // RLE encoded. const char* urtle = @@ -904,8 +968,12 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase }, { "recompact", "recompacts ninja-internal data structures", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact }, + { "restat", "restats all outputs in the build log", + Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolRestat }, { "rules", "list all rules", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRules }, + { "cleandead", "clean built files that are no longer produced by the manifest", + Tool::RUN_AFTER_LOGS, &NinjaMain::ToolCleanDead }, { "urtle", NULL, Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle }, { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } @@ -989,7 +1057,6 @@ bool WarningEnable(const string& name, Options* options) { printf("warning flags:\n" " dupbuild={err,warn} multiple build lines for one target\n" " phonycycle={err,warn} phony build statement references itself\n" -" depfilemulti={err,warn} depfile has multiple output paths on separate lines\n" ); return false; } else if (name == "dupbuild=err") { @@ -1004,11 +1071,9 @@ bool WarningEnable(const string& name, Options* options) { } else if (name == "phonycycle=warn") { options->phony_cycle_should_err = false; return true; - } else if (name == "depfilemulti=err") { - options->depfile_distinct_target_lines_should_err = true; - return true; - } else if (name == "depfilemulti=warn") { - options->depfile_distinct_target_lines_should_err = false; + } else if (name == "depfilemulti=err" || + name == "depfilemulti=warn") { + Warning("deprecated warning 'depfilemulti'"); return true; } else { const char* suggestion = @@ -1030,17 +1095,21 @@ bool NinjaMain::OpenBuildLog(bool recompact_only) { log_path = build_dir_ + "/" + log_path; string err; - if (!build_log_.Load(log_path, &err)) { + const LoadStatus status = build_log_.Load(log_path, &err); + if (status == LOAD_ERROR) { Error("loading build log %s: %s", log_path.c_str(), err.c_str()); return false; } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } if (recompact_only) { + if (status == LOAD_NOT_FOUND) { + return true; + } bool success = build_log_.Recompact(log_path, *this, &err); if (!success) Error("failed recompaction: %s", err.c_str()); @@ -1065,17 +1134,21 @@ bool NinjaMain::OpenDepsLog(bool recompact_only) { path = build_dir_ + "/" + path; string err; - if (!deps_log_.Load(path, &state_, &err)) { + const LoadStatus status = deps_log_.Load(path, &state_, &err); + if (status == LOAD_ERROR) { Error("loading deps log %s: %s", path.c_str(), err.c_str()); return false; } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } if (recompact_only) { + if (status == LOAD_NOT_FOUND) { + return true; + } bool success = deps_log_.Recompact(path, &err); if (!success) Error("failed recompaction: %s", err.c_str()); @@ -1284,11 +1357,6 @@ NORETURN void real_main(int argc, char** argv) { if (exit_code >= 0) exit(exit_code); - if (options.depfile_distinct_target_lines_should_err) { - config.depfile_parser_options.depfile_distinct_target_lines_action_ = - kDepfileDistinctTargetLinesActionError; - } - if (options.working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for |