diff options
58 files changed, 2177 insertions, 709 deletions
@@ -1,5 +1,7 @@ *.pyc *.exe +*.pdb +*.ilk TAGS /build /build.ninja @@ -12,3 +14,4 @@ TAGS /graph.png /doc/manual.html /doc/doxygen +/gtest-1.6.0 diff --git a/HACKING b/HACKING deleted file mode 100644 index 4e01d49..0000000 --- a/HACKING +++ /dev/null @@ -1,76 +0,0 @@ -Adjusting build flags: - CFLAGS=-O3 ./configure.py - and rebuild. - -Building tests requires gtest, to get it: - - On older Ubuntus you can apt-get install libgtest. - - On newer Ubuntus it's only distributed as source; - 1) apt-get install libgtest-dev - 2) ./configure --with-gtest=/usr/src/gtest - - Otherwise you need to download it, unpack it, and pass --with-gtest - as appropriate. - -Test-driven development: - Set your build command to - ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name - now you can repeatedly run that while developing until the tests pass. - Remember to build "all" before committing to verify the other source - still works! - -Testing performance impact of changes: - If you have a Chrome build handy, it's a good test case. - Otherwise, https://github.com/martine/ninja/downloads has a copy of - the Chrome build files (and depfiles). You can untar that, then run - path/to/my/ninja chrome - and compare that against a baseline Ninja. - - There's a script at misc/measure.py that repeatedly runs a command like - the above (to address variance) and summarizes its runtime. E.g. - path/to/misc/measure.py path/to/my/ninja chrome - - For changing the depfile parser, you can also build 'parser_perftest' - and run that directly on some representative input files. - -Coding guidelines: -- Function name are camelcase. -- Member methods are camelcase, expect for trivial getters which are - underscore separated. -- Local variables are underscore separated. -- Member variables are underscore separated and suffixed by an extra underscore. -- Two spaces indentation. -- Opening braces is at the end of line. -- Lines are 80 columns maximum. -- All source files should have the Google Inc. license header. -- Also follow this style: - http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml - -Documentation guidelines: -- Use /// for doxygen. -- Use \a to refer to arguments. -- It's not necessary to document each argument, especially when they're - relatively self-evident (e.g. in CanonicalizePath(string* path, string* err), - the arguments are hopefully obvious) - -Generating the manual: - sudo apt-get install asciidoc --no-install-recommends - ./ninja manual - -Windows development on Linux (this is kind of hacky right now): -- Get the gtest source, unpack it into your source dir -- sudo apt-get install gcc-mingw32 wine -- export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar -- ./configure.py --platform=mingw --host=linux --with-gtest=gtest-1.6.0 -- Build ninja: /path/to/linux/ninja -- Run: ./ninja.exe (implicitly runs through wine(!)) - -Windows development on Windows: -- install mingw, msys, and python -- in the mingw shell, put Python in your path, and: python bootstrap.py -- to reconfigure, run 'python configure.py' -- remember to strip the resulting executable if size matters to you -- you'll need to rename ninja.exe into my-ninja.exe during development, - otherwise ninja won't be able to overwrite itself when building - -Using clang: -- Enable colors manually: - CXX='/path/to/llvm/Release+Asserts/bin/clang++ -fcolor-diagnostics' ./configure.py diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..a777b57 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,167 @@ +## Basic overview + +`./configure.py` generates the `build.ninja` files used to build +ninja. It accepts various flags to adjust build parameters. + +The primary build target of interest is `ninja`, but when hacking on +Ninja your changes should be testable so it's more useful to build +and run `ninja_test` when developing. + +(`./bootstrap.py` creates a bootstrap `ninja` and runs the above +process; it's only necessary to run if you don't have a copy of +`ninja` to build with.) + +### Adjusting build flags + +Build in "debug" mode while developing (disables optimizations and builds +way faster on Windows): + + ./configure.py --debug + +To use clang, set `CXX`: + + CXX=clang++ ./configure.py + +## How to successfully make changes to Ninja + +Github pull requests are convenient for me to merge (I can just click +a button and it's all handled server-side), but I'm also comfortable +accepting pre-github git patches (via `send-email` etc.). + +Good pull requests have all of these attributes: + +* Are scoped to one specific issue +* Include a test to demonstrate their correctness +* Update the docs where relevant +* Match the Ninja coding style (see below) +* Don't include a mess of "oops, fix typo" commits + +These are typically merged without hesitation. If a change is lacking +any of the above I usually will ask you to fix it, though there are +obvious exceptions (fixing typos in comments don't need tests). + +I am very wary of changes that increase the complexity of Ninja (in +particular, new build file syntax or command-line flags) or increase +the maintenance burden of Ninja. Ninja is already successfully in use +by hundreds of developers for large projects and it already achieves +(most of) the goals I set out for it to do. It's probably best to +discuss new feature ideas on the mailing list before I shoot down your +patch. + +## Testing + +### Installing gtest + +The `ninja_test` binary, containing all the tests, depends on the +googletest (gtest) library. + +* On older Ubuntus it'll install as libraries into `/usr/lib`: + + apt-get install libgtest + +* On newer Ubuntus it's only distributed as source + + apt-get install libgtest-dev + ./configure --with-gtest=/usr/src/gtest + +* Otherwise you need to download it, unpack it, and pass + `--with-gtest` to `configure.py`. Get it from [its downloads + page](http://code.google.com/p/googletest/downloads/list); [this + direct download link might work + too](http://googletest.googlecode.com/files/gtest-1.6.0.zip). + +### Test-driven development + +Set your build command to + + ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name + +now you can repeatedly run that while developing until the tests pass +(I frequently set it as my compilation command in Emacs). Remember to +build "all" before committing to verify the other source still works! + +## Testing performance impact of changes + +If you have a Chrome build handy, it's a good test case. Otherwise, +[the github downoads page](https://github.com/martine/ninja/downloads) +has a copy of the Chrome build files (and depfiles). You can untar +that, then run + + path/to/my/ninja chrome + +and compare that against a baseline Ninja. + +There's a script at `misc/measure.py` that repeatedly runs a command like +the above (to address variance) and summarizes its runtime. E.g. + + path/to/misc/measure.py path/to/my/ninja chrome + +For changing the depfile parser, you can also build `parser_perftest` +and run that directly on some representative input files. + +## Coding guidelines + +Generally it's the [Google C++ coding style][], but in brief: + +* Function name are camelcase. +* Member methods are camelcase, expect for trivial getters which are + underscore separated. +* Local variables are underscore separated. +* Member variables are underscore separated and suffixed by an extra + underscore. +* Two spaces indentation. +* Opening braces is at the end of line. +* Lines are 80 columns maximum. +* All source files should have the Google Inc. license header. + +[Google C++ coding style]: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml + +## Documentation + +### Style guidelines + +* Use `///` for doxygen. +* Use `\a` to refer to arguments. +* It's not necessary to document each argument, especially when they're + relatively self-evident (e.g. in `CanonicalizePath(string* path, string* err)`, + the arguments are hopefully obvious) + +### Building the manual + + sudo apt-get install asciidoc --no-install-recommends + ./ninja manual + +### Building the code documentation + + sudo apt-get install doxygen + ./ninja doxygen + +## Building for Windows + +While developing, it's helpful to copy `ninja.exe` to another name like +`n.exe`; otherwise, rebuilds will be unable to write `ninja.exe` because +it's locked while in use. + +### Via Visual Studio + +* Install Visual Studio (Express is fine), [Python for Windows][], + and (if making changes) googletest (see above instructions) +* In a Visual Studio command prompt: `python bootstrap.py` + +[Python for Windows]: http://www.python.org/getit/windows/ + +### Via mingw on Windows (not well supported) + +* Install mingw, msys, and python +* In the mingw shell, put Python in your path, and `python bootstrap.py` +* To reconfigure, run `python configure.py` +* Remember to strip the resulting executable if size matters to you + +### Via mingw on Linux (not well supported) + +* `sudo apt-get install gcc-mingw32 wine` +* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar` +* `./configure.py --platform=mingw --host=linux` +* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` +* Run: `./ninja.exe` (implicitly runs through wine(!)) + @@ -13,3 +13,4 @@ help. There is no installation step. The only file of interest to a user is the resulting ninja binary. +If you're interested in making changes to Ninja, read HACKING.md first. diff --git a/bootstrap.py b/bootstrap.py index 9ac46ba..3032a9b 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -19,6 +19,7 @@ import os import glob import errno import shlex +import shutil import subprocess os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -26,6 +27,8 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = OptionParser() parser.add_option('--verbose', action='store_true', help='enable verbose build',) +parser.add_option('--x64', action='store_true', + help='force 64-bit build (Windows)',) (options, conf_args) = parser.parse_args() def run(*args, **kwargs): @@ -61,7 +64,7 @@ for src in glob.glob('src/*.cc'): continue if sys.platform.startswith('win32'): - if filename == 'subprocess.cc': + if src.endswith('-posix.cc'): continue else: if src.endswith('-win32.cc'): @@ -74,12 +77,20 @@ if sys.platform.startswith('win32'): vcdir = os.environ.get('VCINSTALLDIR') if vcdir: - args = [os.path.join(vcdir, 'bin', 'cl.exe'), '/nologo', '/EHsc', '/DNOMINMAX'] + if options.x64: + cl = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')] + else: + cl = [os.path.join(vcdir, 'bin', 'cl.exe')] + args = cl + ['/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) - args.extend(['-Wno-deprecated', - '-DNINJA_PYTHON="' + sys.executable + '"', - '-DNINJA_BOOTSTRAP']) + cflags.extend(['-Wno-deprecated', + '-DNINJA_PYTHON="' + sys.executable + '"', + '-DNINJA_BOOTSTRAP']) + if sys.platform.startswith('win32'): + cflags.append('-D_WIN32_WINNT=0x0501') + if options.x64: + cflags.append('-m64') args.extend(cflags) args.extend(ldflags) binary = 'ninja.bootstrap' @@ -100,9 +111,30 @@ verbose = [] if options.verbose: verbose = ['-v'] -print 'Building ninja using itself...' -run([sys.executable, 'configure.py'] + conf_args) -run(['./' + binary] + verbose) -os.unlink(binary) - -print 'Done!' +if sys.platform.startswith('win32'): + print 'Building ninja using itself...' + run([sys.executable, 'configure.py', '--with-ninja=%s' % binary] + + conf_args) + run(['./' + binary] + verbose) + + # Copy the new executable over the bootstrap one. + shutil.copyfile('ninja.exe', binary) + + # Clean up. + for obj in glob.glob('*.obj'): + os.unlink(obj) + + print """ +Done! + +Note: to work around Windows file locking, where you can't rebuild an +in-use binary, to run ninja after making any changes to build ninja itself +you should run ninja.bootstrap instead. Your build is also configured to +use ninja.bootstrap.exe as the MSVC helper; see the --with-ninja flag of +the --help output of configure.py.""" +else: + print 'Building ninja using itself...' + run([sys.executable, 'configure.py'] + conf_args) + run(['./' + binary] + verbose) + os.unlink(binary) + print 'Done!' diff --git a/configure.py b/configure.py index 6fb718f..98274e6 100755 --- a/configure.py +++ b/configure.py @@ -27,7 +27,7 @@ sys.path.insert(0, 'misc') import ninja_syntax parser = OptionParser() -platforms = ['linux', 'freebsd', 'mingw', 'windows'] +platforms = ['linux', 'freebsd', 'solaris', 'mingw', 'windows'] profilers = ['gmon', 'pprof'] parser.add_option('--platform', help='target platform (' + '/'.join(platforms) + ')', @@ -45,6 +45,9 @@ parser.add_option('--with-gtest', metavar='PATH', parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) +parser.add_option('--with-ninja', metavar='NAME', + help="name for ninja binary for -t msvc (MSVC only)", + default="ninja") (options, args) = parser.parse_args() if args: print 'ERROR: extra unparsed command-line arguments:', args @@ -57,6 +60,8 @@ if platform is None: platform = 'linux' elif platform.startswith('freebsd'): platform = 'freebsd' + elif platform.startswith('solaris'): + platform = 'solaris' elif platform.startswith('mingw'): platform = 'mingw' elif platform.startswith('win'): @@ -97,7 +102,9 @@ def cxx(name, **kwargs): return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs) def binary(name): if platform in ('mingw', 'windows'): - return name + '.exe' + exe = name + '.exe' + n.build(name, 'phony', exe) + return exe return name n.variable('builddir', 'build') @@ -108,10 +115,20 @@ else: n.variable('ar', configure_env.get('AR', 'ar')) if platform == 'windows': - cflags = ['/nologo', '/Zi', '/W4', '/WX', '/wd4530', '/wd4100', '/wd4706', - '/wd4512', '/wd4800', '/wd4702', '/wd4819', '/GR-', + cflags = ['/nologo', # Don't print startup banner. + '/Zi', # Create pdb with debug info. + '/W4', # Highest warning level. + '/WX', # Warnings as errors. + '/wd4530', '/wd4100', '/wd4706', + '/wd4512', '/wd4800', '/wd4702', '/wd4819', + # Disable warnings about passing "this" during initialization. + '/wd4355', + '/GR-', # Disable RTTI. + # Disable size_t -> int truncation warning. + # We never have strings or arrays larger than 2**31. + '/wd4267', '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS', - "/DNINJA_PYTHON=\"%s\"" % (options.with_python,)] + '/DNINJA_PYTHON="%s"' % options.with_python] ldflags = ['/DEBUG', '/libpath:$builddir'] if not options.debug: cflags += ['/Ox', '/DNDEBUG', '/GL'] @@ -123,13 +140,16 @@ else: '-fno-rtti', '-fno-exceptions', '-fvisibility=hidden', '-pipe', - "-DNINJA_PYTHON=\"%s\"" % options.with_python] + '-DNINJA_PYTHON="%s"' % options.with_python] if options.debug: cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC'] + cflags.remove('-fno-rtti') # Needed for above pedanticness. else: cflags += ['-O2', '-DNDEBUG'] if 'clang' in os.path.basename(CXX): cflags += ['-fcolor-diagnostics'] + if platform == 'mingw': + cflags += ['-D_WIN32_WINNT=0x0501'] ldflags = ['-L$builddir'] libs = [] @@ -150,8 +170,8 @@ else: def shell_escape(str): """Escape str such that it's interpreted as a single argument by the shell.""" # This isn't complete, but it's just enough to make NINJA_PYTHON work. - # TODO: do the appropriate thing for Windows-style cmd here, perhaps by - # just returning the input string. + if platform in ('windows', 'mingw'): + return str if '"' in str: return "'%s'" % str.replace("'", "\\'") return str @@ -165,8 +185,12 @@ n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags)) n.newline() if platform == 'windows': + compiler = '$cxx' + if options.with_ninja: + compiler = ('%s -t msvc -o $out -- $cxx /showIncludes' % + options.with_ninja) n.rule('cxx', - command='$cxx $cflags -c $in /Fo$out', + command='%s $cflags -c $in /Fo$out' % compiler, depfile='$out.d', description='CXX $out') else: @@ -202,7 +226,7 @@ n.newline() objs = [] -if platform not in ('mingw', 'windows'): +if platform not in ('solaris', 'mingw', 'windows'): n.comment('browse_py.h is used to inline browse.py.') n.rule('inline', command='src/inline.sh $varname < $in > $out', @@ -216,12 +240,23 @@ if platform not in ('mingw', 'windows'): n.newline() n.comment('the depfile parser and ninja lexers are generated using re2c.') -n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', - description='RE2C $out') -# Generate the .cc files in the source directory so we can check them in. -n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) -n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) +def has_re2c(): + import subprocess + try: + proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE) + return int(proc.communicate()[0], 10) >= 1103 + except OSError: + return False +if has_re2c(): + n.rule('re2c', + command='re2c -b -i --no-generation-date -o $out $in', + description='RE2C $out') + # Generate the .cc files in the source directory so we can check them in. + n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) + n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) +else: + print ("warning: A compatible version of re2c (>= 0.11.3) was not found; " + "changes to src/*.in.cc will not affect your build.") n.newline() n.comment('Core source files all build into ninja library.') @@ -241,11 +276,16 @@ for name in ['build', 'state', 'util']: objs += cxx(name) -if platform == 'mingw' or platform == 'windows': +if platform in ('mingw', 'windows'): objs += cxx('subprocess-win32') + if platform == 'windows': + objs += cxx('includes_normalize-win32') + objs += cxx('msvc_helper-win32') + objs += cxx('msvc_helper_main-win32') + objs += cxx('minidump-win32') objs += cc('getopt') else: - objs += cxx('subprocess') + objs += cxx('subprocess-posix') if platform == 'windows': ninja_lib = n.build(built('ninja.lib'), 'ar', objs) else: @@ -263,8 +303,6 @@ n.comment('Main executable is library plus main() function.') objs = cxx('ninja') ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) -if 'ninja' not in ninja: - n.build('ninja', 'phony', ninja) n.newline() all_targets += ninja @@ -280,14 +318,14 @@ if options.with_gtest: gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include')) if platform == 'windows': - gtest_cflags = '/nologo /EHsc ' + gtest_all_incs + gtest_cflags = '/nologo /EHsc /Zi ' + gtest_all_incs else: gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs objs += n.build(built('gtest-all' + objext), 'cxx', - os.path.join(path, 'src/gtest-all.cc'), + os.path.join(path, 'src', 'gtest-all.cc'), variables=[('cflags', gtest_cflags)]) objs += n.build(built('gtest_main' + objext), 'cxx', - os.path.join(path, 'src/gtest_main.cc'), + os.path.join(path, 'src', 'gtest_main.cc'), variables=[('cflags', gtest_cflags)]) test_cflags = cflags + ['-DGTEST_HAS_RTTI=0', @@ -311,14 +349,15 @@ for name in ['build_log_test', 'test', 'util_test']: objs += cxx(name, variables=[('cflags', test_cflags)]) +if platform == 'windows': + for name in ['includes_normalize_test', 'msvc_helper_test']: + objs += cxx(name, variables=[('cflags', test_cflags)]) if platform != 'mingw' and platform != 'windows': test_libs.append('-lpthread') ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, variables=[('ldflags', test_ldflags), ('libs', test_libs)]) -if 'ninja_test' not in ninja_test: - n.build('ninja_test', 'phony', ninja_test) n.newline() all_targets += ninja_test @@ -340,7 +379,7 @@ n.newline() n.comment('Generate a graph using the "graph" tool.') n.rule('gendot', - command='./ninja -t graph > $out') + command='./ninja -t graph all > $out') n.rule('gengraph', command='dot -Tpng $in > $out') dot = n.build(built('graph.dot'), 'gendot', ['ninja', 'build.ninja']) @@ -366,7 +405,7 @@ n.rule('doxygen_mainpage', command='$doxygen_mainpage_generator $in > $out', description='DOXYGEN_MAINPAGE $out') mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage', - ['README', 'HACKING', 'COPYING'], + ['README', 'COPYING'], implicit=['$doxygen_mainpage_generator']) n.build('doxygen', 'doxygen', doc('doxygen.config'), implicit=mainpage) @@ -379,13 +418,32 @@ if host != 'mingw': options.with_python, generator=True) n.build('build.ninja', 'configure', - implicit=['configure.py', 'misc/ninja_syntax.py']) + implicit=['configure.py', os.path.normpath('misc/ninja_syntax.py')]) n.newline() -n.comment('Build only the main binary by default.') n.default(ninja) n.newline() +if host == 'linux': + n.comment('Packaging') + n.rule('rpmbuild', + command="rpmbuild \ + --define 'ver git' \ + --define \"rel `git rev-parse --short HEAD`\" \ + --define '_topdir %(pwd)/rpm-build' \ + --define '_builddir %{_topdir}' \ + --define '_rpmdir %{_topdir}' \ + --define '_srcrpmdir %{_topdir}' \ + --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \ + --define '_specdir %{_topdir}' \ + --define '_sourcedir %{_topdir}' \ + --quiet \ + -bb misc/packaging/ninja.spec", + description='Building RPM..') + n.build('rpm', 'rpmbuild', + implicit=['ninja','README', 'COPYING', doc('manual.html')]) + n.newline() + n.build('all', 'phony', all_targets) print 'wrote %s.' % BUILD_FILENAME diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e7983be..03d27df 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -218,6 +218,8 @@ Several placeholders are available: * `%r`: The number of currently running edges. * `%u`: The number of remaining edges to start. * `%f`: The number of finished edges. +* `%o`: Overall rate of finished edges per second +* `%c`: Current rate of finished edges per second (average over builds specified by -j or its default) * `%%`: A plain `%` character. * The default progress status is `"[%s/%t] "` (note the trailing space to separate from the build rule). Another example of possible progress status @@ -278,9 +280,9 @@ built for them. If used like +ninja -t clean -r _rules_+ it removes all files built using the given rules. + -depfiles are not removed. Files created but not referenced in the -graph are not removed. This tool takes in account the +-v+ and the -+-n+ options (note that +-n+ implies +-v+). +Files created but not referenced in the graph are not removed. This +tool takes in account the +-v+ and the +-n+ options (note that +-n+ +implies +-v+). @@ -522,7 +524,9 @@ Lexical syntax Ninja is mostly encoding agnostic, as long as the bytes Ninja cares about (like slashes in paths) are ASCII. This means e.g. UTF-8 or -ISO-8859-1 input files ought to work. +ISO-8859-1 input files ought to work. (To simplify some code, tabs +and carriage returns are currently disallowed; this could be fixed if +it really mattered to you.) Comments begin with `#` and extend to the end of the line. diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index 3ecbcee..66babbe 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -10,8 +10,8 @@ use Python. import textwrap import re -def escape_spaces(word): - return word.replace('$ ','$$ ').replace(' ','$ ') +def escape_path(word): + return word.replace('$ ','$$ ').replace(' ','$ ').replace(':', '$:') class Writer(object): def __init__(self, output, width=78): @@ -53,15 +53,15 @@ class Writer(object): variables=None): outputs = self._as_list(outputs) all_inputs = self._as_list(inputs)[:] - out_outputs = list(map(escape_spaces, outputs)) - all_inputs = list(map(escape_spaces, all_inputs)) + out_outputs = list(map(escape_path, outputs)) + all_inputs = list(map(escape_path, all_inputs)) if implicit: - implicit = map(escape_spaces, self._as_list(implicit)) + implicit = map(escape_path, self._as_list(implicit)) all_inputs.append('|') all_inputs.extend(implicit) if order_only: - order_only = map(escape_spaces, self._as_list(order_only)) + order_only = map(escape_path, self._as_list(order_only)) all_inputs.append('||') all_inputs.extend(order_only) diff --git a/misc/packaging/ninja.spec b/misc/packaging/ninja.spec new file mode 100644 index 0000000..d513c6d --- /dev/null +++ b/misc/packaging/ninja.spec @@ -0,0 +1,33 @@ +Summary: Ninja is a small build system with a focus on speed. +Name: ninja +Version: %{ver} +Release: %{rel}%{?dist} +Group: Development/Tools +License: Apache 2.0 +URL: https://github.com/martine/ninja + +%description +Ninja is yet another build system. It takes as input the interdependencies of files (typically source code and output executables) and +orchestrates building them, quickly. + +Ninja joins a sea of other build systems. Its distinguishing goal is to be fast. It is born from my work on the Chromium browser project, +which has over 30,000 source files and whose other build systems (including one built from custom non-recursive Makefiles) can take ten +seconds to start building after changing one file. Ninja is under a second. + +%build +# Assuming we've bootstrapped already.. +../ninja manual ninja -C .. + +%install +mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_docdir} +cp -p ../ninja %{buildroot}%{_bindir}/ +git log --oneline --pretty=format:'%h: %s (%an, %cd)' --abbrev-commit --all > GITLOG + +%files +%defattr(-, root, root) +%doc GITLOG ../COPYING ../README ../doc/manual.html +%{_bindir}/* + +%clean +mv %{_topdir}/*.rpm .. +rm -rf %{_topdir} diff --git a/src/build.cc b/src/build.cc index 0fc387c..e1aaad1 100644 --- a/src/build.cc +++ b/src/build.cc @@ -26,6 +26,10 @@ #include <sys/time.h> #endif +#if defined(__SVR4) && defined(__sun) +#include <sys/termios.h> +#endif + #include "build_log.h" #include "disk_interface.h" #include "graph.h" @@ -37,7 +41,9 @@ BuildStatus::BuildStatus(const BuildConfig& config) : config_(config), start_time_millis_(GetTimeMillis()), started_edges_(0), finished_edges_(0), total_edges_(0), - have_blank_line_(true), progress_status_format_(NULL) { + have_blank_line_(true), progress_status_format_(NULL), + overall_rate_(), current_rate_(), + current_rate_average_count_(config.parallelism) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -171,6 +177,25 @@ string BuildStatus::FormatProgressStatus(const char* progress_status_format) con out += buf; break; + // Overall finished edges per second. + case 'o': + overall_rate_.UpdateRate(finished_edges_, finished_edges_); + overall_rate_.snprinfRate(buf, "%.1f"); + out += buf; + break; + + // Current rate, average over the last '-j' jobs. + case 'c': + // TODO use sliding window? + if (finished_edges_ > current_rate_.last_update() && + finished_edges_ - current_rate_.last_update() == current_rate_average_count_) { + current_rate_.UpdateRate(current_rate_average_count_, finished_edges_); + current_rate_.Restart(); + } + current_rate_.snprinfRate(buf, "%.0f"); + out += buf; + break; + default: { Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); return ""; @@ -208,50 +233,57 @@ void BuildStatus::PrintStatus(Edge* edge) { #endif } + if (finished_edges_ == 0) { + overall_rate_.Restart(); + current_rate_.Restart(); + } to_print = FormatProgressStatus(progress_status_format_) + to_print; if (smart_terminal_ && !force_full_command) { - const int kMargin = 3; // Space for "...". #ifndef _WIN32 // Limit output to width of the terminal if provided so we don't cause // line-wrapping. winsize size; if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) { - if (to_print.size() + kMargin > size.ws_col) { - int elide_size = (size.ws_col - kMargin) / 2; - to_print = to_print.substr(0, elide_size) - + "..." - + to_print.substr(to_print.size() - elide_size, elide_size); - } + to_print = ElideMiddle(to_print, size.ws_col); } #else // Don't use the full width or console will move to next line. size_t width = static_cast<size_t>(csbi.dwSize.X) - 1; - if (to_print.size() + kMargin > width) { - int elide_size = (width - kMargin) / 2; - to_print = to_print.substr(0, elide_size) - + "..." - + to_print.substr(to_print.size() - elide_size, elide_size); - } + to_print = ElideMiddle(to_print, width); #endif } - printf("%s", to_print.c_str()); - if (smart_terminal_ && !force_full_command) { #ifndef _WIN32 + printf("%s", to_print.c_str()); printf("\x1B[K"); // Clear to end of line. fflush(stdout); have_blank_line_ = false; #else - // Clear to end of line. + // We don't want to have the cursor spamming back and forth, so + // use WriteConsoleOutput instead which updates the contents of + // the buffer, but doesn't move the cursor position. GetConsoleScreenBufferInfo(console_, &csbi); - int num_spaces = csbi.dwSize.X - 1 - csbi.dwCursorPosition.X; - printf("%*s", num_spaces, ""); + COORD buf_size = { csbi.dwSize.X, 1 }; + COORD zero_zero = { 0, 0 }; + SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, + (SHORT)(csbi.dwCursorPosition.X + csbi.dwSize.X - 1), + csbi.dwCursorPosition.Y }; + CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X]; + memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X); + for (int i = 0; i < csbi.dwSize.X; ++i) { + char_data[i].Char.AsciiChar = ' '; + char_data[i].Attributes = csbi.wAttributes; + } + for (size_t i = 0; i < to_print.size(); ++i) + char_data[i].Char.AsciiChar = to_print[i]; + WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target); + delete[] char_data; have_blank_line_ = false; #endif } else { - printf("\n"); + printf("%s\n", to_print.c_str()); } } @@ -378,7 +410,7 @@ void Plan::NodeFinished(Node* node) { } } -void Plan::CleanNode(BuildLog* build_log, Node* node) { +void Plan::CleanNode(DependencyScan* scan, Node* node) { node->set_dirty(false); for (vector<Edge*>::const_iterator ei = node->out_edges().begin(); @@ -394,10 +426,11 @@ void Plan::CleanNode(BuildLog* build_log, Node* node) { end = (*ei)->inputs_.end() - (*ei)->order_only_deps_; if (find_if(begin, end, mem_fun(&Node::dirty)) == end) { // Recompute most_recent_input and command. - TimeStamp most_recent_input = 1; - for (vector<Node*>::iterator ni = begin; ni != end; ++ni) - if ((*ni)->mtime() > most_recent_input) - most_recent_input = (*ni)->mtime(); + Node* most_recent_input = NULL; + for (vector<Node*>::iterator ni = begin; ni != end; ++ni) { + if (!most_recent_input || (*ni)->mtime() > most_recent_input->mtime()) + most_recent_input = *ni; + } string command = (*ei)->EvaluateCommand(true); // Now, recompute the dirty state of each output. @@ -407,12 +440,12 @@ void Plan::CleanNode(BuildLog* build_log, Node* node) { if (!(*ni)->dirty()) continue; - if ((*ei)->RecomputeOutputDirty(build_log, most_recent_input, NULL, command, - *ni)) { + if (scan->RecomputeOutputDirty(*ei, most_recent_input, + command, *ni)) { (*ni)->MarkDirty(); all_outputs_clean = false; } else { - CleanNode(build_log, *ni); + CleanNode(scan, *ni); } } @@ -524,11 +557,11 @@ struct DryRunCommandRunner : public CommandRunner { queue<Edge*> finished_; }; -Builder::Builder(State* state, const BuildConfig& config) - : state_(state), config_(config) { - disk_interface_ = new RealDiskInterface; +Builder::Builder(State* state, const BuildConfig& config, + BuildLog* log, DiskInterface* disk_interface) + : state_(state), config_(config), disk_interface_(disk_interface), + scan_(state, log, disk_interface) { status_ = new BuildStatus(config); - log_ = state->build_log_; } Builder::~Builder() { @@ -576,7 +609,7 @@ Node* Builder::AddTarget(const string& name, string* err) { bool Builder::AddTarget(Node* node, string* err) { node->StatIfNecessary(disk_interface_); if (Edge* in_edge = node->in_edge()) { - if (!in_edge->RecomputeDirty(state_, disk_interface_, err)) + if (!scan_.RecomputeDirty(in_edge, err)) return false; if (in_edge->outputs_ready()) return true; // Nothing to do. @@ -728,7 +761,7 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { // The rule command did not change the output. Propagate the clean // state through the build graph. // Note that this also applies to nonexistent outputs (mtime == 0). - plan_.CleanNode(log_, *i); + plan_.CleanNode(&scan_, *i); node_cleaned = true; } } @@ -767,6 +800,7 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { int start_time, end_time; status_->BuildEdgeFinished(edge, success, output, &start_time, &end_time); - if (success && log_) - log_->RecordCommand(edge, start_time, end_time, restat_mtime); + if (success && scan_.build_log()) + scan_.build_log()->RecordCommand(edge, start_time, end_time, restat_mtime); } + diff --git a/src/build.h b/src/build.h index c9ee4ac..3e7a144 100644 --- a/src/build.h +++ b/src/build.h @@ -21,9 +21,11 @@ #include <queue> #include <vector> #include <memory> -using namespace std; +#include <cstdio> +#include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" +#include "metrics.h" #include "util.h" // int64_t struct BuildLog; @@ -58,7 +60,7 @@ struct Plan { void EdgeFinished(Edge* edge); /// Clean the given node during the build. - void CleanNode(BuildLog* build_log, Node* node); + void CleanNode(DependencyScan* scan, Node* node); /// Number of edges with commands to run. int command_edge_count() const { return command_edges_; } @@ -118,7 +120,8 @@ struct BuildConfig { /// Builder wraps the build process: starting commands, updating status. struct Builder { - Builder(State* state, const BuildConfig& config); + Builder(State* state, const BuildConfig& config, + BuildLog* log, DiskInterface* disk_interface); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -140,15 +143,21 @@ struct Builder { bool StartEdge(Edge* edge, string* err); void FinishEdge(Edge* edge, bool success, const string& output); + /// Used for tests. + void SetBuildLog(BuildLog* log) { + scan_.set_build_log(log); + } + State* state_; const BuildConfig& config_; Plan plan_; - DiskInterface* disk_interface_; auto_ptr<CommandRunner> command_runner_; BuildStatus* status_; - BuildLog* log_; private: + DiskInterface* disk_interface_; + DependencyScan scan_; + // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. Builder(const Builder &other); // DO NOT IMPLEMENT void operator=(const Builder &other); // DO NOT IMPLEMENT @@ -191,9 +200,43 @@ struct BuildStatus { /// The custom progress status format to use. const char* progress_status_format_; + struct RateInfo { + RateInfo() : last_update_(0), rate_(-1) {} + + double rate() const { return rate_; } + int last_update() const { return last_update_; } + void Restart() { return stopwatch_.Restart(); } + + double UpdateRate(int edges, int update_hint) { + if (update_hint != last_update_) { + rate_ = edges / stopwatch_.Elapsed() + 0.5; + last_update_ = update_hint; + } + return rate_; + } + + template<class T> + void snprinfRate(T buf, const char* format) { + if (rate_ == -1) + snprintf(buf, sizeof(buf), "?"); + else + snprintf(buf, sizeof(buf), format, rate_); + } + + private: + Stopwatch stopwatch_; + int last_update_; + double rate_; + }; + + mutable RateInfo overall_rate_; + mutable RateInfo current_rate_; + const int current_rate_average_count_; + #ifdef _WIN32 void* console_; #endif }; #endif // NINJA_BUILD_H_ + diff --git a/src/build_log.cc b/src/build_log.cc index 02a9fb5..a633892 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -49,7 +49,7 @@ const int kCurrentVersion = 5; #define BIG_CONSTANT(x) (x##LLU) #endif // !defined(_MSC_VER) inline -uint64_t MurmurHash64A(const void* key, int len) { +uint64_t MurmurHash64A(const void* key, size_t len) { static const uint64_t seed = 0xDECAFBADDECAFBADull; const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); const int r = 47; @@ -58,11 +58,11 @@ uint64_t MurmurHash64A(const void* key, int len) { const uint64_t * end = data + (len/8); while(data != end) { uint64_t k = *data++; - k *= m; - k ^= k >> r; - k *= m; + k *= m; + k ^= k >> r; + k *= m; h ^= k; - h *= m; + h *= m; } const unsigned char* data2 = (const unsigned char*)data; switch(len & 7) @@ -80,7 +80,7 @@ uint64_t MurmurHash64A(const void* key, int len) { h *= m; h ^= h >> r; return h; -} +} #undef BIG_CONSTANT @@ -92,16 +92,13 @@ uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { } BuildLog::BuildLog() - : log_file_(NULL), config_(NULL), needs_recompaction_(false) {} + : log_file_(NULL), needs_recompaction_(false) {} BuildLog::~BuildLog() { Close(); } bool BuildLog::OpenForWrite(const string& path, string* err) { - if (config_ && config_->dry_run) - return true; // Do nothing, report success. - if (needs_recompaction_) { Close(); if (!Recompact(path, err)) @@ -136,14 +133,14 @@ void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, for (vector<Node*>::iterator out = edge->outputs_.begin(); out != edge->outputs_.end(); ++out) { const string& path = (*out)->path(); - Log::iterator i = log_.find(path); + Entries::iterator i = entries_.find(path); LogEntry* log_entry; - if (i != log_.end()) { + if (i != entries_.end()) { log_entry = i->second; } else { log_entry = new LogEntry; log_entry->output = path; - log_.insert(Log::value_type(log_entry->output, log_entry)); + entries_.insert(Entries::value_type(log_entry->output, log_entry)); } log_entry->command_hash = LogEntry::HashCommand(command); log_entry->start_time = start_time; @@ -164,7 +161,9 @@ void BuildLog::Close() { class LineReader { public: explicit LineReader(FILE* file) - : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {} + : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { + memset(buf_, 0, sizeof(buf_)); + } // Reads a \n-terminated line from the file passed to the constructor. // On return, *line_start points to the beginning of the next line, and @@ -233,9 +232,13 @@ bool BuildLog::Load(const string& path, string* err) { sscanf(line_start, kFileSignature, &log_version); if (log_version < kOldestSupportedVersion) { - *err = "unable to extract version from build log, perhaps due to " - "being too old; you must clobber your build output and rebuild"; - return false; + *err = ("build log version invalid, perhaps due to being too old; " + "starting over"); + fclose(file); + unlink(path.c_str()); + // Don't report this as a failure. An empty build log will cause + // us to rebuild the outputs anyway. + return true; } } @@ -280,13 +283,13 @@ bool BuildLog::Load(const string& path, string* err) { end = line_end; LogEntry* entry; - Log::iterator i = log_.find(output); - if (i != log_.end()) { + Entries::iterator i = entries_.find(output); + if (i != entries_.end()) { entry = i->second; } else { entry = new LogEntry; entry->output = output; - log_.insert(Log::value_type(entry->output, entry)); + entries_.insert(Entries::value_type(entry->output, entry)); ++unique_entry_count; } ++total_entry_count; @@ -325,8 +328,8 @@ bool BuildLog::Load(const string& path, string* err) { } BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { - Log::iterator i = log_.find(path); - if (i != log_.end()) + Entries::iterator i = entries_.find(path); + if (i != entries_.end()) return i->second; return NULL; } @@ -353,7 +356,7 @@ bool BuildLog::Recompact(const string& path, string* err) { return false; } - for (Log::iterator i = log_.begin(); i != log_.end(); ++i) { + for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { WriteEntry(f, *i->second); } diff --git a/src/build_log.h b/src/build_log.h index d3994ff..4141ff3 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -22,24 +22,21 @@ using namespace std; #include "hash_map.h" #include "timestamp.h" -#include "util.h" +#include "util.h" // uint64_t -struct BuildConfig; struct Edge; /// Store a log of every command ran for every build. /// It has a few uses: /// -/// 1) historical command lines for output files, so we know +/// 1) (hashes of) command lines for existing output files, so we know /// when we need to rebuild due to the command changing -/// 2) historical timing information -/// 3) maybe we can generate some sort of build overview output -/// from it +/// 2) timing information, perhaps for generating reports +/// 3) restat information struct BuildLog { BuildLog(); ~BuildLog(); - void SetConfig(BuildConfig* config) { config_ = config; } bool OpenForWrite(const string& path, string* err); void RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp restat_mtime = 0); @@ -74,13 +71,12 @@ struct BuildLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); - typedef ExternalStringHashMap<LogEntry*>::Type Log; - const Log& log() const { return log_; } + typedef ExternalStringHashMap<LogEntry*>::Type Entries; + const Entries& entries() const { return entries_; } private: - Log log_; + Entries entries_; FILE* log_file_; - BuildConfig* config_; bool needs_recompaction_; }; diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index 02f4c60..a09beb8 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -20,6 +20,11 @@ #include "manifest_parser.h" #include "state.h" #include "util.h" +#include "metrics.h" + +#ifndef _WIN32 +#include <unistd.h> +#endif const char kTestFilename[] = "BuildLogPerfTest-tempfile"; diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 9fb42c9..a6c2a86 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -20,9 +20,7 @@ #ifdef _WIN32 #include <fcntl.h> #include <share.h> -#endif - -#ifdef linux +#else #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> @@ -32,6 +30,8 @@ const char kTestFilename[] = "BuildLogTest-tempfile"; struct BuildLogTest : public StateTestWithBuiltinRules { virtual void SetUp() { + // In case a crashing test left a stale file behind. + unlink(kTestFilename); } virtual void TearDown() { unlink(kTestFilename); @@ -55,8 +55,8 @@ TEST_F(BuildLogTest, WriteRead) { EXPECT_TRUE(log2.Load(kTestFilename, &err)); ASSERT_EQ("", err); - ASSERT_EQ(2u, log1.log().size()); - ASSERT_EQ(2u, log2.log().size()); + ASSERT_EQ(2u, log1.entries().size()); + ASSERT_EQ(2u, log2.entries().size()); BuildLog::LogEntry* e1 = log1.LookupByOutput("out"); ASSERT_TRUE(e1); BuildLog::LogEntry* e2 = log2.LookupByOutput("out"); @@ -131,8 +131,16 @@ TEST_F(BuildLogTest, Truncate) { ASSERT_GT(statbuf.st_size, 0); // For all possible truncations of the input file, assert that we don't - // crash or report an error when parsing. + // crash when parsing. for (off_t size = statbuf.st_size; size > 0; --size) { + BuildLog log2; + string err; + EXPECT_TRUE(log2.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + log2.RecordCommand(state_.edges_[0], 15, 18); + log2.RecordCommand(state_.edges_[1], 20, 25); + log2.Close(); + #ifndef _WIN32 ASSERT_EQ(0, truncate(kTestFilename, size)); #else @@ -142,9 +150,9 @@ TEST_F(BuildLogTest, Truncate) { _close(fh); #endif - BuildLog log2; - EXPECT_TRUE(log2.Load(kTestFilename, &err)); - ASSERT_EQ("", err); + BuildLog log3; + err.clear(); + ASSERT_TRUE(log3.Load(kTestFilename, &err) || !err.empty()); } } @@ -156,7 +164,7 @@ TEST_F(BuildLogTest, ObsoleteOldVersion) { string err; BuildLog log; - EXPECT_FALSE(log.Load(kTestFilename, &err)); + EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_NE(err.find("version"), string::npos); } diff --git a/src/build_test.cc b/src/build_test.cc index a3f345b..859e758 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -178,9 +178,9 @@ TEST_F(PlanTest, DependencyCycle) { struct BuildTest : public StateTestWithBuiltinRules, public CommandRunner { - BuildTest() : config_(MakeConfig()), builder_(&state_, config_), now_(1), - last_command_(NULL), status_(config_) { - builder_.disk_interface_ = &fs_; + BuildTest() : config_(MakeConfig()), + builder_(&state_, config_, NULL, &fs_), + now_(1), last_command_(NULL), status_(config_) { builder_.command_runner_.reset(this); AssertParse(&state_, "build cat1: cat in1\n" @@ -212,11 +212,10 @@ struct BuildTest : public StateTestWithBuiltinRules, } BuildConfig config_; + VirtualFileSystem fs_; Builder builder_; int now_; - VirtualFileSystem fs_; - vector<string> commands_ran_; Edge* last_command_; BuildStatus status_; @@ -719,12 +718,37 @@ TEST_F(BuildTest, SwallowFailuresLimit) { struct BuildWithLogTest : public BuildTest { BuildWithLogTest() { - state_.build_log_ = builder_.log_ = &build_log_; + builder_.SetBuildLog(&build_log_); } BuildLog build_log_; }; +TEST_F(BuildWithLogTest, NotInLogButOnDisk) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n" +" command = cc\n" +"build out1: cc in\n")); + + // Create input/output that would be considered up to date when + // not considering the command line hash. + fs_.Create("in", now_, ""); + fs_.Create("out1", now_, ""); + string err; + + // Because it's not in the log, it should not be up-to-date until + // we build again. + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + commands_ran_.clear(); + state_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + TEST_F(BuildWithLogTest, RestatTest) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule true\n" @@ -745,9 +769,22 @@ TEST_F(BuildWithLogTest, RestatTest) { fs_.Create("in", now_, ""); + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause out3 to rebuild + // regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out3", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + commands_ran_.clear(); + state_.Reset(); + + now_++; + + fs_.Create("in", now_, ""); // "cc" touches out1, so we should build out2. But because "true" does not // touch out2, we should cancel the build of out3. - string err; EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); @@ -792,10 +829,24 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { fs_.Create("in", now_, ""); fs_.Create("out2", now_, ""); + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause out2 to rebuild + // regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + commands_ran_.clear(); + state_.Reset(); + + now_++; + fs_.Create("in", now_, ""); + fs_.Create("out2", now_, ""); + // Run a build, expect only the first command to run. // It doesn't touch its output (due to being the "true" command), so // we shouldn't run the dependent build. - string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index 6248937..59bd18f 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -16,6 +16,7 @@ #include <string.h> #include "util.h" +#include "metrics.h" const char kPath[] = "../../third_party/WebKit/Source/WebCore/" @@ -26,7 +27,7 @@ int main() { string err; char buf[200]; - int len = strlen(kPath); + size_t len = strlen(kPath); strcpy(buf, kPath); for (int j = 0; j < 5; ++j) { diff --git a/src/clean.h b/src/clean.h index 4d9b4e6..5938dff 100644 --- a/src/clean.h +++ b/src/clean.h @@ -14,7 +14,6 @@ #ifndef NINJA_CLEAN_H_ #define NINJA_CLEAN_H_ -#pragma once #include <set> #include <string> diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 54b934c..6887c91 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -56,8 +56,8 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 0, 0, 0, - 0, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 0, 0, 128, 0, 0, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 128, @@ -84,27 +84,35 @@ bool DepfileParser::Parse(string* content, string* err) { }; yych = *in; - if (yych <= '[') { + if (yych <= 'Z') { if (yych <= '*') { if (yych <= 0x00) goto yy6; if (yych <= '\'') goto yy8; if (yych <= ')') goto yy4; goto yy8; } else { - if (yych <= ':') goto yy4; - if (yych <= '@') goto yy8; - if (yych <= 'Z') goto yy4; - goto yy8; + if (yych <= '<') { + if (yych <= ':') goto yy4; + goto yy8; + } else { + if (yych <= '=') goto yy4; + if (yych <= '?') goto yy8; + goto yy4; + } } } else { - if (yych <= '`') { - if (yych <= '\\') goto yy2; - if (yych == '_') goto yy4; - goto yy8; + if (yych <= '_') { + if (yych == '\\') goto yy2; + if (yych <= '^') goto yy8; + goto yy4; } else { - if (yych <= 'z') goto yy4; - if (yych == '~') goto yy4; - goto yy8; + if (yych <= 'z') { + if (yych <= '`') goto yy8; + goto yy4; + } else { + if (yych == '~') goto yy4; + goto yy8; + } } } yy2: @@ -141,7 +149,7 @@ yy4: yy5: { // Got a span of plain text. - int len = in - start; + int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. if (out < start) memmove(out, start, len); @@ -183,7 +191,7 @@ yy13: } - int len = out - filename; + int len = (int)(out - filename); const bool is_target = parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 8c415b9..1d4a177 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -68,9 +68,9 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~()-]+ { + [a-zA-Z0-9+,/_:.~()@=-]+ { // Got a span of plain text. - int len = in - start; + int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. if (out < start) memmove(out, start, len); @@ -88,7 +88,7 @@ bool DepfileParser::Parse(string* content, string* err) { */ } - int len = out - filename; + int len = (int)(out - filename); const bool is_target = parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index fd76ae7..93d42db 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -106,12 +106,17 @@ TEST_F(DepfileParserTest, Escapes) { TEST_F(DepfileParserTest, SpecialChars) { string err; EXPECT_TRUE(Parse( -"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h:", +"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" +" en@quot.header~ t+t-x=1", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", parser_.out_.AsString()); - ASSERT_EQ(0u, parser_.ins_.size()); + ASSERT_EQ(2u, parser_.ins_.size()); + EXPECT_EQ("en@quot.header~", + parser_.ins_[0].AsString()); + EXPECT_EQ("t+t-x=1", + parser_.ins_[1].AsString()); } TEST_F(DepfileParserTest, UnifyMultipleOutputs) { @@ -130,16 +135,3 @@ TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { string err; EXPECT_FALSE(Parse("foo bar: x y z", &err)); } - -TEST_F(DepfileParserTest, Tilde) { - string err; - EXPECT_TRUE(Parse( -"foo~.o: foo~.c", - &err)); - ASSERT_EQ("", err); - EXPECT_EQ("foo~.o", - parser_.out_.AsString()); - ASSERT_EQ(1u, parser_.ins_.size()); - EXPECT_EQ("foo~.c", - parser_.ins_[0].AsString()); -} diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 74f33c4..515ff59 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -18,9 +18,11 @@ #include <stdio.h> #include <string.h> #include <sys/stat.h> +#include <sys/types.h> #ifdef _WIN32 #include <windows.h> +#include <direct.h> // _mkdir #endif #include "util.h" @@ -42,6 +44,14 @@ string DirName(const string& path) { return path.substr(0, slash_pos); } +int MakeDir(const string& path) { +#ifdef _WIN32 + return _mkdir(path.c_str()); +#else + return mkdir(path.c_str(), 0777); +#endif +} + } // namespace // DiskInterface --------------------------------------------------------------- diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 985e991..32fe9cb 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -105,6 +105,8 @@ TEST_F(DiskInterfaceTest, RemoveFile) { struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { + StatTest() : scan_(&state_, NULL, this) {} + // DiskInterface implementation. virtual TimeStamp Stat(const string& path); virtual bool WriteFile(const string& path, const string& contents) { @@ -124,6 +126,7 @@ struct StatTest : public StateTestWithBuiltinRules, return 0; } + DependencyScan scan_; map<string, TimeStamp> mtimes_; vector<string> stats_; }; @@ -143,7 +146,7 @@ TEST_F(StatTest, Simple) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(2u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_EQ("in", stats_[1]); @@ -157,7 +160,7 @@ TEST_F(StatTest, TwoStep) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(3u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_TRUE(GetNode("out")->dirty()); @@ -175,7 +178,7 @@ TEST_F(StatTest, Tree) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(1u + 6u, stats_.size()); ASSERT_EQ("mid1", stats_[1]); ASSERT_TRUE(GetNode("mid1")->dirty()); @@ -194,7 +197,7 @@ TEST_F(StatTest, Middle) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_FALSE(GetNode("in")->dirty()); ASSERT_TRUE(GetNode("mid")->dirty()); ASSERT_TRUE(GetNode("out")->dirty()); diff --git a/src/getopt.c b/src/getopt.c index 1e3c09d..75ef99c 100644 --- a/src/getopt.c +++ b/src/getopt.c @@ -91,10 +91,6 @@ gpietsch@comcast.net #include "getopt.h" #endif -#ifdef _WIN32 -#pragma warning(disable: 4701) -#endif - /* macros */ /* types */ @@ -159,7 +155,7 @@ getopt_internal (int argc, char **argv, char *shortopts, char *possible_arg = NULL; int longopt_match = -1; int has_arg = -1; - char *cp; + char *cp = NULL; int arg_next = 0; /* first, deal with silly parameters and easy stuff */ @@ -255,7 +251,7 @@ getopt_internal (int argc, char **argv, char *shortopts, longopts[optindex].name, match_chars) == 0) { /* do we have an exact match? */ - if (match_chars == (int) (strlen (longopts[optindex].name))) + if (match_chars == strlen (longopts[optindex].name)) { longopt_match = optindex; break; diff --git a/src/graph.cc b/src/graph.cc index 071a3b6..6ae324d 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -32,23 +32,27 @@ bool Node::Stat(DiskInterface* disk_interface) { return mtime_ > 0; } -bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, - string* err) { +bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { bool dirty = false; - outputs_ready_ = true; - - if (!rule_->depfile().empty()) { - if (!LoadDepFile(state, disk_interface, err)) - return false; + edge->outputs_ready_ = true; + + if (!edge->rule_->depfile().empty()) { + if (!LoadDepFile(edge, err)) { + if (!err->empty()) + return false; + EXPLAIN("Edge targets are dirty because depfile '%s' is missing", + edge->EvaluateDepFile().c_str()); + dirty = true; + } } // Visit all inputs; we're dirty if any of the inputs are dirty. - TimeStamp most_recent_input = 1; - Node* most_recent_node = NULL; - for (vector<Node*>::iterator i = inputs_.begin(); i != inputs_.end(); ++i) { - if ((*i)->StatIfNecessary(disk_interface)) { - if (Edge* edge = (*i)->in_edge()) { - if (!edge->RecomputeDirty(state, disk_interface, err)) + Node* most_recent_input = NULL; + for (vector<Node*>::iterator i = edge->inputs_.begin(); + i != edge->inputs_.end(); ++i) { + if ((*i)->StatIfNecessary(disk_interface_)) { + if (Edge* in_edge = (*i)->in_edge()) { + if (!RecomputeDirty(in_edge, err)) return false; } else { // This input has no in-edge; it is dirty if it is missing. @@ -59,21 +63,20 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, } // If an input is not ready, neither are our outputs. - if (Edge* edge = (*i)->in_edge()) { - if (!edge->outputs_ready_) - outputs_ready_ = false; + if (Edge* in_edge = (*i)->in_edge()) { + if (!in_edge->outputs_ready_) + edge->outputs_ready_ = false; } - if (!is_order_only(i - inputs_.begin())) { + if (!edge->is_order_only(i - edge->inputs_.begin())) { // If a regular input is dirty (or missing), we're dirty. // Otherwise consider mtime. if ((*i)->dirty()) { EXPLAIN("%s is dirty", (*i)->path().c_str()); dirty = true; } else { - if ((*i)->mtime() > most_recent_input) { - most_recent_input = (*i)->mtime(); - most_recent_node = *i; + if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) { + most_recent_input = *i; } } } @@ -82,13 +85,12 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // We may also be dirty due to output state: missing outputs, out of // date outputs, etc. Visit all outputs and determine whether they're dirty. if (!dirty) { - BuildLog* build_log = state ? state->build_log_ : 0; - string command = EvaluateCommand(true); + string command = edge->EvaluateCommand(true); - for (vector<Node*>::iterator i = outputs_.begin(); - i != outputs_.end(); ++i) { - (*i)->StatIfNecessary(disk_interface); - if (RecomputeOutputDirty(build_log, most_recent_input, most_recent_node, command, *i)) { + for (vector<Node*>::iterator i = edge->outputs_.begin(); + i != edge->outputs_.end(); ++i) { + (*i)->StatIfNecessary(disk_interface_); + if (RecomputeOutputDirty(edge, most_recent_input, command, *i)) { dirty = true; break; } @@ -97,30 +99,32 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // Finally, visit each output to mark off that we've visited it, and update // their dirty state if necessary. - for (vector<Node*>::iterator i = outputs_.begin(); i != outputs_.end(); ++i) { - (*i)->StatIfNecessary(disk_interface); + for (vector<Node*>::iterator i = edge->outputs_.begin(); + i != edge->outputs_.end(); ++i) { + (*i)->StatIfNecessary(disk_interface_); if (dirty) (*i)->MarkDirty(); } - // If we're dirty, our outputs are normally not ready. (It's possible to be - // clean but still not be ready in the presence of order-only inputs.) - // But phony edges with no inputs have nothing to do, so are always ready. - if (dirty && !(is_phony() && inputs_.empty())) - outputs_ready_ = false; + // If an edge is dirty, its outputs are normally not ready. (It's + // possible to be clean but still not be ready in the presence of + // order-only inputs.) + // But phony edges with no inputs have nothing to do, so are always + // ready. + if (dirty && !(edge->is_phony() && edge->inputs_.empty())) + edge->outputs_ready_ = false; return true; } -bool Edge::RecomputeOutputDirty(BuildLog* build_log, - TimeStamp most_recent_input, - Node* most_recent_node, - const string& command, - Node* output) { - if (is_phony()) { +bool DependencyScan::RecomputeOutputDirty(Edge* edge, + Node* most_recent_input, + const string& command, + Node* output) { + if (edge->is_phony()) { // Phony edges don't write any output. Outputs are only dirty if // there are no inputs and we're missing the output. - return inputs_.empty() && !output->exists(); + return edge->inputs_.empty() && !output->exists(); } BuildLog::LogEntry* entry = 0; @@ -132,22 +136,24 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, } // Dirty if the output is older than the input. - if (output->mtime() < most_recent_input) { + if (most_recent_input && output->mtime() < most_recent_input->mtime()) { // If this is a restat rule, we may have cleaned the output with a restat // rule in a previous run and stored the most recent input mtime in the // build log. Use that mtime instead, so that the file will only be // considered dirty if an input was modified since the previous run. - if (rule_->restat() && build_log && - (entry = build_log->LookupByOutput(output->path()))) { - if (entry->restat_mtime < most_recent_input) { - EXPLAIN("restat of output %s older than inputs", output->path().c_str()); + TimeStamp most_recent_stamp = most_recent_input->mtime(); + if (edge->rule_->restat() && build_log() && + (entry = build_log()->LookupByOutput(output->path()))) { + if (entry->restat_mtime < most_recent_stamp) { + EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)", + output->path().c_str(), most_recent_input->path().c_str(), + entry->restat_mtime, most_recent_stamp); return true; } } else { EXPLAIN("output %s older than most recent input %s (%d vs %d)", - output->path().c_str(), - most_recent_node ? most_recent_node->path().c_str() : "", - output->mtime(), most_recent_input); + output->path().c_str(), most_recent_input->path().c_str(), + output->mtime(), most_recent_stamp); return true; } } @@ -155,10 +161,15 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. - if (!rule_->generator() && build_log && - (entry || (entry = build_log->LookupByOutput(output->path())))) { - if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { - EXPLAIN("command line changed for %s", output->path().c_str()); + if (!edge->rule_->generator() && build_log()) { + if (entry || (entry = build_log()->LookupByOutput(output->path()))) { + if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { + EXPLAIN("command line changed for %s", output->path().c_str()); + return true; + } + } + if (!entry) { + EXPLAIN("command line not found in log for %s", output->path().c_str()); return true; } } @@ -259,15 +270,15 @@ string Edge::GetRspFileContent() { return rule_->rspfile_content().Evaluate(&env); } -bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, - string* err) { +bool DependencyScan::LoadDepFile(Edge* edge, string* err) { METRIC_RECORD("depfile load"); - string path = EvaluateDepFile(); - string content = disk_interface->ReadFile(path, err); + string path = edge->EvaluateDepFile(); + string content = disk_interface_->ReadFile(path, err); if (!err->empty()) return false; + // On a missing depfile: return false and empty *err. if (content.empty()) - return true; + return false; DepfileParser depfile; string depfile_err; @@ -276,18 +287,21 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, return false; } - // Check that this depfile matches our output. - StringPiece opath = StringPiece(outputs_[0]->path()); + // Check that this depfile matches the edge's output. + Node* first_output = edge->outputs_[0]; + StringPiece opath = StringPiece(first_output->path()); if (opath != depfile.out_) { *err = "expected depfile '" + path + "' to mention '" + - outputs_[0]->path() + "', got '" + depfile.out_.AsString() + "'"; + first_output->path() + "', got '" + depfile.out_.AsString() + "'"; return false; } - inputs_.insert(inputs_.end() - order_only_deps_, depfile.ins_.size(), 0); - implicit_deps_ += depfile.ins_.size(); + // Preallocate space in edge->inputs_ to be filled in below. + edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, + depfile.ins_.size(), 0); + edge->implicit_deps_ += depfile.ins_.size(); vector<Node*>::iterator implicit_dep = - inputs_.end() - order_only_deps_ - depfile.ins_.size(); + edge->inputs_.end() - edge->order_only_deps_ - depfile.ins_.size(); // Add all its in-edges. for (vector<StringPiece>::iterator i = depfile.ins_.begin(); @@ -295,15 +309,15 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err)) return false; - Node* node = state->GetNode(*i); + Node* node = state_->GetNode(*i); *implicit_dep = node; - node->AddOutEdge(this); + node->AddOutEdge(edge); // If we don't have a edge that generates this input already, // create one; this makes us not abort if the input is missing, // but instead will rebuild in that circumstance. if (!node->in_edge()) { - Edge* phony_edge = state->AddEdge(&State::kPhonyRule); + Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); node->set_in_edge(phony_edge); phony_edge->outputs_.push_back(node); diff --git a/src/graph.h b/src/graph.h index 0ef4f3f..272fcb9 100644 --- a/src/graph.h +++ b/src/graph.h @@ -111,12 +111,14 @@ struct Rule { bool restat() const { return restat_; } const EvalString& command() const { return command_; } - EvalString& command() { return command_; } const EvalString& description() const { return description_; } const EvalString& depfile() const { return depfile_; } const EvalString& rspfile() const { return rspfile_; } const EvalString& rspfile_content() const { return rspfile_content_; } + /// Used by a test. + void set_command(const EvalString& command) { command_ = command; } + private: // Allow the parsers to reach into this object and fill out its fields. friend struct ManifestParser; @@ -142,39 +144,25 @@ struct Edge { Edge() : rule_(NULL), env_(NULL), outputs_ready_(false), implicit_deps_(0), order_only_deps_(0) {} - /// Examine inputs, outputs, and command lines to judge whether this edge - /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| - /// state accordingly. - /// Returns false on failure. - bool RecomputeDirty(State* state, DiskInterface* disk_interface, string* err); - - /// Recompute whether a given single output should be marked dirty. - /// Returns true if so. - bool RecomputeOutputDirty(BuildLog* build_log, TimeStamp most_recent_input, - Node* most_recent_node, const string& command, - Node* output); - /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; /// Expand all variables in a command and return it as a string. - /// If incl_rsp_file is enabled, the string will also contain the + /// If incl_rsp_file is enabled, the string will also contain the /// full contents of a response file (if applicable) string EvaluateCommand(bool incl_rsp_file = false); // XXX move to env, take env ptr string EvaluateDepFile(); string GetDescription(); - + /// Does the edge use a response file? bool HasRspFile(); - + /// Get the path to the response file string GetRspFile(); /// Get the contents of the response file string GetRspFileContent(); - bool LoadDepFile(State* state, DiskInterface* disk_interface, string* err); - void Dump(const char* prefix="") const; const Rule* rule_; @@ -199,15 +187,50 @@ struct Edge { // pointer...) int implicit_deps_; int order_only_deps_; - bool is_implicit(int index) { - return index >= ((int)inputs_.size()) - order_only_deps_ - implicit_deps_ && + bool is_implicit(size_t index) { + return index >= inputs_.size() - order_only_deps_ - implicit_deps_ && !is_order_only(index); } - bool is_order_only(int index) { - return index >= ((int)inputs_.size()) - order_only_deps_; + bool is_order_only(size_t index) { + return index >= inputs_.size() - order_only_deps_; } bool is_phony() const; }; + +/// DependencyScan manages the process of scanning the files in a graph +/// and updating the dirty/outputs_ready state of all the nodes and edges. +struct DependencyScan { + DependencyScan(State* state, BuildLog* build_log, + DiskInterface* disk_interface) + : state_(state), build_log_(build_log), + disk_interface_(disk_interface) {} + + /// Examine inputs, outputs, and command lines to judge whether an edge + /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| + /// state accordingly. + /// Returns false on failure. + bool RecomputeDirty(Edge* edge, string* err); + + /// Recompute whether a given single output should be marked dirty. + /// Returns true if so. + bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input, + const string& command, Node* output); + + bool LoadDepFile(Edge* edge, string* err); + + BuildLog* build_log() const { + return build_log_; + } + void set_build_log(BuildLog* log) { + build_log_ = log; + } + + private: + State* state_; + BuildLog* build_log_; + DiskInterface* disk_interface_; +}; + #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index 07a1936..5b25c2f 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -17,7 +17,10 @@ #include "test.h" struct GraphTest : public StateTestWithBuiltinRules { + GraphTest() : scan_(&state_, NULL, &fs_) {} + VirtualFileSystem fs_; + DependencyScan scan_; }; TEST_F(GraphTest, MissingImplicit) { @@ -28,7 +31,7 @@ TEST_F(GraphTest, MissingImplicit) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // A missing implicit dep *should* make the output dirty. @@ -46,7 +49,7 @@ TEST_F(GraphTest, ModifiedImplicit) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // A modified implicit dep should make the output dirty. @@ -66,7 +69,7 @@ TEST_F(GraphTest, FunkyMakefilePath) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // implicit.h has changed, though our depfile refers to it with a @@ -89,7 +92,7 @@ TEST_F(GraphTest, ExplicitImplicit) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // We have both an implicit and an explicit dep on implicit.h. @@ -110,7 +113,7 @@ TEST_F(GraphTest, PathWithCurrentDirectory) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -140,3 +143,47 @@ TEST_F(GraphTest, VarInOutQuoteSpaces) { EXPECT_EQ("cat nospace \"with space\" nospace2 > \"a b\"", edge->EvaluateCommand()); } + +// Regression test for https://github.com/martine/ninja/issues/380 +TEST_F(GraphTest, DepfileWithCanonicalizablePath) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule catdep\n" +" depfile = $out.d\n" +" command = cat $in > $out\n" +"build ./out.o: catdep ./foo.cc\n")); + fs_.Create("foo.cc", 1, ""); + fs_.Create("out.o.d", 1, "out.o: bar/../foo.cc\n"); + fs_.Create("out.o", 1, ""); + + Edge* edge = GetNode("out.o")->in_edge(); + string err; + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); + ASSERT_EQ("", err); + + EXPECT_FALSE(GetNode("out.o")->dirty()); +} + +// Regression test for https://github.com/martine/ninja/issues/404 +TEST_F(GraphTest, DepfileRemoved) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule catdep\n" +" depfile = $out.d\n" +" command = cat $in > $out\n" +"build ./out.o: catdep ./foo.cc\n")); + fs_.Create("foo.h", 1, ""); + fs_.Create("foo.cc", 1, ""); + fs_.Create("out.o.d", 2, "out.o: foo.h\n"); + fs_.Create("out.o", 2, ""); + + Edge* edge = GetNode("out.o")->in_edge(); + string err; + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(GetNode("out.o")->dirty()); + + state_.Reset(); + fs_.RemoveFile("out.o.d"); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(GetNode("out.o")->dirty()); +} diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc index 6736109..d0eabde 100644 --- a/src/hash_collision_bench.cc +++ b/src/hash_collision_bench.cc @@ -14,6 +14,11 @@ #include "build_log.h" +#include <algorithm> +using namespace std; + +#include <time.h> + int random(int low, int high) { return int(low + (rand() / double(RAND_MAX)) * (high - low) + 0.5); } @@ -22,7 +27,7 @@ void RandomCommand(char** s) { int len = random(5, 100); *s = new char[len]; for (int i = 0; i < len; ++i) - (*s)[i] = random(32, 127); + (*s)[i] = (char)random(32, 127); } int main() { @@ -32,7 +37,7 @@ int main() { char** commands = new char*[N]; pair<uint64_t, int>* hashes = new pair<uint64_t, int>[N]; - srand(time(NULL)); + srand((int)time(NULL)); for (int i = 0; i < N; ++i) { RandomCommand(&commands[i]); diff --git a/src/hash_map.h b/src/hash_map.h index 88c2681..9904fb8 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -19,7 +19,7 @@ // MurmurHash2, by Austin Appleby static inline -unsigned int MurmurHash2(const void* key, int len) { +unsigned int MurmurHash2(const void* key, size_t len) { static const unsigned int seed = 0xDECAFBAD; const unsigned int m = 0x5bd1e995; const int r = 24; diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc new file mode 100644 index 0000000..4bc8756 --- /dev/null +++ b/src/includes_normalize-win32.cc @@ -0,0 +1,115 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "includes_normalize.h" + +#include "string_piece.h" +#include "util.h" + +#include <algorithm> +#include <iterator> +#include <sstream> + +#include <windows.h> + +namespace { + +/// Return true if paths a and b are on the same Windows drive. +bool SameDrive(StringPiece a, StringPiece b) { + char a_absolute[_MAX_PATH]; + char b_absolute[_MAX_PATH]; + GetFullPathName(a.AsString().c_str(), sizeof(a_absolute), a_absolute, NULL); + GetFullPathName(b.AsString().c_str(), sizeof(b_absolute), b_absolute, NULL); + char a_drive[_MAX_DIR]; + char b_drive[_MAX_DIR]; + _splitpath(a_absolute, a_drive, NULL, NULL, NULL); + _splitpath(b_absolute, b_drive, NULL, NULL, NULL); + return _stricmp(a_drive, b_drive) == 0; +} + +} // anonymous namespace + +string IncludesNormalize::Join(const vector<string>& list, char sep) { + string ret; + for (size_t i = 0; i < list.size(); ++i) { + ret += list[i]; + if (i != list.size() - 1) + ret += sep; + } + return ret; +} + +vector<string> IncludesNormalize::Split(const string& input, char sep) { + vector<string> elems; + stringstream ss(input); + string item; + while (getline(ss, item, sep)) + elems.push_back(item); + return elems; +} + +string IncludesNormalize::ToLower(const string& s) { + string ret; + transform(s.begin(), s.end(), back_inserter(ret), ::tolower); + return ret; +} + +string IncludesNormalize::AbsPath(StringPiece s) { + char result[_MAX_PATH]; + GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + return result; +} + +string IncludesNormalize::Relativize(StringPiece path, const string& start) { + vector<string> start_list = Split(AbsPath(start), '\\'); + vector<string> path_list = Split(AbsPath(path), '\\'); + int i; + for (i = 0; i < static_cast<int>(min(start_list.size(), path_list.size())); + ++i) { + if (ToLower(start_list[i]) != ToLower(path_list[i])) + break; + } + + vector<string> rel_list; + for (int j = 0; j < static_cast<int>(start_list.size() - i); ++j) + rel_list.push_back(".."); + for (int j = i; j < static_cast<int>(path_list.size()); ++j) + rel_list.push_back(path_list[j]); + if (rel_list.size() == 0) + return "."; + return Join(rel_list, '\\'); +} + +string IncludesNormalize::Normalize(const string& input, + const char* relative_to) { + char copy[_MAX_PATH]; + size_t len = input.size(); + strncpy(copy, input.c_str(), input.size() + 1); + for (size_t j = 0; j < len; ++j) + if (copy[j] == '/') + copy[j] = '\\'; + string err; + if (!CanonicalizePath(copy, &len, &err)) { + Warning("couldn't canonicalize '%s: %s\n", input.c_str(), err.c_str()); + } + string curdir; + if (!relative_to) { + curdir = AbsPath("."); + relative_to = curdir.c_str(); + } + StringPiece partially_fixed(copy, len); + if (!SameDrive(partially_fixed, relative_to)) + return ToLower(partially_fixed.AsString()); + return ToLower(Relativize(partially_fixed, relative_to)); +} diff --git a/src/includes_normalize.h b/src/includes_normalize.h new file mode 100644 index 0000000..43527af --- /dev/null +++ b/src/includes_normalize.h @@ -0,0 +1,35 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> +#include <vector> +using namespace std; + +struct StringPiece; + +/// Utility functions for normalizing include paths on Windows. +/// TODO: this likely duplicates functionality of CanonicalizePath; refactor. +struct IncludesNormalize { + // Internal utilities made available for testing, maybe useful otherwise. + static string Join(const vector<string>& list, char sep); + static vector<string> Split(const string& input, char sep); + static string ToLower(const string& s); + static string AbsPath(StringPiece s); + static string Relativize(StringPiece path, const string& start); + + /// Normalize by fixing slashes style, fixing redundant .. and . and makes the + /// path relative to |relative_to|. Case is normalized to lowercase on + /// Windows too. + static string Normalize(const string& input, const char* relative_to); +}; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc new file mode 100644 index 0000000..77b5b3b --- /dev/null +++ b/src/includes_normalize_test.cc @@ -0,0 +1,98 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "includes_normalize.h" + +#include <gtest/gtest.h> + +#include "test.h" +#include "util.h" + +TEST(IncludesNormalize, Simple) { + EXPECT_EQ("b", IncludesNormalize::Normalize("a\\..\\b", NULL)); + EXPECT_EQ("b", IncludesNormalize::Normalize("a\\../b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\.\\b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); +} + +namespace { + +string GetCurDir() { + char buf[_MAX_PATH]; + _getcwd(buf, sizeof(buf)); + vector<string> parts = IncludesNormalize::Split(string(buf), '\\'); + return parts[parts.size() - 1]; +} + +} // namespace + +TEST(IncludesNormalize, WithRelative) { + string currentdir = IncludesNormalize::ToLower(GetCurDir()); + EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); + EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); + EXPECT_EQ(string("..\\") + currentdir + string("\\a"), + IncludesNormalize::Normalize("a", "../b")); + EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), + IncludesNormalize::Normalize("a/b", "../c")); + EXPECT_EQ("..\\..\\a", IncludesNormalize::Normalize("a", "b/c")); + EXPECT_EQ(".", IncludesNormalize::Normalize("a", "a")); +} + +TEST(IncludesNormalize, Case) { + EXPECT_EQ("b", IncludesNormalize::Normalize("Abc\\..\\b", NULL)); + EXPECT_EQ("bdef", IncludesNormalize::Normalize("Abc\\..\\BdEf", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\.\\b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\./b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\.\\B", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\./B", NULL)); +} + +TEST(IncludesNormalize, Join) { + vector<string> x; + EXPECT_EQ("", IncludesNormalize::Join(x, ':')); + x.push_back("alpha"); + EXPECT_EQ("alpha", IncludesNormalize::Join(x, ':')); + x.push_back("beta"); + x.push_back("gamma"); + EXPECT_EQ("alpha:beta:gamma", IncludesNormalize::Join(x, ':')); +} + +TEST(IncludesNormalize, Split) { + EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), ':')); + EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), ':')); + EXPECT_EQ("a:b:c", IncludesNormalize::Join(IncludesNormalize::Split("a/b/c", '/'), ':')); +} + +TEST(IncludesNormalize, ToLower) { + EXPECT_EQ("", IncludesNormalize::ToLower("")); + EXPECT_EQ("stuff", IncludesNormalize::ToLower("Stuff")); + EXPECT_EQ("stuff and things", IncludesNormalize::ToLower("Stuff AND thINGS")); + EXPECT_EQ("stuff 3and thin43gs", IncludesNormalize::ToLower("Stuff 3AND thIN43GS")); +} + +TEST(IncludesNormalize, DifferentDrive) { + EXPECT_EQ("stuff.h", + IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("p:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "c:\\vs08")); + EXPECT_EQ("p:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "D:\\stuff/things")); + EXPECT_EQ("p:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); + // TODO: this fails; fix it. + //EXPECT_EQ("P:\\wee\\stuff.h", + // IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", "D:\\stuff/things")); +} diff --git a/src/lexer.cc b/src/lexer.cc index b3efe22..5d7d185 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -30,7 +30,7 @@ bool Lexer::Error(const string& message, string* err) { context = p + 1; } } - int col = last_token_ ? last_token_ - context : 0; + int col = last_token_ ? (int)(last_token_ - context) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); @@ -90,24 +90,25 @@ const char* Lexer::TokenName(Token t) { return NULL; // not reached } -const char* Lexer::TokenErrorHint(Token t) { - switch (t) { - case ERROR: return ""; - case BUILD: return ""; - case COLON: return " ($ also escapes ':')"; - case DEFAULT: return ""; - case EQUALS: return ""; - case IDENT: return ""; - case INCLUDE: return ""; - case INDENT: return ""; - case NEWLINE: return ""; - case PIPE2: return ""; - case PIPE: return ""; - case RULE: return ""; - case SUBNINJA: return ""; - case TEOF: return ""; +const char* Lexer::TokenErrorHint(Token expected) { + switch (expected) { + case COLON: + return " ($ also escapes ':')"; + default: + return ""; + } +} + +string Lexer::DescribeLastError() { + if (last_token_) { + switch (last_token_[0]) { + case '\r': + return "carriage returns are not allowed, use newlines"; + case '\t': + return "tabs are not allowed, use spaces"; + } } - return ""; + return "lexing error"; } void Lexer::UnreadToken() { @@ -127,7 +128,7 @@ Lexer::Token Lexer::ReadToken() { unsigned int yyaccept = 0; static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 64, 64, 64, + 64, 64, 0, 64, 64, 0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 192, 64, 64, 64, 64, 64, 64, 64, @@ -216,7 +217,8 @@ yy3: yy4: yyaccept = 1; yych = *(q = ++p); - if (yych >= 0x01) goto yy60; + if (yych <= 0x00) goto yy5; + if (yych != '\r') goto yy60; yy5: { token = ERROR; break; } yy6: @@ -354,7 +356,9 @@ yy60: if (yybm[0+yych] & 64) { goto yy59; } - if (yych >= 0x01) goto yy62; + if (yych <= 0x00) goto yy61; + if (yych <= '\f') goto yy62; +yy61: p = q; if (yyaccept <= 0) { goto yy3; @@ -576,7 +580,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { unsigned char yych; static const unsigned char yybm[] = { 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 16, 128, 128, 128, 0, 128, 128, 128, @@ -609,24 +613,25 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { 128, 128, 128, 128, 128, 128, 128, 128, }; yych = *p; - if (yych <= '#') { + if (yych <= ' ') { if (yych <= '\n') { if (yych <= 0x00) goto yy96; if (yych >= '\n') goto yy92; } else { - if (yych == ' ') goto yy92; + if (yych == '\r') goto yy98; + if (yych >= ' ') goto yy92; } } else { - if (yych <= ':') { - if (yych <= '$') goto yy94; - if (yych >= ':') goto yy92; + if (yych <= '9') { + if (yych == '$') goto yy94; } else { + if (yych <= ':') goto yy92; if (yych == '|') goto yy92; } } ++p; yych = *p; - goto yy120; + goto yy121; yy91: { eval->AddText(StringPiece(start, p - start)); @@ -649,42 +654,43 @@ yy94: ++p; if ((yych = *p) <= '/') { if (yych <= ' ') { - if (yych == '\n') goto yy109; - if (yych <= 0x1F) goto yy98; - goto yy100; + if (yych == '\n') goto yy110; + if (yych <= 0x1F) goto yy99; + goto yy101; } else { if (yych <= '$') { - if (yych <= '#') goto yy98; - goto yy102; + if (yych <= '#') goto yy99; + goto yy103; } else { - if (yych == '-') goto yy104; - goto yy98; + if (yych == '-') goto yy105; + goto yy99; } } } else { if (yych <= '^') { if (yych <= ':') { - if (yych <= '9') goto yy104; - goto yy106; + if (yych <= '9') goto yy105; + goto yy107; } else { - if (yych <= '@') goto yy98; - if (yych <= 'Z') goto yy104; - goto yy98; + if (yych <= '@') goto yy99; + if (yych <= 'Z') goto yy105; + goto yy99; } } else { if (yych <= '`') { - if (yych <= '_') goto yy104; - goto yy98; + if (yych <= '_') goto yy105; + goto yy99; } else { - if (yych <= 'z') goto yy104; - if (yych <= '{') goto yy108; - goto yy98; + if (yych <= 'z') goto yy105; + if (yych <= '{') goto yy109; + goto yy99; } } } +yy95: { last_token_ = start; - return Error("lexing error", err); + return Error(DescribeLastError(), err); } yy96: ++p; @@ -693,83 +699,86 @@ yy96: return Error("unexpected EOF", err); } yy98: - ++p; + yych = *++p; + goto yy95; yy99: + ++p; +yy100: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy100: +yy101: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy102: +yy103: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy104: +yy105: ++p; yych = *p; - goto yy118; -yy105: + goto yy119; +yy106: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy106: +yy107: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy108: +yy109: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy112; + goto yy113; } - goto yy99; -yy109: + goto yy100; +yy110: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy109; + goto yy110; } { continue; } -yy112: +yy113: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy112; + goto yy113; } - if (yych == '}') goto yy115; + if (yych == '}') goto yy116; p = q; - goto yy99; -yy115: + goto yy100; +yy116: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy117: +yy118: ++p; yych = *p; -yy118: +yy119: if (yybm[0+yych] & 64) { - goto yy117; + goto yy118; } - goto yy105; -yy119: + goto yy106; +yy120: ++p; yych = *p; -yy120: +yy121: if (yybm[0+yych] & 128) { - goto yy119; + goto yy120; } goto yy91; } diff --git a/src/lexer.h b/src/lexer.h index 19008d7..03c59f2 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -49,9 +49,12 @@ struct Lexer { /// Return a human-readable form of a token, used in error messages. static const char* TokenName(Token t); - /// Return a human-readable token hint, used in error messages. - static const char* TokenErrorHint(Token t); + static const char* TokenErrorHint(Token expected); + + /// If the last token read was an ERROR token, provide more info + /// or the empty string. + string DescribeLastError(); /// Start parsing some input. void Start(StringPiece filename, StringPiece input); diff --git a/src/lexer.in.cc b/src/lexer.in.cc index e478921..7ae9c61 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -29,7 +29,7 @@ bool Lexer::Error(const string& message, string* err) { context = p + 1; } } - int col = last_token_ ? last_token_ - context : 0; + int col = last_token_ ? (int)(last_token_ - context) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); @@ -89,24 +89,25 @@ const char* Lexer::TokenName(Token t) { return NULL; // not reached } -const char* Lexer::TokenErrorHint(Token t) { - switch (t) { - case ERROR: return ""; - case BUILD: return ""; - case COLON: return " ($ also escapes ':')"; - case DEFAULT: return ""; - case EQUALS: return ""; - case IDENT: return ""; - case INCLUDE: return ""; - case INDENT: return ""; - case NEWLINE: return ""; - case PIPE2: return ""; - case PIPE: return ""; - case RULE: return ""; - case SUBNINJA: return ""; - case TEOF: return ""; +const char* Lexer::TokenErrorHint(Token expected) { + switch (expected) { + case COLON: + return " ($ also escapes ':')"; + default: + return ""; + } +} + +string Lexer::DescribeLastError() { + if (last_token_) { + switch (last_token_[0]) { + case '\r': + return "carriage returns are not allowed, use newlines"; + case '\t': + return "tabs are not allowed, use spaces"; + } } - return ""; + return "lexing error"; } void Lexer::UnreadToken() { @@ -130,7 +131,7 @@ Lexer::Token Lexer::ReadToken() { simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+; - [ ]*"#"[^\000\n]*"\n" { continue; } + [ ]*"#"[^\000\r\n]*"\n" { continue; } [ ]*[\n] { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } @@ -200,7 +201,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { for (;;) { start = p; /*!re2c - [^$ :\n|\000]+ { + [^$ :\r\n|\000]+ { eval->AddText(StringPiece(start, p - start)); continue; } @@ -248,7 +249,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { } [^] { last_token_ = start; - return Error("lexing error", err); + return Error(DescribeLastError(), err); } */ } diff --git a/src/lexer_test.cc b/src/lexer_test.cc index 5795e5e..e8a1642 100644 --- a/src/lexer_test.cc +++ b/src/lexer_test.cc @@ -85,3 +85,13 @@ TEST(Lexer, CommentEOF) { Lexer::Token token = lexer.ReadToken(); EXPECT_EQ(Lexer::ERROR, token); } + +TEST(Lexer, Tabs) { + // Verify we print a useful error on a disallowed character. + Lexer lexer(" \tfoobar"); + Lexer::Token token = lexer.ReadToken(); + EXPECT_EQ(Lexer::INDENT, token); + token = lexer.ReadToken(); + EXPECT_EQ(Lexer::ERROR, token); + EXPECT_EQ("tabs are not allowed, use spaces", lexer.DescribeLastError()); +} diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 057e12c..405e244 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -76,8 +76,9 @@ bool ManifestParser::Parse(const string& filename, const string& input, if (!ParseFileInclude(true, err)) return false; break; - case Lexer::ERROR: - return lexer_.Error("lexing error", err); + case Lexer::ERROR: { + return lexer_.Error(lexer_.DescribeLastError(), err); + } case Lexer::TEOF: return true; case Lexer::NEWLINE: diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 9c6644c..8b00efb 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -64,6 +64,20 @@ TEST_F(ParserTest, Rules) { EXPECT_EQ("[cat ][$in][ > ][$out]", rule->command().Serialize()); } +TEST_F(ParserTest, RuleAttributes) { + // Check that all of the allowed rule attributes are parsed ok. + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = a\n" +" depfile = a\n" +" description = a\n" +" generator = a\n" +" restat = a\n" +" rspfile = a\n" +" rspfile_content = a\n" +)); +} + TEST_F(ParserTest, IgnoreIndentedComments) { ASSERT_NO_FATAL_FAILURE(AssertParse( " #indented comment\n" @@ -682,3 +696,23 @@ TEST_F(ParserTest, UTF8) { " command = true\n" " description = compilaci\xC3\xB3\n")); } + +// We might want to eventually allow CRLF to be nice to Windows developers, +// but for now just verify we error out with a nice message. +TEST_F(ParserTest, CRLF) { + State state; + ManifestParser parser(&state, NULL); + string err; + + EXPECT_FALSE(parser.ParseTest("# comment with crlf\r\n", + &err)); + EXPECT_EQ("input:1: lexing error\n", + err); + + EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", + &err)); + EXPECT_EQ("input:2: carriage returns are not allowed, use newlines\n" + "bar = bar\r\n" + " ^ near here", + err); +} diff --git a/src/metrics.cc b/src/metrics.cc index fb44868..ca4f97a 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -72,6 +72,7 @@ int64_t TimerToMicros(int64_t dt) { } // anonymous namespace + ScopedMetric::ScopedMetric(Metric* metric) { metric_ = metric; if (!metric_) @@ -113,3 +114,12 @@ void Metrics::Report() { metric->count, avg, total); } } + +uint64_t Stopwatch::Now() const { + return TimerToMicros(HighResTimer()); +} + +int64_t GetTimeMillis() { + return TimerToMicros(HighResTimer()) / 1000; +} + diff --git a/src/metrics.h b/src/metrics.h index af6e9a2..f5ac0de 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -33,6 +33,7 @@ struct Metric { int64_t sum; }; + /// A scoped object for recording a metric across the body of a function. /// Used by the METRIC_RECORD macro. struct ScopedMetric { @@ -57,6 +58,30 @@ private: vector<Metric*> metrics_; }; +/// Get the current time as relative to some epoch. +/// Epoch varies between platforms; only useful for measuring elapsed +/// time. +int64_t GetTimeMillis(); + + +/// A simple stopwatch which retruns the time +// in seconds since Restart() was called +class Stopwatch +{ +public: + Stopwatch() : started_(0) {} + + /// Seconds since Restart() call + double Elapsed() const { return 1e-6 * static_cast<double>(Now() - started_); } + + void Restart() { started_ = Now(); } + +private: + uint64_t started_; + uint64_t Now() const; +}; + + /// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top /// of a function to get timing stats recorded for each call of the function. #define METRIC_RECORD(name) \ diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc new file mode 100644 index 0000000..c79ec0e --- /dev/null +++ b/src/minidump-win32.cc @@ -0,0 +1,88 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_BOOTSTRAP + +#include <windows.h> +#include <DbgHelp.h> + + +#include "util.h" + +typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( + IN HANDLE, + IN DWORD, + IN HANDLE, + IN MINIDUMP_TYPE, + IN CONST PMINIDUMP_EXCEPTION_INFORMATION, OPTIONAL + IN CONST PMINIDUMP_USER_STREAM_INFORMATION, OPTIONAL + IN CONST PMINIDUMP_CALLBACK_INFORMATION OPTIONAL + ); + +/// Creates a windows minidump in temp folder. +void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { + char temp_path[MAX_PATH]; + GetTempPath(sizeof(temp_path), temp_path); + char temp_file[MAX_PATH]; + sprintf(temp_file, "%s\\ninja_crash_dump_%d.dmp", + temp_path, GetCurrentProcessId()); + + // Delete any previous minidump of the same name. + DeleteFile(temp_file); + + // Load DbgHelp.dll dynamically, as library is not present on all + // Windows versions. + HMODULE dbghelp = LoadLibrary("dbghelp.dll"); + if (dbghelp == NULL) { + Error("failed to create minidump: LoadLibrary('dbghelp.dll'): %s", + GetLastErrorString().c_str()); + return; + } + + MiniDumpWriteDumpFunc mini_dump_write_dump = + (MiniDumpWriteDumpFunc)GetProcAddress(dbghelp, "MiniDumpWriteDump"); + if (mini_dump_write_dump == NULL) { + Error("failed to create minidump: GetProcAddress('MiniDumpWriteDump'): %s", + GetLastErrorString().c_str()); + return; + } + + HANDLE hFile = CreateFileA(temp_file, GENERIC_READ | GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == NULL) { + Error("failed to create minidump: CreateFileA(%s): %s", + temp_file, GetLastErrorString().c_str()); + return; + } + + MINIDUMP_EXCEPTION_INFORMATION mdei; + mdei.ThreadId = GetCurrentThreadId(); + mdei.ExceptionPointers = pep; + mdei.ClientPointers = FALSE; + MINIDUMP_TYPE mdt = (MINIDUMP_TYPE) (MiniDumpWithDataSegs | + MiniDumpWithHandleData); + + BOOL rv = mini_dump_write_dump(GetCurrentProcess(), GetCurrentProcessId(), + hFile, mdt, (pep != 0) ? &mdei : 0, 0, 0); + CloseHandle(hFile); + + if (!rv) { + Error("MiniDumpWriteDump failed: %s", GetLastErrorString().c_str()); + return; + } + + Warning("minidump created: %s", temp_file); +} + +#endif // NINJA_BOOTSTRAP diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc new file mode 100644 index 0000000..8e440fe --- /dev/null +++ b/src/msvc_helper-win32.cc @@ -0,0 +1,164 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include <string.h> +#include <windows.h> + +#include "includes_normalize.h" +#include "util.h" + +namespace { + +/// Return true if \a input ends with \a needle. +bool EndsWith(const string& input, const string& needle) { + return (input.size() >= needle.size() && + input.substr(input.size() - needle.size()) == needle); +} + +} // anonymous namespace + +// static +string CLWrapper::FilterShowIncludes(const string& line) { + static const char kMagicPrefix[] = "Note: including file: "; + const char* in = line.c_str(); + const char* end = in + line.size(); + + if (end - in > (int)sizeof(kMagicPrefix) - 1 && + memcmp(in, kMagicPrefix, sizeof(kMagicPrefix) - 1) == 0) { + in += sizeof(kMagicPrefix) - 1; + while (*in == ' ') + ++in; + return line.substr(in - line.c_str()); + } + return ""; +} + +// static +bool CLWrapper::IsSystemInclude(const string& path) { + // TODO: this is a heuristic, perhaps there's a better way? + return (path.find("program files") != string::npos || + path.find("microsoft visual studio") != string::npos); +} + +// static +bool CLWrapper::FilterInputFilename(const string& line) { + // TODO: other extensions, like .asm? + return EndsWith(line, ".c") || + EndsWith(line, ".cc") || + EndsWith(line, ".cxx") || + EndsWith(line, ".cpp"); +} + +int CLWrapper::Run(const string& command, string* extra_output) { + SECURITY_ATTRIBUTES security_attributes = {}; + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = TRUE; + + // Must be inheritable so subprocesses can dup to children. + HANDLE nul = CreateFile("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); + if (nul == INVALID_HANDLE_VALUE) + Fatal("couldn't open nul"); + + HANDLE stdout_read, stdout_write; + if (!CreatePipe(&stdout_read, &stdout_write, &security_attributes, 0)) + Win32Fatal("CreatePipe"); + + if (!SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0)) + Win32Fatal("SetHandleInformation"); + + PROCESS_INFORMATION process_info = {}; + STARTUPINFO startup_info = {}; + startup_info.cb = sizeof(STARTUPINFO); + startup_info.hStdInput = nul; + startup_info.hStdError = stdout_write; + startup_info.hStdOutput = stdout_write; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL, + /* inherit handles */ TRUE, 0, + env_block_, NULL, + &startup_info, &process_info)) { + Win32Fatal("CreateProcess"); + } + + if (!CloseHandle(nul) || + !CloseHandle(stdout_write)) { + Win32Fatal("CloseHandle"); + } + + // Read output of the subprocess and parse it. + string output; + DWORD read_len = 1; + while (read_len) { + char buf[64 << 10]; + read_len = 0; + if (!::ReadFile(stdout_read, buf, sizeof(buf), &read_len, NULL) && + GetLastError() != ERROR_BROKEN_PIPE) { + Win32Fatal("ReadFile"); + } + output.append(buf, read_len); + + // Loop over all lines in the output and process them. + for (;;) { + size_t ofs = output.find_first_of("\r\n"); + if (ofs == string::npos) + break; + string line = output.substr(0, ofs); + + string include = FilterShowIncludes(line); + if (!include.empty()) { + include = IncludesNormalize::Normalize(include, NULL); + if (!IsSystemInclude(include)) + includes_.push_back(include); + } else if (FilterInputFilename(line)) { + // Drop it. + // TODO: if we support compiling multiple output files in a single + // cl.exe invocation, we should stash the filename. + } else { + if (extra_output) { + extra_output->append(line); + extra_output->append("\n"); + } else { + printf("%s\n", line.c_str()); + } + } + + if (ofs < output.size() && output[ofs] == '\r') + ++ofs; + if (ofs < output.size() && output[ofs] == '\n') + ++ofs; + output = output.substr(ofs); + } + } + + if (WaitForSingleObject(process_info.hProcess, INFINITE) == WAIT_FAILED) + Win32Fatal("WaitForSingleObject"); + + DWORD exit_code = 0; + if (!GetExitCodeProcess(process_info.hProcess, &exit_code)) + Win32Fatal("GetExitCodeProcess"); + + if (!CloseHandle(stdout_read) || + !CloseHandle(process_info.hProcess) || + !CloseHandle(process_info.hThread)) { + Win32Fatal("CloseHandle"); + } + + return exit_code; +} diff --git a/src/msvc_helper.h b/src/msvc_helper.h new file mode 100644 index 0000000..f623520 --- /dev/null +++ b/src/msvc_helper.h @@ -0,0 +1,54 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> +#include <vector> +using namespace std; + +/// Visual Studio's cl.exe requires some massaging to work with Ninja; +/// for example, it emits include information on stderr in a funny +/// format when building with /showIncludes. This class wraps a CL +/// process and parses that output to extract the file list. +struct CLWrapper { + CLWrapper() : env_block_(NULL) {} + + /// Set the environment block (as suitable for CreateProcess) to be used + /// by Run(). + void SetEnvBlock(void* env_block) { env_block_ = env_block; } + + /// Start a process and parse its output. Returns its exit code. + /// Any non-parsed output is buffered into \a extra_output if provided, + /// otherwise it is printed to stdout while the process runs. + /// Crashes (calls Fatal()) on error. + int Run(const string& command, string* extra_output=NULL); + + /// Parse a line of cl.exe output and extract /showIncludes info. + /// If a dependency is extracted, returns a nonempty string. + /// Exposed for testing. + static string FilterShowIncludes(const string& line); + + /// Return true if a mentioned include file is a system path. + /// Expects the path to already by normalized (including lower case). + /// Filtering these out reduces dependency information considerably. + static bool IsSystemInclude(const string& path); + + /// Parse a line of cl.exe output and return true if it looks like + /// it's printing an input filename. This is a heuristic but it appears + /// to be the best we can do. + /// Exposed for testing. + static bool FilterInputFilename(const string& line); + + void* env_block_; + vector<string> includes_; +}; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc new file mode 100644 index 0000000..0c8db37 --- /dev/null +++ b/src/msvc_helper_main-win32.cc @@ -0,0 +1,115 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include <windows.h> + +#include "util.h" + +#include "getopt.h" + +namespace { + +void Usage() { + printf( +"usage: ninja -t msvc [options] -- cl.exe /showIncludes /otherArgs\n" +"options:\n" +" -e ENVFILE load environment block from ENVFILE as environment\n" +" -r BASE normalize paths and make relative to BASE before output\n" +" -o FILE write output dependency information to FILE.d\n" + ); +} + +void PushPathIntoEnvironment(const string& env_block) { + const char* as_str = env_block.c_str(); + while (as_str[0]) { + if (_strnicmp(as_str, "path=", 5) == 0) { + _putenv(as_str); + return; + } else { + as_str = &as_str[strlen(as_str) + 1]; + } + } +} + +} // anonymous namespace + +int MSVCHelperMain(int argc, char** argv) { + const char* output_filename = NULL; + const char* relative_to = NULL; + const char* envfile = NULL; + + const option kLongOptions[] = { + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + int opt; + while ((opt = getopt_long(argc, argv, "e:o:r:h", kLongOptions, NULL)) != -1) { + switch (opt) { + case 'e': + envfile = optarg; + break; + case 'o': + output_filename = optarg; + break; + case 'r': + relative_to = optarg; + break; + case 'h': + default: + Usage(); + return 0; + } + } + + if (!output_filename) { + Usage(); + Fatal("-o required"); + } + + string env; + if (envfile) { + string err; + if (ReadFile(envfile, &env, &err) != 0) + Fatal("couldn't open %s: %s", envfile, err.c_str()); + PushPathIntoEnvironment(env); + } + + char* command = GetCommandLine(); + command = strstr(command, " -- "); + if (!command) { + Fatal("expected command line to end with \" -- command args\""); + } + command += 4; + + CLWrapper cl; + if (!env.empty()) + cl.SetEnvBlock((void*)env.data()); + int exit_code = cl.Run(command); + + string depfile = string(output_filename) + ".d"; + FILE* output = fopen(depfile.c_str(), "w"); + if (!output) { + Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str()); + } + fprintf(output, "%s: ", output_filename); + for (vector<string>::iterator i = cl.includes_.begin(); + i != cl.includes_.end(); ++i) { + fprintf(output, "%s\n", i->c_str()); + } + fclose(output); + + return exit_code; +} diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc new file mode 100644 index 0000000..29fefd4 --- /dev/null +++ b/src/msvc_helper_test.cc @@ -0,0 +1,83 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include <gtest/gtest.h> + +#include "test.h" +#include "util.h" + +TEST(MSVCHelperTest, ShowIncludes) { + ASSERT_EQ("", CLWrapper::FilterShowIncludes("")); + + ASSERT_EQ("", CLWrapper::FilterShowIncludes("Sample compiler output")); + ASSERT_EQ("c:\\Some Files\\foobar.h", + CLWrapper::FilterShowIncludes("Note: including file: " + "c:\\Some Files\\foobar.h")); + ASSERT_EQ("c:\\initspaces.h", + CLWrapper::FilterShowIncludes("Note: including file: " + "c:\\initspaces.h")); +} + +TEST(MSVCHelperTest, FilterInputFilename) { + ASSERT_TRUE(CLWrapper::FilterInputFilename("foobar.cc")); + ASSERT_TRUE(CLWrapper::FilterInputFilename("foo bar.cc")); + ASSERT_TRUE(CLWrapper::FilterInputFilename("baz.c")); + + ASSERT_FALSE(CLWrapper::FilterInputFilename( + "src\\cl_helper.cc(166) : fatal error C1075: end " + "of file found ...")); +} + +TEST(MSVCHelperTest, Run) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo foo&& echo Note: including file: foo.h&&echo bar\"", + &output); + ASSERT_EQ("foo\nbar\n", output); + ASSERT_EQ(1u, cl.includes_.size()); + ASSERT_EQ("foo.h", cl.includes_[0]); +} + +TEST(MSVCHelperTest, RunFilenameFilter) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo foo.cc&& echo cl: warning\"", + &output); + ASSERT_EQ("cl: warning\n", output); +} + +TEST(MSVCHelperTest, RunSystemInclude) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo Note: including file: c:\\Program Files\\foo.h&&" + "echo Note: including file: d:\\Microsoft Visual Studio\\bar.h&&" + "echo Note: including file: path.h\"", + &output); + // We should have dropped the first two includes because they look like + // system headers. + ASSERT_EQ("", output); + ASSERT_EQ(1u, cl.includes_.size()); + ASSERT_EQ("path.h", cl.includes_[0]); +} + +TEST(MSVCHelperTest, EnvBlock) { + char env_block[] = "foo=bar\0"; + CLWrapper cl; + cl.SetEnvBlock(env_block); + string output; + cl.Run("cmd /c \"echo foo is %foo%", &output); + ASSERT_EQ("foo is bar\n", output); +} diff --git a/src/ninja.cc b/src/ninja.cc index 4ce16e7..5a3c530 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -19,12 +19,6 @@ #include <sys/stat.h> #include <sys/types.h> -#if defined(__APPLE__) || defined(__FreeBSD__) -#include <sys/sysctl.h> -#elif defined(linux) -#include <sys/sysinfo.h> -#endif - #ifdef _WIN32 #include "getopt.h" #include <direct.h> @@ -38,6 +32,7 @@ #include "build.h" #include "build_log.h" #include "clean.h" +#include "disk_interface.h" #include "edit_distance.h" #include "explain.h" #include "graph.h" @@ -47,11 +42,14 @@ #include "state.h" #include "util.h" +// Defined in msvc_helper_main-win32.cc. +int MSVCHelperMain(int argc, char** argv); + namespace { /// The version number of the current Ninja release. This will always /// be "git" on trunk. -const char* kVersion = "120715"; +const char* kVersion = "1.0.0"; /// Global information passed into subtools. struct Globals { @@ -68,61 +66,67 @@ struct Globals { /// Command line used to run Ninja. const char* ninja_command; - /// Build configuration (e.g. parallelism). - BuildConfig config; + /// Build configuration set from flags (e.g. parallelism). + BuildConfig* config; /// Loaded state (rules, nodes). This is a pointer so it can be reset. State* state; }; +/// The type of functions that are the entry points to tools (subcommands). +typedef int (*ToolFunc)(Globals*, int, char**); + +/// Subtools, accessible via "-t foo". +struct Tool { + /// Short name of the tool. + const char* name; + + /// Description (shown in "-t list"). + const char* desc; + + /// When to run the tool. + enum { + /// Run after parsing the command-line flags (as early as possible). + RUN_AFTER_FLAGS, + + /// Run after loading build.ninja. + RUN_AFTER_LOAD, + } when; + + /// Implementation of the tool. + ToolFunc func; +}; + /// Print usage information. void Usage(const BuildConfig& config) { fprintf(stderr, "usage: ninja [options] [targets...]\n" "\n" "if targets are unspecified, builds the 'default' target (see manual).\n" -"targets are paths, with additional special syntax:\n" -" 'target^' means 'the first output that uses target'.\n" -" example: 'ninja foo.cc^' will likely build foo.o.\n" "\n" "options:\n" +" --version print ninja version (\"%s\")\n" +"\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" -" -V print ninja version (\"%s\")\n" "\n" " -j N run N jobs in parallel [default=%d]\n" " -l N do not start new jobs if the load average is greater than N\n" +#ifdef _WIN32 +" (not yet implemented on Windows)\n" +#endif " -k N keep going until N jobs fail [default=1]\n" -" -n dry run (don't run commands but pretend they succeeded)\n" +" -n dry run (don't run commands but act like they succeeded)\n" " -v show all command lines while building\n" "\n" " -d MODE enable debugging (use -d list to list modes)\n" -" -t TOOL run a subtool\n" -" use '-t list' to list subtools.\n" -" terminates toplevel options; further flags are passed to the tool.\n", +" -t TOOL run a subtool (use -t list to list subtools)\n" +" terminates toplevel options; further flags are passed to the tool\n", kVersion, config.parallelism); } /// Choose a default value for the -j (parallelism) flag. int GuessParallelism() { - int processors = 0; - -#if defined(linux) - processors = get_nprocs(); -#elif defined(__APPLE__) || defined(__FreeBSD__) - size_t processors_size = sizeof(processors); - int name[] = {CTL_HW, HW_NCPU}; - if (sysctl(name, sizeof(name) / sizeof(int), - &processors, &processors_size, - NULL, 0) < 0) { - processors = 1; - } -#elif defined(_WIN32) - SYSTEM_INFO info; - GetSystemInfo(&info); - processors = info.dwNumberOfProcessors; -#endif - - switch (processors) { + switch (int processors = GetProcessorCount()) { case 0: case 1: return 2; @@ -143,22 +147,20 @@ struct RealFileReader : public ManifestParser::FileReader { /// Rebuild the build manifest, if necessary. /// Returns true if the manifest was rebuilt. -bool RebuildManifest(State* state, const BuildConfig& config, - const char* input_file, string* err) { +bool RebuildManifest(Builder* builder, const char* input_file, string* err) { string path = input_file; if (!CanonicalizePath(&path, err)) return false; - Node* node = state->LookupNode(path); + Node* node = builder->state_->LookupNode(path); if (!node) return false; - Builder manifest_builder(state, config); - if (!manifest_builder.AddTarget(node, err)) + if (!builder->AddTarget(node, err)) return false; - if (manifest_builder.AlreadyUpToDate()) + if (builder->AlreadyUpToDate()) return false; // Not an error, but we didn't rebuild. - if (!manifest_builder.Build(err)) + if (!builder->Build(err)) return false; // The manifest was only rebuilt if it is now dirty (it may have been cleaned @@ -170,51 +172,50 @@ bool CollectTargetsFromArgs(State* state, int argc, char* argv[], vector<Node*>* targets, string* err) { if (argc == 0) { *targets = state->DefaultNodes(err); - if (!err->empty()) + return err->empty(); + } + + for (int i = 0; i < argc; ++i) { + string path = argv[i]; + if (!CanonicalizePath(&path, err)) return false; - } else { - for (int i = 0; i < argc; ++i) { - string path = argv[i]; - if (!CanonicalizePath(&path, err)) - return false; - - // Special syntax: "foo.cc^" means "the first output of foo.cc". - bool first_dependent = false; - if (!path.empty() && path[path.size() - 1] == '^') { - path.resize(path.size() - 1); - first_dependent = true; - } - Node* node = state->LookupNode(path); - if (node) { - if (first_dependent) { - if (node->out_edges().empty()) { - *err = "'" + path + "' has no out edge"; - return false; - } - Edge* edge = node->out_edges()[0]; - if (edge->outputs_.empty()) { - edge->Dump(); - Fatal("edge has no outputs"); - } - node = edge->outputs_[0]; + // Special syntax: "foo.cc^" means "the first output of foo.cc". + bool first_dependent = false; + if (!path.empty() && path[path.size() - 1] == '^') { + path.resize(path.size() - 1); + first_dependent = true; + } + + Node* node = state->LookupNode(path); + if (node) { + if (first_dependent) { + if (node->out_edges().empty()) { + *err = "'" + path + "' has no out edge"; + return false; + } + Edge* edge = node->out_edges()[0]; + if (edge->outputs_.empty()) { + edge->Dump(); + Fatal("edge has no outputs"); } - targets->push_back(node); + node = edge->outputs_[0]; + } + targets->push_back(node); + } else { + *err = "unknown target '" + path + "'"; + + if (path == "clean") { + *err += ", did you mean 'ninja -t clean'?"; + } else if (path == "help") { + *err += ", did you mean 'ninja -h'?"; } else { - *err = "unknown target '" + path + "'"; - - if (path == "clean") { - *err += ", did you mean 'ninja -t clean'?"; - } else if (path == "help") { - *err += ", did you mean 'ninja -h'?"; - } else { - Node* suggestion = state->SpellcheckNode(path); - if (suggestion) { - *err += ", did you mean '" + suggestion->path() + "'?"; - } + Node* suggestion = state->SpellcheckNode(path); + if (suggestion) { + *err += ", did you mean '" + suggestion->path() + "'?"; } - return false; } + return false; } } return true; @@ -291,6 +292,16 @@ int ToolBrowse(Globals* globals, int argc, char* argv[]) { } #endif // _WIN32 +#if defined(WIN32) +int ToolMSVC(Globals* globals, int argc, char* argv[]) { + // Reset getopt: push one argument onto the front of argv, reset optind. + argc++; + argv--; + optind = 0; + return MSVCHelperMain(argc, argv); +} +#endif + int ToolTargetsList(const vector<Node*>& nodes, int depth, int indent) { for (vector<Node*>::const_iterator n = nodes.begin(); n != nodes.end(); @@ -481,7 +492,7 @@ int ToolClean(Globals* globals, int argc, char* argv[]) { return 1; } - Cleaner cleaner(globals->state, globals->config); + Cleaner cleaner(globals->state, *globals->config); if (argc >= 1) { if (clean_rules) return cleaner.CleanRules(argc, argv); @@ -492,7 +503,7 @@ int ToolClean(Globals* globals, int argc, char* argv[]) { } } -void ToolUrtle() { +int ToolUrtle(Globals* globals, int argc, char** argv) { // RLE encoded. const char* urtle = " 13 ,3;2!2;\n8 ,;<11!;\n5 `'<10!(2`'2!\n11 ,6;, `\\. `\\9 .,c13$ec,.\n6 " @@ -515,58 +526,63 @@ void ToolUrtle() { count = 0; } } + return 0; } -int RunTool(const string& tool, Globals* globals, int argc, char** argv) { - typedef int (*ToolFunc)(Globals*, int, char**); - struct Tool { - const char* name; - const char* desc; - ToolFunc func; - } tools[] = { +/// Find the function to execute for \a tool_name and return it via \a func. +/// If there is no tool to run (e.g.: unknown tool), returns an exit code. +int ChooseTool(const string& tool_name, const Tool** tool_out) { + static const Tool kTools[] = { #if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) { "browse", "browse dependency graph in a web browser", - ToolBrowse }, + Tool::RUN_AFTER_LOAD, ToolBrowse }, +#endif +#if defined(WIN32) + { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", + Tool::RUN_AFTER_FLAGS, ToolMSVC }, #endif { "clean", "clean built files", - ToolClean }, + Tool::RUN_AFTER_LOAD, ToolClean }, { "commands", "list all commands required to rebuild given targets", - ToolCommands }, + Tool::RUN_AFTER_LOAD, ToolCommands }, { "graph", "output graphviz dot file for targets", - ToolGraph }, + Tool::RUN_AFTER_LOAD, ToolGraph }, { "query", "show inputs/outputs for a path", - ToolQuery }, + Tool::RUN_AFTER_LOAD, ToolQuery }, { "rules", "list all rules", - ToolRules }, + Tool::RUN_AFTER_LOAD, ToolRules }, { "targets", "list targets by their rule or depth in the DAG", - ToolTargets }, - { NULL, NULL, NULL } + Tool::RUN_AFTER_LOAD, ToolTargets }, + { "urtle", NULL, + Tool::RUN_AFTER_FLAGS, ToolUrtle }, + { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } }; - if (tool == "list") { + if (tool_name == "list") { printf("ninja subtools:\n"); - for (int i = 0; tools[i].name; ++i) { - printf("%10s %s\n", tools[i].name, tools[i].desc); + for (const Tool* tool = &kTools[0]; tool->name; ++tool) { + if (tool->desc) + printf("%10s %s\n", tool->name, tool->desc); } return 0; - } else if (tool == "urtle") { - ToolUrtle(); - return 0; } - for (int i = 0; tools[i].name; ++i) { - if (tool == tools[i].name) - return tools[i].func(globals, argc, argv); + for (const Tool* tool = &kTools[0]; tool->name; ++tool) { + if (tool->name == tool_name) { + *tool_out = tool; + return 0; + } } vector<const char*> words; - for (int i = 0; tools[i].name; ++i) - words.push_back(tools[i].name); - const char* suggestion = SpellcheckStringV(tool, words); + for (const Tool* tool = &kTools[0]; tool->name; ++tool) + words.push_back(tool->name); + const char* suggestion = SpellcheckStringV(tool_name, words); if (suggestion) { - Error("unknown tool '%s', did you mean '%s'?", tool.c_str(), suggestion); + Error("unknown tool '%s', did you mean '%s'?", + tool_name.c_str(), suggestion); } else { - Error("unknown tool '%s'", tool.c_str()); + Error("unknown tool '%s'", tool_name.c_str()); } return 1; } @@ -592,17 +608,63 @@ bool DebugEnable(const string& name, Globals* globals) { } } -int RunBuild(Globals* globals, int argc, char** argv) { +bool OpenLog(BuildLog* build_log, Globals* globals, + DiskInterface* disk_interface) { + const string build_dir = + globals->state->bindings_.LookupVariable("builddir"); + const char* kLogPath = ".ninja_log"; + string log_path = kLogPath; + if (!build_dir.empty()) { + log_path = build_dir + "/" + kLogPath; + if (!disk_interface->MakeDirs(log_path) && errno != EEXIST) { + Error("creating build directory %s: %s", + build_dir.c_str(), strerror(errno)); + return false; + } + } + + string err; + if (!build_log->Load(log_path, &err)) { + Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + return false; + } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning true. + Warning("%s", err.c_str()); + err.clear(); + } + + if (!globals->config->dry_run) { + if (!build_log->OpenForWrite(log_path, &err)) { + Error("opening build log: %s", err.c_str()); + return false; + } + } + + return true; +} + +/// Dump the output requested by '-d stats'. +void DumpMetrics(Globals* globals) { + g_metrics->Report(); + + printf("\n"); + int count = (int)globals->state->paths_.size(); + int buckets = (int)globals->state->paths_.bucket_count(); + printf("path->node hash load %.2f (%d entries / %d buckets)\n", + count / (double) buckets, count, buckets); +} + +int RunBuild(Builder* builder, int argc, char** argv) { string err; vector<Node*> targets; - if (!CollectTargetsFromArgs(globals->state, argc, argv, &targets, &err)) { + if (!CollectTargetsFromArgs(builder->state_, argc, argv, &targets, &err)) { Error("%s", err.c_str()); return 1; } - Builder builder(globals->state, globals->config); for (size_t i = 0; i < targets.size(); ++i) { - if (!builder.AddTarget(targets[i], &err)) { + if (!builder->AddTarget(targets[i], &err)) { if (!err.empty()) { Error("%s", err.c_str()); return 1; @@ -613,12 +675,12 @@ int RunBuild(Globals* globals, int argc, char** argv) { } } - if (builder.AlreadyUpToDate()) { + if (builder->AlreadyUpToDate()) { printf("ninja: no work to do.\n"); return 0; } - if (!builder.Build(&err)) { + if (!builder->Build(&err)) { printf("ninja: build stopped: %s.\n", err.c_str()); return 1; } @@ -626,26 +688,58 @@ int RunBuild(Globals* globals, int argc, char** argv) { return 0; } -} // anonymous namespace +#ifdef _MSC_VER -int main(int argc, char** argv) { +} // anonymous namespace + +// Defined in minidump-win32.cc. +void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep); + +namespace { + +/// This handler processes fatal crashes that you can't catch +/// Test example: C++ exception in a stack-unwind-block +/// Real-world example: ninja launched a compiler to process a tricky +/// C++ input file. The compiler got itself into a state where it +/// generated 3 GB of output and caused ninja to crash. +void TerminateHandler() { + CreateWin32MiniDump(NULL); + Fatal("terminate handler called"); +} + +/// On Windows, we want to prevent error dialogs in case of exceptions. +/// This function handles the exception, and writes a minidump. +int ExceptionFilter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { + Error("exception: 0x%X", code); // e.g. EXCEPTION_ACCESS_VIOLATION + fflush(stderr); + CreateWin32MiniDump(ep); + return EXCEPTION_EXECUTE_HANDLER; +} + +#endif // _MSC_VER + +int NinjaMain(int argc, char** argv) { + BuildConfig config; Globals globals; globals.ninja_command = argv[0]; + globals.config = &config; const char* input_file = "build.ninja"; const char* working_dir = NULL; - string tool; + string tool_name; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); - globals.config.parallelism = GuessParallelism(); + config.parallelism = GuessParallelism(); + enum { OPT_VERSION = 1 }; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, OPT_VERSION }, { NULL, 0, NULL, 0 } }; int opt; - while (tool.empty() && + while (tool_name.empty() && (opt = getopt_long(argc, argv, "d:f:hj:k:l:nt:vC:V", kLongOptions, NULL)) != -1) { switch (opt) { @@ -657,14 +751,14 @@ int main(int argc, char** argv) { input_file = optarg; break; case 'j': - globals.config.parallelism = atoi(optarg); + config.parallelism = atoi(optarg); break; case 'l': { char* end; double value = strtod(optarg, &end); if (end == optarg) Fatal("-l parameter not numeric: did you mean -l 0.0?"); - globals.config.max_load_average = value; + config.max_load_average = value; break; } case 'k': { @@ -676,43 +770,54 @@ int main(int argc, char** argv) { // We want to go until N jobs fail, which means we should allow // N failures and then stop. For N <= 0, INT_MAX is close enough // to infinite for most sane builds. - globals.config.failures_allowed = value > 0 ? value : INT_MAX; + config.failures_allowed = value > 0 ? value : INT_MAX; break; } case 'n': - globals.config.dry_run = true; + config.dry_run = true; break; case 'v': - globals.config.verbosity = BuildConfig::VERBOSE; + config.verbosity = BuildConfig::VERBOSE; break; case 't': - tool = optarg; + tool_name = optarg; break; case 'C': working_dir = optarg; break; - case 'V': + case OPT_VERSION: printf("%s\n", kVersion); return 0; case 'h': default: - Usage(globals.config); + Usage(config); return 1; } } argv += optind; argc -= optind; + // If specified, select a tool as early as possible, so commands like + // -t list can run before we attempt to load build.ninja etc. + const Tool* tool = NULL; + if (!tool_name.empty()) { + int exit_code = ChooseTool(tool_name, &tool); + if (!tool) + return exit_code; + } + + if (tool && tool->when == Tool::RUN_AFTER_FLAGS) + return tool->func(&globals, argc, argv); + if (working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for // subsequent commands. - printf("ninja: Entering directory `%s'\n", working_dir); -#ifdef _WIN32 - if (_chdir(working_dir) < 0) { -#else + // Don't print this if a tool is being used, so that tool output + // can be piped into a file without this string showing up. + if (!tool) + printf("ninja: Entering directory `%s'\n", working_dir); if (chdir(working_dir) < 0) { -#endif Fatal("chdir to '%s' - %s", working_dir, strerror(errno)); } } @@ -720,6 +825,7 @@ int main(int argc, char** argv) { bool rebuilt_manifest = false; reload: + RealDiskInterface disk_interface; RealFileReader file_reader; ManifestParser parser(globals.state, &file_reader); string err; @@ -728,38 +834,18 @@ reload: return 1; } - if (!tool.empty()) - return RunTool(tool, &globals, argc, argv); + if (tool && tool->when == Tool::RUN_AFTER_LOAD) + return tool->func(&globals, argc, argv); BuildLog build_log; - build_log.SetConfig(&globals.config); - globals.state->build_log_ = &build_log; - - const string build_dir = globals.state->bindings_.LookupVariable("builddir"); - const char* kLogPath = ".ninja_log"; - string log_path = kLogPath; - if (!build_dir.empty()) { - if (MakeDir(build_dir) < 0 && errno != EEXIST) { - Error("creating build directory %s: %s", - build_dir.c_str(), strerror(errno)); - return 1; - } - log_path = build_dir + "/" + kLogPath; - } - - if (!build_log.Load(log_path, &err)) { - Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + if (!OpenLog(&build_log, &globals, &disk_interface)) return 1; - } - - if (!build_log.OpenForWrite(log_path, &err)) { - Error("opening build log: %s", err.c_str()); - return 1; - } if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. - if (RebuildManifest(globals.state, globals.config, input_file, &err)) { + Builder manifest_builder(globals.state, config, &build_log, + &disk_interface); + if (RebuildManifest(&manifest_builder, input_file, &err)) { rebuilt_manifest = true; globals.ResetState(); goto reload; @@ -769,20 +855,31 @@ reload: } } - int result = RunBuild(&globals, argc, argv); - if (g_metrics) { - g_metrics->Report(); + Builder builder(globals.state, config, &build_log, &disk_interface); + int result = RunBuild(&builder, argc, argv); + if (g_metrics) + DumpMetrics(&globals); + return result; +} - printf("\n"); - int count = (int)globals.state->paths_.size(); - int buckets = -#ifdef _MSC_VER - (int)globals.state->paths_.comp.bucket_size; +} // anonymous namespace + +int main(int argc, char** argv) { +#if !defined(NINJA_BOOTSTRAP) && defined(_MSC_VER) + // Set a handler to catch crashes not caught by the __try..__except + // block (e.g. an exception in a stack-unwind-block). + set_terminate(TerminateHandler); + __try { + // Running inside __try ... __except suppresses any Windows error + // dialogs for errors such as bad_alloc. + return NinjaMain(argc, argv); + } + __except(ExceptionFilter(GetExceptionCode(), GetExceptionInformation())) { + // Common error situations return exitCode=1. 2 was chosen to + // indicate a more serious problem. + return 2; + } #else - (int)globals.state->paths_.bucket_count(); + return NinjaMain(argc, argv); #endif - printf("path->node hash load %.2f (%d entries / %d buckets)\n", - count / (double) buckets, count, buckets); - } - return result; } diff --git a/src/parser_perftest.cc b/src/parser_perftest.cc index b1d5bf0..b215221 100644 --- a/src/parser_perftest.cc +++ b/src/parser_perftest.cc @@ -17,6 +17,7 @@ #include "depfile_parser.h" #include "util.h" +#include "metrics.h" int main(int argc, char* argv[]) { if (argc < 2) { diff --git a/src/state.cc b/src/state.cc index 9f38fe9..4c7168b 100644 --- a/src/state.cc +++ b/src/state.cc @@ -24,7 +24,7 @@ const Rule State::kPhonyRule("phony"); -State::State() : build_log_(NULL) { +State::State() { AddRule(&kPhonyRule); } diff --git a/src/state.h b/src/state.h index 23ca12b..026acf3 100644 --- a/src/state.h +++ b/src/state.h @@ -14,7 +14,6 @@ #ifndef NINJA_STATE_H_ #define NINJA_STATE_H_ -#pragma once #include <map> #include <string> @@ -24,7 +23,6 @@ using namespace std; #include "eval_env.h" #include "hash_map.h" -struct BuildLog; struct Edge; struct Node; struct Rule; @@ -72,7 +70,6 @@ struct State { BindingEnv bindings_; vector<Node*> defaults_; - BuildLog* build_log_; }; #endif // NINJA_STATE_H_ diff --git a/src/state_test.cc b/src/state_test.cc index 354468b..bc24edd 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -22,11 +22,14 @@ namespace { TEST(State, Basic) { State state; + EvalString command; + command.AddText("cat "); + command.AddSpecial("in"); + command.AddText(" > "); + command.AddSpecial("out"); + Rule* rule = new Rule("cat"); - rule->command().AddText("cat "); - rule->command().AddSpecial("in"); - rule->command().AddText(" > "); - rule->command().AddSpecial("out"); + rule->set_command(command); state.AddRule(rule); Edge* edge = state.AddEdge(rule); diff --git a/src/string_piece.h b/src/string_piece.h index ad1153e..b1bf105 100644 --- a/src/string_piece.h +++ b/src/string_piece.h @@ -31,7 +31,7 @@ struct StringPiece { StringPiece(const string& str) : str_(str.data()), len_(str.size()) {} StringPiece(const char* str) : str_(str), len_(strlen(str)) {} - StringPiece(const char* str, int len) : str_(str), len_(len) {} + StringPiece(const char* str, size_t len) : str_(str), len_(len) {} bool operator==(const StringPiece& other) const { return len_ == other.len_ && memcmp(str_, other.str_, len_) == 0; @@ -47,7 +47,7 @@ struct StringPiece { } const char* str_; - int len_; + size_t len_; }; #endif // NINJA_STRINGPIECE_H_ diff --git a/src/subprocess.cc b/src/subprocess-posix.cc index 1c47fd1..1c47fd1 100644 --- a/src/subprocess.cc +++ b/src/subprocess-posix.cc diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 3484e5f..4b103a5 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -20,14 +20,6 @@ #include "util.h" -namespace { - -void Win32Fatal(const char* function) { - Fatal("%s: %s", function, GetLastErrorString().c_str()); -} - -} // anonymous namespace - Subprocess::Subprocess() : child_(NULL) , overlapped_(), is_reading_(false) { } @@ -44,7 +36,7 @@ Subprocess::~Subprocess() { HANDLE Subprocess::SetupPipe(HANDLE ioport) { char pipe_name[100]; snprintf(pipe_name, sizeof(pipe_name), - "\\\\.\\pipe\\ninja_pid%u_sp%p", GetCurrentProcessId(), this); + "\\\\.\\pipe\\ninja_pid%lu_sp%p", GetCurrentProcessId(), this); pipe_ = ::CreateNamedPipeA(pipe_name, PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, @@ -271,9 +263,12 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) { - if ((*i)->child_) - if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, GetProcessId((*i)->child_))) + if ((*i)->child_) { + if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, + GetProcessId((*i)->child_))) { Win32Fatal("GenerateConsoleCtrlEvent"); + } + } } for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index c155012..d89525e 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -16,6 +16,13 @@ #include "test.h" +#ifndef _WIN32 +// SetWithLots need setrlimit. +#include <sys/time.h> +#include <sys/resource.h> +#include <unistd.h> +#endif + namespace { #ifdef _WIN32 @@ -142,6 +149,8 @@ TEST_F(SubprocessTest, SetWithMulti) { } } +// OS X's process limit is less than 1025 by default +// (|sysctl kern.maxprocperuid| is 709 on 10.7 and 10.8 and less prior to that). #ifdef linux TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer diff --git a/src/util.cc b/src/util.cc index d8d7fb3..0feb99d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -34,8 +34,13 @@ #include <vector> -#ifdef _WIN32 -#include <direct.h> // _mkdir +#if defined(__APPLE__) || defined(__FreeBSD__) +#include <sys/sysctl.h> +#elif defined(__SVR4) && defined(__sun) +#include <unistd.h> +#include <sys/loadavg.h> +#elif defined(linux) +#include <sys/sysinfo.h> #endif #include "edit_distance.h" @@ -43,7 +48,7 @@ void Fatal(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: FATAL: "); + fprintf(stderr, "ninja: fatal: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -61,7 +66,7 @@ void Fatal(const char* msg, ...) { void Warning(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: WARNING: "); + fprintf(stderr, "ninja: warning: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -70,7 +75,7 @@ void Warning(const char* msg, ...) { void Error(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: ERROR: "); + fprintf(stderr, "ninja: error: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -79,7 +84,7 @@ void Error(const char* msg, ...) { bool CanonicalizePath(string* path, string* err) { METRIC_RECORD("canonicalize str"); - int len = path->size(); + size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; @@ -89,7 +94,7 @@ bool CanonicalizePath(string* path, string* err) { return true; } -bool CanonicalizePath(char* path, int* len, string* err) { +bool CanonicalizePath(char* path, size_t* len, string* err) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. METRIC_RECORD("canonicalize path"); @@ -108,8 +113,19 @@ bool CanonicalizePath(char* path, int* len, string* err) { const char* end = start + *len; if (*src == '/') { +#ifdef _WIN32 + // network path starts with // + if (*len > 1 && *(src + 1) == '/') { + src += 2; + dst += 2; + } else { + ++src; + ++dst; + } +#else ++src; ++dst; +#endif } while (src < end) { @@ -157,14 +173,6 @@ bool CanonicalizePath(char* path, int* len, string* err) { return true; } -int MakeDir(const string& path) { -#ifdef _WIN32 - return _mkdir(path.c_str()); -#else - return mkdir(path.c_str(), 0777); -#endif -} - int ReadFile(const string& path, string* contents, string* err) { FILE* f = fopen(path.c_str(), "r"); if (!f) { @@ -204,16 +212,6 @@ void SetCloseOnExec(int fd) { #endif // ! _WIN32 } -int64_t GetTimeMillis() { -#ifdef _WIN32 - // GetTickCount64 is only available on Vista or later. - return GetTickCount(); -#else - timeval now; - gettimeofday(&now, NULL); - return ((int64_t)now.tv_sec * 1000) + (now.tv_usec / 1000); -#endif -} const char* SpellcheckStringV(const string& text, const vector<const char*>& words) { @@ -263,6 +261,10 @@ string GetLastErrorString() { LocalFree(msg_buf); return msg; } + +void Win32Fatal(const char* function) { + Fatal("%s: %s", function, GetLastErrorString().c_str()); +} #endif static bool islatinalpha(int c) { @@ -293,18 +295,46 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } +#if defined(linux) +int GetProcessorCount() { + return get_nprocs(); +} +#elif defined(__APPLE__) || defined(__FreeBSD__) +int GetProcessorCount() { + int processors; + size_t processors_size = sizeof(processors); + int name[] = {CTL_HW, HW_NCPU}; + if (sysctl(name, sizeof(name) / sizeof(int), + &processors, &processors_size, + NULL, 0) < 0) { + return 0; + } + return processors; +} +#elif defined(_WIN32) +int GetProcessorCount() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return info.dwNumberOfProcessors; +} +#else +// This is what get_nprocs() should be doing in the Linux implementation +// above, but in a more standard way. +int GetProcessorCount() { + return sysconf(_SC_NPROCESSORS_ONLN); +} +#endif + #ifdef _WIN32 -static double GetLoadAverage_win32() -{ +double GetLoadAverage() { // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. + // Remember to also update Usage() when this is fixed. return -0.0f; } #else -static double GetLoadAverage_unix() -{ +double GetLoadAverage() { double loadavg[3] = { 0.0f, 0.0f, 0.0f }; - if (getloadavg(loadavg, 3) < 0) - { + if (getloadavg(loadavg, 3) < 0) { // Maybe we should return an error here or the availability of // getloadavg(3) should be checked when ninja is configured. return -0.0f; @@ -313,11 +343,14 @@ static double GetLoadAverage_unix() } #endif // _WIN32 -double GetLoadAverage() -{ -#ifdef _WIN32 - return GetLoadAverage_win32(); -#else - return GetLoadAverage_unix(); -#endif // _WIN32 +string ElideMiddle(const string& str, size_t width) { + const int kMargin = 3; // Space for "...". + string result = str; + if (result.size() + kMargin > width) { + size_t elide_size = (width - kMargin) / 2; + result = result.substr(0, elide_size) + + "..." + + result.substr(result.size() - elide_size, elide_size); + } + return result; } @@ -14,7 +14,6 @@ #ifndef NINJA_UTIL_H_ #define NINJA_UTIL_H_ -#pragma once #ifdef _WIN32 #include "win32port.h" @@ -26,8 +25,6 @@ #include <vector> using namespace std; -#define NINJA_UNUSED_ARG(arg_name) (void)arg_name; - /// Log a fatal message and exit. void Fatal(const char* msg, ...); @@ -40,24 +37,16 @@ void Error(const char* msg, ...); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". bool CanonicalizePath(string* path, string* err); -bool CanonicalizePath(char* path, int* len, string* err); - -/// Create a directory (mode 0777 on Unix). -/// Portability abstraction. -int MakeDir(const string& path); +bool CanonicalizePath(char* path, size_t* len, string* err); -/// Read a file to a string. +/// Read a file to a string (in text mode: with CRLF conversion +/// on Windows). /// Returns -errno and fills in \a err on error. int ReadFile(const string& path, string* contents, string* err); /// Mark a file descriptor to not be inherited on exec()s. void SetCloseOnExec(int fd); -/// Get the current time as relative to some epoch. -/// Epoch varies between platforms; only useful for measuring elapsed -/// time. -int64_t GetTimeMillis(); - /// Given a misspelled string and a list of correct spellings, returns /// the closest match or NULL if there is no close enough match. const char* SpellcheckStringV(const string& text, const vector<const char*>& words); @@ -68,10 +57,18 @@ const char* SpellcheckString(const string& text, ...); /// Removes all Ansi escape codes (http://www.termsys.demon.co.uk/vtansi.htm). string StripAnsiEscapeCodes(const string& in); +/// @return the number of processors on the machine. Useful for an initial +/// guess for how many jobs to run in parallel. @return 0 on error. +int GetProcessorCount(); + /// @return the load average of the machine. A negative value is returned /// on error. double GetLoadAverage(); +/// Elide the given string @a str with '...' in the middle if the length +/// exceeds @a width. +string ElideMiddle(const string& str, size_t width); + #ifdef _MSC_VER #define snprintf _snprintf #define fileno _fileno @@ -83,6 +80,9 @@ double GetLoadAverage(); #ifdef _WIN32 /// Convert the value returned by GetLastError() into a string. string GetLastErrorString(); + +/// Calls Fatal() with a function name and GetLastErrorString. +void Win32Fatal(const char* function); #endif #endif // NINJA_UTIL_H_ diff --git a/src/util_test.cc b/src/util_test.cc index 23d4b83..4776546 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -66,6 +66,22 @@ TEST(CanonicalizePath, PathSamples) { path = "foo/.hidden_bar"; EXPECT_TRUE(CanonicalizePath(&path, &err)); EXPECT_EQ("foo/.hidden_bar", path); + + path = "/foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("/foo", path); + + path = "//foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); +#ifdef _WIN32 + EXPECT_EQ("//foo", path); +#else + EXPECT_EQ("/foo", path); +#endif + + path = "/"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("", path); } TEST(CanonicalizePath, EmptyResult) { @@ -105,18 +121,18 @@ TEST(CanonicalizePath, AbsolutePath) { TEST(CanonicalizePath, NotNullTerminated) { string path; string err; - int len; + size_t len; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); - EXPECT_EQ(strlen("foo"), static_cast<size_t>(len)); + EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); path = "foo/../file bar/."; len = strlen("foo/../file"); EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); - EXPECT_EQ(strlen("file"), static_cast<size_t>(len)); + EXPECT_EQ(strlen("file"), len); EXPECT_EQ("file ./file bar/.", string(path)); } @@ -136,3 +152,14 @@ TEST(StripAnsiEscapeCodes, StripColors) { EXPECT_EQ("affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]", stripped); } + +TEST(ElideMiddle, NothingToElide) { + string input = "Nothing to elide in this short string."; + EXPECT_EQ(input, ElideMiddle(input, 80)); +} + +TEST(ElideMiddle, ElideInTheMiddle) { + string input = "01234567890123456789"; + string elided = ElideMiddle(input, 10); + EXPECT_EQ("012...789", elided); +} diff --git a/src/win32port.h b/src/win32port.h index 8b42b38..ce3c949 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -14,8 +14,9 @@ #ifndef NINJA_WIN32PORT_H_ #define NINJA_WIN32PORT_H_ -#pragma once +typedef signed short int16_t; +typedef unsigned short uint16_t; /// A 64-bit integer type typedef signed long long int64_t; typedef unsigned long long uint64_t; |