diff options
author | Mike Grundy <michael.grundy@10gen.com> | 2016-03-10 12:20:59 -0500 |
---|---|---|
committer | Mike Grundy <michael.grundy@10gen.com> | 2016-03-18 16:57:23 -0400 |
commit | d82956a6fe2e7d5dd6216e66e95900d73dbf2e0c (patch) | |
tree | 7152a153bb9a50d13e1afaea80fd1a1a45c481fa /buildscripts/eslint.py | |
parent | fde6349069e18220e5d417f9d12dffab36b8942a (diff) | |
download | mongo-d82956a6fe2e7d5dd6216e66e95900d73dbf2e0c.tar.gz |
SERVER-22338 Integrate JavaScript linting into scons lint target
Diffstat (limited to 'buildscripts/eslint.py')
-rwxr-xr-x | buildscripts/eslint.py | 553 |
1 files changed, 553 insertions, 0 deletions
diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py new file mode 100755 index 00000000000..1c706edbc61 --- /dev/null +++ b/buildscripts/eslint.py @@ -0,0 +1,553 @@ +#!/usr/bin/env python +""" +eslint.py + Will download a prebuilt ESLint binary if necessary (i.e. it isn't installed, isn't in the current + path, or is the wrong version). It works in much the same way as clang_format.py. In lint mode, it + will lint the files or directory paths passed. In lint-patch mode, for upload.py, it will see if + there are any candidate files in the supplied patch. Fix mode will run ESLint with the --fix + option, and that will update the files with missing semicolons and similar repairable issues. + There is also a -d mode that assumes you only want to run one copy of ESLint per file / directory + parameter supplied. This lets ESLint search for candidate files to lint. +""" +import Queue +import itertools +import os +import re +import shutil +import string +import subprocess +import sys +import tarfile +import tempfile +import threading +import time +import urllib +from distutils import spawn +from multiprocessing import cpu_count +from optparse import OptionParser + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) + +from buildscripts.resmokelib.utils import globstar +from buildscripts import moduleconfig + + +############################################################################## +# +# Constants for ESLint +# +# + +# Expected version of ESLint. +ESLINT_VERSION = "2.3.0" + +# Name of ESLint as a binary. +ESLINT_PROGNAME = "eslint" + +# URL location of our provided ESLint binaries. +ESLINT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \ + ESLINT_VERSION + "-linux.tar.gz" +ESLINT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \ + ESLINT_VERSION + "-darwin.tar.gz" + +# Path in the tarball to the ESLint binary. +ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$platform-$arch") + +# Path to the modules in the mongodb source tree. +# Has to match the string in SConstruct. +MODULE_DIR = "src/mongo/db/modules" + +# Copied from python 2.7 version of subprocess.py +# Exception classes used by this module. +class CalledProcessError(Exception): + """This exception is raised when a process run by check_call() or + check_output() returns a non-zero exit status. + The exit status will be stored in the returncode attribute; + check_output() will also store the output in the output attribute. + """ + def __init__(self, returncode, cmd, output=None): + self.returncode = returncode + self.cmd = cmd + self.output = output + def __str__(self): + return ("Command '%s' returned non-zero exit status %d with output %s" % + (self.cmd, self.returncode, self.output)) + + +# Copied from python 2.7 version of subprocess.py +def check_output(*popenargs, **kwargs): + r"""Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. + + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd, output) + return output + +def callo(args): + """Call a program, and capture its output + """ + return check_output(args) + +def extract_eslint(tar_path, target_file): + tarfp = tarfile.open(tar_path) + for name in tarfp.getnames(): + if name == target_file: + tarfp.extract(name) + tarfp.close() + +def get_eslint_from_cache(dest_file, platform, arch): + """Get ESLint binary from mongodb's cache + """ + # Get URL + if platform == "Linux": + url = ESLINT_HTTP_LINUX_CACHE + elif platform == "Darwin": + url = ESLINT_HTTP_DARWIN_CACHE + else: + raise ValueError('ESLint is not available as a binary for ' + platform) + + dest_dir = tempfile.gettempdir() + temp_tar_file = os.path.join(dest_dir, "temp.tar.gz") + + # Download the file + print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, + url, temp_tar_file)) + urllib.urlretrieve(url, temp_tar_file) + + eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch) + extract_eslint(temp_tar_file, eslint_distfile) + shutil.move(eslint_distfile, dest_file) + + +class ESLint(object): + """Class encapsulates finding a suitable copy of ESLint, and linting an individual file + """ + def __init__(self, path, cache_dir): + eslint_progname = ESLINT_PROGNAME + + # Initialize ESLint configuration information + if sys.platform.startswith("linux"): + self.arch = "x86_64" + self.tar_path = None + elif sys.platform == "darwin": + self.arch = "x86_64" + self.tar_path = None + + self.path = None + + # Find ESLint now + if path is not None: + if os.path.isfile(path): + self.path = path + else: + print("WARNING: Could not find ESLint at %s" % (path)) + + # Check the environment variable + if "MONGO_ESLINT" in os.environ: + self.path = os.environ["MONGO_ESLINT"] + + if self.path and not self._validate_version(warn=True): + self.path = None + + # Check the user's PATH environment variable now + if self.path is None: + self.path = spawn.find_executable(eslint_progname) + + if self.path and not self._validate_version(warn=True): + self.path = None + + # Have not found it yet, download it from the web + if self.path is None: + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + + self.path = os.path.join(cache_dir, eslint_progname) + + if not os.path.isfile(self.path): + if sys.platform.startswith("linux"): + get_eslint_from_cache(self.path, "Linux", self.arch) + elif sys.platform == "darwin": + get_eslint_from_cache(self.path, "Darwin", self.arch) + else: + print("ERROR: eslint.py does not support downloading ESLint " + + "on this platform, please install ESLint " + ESLINT_VERSION) + # Validate we have the correct version + if not self._validate_version(): + raise ValueError('correct version of ESLint was not found.') + + self.print_lock = threading.Lock() + + def _validate_version(self, warn=False): + """Validate ESLint is the expected version + """ + esl_version = callo([self.path, "--version"]).rstrip() + # Ignore the leading v in the version string. + if ESLINT_VERSION == esl_version[1:]: + return True + + if warn: + print("WARNING: eslint found in path, but incorrect version found at " + + self.path + " with version: " + esl_version) + return False + + def _lint(self, file_name, print_diff): + """Check the specified file for linting errors + """ + # ESLint returns non-zero on a linting error. That's all we care about + # so only enter the printing logic if we have an error. + try: + eslint_output = callo([self.path, "-f", "unix", file_name]) + except CalledProcessError as e: + if print_diff: + # Take a lock to ensure error messages do not get mixed when printed to the screen + with self.print_lock: + print("ERROR: ESLint found errors in " + file_name) + print(e.output) + return False + except: + print("ERROR: ESLint process threw unexpected error", sys.exc_info()[0]) + return False + + return True + + def lint(self, file_name): + """Check the specified file has no linting errors + """ + return self._lint(file_name, print_diff=True) + + def autofix(self, file_name): + """ Run ESLint in fix mode. + """ + return not subprocess.call([self.path, "--fix", file_name]) + +def parallel_process(items, func): + """Run a set of work items to completion + """ + try: + cpus = cpu_count() + except NotImplementedError: + cpus = 1 + + task_queue = Queue.Queue() + + # Use a list so that worker function will capture this variable + pp_event = threading.Event() + pp_result = [True] + pp_lock = threading.Lock() + + def worker(): + """Worker thread to process work items in parallel + """ + while not pp_event.is_set(): + try: + item = task_queue.get_nowait() + except Queue.Empty: + # if the queue is empty, exit the worker thread + pp_event.set() + return + + try: + ret = func(item) + finally: + # Tell the queue we finished with the item + task_queue.task_done() + + # Return early if we fail, and signal we are done + if not ret: + with pp_lock: + pp_result[0] = False + + pp_event.set() + return + + # Enqueue all the work we want to process + for item in items: + task_queue.put(item) + + # Process all the work + threads = [] + for cpu in range(cpus): + thread = threading.Thread(target=worker) + + thread.daemon = True + thread.start() + threads.append(thread) + + # Wait for the threads to finish + # Loop with a timeout so that we can process Ctrl-C interrupts + # Note: On Python 2.6 wait always returns None so we check is_set also, + # This works because we only set the event once, and never reset it + while not pp_event.wait(1) and not pp_event.is_set(): + time.sleep(1) + + for thread in threads: + thread.join() + return pp_result[0] + +def get_base_dir(): + """Get the base directory for mongo repo. + This script assumes that it is running in buildscripts/, and uses + that to find the base directory. + """ + try: + return subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).rstrip() + except: + # We are not in a valid git directory. Use the script path instead. + return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +def get_repos(): + """Get a list of linked repos and directories to run ESLint on. + """ + base_dir = get_base_dir() + + # Get a list of modules + # TODO: how do we filter rocks, does it matter? + mongo_modules = moduleconfig.discover_module_directories( + os.path.join(base_dir, MODULE_DIR), None) + + paths = [os.path.join(base_dir, MODULE_DIR, m) for m in mongo_modules] + + paths.append(base_dir) + + return [Repo(p) for p in paths] + + +class Repo(object): + """Class encapsulates all knowledge about a git repository, and its metadata + to run ESLint. + """ + def __init__(self, path): + self.path = path + + # Get candidate files + self.candidate_files = self.get_candidate_files() + + self.root = self._get_root() + + def _callgito(self, args): + """Call git for this repository + """ + # These two flags are the equivalent of -C in newer versions of Git + # but we use these to support versions back to ~1.8 + return callo(['git', '--git-dir', os.path.join(self.path, ".git"), + '--work-tree', self.path] + args) + + def _get_local_dir(self, path): + """Get a directory path relative to the git root directory + """ + if os.path.isabs(path): + return os.path.relpath(path, self.root) + return path + + def get_candidates(self, candidates): + """Get the set of candidate files to check by doing an intersection + between the input list, and the list of candidates in the repository + + Returns the full path to the files for ESLint to consume. + """ + # NOTE: Files may have an absolute root (i.e. leading /) + + if candidates is not None and len(candidates) > 0: + candidates = [self._get_local_dir(f) for f in candidates] + valid_files = list(set(candidates).intersection(self.get_candidate_files())) + else: + valid_files = list(self.get_candidate_files()) + + # Get the full file names here + valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files] + return valid_files + + def _get_root(self): + """Gets the root directory for this repository from git + """ + gito = self._callgito(['rev-parse', '--show-toplevel']) + + return gito.rstrip() + + def get_candidate_files(self): + """Query git to get a list of all files in the repo to consider for analysis + """ + gito = self._callgito(["ls-files"]) + + # This allows us to pick all the interesting files + # in the mongo and mongo-enterprise repos + file_list = [line.rstrip() + for line in gito.splitlines() + if "src/mongo" in line or "jstests" in line] + + files_match = re.compile('\\.js$') + + file_list = [a for a in file_list if files_match.search(a)] + + return file_list + + +def expand_file_string(glob_pattern): + """Expand a string that represents a set of files + """ + return [os.path.abspath(f) for f in globstar.iglob(glob_pattern)] + +def get_files_to_check(files): + """Filter the specified list of files to check down to the actual + list of files that need to be checked.""" + candidates = [] + + # Get a list of candidate_files + candidates = [expand_file_string(f) for f in files] + candidates = list(itertools.chain.from_iterable(candidates)) + + repos = get_repos() + + valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos])) + + return valid_files + +def get_files_to_check_from_patch(patches): + """Take a patch file generated by git diff, and scan the patch for a list of files to check. + """ + candidates = [] + + # Get a list of candidate_files + check = re.compile(r"^diff --git a\/([a-z\/\.\-_0-9]+) b\/[a-z\/\.\-_0-9]+") + + lines = [] + for patch in patches: + with open(patch, "rb") as infile: + lines += infile.readlines() + + candidates = [check.match(line).group(1) for line in lines if check.match(line)] + + repos = get_repos() + + valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos])) + + return valid_files + +def _get_build_dir(): + """Get the location of the scons build directory in case we need to download ESLint + """ + return os.path.join(get_base_dir(), "build") + +def _lint_files(eslint, files): + """Lint a list of files with ESLint + """ + eslint = ESLint(eslint, _get_build_dir()) + + lint_clean = parallel_process([os.path.abspath(f) for f in files], eslint.lint) + + if not lint_clean: + print("ERROR: ESLint found errors. Run ESLint manually to see errors in "\ + "files that were skipped") + sys.exit(1) + +def lint_patch(eslint, infile): + """Lint patch command entry point + """ + files = get_files_to_check_from_patch(infile) + + # Patch may have files that we do not want to check which is fine + if files: + return _lint_files(eslint, files) + return True + +def lint(eslint, dirmode, glob): + """Lint files command entry point + """ + if dirmode and glob: + files = glob + else: + files = get_files_to_check(glob) + + _lint_files(eslint, files) + + return True + +def _autofix_files(eslint, files): + """Auto-fix the specified files with ESLint. + """ + eslint = ESLint(eslint, _get_build_dir()) + + autofix_clean = parallel_process([os.path.abspath(f) for f in files], eslint.autofix) + + if not autofix_clean: + print("ERROR: failed to auto-fix files") + return False + +def autofix_func(eslint, dirmode, glob): + """Auto-fix files command entry point + """ + if dirmode: + files = glob + else: + files = get_files_to_check(glob) + + return _autofix_files(eslint, files) + + +def main(): + """Main entry point + """ + success = False + usage = "%prog [-e <eslint>] [-d] lint|lint-patch|fix [glob patterns] " + description = "lint runs ESLint on provided patterns or all .js files under jstests/ "\ + "and src/mongo. lint-patch runs ESLint against .js files modified in the "\ + "patch file (for upload.py). fix runs ESLint with --fix on provided patterns "\ + "or files under jstests/ and src/mongo." + epilog ="*Unless you specify -d a separate ESLint process will be launched for every file" + parser = OptionParser() + parser = OptionParser(usage=usage, description=description, epilog=epilog) + parser.add_option("-e", "--eslint", type="string", dest="eslint", + help="Fully qualified path to eslint executable",) + parser.add_option("-d", "--dirmode", action="store_true", default=True, dest="dirmode", + help="Considers the glob patterns as directories and runs ESLint process " \ + "against each pattern",) + + (options, args) = parser.parse_args(args=sys.argv) + + if len(args) > 1: + command = args[1] + + searchlist = args[2:] + if not searchlist: + searchlist = ["jstests/", "src/mongo/"] + + if command == "lint": + success = lint(options.eslint, options.dirmode, searchlist) + elif command == "lint-patch": + success = lint_patch(options.eslint, searchlist) + elif command == "fix": + success = autofix_func(options.eslint, options.dirmode, searchlist) + else: + parser.print_help() + else: + parser.print_help() + + sys.exit(0 if success else 1) +if __name__ == "__main__": + main() |