SERVER-22338 Integrate JavaScript linting into scons lint target

1 files changed, 553 insertions, 0 deletions

diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py
new file mode 100755
index 00000000000..1c706edbc61
--- /dev/null
+++ b/buildscripts/eslint.py
@@ -0,0 +1,553 @@
+#!/usr/bin/env python
+"""
+eslint.py
+ Will download a prebuilt ESLint binary if necessary (i.e. it isn't installed, isn't in the current
+ path, or is the wrong version). It works in much the same way as clang_format.py. In lint mode, it
+ will lint the files or directory paths passed. In lint-patch mode, for upload.py, it will see if
+ there are any candidate files in the supplied patch. Fix mode will run ESLint with the --fix
+ option, and that will update the files with missing semicolons and similar repairable issues.
+ There is also a -d mode that assumes you only want to run one copy of ESLint per file / directory
+ parameter supplied. This lets ESLint search for candidate files to lint.
+"""
+import Queue
+import itertools
+import os
+import re
+import shutil
+import string
+import subprocess
+import sys
+import tarfile
+import tempfile
+import threading
+import time
+import urllib
+from distutils import spawn
+from multiprocessing import cpu_count
+from optparse import OptionParser
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
+
+from buildscripts.resmokelib.utils import globstar
+from buildscripts import moduleconfig
+
+
+##############################################################################
+#
+# Constants for ESLint
+#
+#
+
+# Expected version of ESLint.
+ESLINT_VERSION = "2.3.0"
+
+# Name of ESLint as a binary.
+ESLINT_PROGNAME = "eslint"
+
+# URL location of our provided ESLint binaries.
+ESLINT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \
+                           ESLINT_VERSION + "-linux.tar.gz"
+ESLINT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \
+                            ESLINT_VERSION + "-darwin.tar.gz"
+
+# Path in the tarball to the ESLint binary.
+ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$platform-$arch")
+
+# Path to the modules in the mongodb source tree.
+# Has to match the string in SConstruct.
+MODULE_DIR = "src/mongo/db/modules"
+
+# Copied from python 2.7 version of subprocess.py
+# Exception classes used by this module.
+class CalledProcessError(Exception):
+    """This exception is raised when a process run by check_call() or
+    check_output() returns a non-zero exit status.
+    The exit status will be stored in the returncode attribute;
+    check_output() will also store the output in the output attribute.
+    """
+    def __init__(self, returncode, cmd, output=None):
+        self.returncode = returncode
+        self.cmd = cmd
+        self.output = output
+    def __str__(self):
+        return ("Command '%s' returned non-zero exit status %d with output %s" %
+            (self.cmd, self.returncode, self.output))
+
+
+# Copied from python 2.7 version of subprocess.py
+def check_output(*popenargs, **kwargs):
+    r"""Run command with arguments and return its output as a byte string.
+
+    If the exit code was non-zero it raises a CalledProcessError.  The
+    CalledProcessError object will have the return code in the returncode
+    attribute and output in the output attribute.
+
+    The arguments are the same as for the Popen constructor.  Example:
+
+    >>> check_output(["ls", "-l", "/dev/null"])
+    'crw-rw-rw- 1 root root 1, 3 Oct 18  2007 /dev/null\n'
+
+    The stdout argument is not allowed as it is used internally.
+    To capture standard error in the result, use stderr=STDOUT.
+
+    >>> check_output(["/bin/sh", "-c",
+    ...               "ls -l non_existent_file ; exit 0"],
+    ...              stderr=STDOUT)
+    'ls: non_existent_file: No such file or directory\n'
+    """
+    if 'stdout' in kwargs:
+        raise ValueError('stdout argument not allowed, it will be overridden.')
+    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    output, unused_err = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        cmd = kwargs.get("args")
+        if cmd is None:
+            cmd = popenargs[0]
+        raise CalledProcessError(retcode, cmd, output)
+    return output
+
+def callo(args):
+    """Call a program, and capture its output
+    """
+    return check_output(args)
+
+def extract_eslint(tar_path, target_file):
+    tarfp = tarfile.open(tar_path)
+    for name in tarfp.getnames():
+        if name == target_file:
+            tarfp.extract(name)
+    tarfp.close()
+
+def get_eslint_from_cache(dest_file, platform, arch):
+    """Get ESLint binary from mongodb's cache
+    """
+    # Get URL
+    if platform == "Linux":
+        url = ESLINT_HTTP_LINUX_CACHE
+    elif platform == "Darwin":
+        url = ESLINT_HTTP_DARWIN_CACHE
+    else:
+        raise ValueError('ESLint is not available as a binary for ' + platform)
+
+    dest_dir = tempfile.gettempdir()
+    temp_tar_file = os.path.join(dest_dir, "temp.tar.gz")
+
+    # Download the file
+    print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION,
+                                                           url, temp_tar_file))
+    urllib.urlretrieve(url, temp_tar_file)
+
+    eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch)
+    extract_eslint(temp_tar_file, eslint_distfile)
+    shutil.move(eslint_distfile, dest_file)
+
+
+class ESLint(object):
+    """Class encapsulates finding a suitable copy of ESLint, and linting an individual file
+    """
+    def __init__(self, path, cache_dir):
+        eslint_progname = ESLINT_PROGNAME
+
+        # Initialize ESLint configuration information
+        if sys.platform.startswith("linux"):
+            self.arch = "x86_64"
+            self.tar_path = None
+        elif sys.platform == "darwin":
+            self.arch = "x86_64"
+            self.tar_path = None
+
+        self.path = None
+
+        # Find ESLint now
+        if path is not None:
+            if os.path.isfile(path):
+                self.path = path
+            else:
+                print("WARNING: Could not find ESLint at %s" % (path))
+
+        # Check the environment variable
+        if "MONGO_ESLINT" in os.environ:
+            self.path = os.environ["MONGO_ESLINT"]
+
+            if self.path and not self._validate_version(warn=True):
+                self.path = None
+
+        # Check the user's PATH environment variable now
+        if self.path is None:
+            self.path = spawn.find_executable(eslint_progname)
+
+            if self.path and not self._validate_version(warn=True):
+                self.path = None
+
+        # Have not found it yet, download it from the web
+        if self.path is None:
+            if not os.path.isdir(cache_dir):
+                os.makedirs(cache_dir)
+
+            self.path = os.path.join(cache_dir, eslint_progname)
+
+            if not os.path.isfile(self.path):
+                if sys.platform.startswith("linux"):
+                    get_eslint_from_cache(self.path, "Linux", self.arch)
+                elif sys.platform == "darwin":
+                    get_eslint_from_cache(self.path, "Darwin", self.arch)
+                else:
+                    print("ERROR: eslint.py does not support downloading ESLint " +
+                          "on this platform, please install ESLint " + ESLINT_VERSION)
+        # Validate we have the correct version
+        if not self._validate_version():
+            raise ValueError('correct version of ESLint was not found.')
+
+        self.print_lock = threading.Lock()
+
+    def _validate_version(self, warn=False):
+        """Validate ESLint is the expected version
+        """
+        esl_version = callo([self.path, "--version"]).rstrip()
+        # Ignore the leading v in the version string.
+        if ESLINT_VERSION == esl_version[1:]:
+            return True
+
+        if warn:
+            print("WARNING: eslint found in path, but incorrect version found at " +
+                  self.path + " with version: " + esl_version)
+        return False
+
+    def _lint(self, file_name, print_diff):
+        """Check the specified file for linting errors
+        """
+        # ESLint returns non-zero on a linting error. That's all we care about
+        # so only enter the printing logic if we have an error.
+        try:
+            eslint_output = callo([self.path, "-f", "unix", file_name])
+        except CalledProcessError as e:
+            if print_diff:
+                # Take a lock to ensure error messages do not get mixed when printed to the screen
+                with self.print_lock:
+                    print("ERROR: ESLint found errors in " + file_name)
+                    print(e.output)
+            return False
+        except:
+            print("ERROR: ESLint process threw unexpected error", sys.exc_info()[0])
+            return False
+
+        return True
+
+    def lint(self, file_name):
+        """Check the specified file has no linting errors
+        """
+        return self._lint(file_name, print_diff=True)
+
+    def autofix(self, file_name):
+        """ Run ESLint in fix mode.
+        """
+        return not subprocess.call([self.path, "--fix", file_name])
+
+def parallel_process(items, func):
+    """Run a set of work items to completion
+    """
+    try:
+        cpus = cpu_count()
+    except NotImplementedError:
+        cpus = 1
+
+    task_queue = Queue.Queue()
+
+    # Use a list so that worker function will capture this variable
+    pp_event = threading.Event()
+    pp_result = [True]
+    pp_lock = threading.Lock()
+
+    def worker():
+        """Worker thread to process work items in parallel
+        """
+        while not pp_event.is_set():
+            try:
+                item = task_queue.get_nowait()
+            except Queue.Empty:
+                # if the queue is empty, exit the worker thread
+                pp_event.set()
+                return
+
+            try:
+                ret = func(item)
+            finally:
+                # Tell the queue we finished with the item
+                task_queue.task_done()
+
+            # Return early if we fail, and signal we are done
+            if not ret:
+                with pp_lock:
+                    pp_result[0] = False
+
+                pp_event.set()
+                return
+
+    # Enqueue all the work we want to process
+    for item in items:
+        task_queue.put(item)
+
+    # Process all the work
+    threads = []
+    for cpu in range(cpus):
+        thread = threading.Thread(target=worker)
+
+        thread.daemon = True
+        thread.start()
+        threads.append(thread)
+
+    # Wait for the threads to finish
+    # Loop with a timeout so that we can process Ctrl-C interrupts
+    # Note: On Python 2.6 wait always returns None so we check is_set also,
+    #  This works because we only set the event once, and never reset it
+    while not pp_event.wait(1) and not pp_event.is_set():
+        time.sleep(1)
+
+    for thread in threads:
+        thread.join()
+    return pp_result[0]
+
+def get_base_dir():
+    """Get the base directory for mongo repo.
+        This script assumes that it is running in buildscripts/, and uses
+        that to find the base directory.
+    """
+    try:
+        return subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).rstrip()
+    except:
+        # We are not in a valid git directory. Use the script path instead.
+        return os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+def get_repos():
+    """Get a list of linked repos and directories to run ESLint on.
+    """
+    base_dir = get_base_dir()
+
+    # Get a list of modules
+    # TODO: how do we filter rocks, does it matter?
+    mongo_modules = moduleconfig.discover_module_directories(
+                        os.path.join(base_dir, MODULE_DIR), None)
+
+    paths = [os.path.join(base_dir, MODULE_DIR, m) for m in mongo_modules]
+
+    paths.append(base_dir)
+
+    return [Repo(p) for p in paths]
+
+
+class Repo(object):
+    """Class encapsulates all knowledge about a git repository, and its metadata
+        to run ESLint.
+    """
+    def __init__(self, path):
+        self.path = path
+
+        # Get candidate files
+        self.candidate_files = self.get_candidate_files()
+
+        self.root = self._get_root()
+
+    def _callgito(self, args):
+        """Call git for this repository
+        """
+        # These two flags are the equivalent of -C in newer versions of Git
+        # but we use these to support versions back to ~1.8
+        return callo(['git', '--git-dir', os.path.join(self.path, ".git"),
+                        '--work-tree', self.path] + args)
+
+    def _get_local_dir(self, path):
+        """Get a directory path relative to the git root directory
+        """
+        if os.path.isabs(path):
+            return os.path.relpath(path, self.root)
+        return path
+
+    def get_candidates(self, candidates):
+        """Get the set of candidate files to check by doing an intersection
+        between the input list, and the list of candidates in the repository
+
+        Returns the full path to the files for ESLint to consume.
+        """
+        # NOTE: Files may have an absolute root (i.e. leading /)
+
+        if candidates is not None and len(candidates) > 0:
+            candidates = [self._get_local_dir(f) for f in candidates]
+            valid_files = list(set(candidates).intersection(self.get_candidate_files()))
+        else:
+            valid_files = list(self.get_candidate_files())
+
+        # Get the full file names here
+        valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files]
+        return valid_files
+
+    def _get_root(self):
+        """Gets the root directory for this repository from git
+        """
+        gito = self._callgito(['rev-parse', '--show-toplevel'])
+
+        return gito.rstrip()
+
+    def get_candidate_files(self):
+        """Query git to get a list of all files in the repo to consider for analysis
+        """
+        gito = self._callgito(["ls-files"])
+
+        # This allows us to pick all the interesting files
+        # in the mongo and mongo-enterprise repos
+        file_list = [line.rstrip()
+                     for line in gito.splitlines()
+                     if "src/mongo" in line or "jstests" in line]
+
+        files_match = re.compile('\\.js$')
+
+        file_list = [a for a in file_list if files_match.search(a)]
+
+        return file_list
+
+
+def expand_file_string(glob_pattern):
+    """Expand a string that represents a set of files
+    """
+    return [os.path.abspath(f) for f in globstar.iglob(glob_pattern)]
+
+def get_files_to_check(files):
+    """Filter the specified list of files to check down to the actual
+        list of files that need to be checked."""
+    candidates = []
+
+    # Get a list of candidate_files
+    candidates = [expand_file_string(f) for f in files]
+    candidates = list(itertools.chain.from_iterable(candidates))
+
+    repos = get_repos()
+
+    valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos]))
+
+    return valid_files
+
+def get_files_to_check_from_patch(patches):
+    """Take a patch file generated by git diff, and scan the patch for a list of files to check.
+    """
+    candidates = []
+
+    # Get a list of candidate_files
+    check = re.compile(r"^diff --git a\/([a-z\/\.\-_0-9]+) b\/[a-z\/\.\-_0-9]+")
+
+    lines = []
+    for patch in patches:
+        with open(patch, "rb") as infile:
+            lines += infile.readlines()
+
+    candidates = [check.match(line).group(1) for line in lines if check.match(line)]
+
+    repos = get_repos()
+
+    valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos]))
+
+    return valid_files
+
+def _get_build_dir():
+    """Get the location of the scons build directory in case we need to download ESLint
+    """
+    return os.path.join(get_base_dir(), "build")
+
+def _lint_files(eslint, files):
+    """Lint a list of files with ESLint
+    """
+    eslint = ESLint(eslint, _get_build_dir())
+
+    lint_clean = parallel_process([os.path.abspath(f) for f in files], eslint.lint)
+
+    if not lint_clean:
+        print("ERROR: ESLint found errors. Run ESLint manually to see errors in "\
+              "files that were skipped")
+        sys.exit(1)
+
+def lint_patch(eslint, infile):
+    """Lint patch command entry point
+    """
+    files = get_files_to_check_from_patch(infile)
+
+    # Patch may have files that we do not want to check which is fine
+    if files:
+        return _lint_files(eslint, files)
+    return True
+
+def lint(eslint, dirmode, glob):
+    """Lint files command entry point
+    """
+    if dirmode and glob:
+        files = glob
+    else:
+        files = get_files_to_check(glob)
+
+    _lint_files(eslint, files)
+
+    return True
+
+def _autofix_files(eslint, files):
+    """Auto-fix the specified files with ESLint.
+    """
+    eslint = ESLint(eslint, _get_build_dir())
+
+    autofix_clean = parallel_process([os.path.abspath(f) for f in files], eslint.autofix)
+
+    if not autofix_clean:
+        print("ERROR: failed to auto-fix files")
+        return False
+
+def autofix_func(eslint, dirmode, glob):
+    """Auto-fix files command entry point
+    """
+    if dirmode:
+        files = glob
+    else:
+        files = get_files_to_check(glob)
+
+    return _autofix_files(eslint, files)
+
+
+def main():
+    """Main entry point
+    """
+    success = False
+    usage = "%prog [-e <eslint>] [-d] lint|lint-patch|fix [glob patterns] "
+    description = "lint runs ESLint on provided patterns or all .js files under jstests/ "\
+                  "and src/mongo. lint-patch runs ESLint against .js files modified in the "\
+                  "patch file (for upload.py). fix runs ESLint with --fix on provided patterns "\
+                  "or files under jstests/ and src/mongo."
+    epilog ="*Unless you specify -d a separate ESLint process will be launched for every file"
+    parser = OptionParser()
+    parser = OptionParser(usage=usage, description=description, epilog=epilog)
+    parser.add_option("-e", "--eslint", type="string", dest="eslint",
+                      help="Fully qualified path to eslint executable",)
+    parser.add_option("-d", "--dirmode", action="store_true", default=True, dest="dirmode",
+                      help="Considers the glob patterns as directories and runs ESLint process " \
+                           "against each pattern",)
+
+    (options, args) = parser.parse_args(args=sys.argv)
+
+    if len(args) > 1:
+        command = args[1]
+        
+        searchlist = args[2:]
+        if not searchlist:
+            searchlist = ["jstests/", "src/mongo/"]
+
+        if command == "lint":
+            success = lint(options.eslint, options.dirmode, searchlist)
+        elif command == "lint-patch":
+            success = lint_patch(options.eslint, searchlist)
+        elif command == "fix":
+            success = autofix_func(options.eslint, options.dirmode, searchlist)
+        else:
+            parser.print_help()
+    else:
+        parser.print_help()
+
+    sys.exit(0 if success else 1)
+if __name__ == "__main__":
+    main()