Introduce automated validation for test format (#994)

This script is intended to identify common test file formatting errors
prior to their acceptance into the project. It is designed to support
future extensions for additional validation rules.

37 files changed, 571 insertions, 0 deletions

diff --git a/tools/lint/__init__.py b/tools/lint/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tools/lint/__init__.py
diff --git a/tools/lint/lib/__init__.py b/tools/lint/lib/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tools/lint/lib/__init__.py
diff --git a/tools/lint/lib/check.py b/tools/lint/lib/check.py
new file mode 100644
index 000000000..313072822
--- /dev/null
+++ b/tools/lint/lib/check.py
@@ -0,0 +1,6 @@
+class Check(object):
+    '''Base class for  defining linting checks.'''
+    ID = None
+
+    def run(self, name, meta, source):
+        return True
diff --git a/tools/lint/lib/checks/__init__.py b/tools/lint/lib/checks/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tools/lint/lib/checks/__init__.py
diff --git a/tools/lint/lib/checks/frontmatter.py b/tools/lint/lib/checks/frontmatter.py
new file mode 100644
index 000000000..a528f746a
--- /dev/null
+++ b/tools/lint/lib/checks/frontmatter.py
@@ -0,0 +1,42 @@
+from ..check import Check
+
+_REQUIRED_FIELDS = set(['description'])
+_OPTIONAL_FIELDS = set([
+    'author', 'es5id', 'es6id', 'esid', 'features', 'flags', 'includes',
+    'info', 'negative', 'timeout'
+])
+_VALID_FIELDS = _REQUIRED_FIELDS | _OPTIONAL_FIELDS
+
+class CheckFrontmatter(Check):
+    '''Ensure tests have the expected YAML-formatted metadata.'''
+    ID = 'FRONTMATTER'
+
+    def run(self, name, meta, source):
+        if name.endswith('_FIXTURE.js'):
+            if meta is not None:
+                return '"Fixture" files cannot specify metadata'
+            return
+
+        if meta is None:
+            return 'No valid YAML-formatted frontmatter'
+
+        fields = set(meta.keys())
+
+        missing = _REQUIRED_FIELDS - fields
+        if len(missing) > 0:
+            return 'Required fields missing: %s' % ', '.join(list(missing))
+
+        unrecognized = fields - _VALID_FIELDS
+        if len(unrecognized) > 0:
+            return 'Unrecognized fields: %s' % ', '.join(list(unrecognized))
+
+        if 'negative' in meta:
+            negative = meta['negative']
+            if not isinstance(negative, dict):
+                return '"negative" must be a dictionary with fields "type" and "phase"'
+
+            if not 'type' in negative:
+                return '"negative" must specify a "type" field'
+
+            if not 'phase' in negative:
+                return '"negative" must specify a "phase" field'
diff --git a/tools/lint/lib/checks/license.py b/tools/lint/lib/checks/license.py
new file mode 100644
index 000000000..364393963
--- /dev/null
+++ b/tools/lint/lib/checks/license.py
@@ -0,0 +1,43 @@
+import re
+
+from ..check import Check
+
+_MIN_YEAR = 2009
+_MAX_YEAR = 2030
+
+_LICENSE_PATTERN = re.compile(
+    r'\/\/ Copyright( \([cC]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
+    r'(' +
+        r'\/\/ (' +
+            r'This code is governed by the( BSD)? license found in the LICENSE file\.' +
+            r'|' +
+            r'See LICENSE for details' +
+        r')' +
+        r'|' +
+        r'\/\/ Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
+        r'\/\/ found in the LICENSE file\.' +
+        r'|' +
+        r'\/\/ See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
+    r')', re.IGNORECASE)
+
+class CheckLicense(Check):
+    '''Ensure tests declare valid license information.'''
+    ID = 'LICENSE'
+
+    def run(self, name, meta, source):
+        if meta and 'flags' in meta and 'generated' in meta['flags']:
+            return
+
+        match = _LICENSE_PATTERN.search(source)
+
+        if not match:
+            return 'No license information found.'
+
+        year_str = match.group(2)
+        try:
+            year = int(year_str)
+
+            if year < _MIN_YEAR or year > _MAX_YEAR:
+                raise ValueError()
+        except ValueError:
+            return 'Invalid year: %s' % year_str
diff --git a/tools/lint/lib/collect_files.py b/tools/lint/lib/collect_files.py
new file mode 100644
index 000000000..e94b2880f
--- /dev/null
+++ b/tools/lint/lib/collect_files.py
@@ -0,0 +1,20 @@
+import os
+
+def collect_files(path):
+    '''Given a path to a file, yield that path. Given a path to a directory,
+    yield the path of all files within that directory recursively, omitting any
+    that begin with a period (.) character.'''
+
+    if os.path.isfile(path):
+        yield path
+        return
+
+    if not os.path.isdir(path):
+        raise ValueError('Not found: "%s"' % path)
+
+    for root, dirs, file_names in os.walk(path):
+        for file_name in file_names:
+            if file_name.startswith('.'):
+                continue
+
+            yield os.path.join(root, file_name)
diff --git a/tools/lint/lib/eprint.py b/tools/lint/lib/eprint.py
new file mode 100644
index 000000000..d38678e23
--- /dev/null
+++ b/tools/lint/lib/eprint.py
@@ -0,0 +1,5 @@
+from __future__ import print_function
+import sys
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
diff --git a/tools/lint/lib/frontmatter.py b/tools/lint/lib/frontmatter.py
new file mode 100644
index 000000000..34edd1eac
--- /dev/null
+++ b/tools/lint/lib/frontmatter.py
@@ -0,0 +1,16 @@
+import re
+import yaml
+
+def parse(src):
+    '''Parse the YAML-formatted metadata found in a given string of source
+    code. Tolerate missing or invalid metadata; those conditions are handled by
+    a dedicated "Check" instance.'''
+
+    match = re.search(r'/\*---(.*)---\*/', src, re.DOTALL)
+    if not match:
+        return None
+
+    try:
+        return yaml.load(match.group(1))
+    except (yaml.scanner.ScannerError, yaml.parser.ParserError):
+        return None
diff --git a/tools/lint/lib/whitelist.py b/tools/lint/lib/whitelist.py
new file mode 100644
index 000000000..fc8b649ac
--- /dev/null
+++ b/tools/lint/lib/whitelist.py
@@ -0,0 +1,24 @@
+def parse(handle):
+    '''Parse the contents of the provided file descriptor as a linting
+    whitelist file. Return a dictionary whose keys are test file names and
+    whose values are Python sets of "Check" ID strings.'''
+
+    whitelist = dict()
+
+    for line in handle:
+        if line.startswith('#'):
+            continue
+
+        parts = line.split()
+        file_name = parts[0]
+        check_names = set(parts[1:])
+
+        assert file_name not in whitelist, (
+            'Whitelist should have a single entry for each file')
+
+        assert len(check_names) > 0, (
+            'Each whitelist entry should specify at least on check')
+
+        whitelist[file_name] = check_names
+
+    return whitelist
diff --git a/tools/lint/lint.py b/tools/lint/lint.py
new file mode 100755
index 000000000..b26569516
--- /dev/null
+++ b/tools/lint/lint.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# Copyright (C) 2017 Mike Pennisi. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import argparse
+import sys
+
+from lib.collect_files import collect_files
+from lib.checks.frontmatter import CheckFrontmatter
+from lib.checks.license import CheckLicense
+from lib.eprint import eprint
+import lib.frontmatter
+import lib.whitelist
+
+parser = argparse.ArgumentParser(description='Test262 linting tool')
+parser.add_argument('--whitelist',
+        type=argparse.FileType('r'),
+        help='file containing expected linting errors')
+parser.add_argument('path',
+        nargs='+',
+        help='file name or directory of files to lint')
+
+checks = [CheckFrontmatter(), CheckLicense()]
+
+def lint(file_names):
+    errors = dict()
+
+    for file_name in file_names:
+        with open(file_name, 'r') as f:
+            content = f.read()
+        meta = lib.frontmatter.parse(content)
+        for check in checks:
+            error = check.run(file_name, meta, content)
+
+            if error is not None:
+                if file_name not in errors:
+                    errors[file_name] = dict()
+                errors[file_name][check.ID] = error
+
+    return errors
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    if args.whitelist:
+        whitelist = lib.whitelist.parse(args.whitelist)
+    else:
+        whitelist = dict()
+
+    files = [path for _path in args.path for path in collect_files(_path)]
+    file_count = len(files)
+    print 'Linting %s file%s.' % (file_count, 's' if file_count != 1 else '')
+
+    all_errors = lint(files)
+    unexpected_errors = dict(all_errors)
+
+    for file_name, failures in all_errors.iteritems():
+        if file_name not in whitelist:
+            continue
+        if set(failures.keys()) == whitelist[file_name]:
+            del unexpected_errors[file_name]
+
+    error_count = len(unexpected_errors)
+    s = 's' if error_count != 1 else ''
+
+    print 'Linting complete. %s error%s found.' % (error_count, s)
+
+    if error_count == 0:
+        sys.exit(0)
+
+    for file_name, failures in unexpected_errors.iteritems():
+        for ID, message in failures.iteritems():
+            eprint('%s: %s - %s' % (file_name, ID, message))
+
+    sys.exit(1)
diff --git a/tools/lint/requirements.txt b/tools/lint/requirements.txt
new file mode 100644
index 000000000..efb082d8d
--- /dev/null
+++ b/tools/lint/requirements.txt
@@ -0,0 +1 @@
+PyYAML==3.11
diff --git a/tools/lint/test/fixtures/frontmatter_invalid_yaml.js b/tools/lint/test/fixtures/frontmatter_invalid_yaml.js
new file mode 100644
index 000000000..9209b8247
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_invalid_yaml.js
@@ -0,0 +1,15 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+description: Applied to a "covered" YieldExpression
+info: This is some information
+features: [generators
+---*/
+
+function* g() {
+  yield 23;
+}
diff --git a/tools/lint/test/fixtures/frontmatter_missing_desc.js b/tools/lint/test/fixtures/frontmatter_missing_desc.js
new file mode 100644
index 000000000..03f705e0a
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_missing_desc.js
@@ -0,0 +1,13 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+info: This is some information
+---*/
+
+function* g() {
+  yield 23;
+}
diff --git a/tools/lint/test/fixtures/frontmatter_module_FIXTURE.js b/tools/lint/test/fixtures/frontmatter_module_FIXTURE.js
new file mode 100644
index 000000000..6c6a408e6
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_module_FIXTURE.js
@@ -0,0 +1,7 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+
+function* g() {
+  yield 23;
+}
diff --git a/tools/lint/test/fixtures/frontmatter_module_with_meta_FIXTURE.js b/tools/lint/test/fixtures/frontmatter_module_with_meta_FIXTURE.js
new file mode 100644
index 000000000..35fbf4d1a
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_module_with_meta_FIXTURE.js
@@ -0,0 +1,15 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+description: Applied to a "covered" YieldExpression
+info: This is some information
+features: [generators]
+---*/
+
+function* g() {
+  yield 23;
+}
diff --git a/tools/lint/test/fixtures/frontmatter_negative_missing_phase.js b/tools/lint/test/fixtures/frontmatter_negative_missing_phase.js
new file mode 100644
index 000000000..8315882ca
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_negative_missing_phase.js
@@ -0,0 +1,12 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  type: SyntaxError
+---*/
+
+!!!
diff --git a/tools/lint/test/fixtures/frontmatter_negative_missing_type.js b/tools/lint/test/fixtures/frontmatter_negative_missing_type.js
new file mode 100644
index 000000000..6e3ec6551
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_negative_missing_type.js
@@ -0,0 +1,12 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  phase: early
+---*/
+
+!!!
diff --git a/tools/lint/test/fixtures/frontmatter_negative_string.js b/tools/lint/test/fixtures/frontmatter_negative_string.js
new file mode 100644
index 000000000..b11d19a3e
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_negative_string.js
@@ -0,0 +1,11 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative: SyntaxError
+---*/
+
+!!!
diff --git a/tools/lint/test/fixtures/frontmatter_negative_valid.js b/tools/lint/test/fixtures/frontmatter_negative_valid.js
new file mode 100644
index 000000000..f1b5cc05d
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_negative_valid.js
@@ -0,0 +1,12 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  type: SyntaxError
+  phase: early
+---*/
+
+!!!
diff --git a/tools/lint/test/fixtures/frontmatter_omitted.js b/tools/lint/test/fixtures/frontmatter_omitted.js
new file mode 100644
index 000000000..7b3b31c64
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_omitted.js
@@ -0,0 +1,8 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+
+function* g() {
+  yield 23;
+}
diff --git a/tools/lint/test/fixtures/frontmatter_unrecognized.js b/tools/lint/test/fixtures/frontmatter_unrecognized.js
new file mode 100644
index 000000000..48b0c857d
--- /dev/null
+++ b/tools/lint/test/fixtures/frontmatter_unrecognized.js
@@ -0,0 +1,15 @@
+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+description: Applied to a "covered" YieldExpression
+info: This is some information
+unrecognized_attr: foo
+---*/
+
+function* g() {
+  yield 23;
+}
diff --git a/tools/lint/test/fixtures/license_alternate_1.js b/tools/lint/test/fixtures/license_alternate_1.js
new file mode 100644
index 000000000..8283f80e6
--- /dev/null
+++ b/tools/lint/test/fixtures/license_alternate_1.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// Copyright (c) 2017 Mike Pennisi.  All rights reserved.
+// This code is governed by the license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_alternate_2.js b/tools/lint/test/fixtures/license_alternate_2.js
new file mode 100644
index 000000000..345f2c83c
--- /dev/null
+++ b/tools/lint/test/fixtures/license_alternate_2.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// Copyright 2017 Mike Pennisi.  All rights reserved.
+// See LICENSE for details.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_alternate_3.js b/tools/lint/test/fixtures/license_alternate_3.js
new file mode 100644
index 000000000..c66cd3e66
--- /dev/null
+++ b/tools/lint/test/fixtures/license_alternate_3.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// copyright (c) 2017 mike pennisi. all rights reserved.
+// this code is governed by the bsd license found in the license file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_alternate_4,js b/tools/lint/test/fixtures/license_alternate_4,js
new file mode 100644
index 000000000..684f0b197
--- /dev/null
+++ b/tools/lint/test/fixtures/license_alternate_4,js
@@ -0,0 +1,10 @@
+^ expected errors | v input
+// Copyright (c) 2017 Mike Pennisi. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_alternate_5.js b/tools/lint/test/fixtures/license_alternate_5.js
new file mode 100644
index 000000000..27605c320
--- /dev/null
+++ b/tools/lint/test/fixtures/license_alternate_5.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// See LICENSE or https://github.com/tc39/test262/blob/master/LICENSE
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_generated.js b/tools/lint/test/fixtures/license_generated.js
new file mode 100644
index 000000000..67aaf0126
--- /dev/null
+++ b/tools/lint/test/fixtures/license_generated.js
@@ -0,0 +1,11 @@
+^ expected errors | v input
+// This file was procedurally generated from the following sources:
+// - foo
+// - bar
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Applied to a "covered" YieldExpression
+flags: [class, generated]
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_invalid_year.js b/tools/lint/test/fixtures/license_invalid_year.js
new file mode 100644
index 000000000..abe53b869
--- /dev/null
+++ b/tools/lint/test/fixtures/license_invalid_year.js
@@ -0,0 +1,10 @@
+LICENSE
+^ expected errors | v input
+// Copyright (C) 2199 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Applied to a "covered" YieldExpression
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/license_missing.js b/tools/lint/test/fixtures/license_missing.js
new file mode 100644
index 000000000..3d90cf27f
--- /dev/null
+++ b/tools/lint/test/fixtures/license_missing.js
@@ -0,0 +1,8 @@
+LICENSE
+^ expected errors | v input
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Applied to a "covered" YieldExpression
+---*/
+
+void 0;
diff --git a/tools/lint/test/fixtures/valid_es5id.js b/tools/lint/test/fixtures/valid_es5id.js
new file mode 100644
index 000000000..2ca0a4d8d
--- /dev/null
+++ b/tools/lint/test/fixtures/valid_es5id.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+es5id: 12.14.1
+description: Minimal test
+---*/
+
+function f() {}
diff --git a/tools/lint/test/fixtures/valid_es6id.js b/tools/lint/test/fixtures/valid_es6id.js
new file mode 100644
index 000000000..4d71e162f
--- /dev/null
+++ b/tools/lint/test/fixtures/valid_es6id.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+es6id: 12.14.1
+description: Minimal test
+---*/
+
+function f() {}
diff --git a/tools/lint/test/fixtures/valid_esid.js b/tools/lint/test/fixtures/valid_esid.js
new file mode 100644
index 000000000..6b3b35c92
--- /dev/null
+++ b/tools/lint/test/fixtures/valid_esid.js
@@ -0,0 +1,9 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+---*/
+
+function f() {}
diff --git a/tools/lint/test/run.py b/tools/lint/test/run.py
new file mode 100755
index 000000000..9c83c7945
--- /dev/null
+++ b/tools/lint/test/run.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# Copyright (C) 2017 Mike Pennisi. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import shutil, subprocess, sys, os, unittest, tempfile
+
+testDir = os.path.dirname(os.path.relpath(__file__))
+OUT_DIR = os.path.join(testDir, 'out')
+ex = os.path.join(testDir, '..', 'lint.py')
+
+class TestLinter(unittest.TestCase):
+    maxDiff = None
+
+    def fixture(self, name, content):
+        fspath = os.path.join(OUT_DIR, name)
+        with open(fspath, 'w') as f:
+            f.write(content)
+        return fspath
+
+    def lint(self, args):
+        args[:0] = [ex]
+        sp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = sp.communicate()
+        return dict(stdout=stdout, stderr=stderr, returncode=sp.returncode)
+
+    def setUp(self):
+        os.mkdir(OUT_DIR)
+
+    def tearDown(self):
+        shutil.rmtree(OUT_DIR, ignore_errors=True)
+
+    def test_no_file(self):
+        result = self.lint(['non-existent-file.js'])
+        self.assertNotEqual(result["returncode"], 0)
+
+    def test_whitelist_single(self):
+        test_content = ('// Copyright (C) 2017 Mike Pennisi. All rights reserved.\n' +
+            '// This code is governed by the BSD license found in the LICENSE file.')
+        test_file = self.fixture('input.js', test_content)
+        whitelist_content = test_file + ' FRONTMATTER'
+        whitelist_file = self.fixture('lint.whitelist', whitelist_content)
+
+        result = self.lint([test_file])
+
+        self.assertNotEqual(result['returncode'], 0)
+
+        result = self.lint(['--whitelist', whitelist_file, test_file])
+
+        self.assertEqual(result['returncode'], 0)
+
+    def test_whitelist_comment(self):
+        test_content = ('// Copyright (C) 2017 Mike Pennisi. All rights reserved.\n' +
+            '// This code is governed by the BSD license found in the LICENSE file.')
+        test_file = self.fixture('input.js', test_content)
+        whitelist_content = ('# One comment\n' +
+            '# Another comment\n' +
+            test_file + ' FRONTMATTER')
+        whitelist_file = self.fixture('lint.whitelist', whitelist_content)
+
+        result = self.lint([test_file])
+
+        self.assertNotEqual(result['returncode'], 0)
+
+        result = self.lint(['--whitelist', whitelist_file, test_file])
+
+        self.assertEqual(result['returncode'], 0)
+
+def create_file_test(name, fspath):
+    '''Dynamically generate a function that may be used as a test method with
+    the Python `unittest` module.'''
+
+    def test(self):
+        with open(fspath, 'r') as f:
+            contents = f.read()
+        expected, input = contents.split('^ expected errors | v input\n')
+        expected = expected.split()
+        tmp_file = self.fixture(name, input)
+        result = self.lint([tmp_file])
+        if len(expected) == 0:
+            self.assertEqual(result['returncode'], 0)
+            self.assertEqual(result['stderr'], '')
+        else:
+            self.assertNotEqual(result['returncode'], 0)
+            for err in expected:
+                self.assertIn(err, result['stderr'])
+
+    return test
+
+dirname = os.path.join(os.path.abspath(testDir), 'fixtures')
+for file_name in os.listdir(dirname):
+    full_path = os.path.join(dirname, file_name)
+    if not os.path.isfile(full_path) or file_name.startswith('.'):
+        continue
+
+    t = create_file_test(file_name, full_path)
+    t.__name__ = 'test_' + file_name
+    setattr(TestLinter, t.__name__, t)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tools/scripts/ci.sh b/tools/scripts/ci_build.sh
index 478880c2a..478880c2a 100644..100755
--- a/tools/scripts/ci.sh
+++ b/tools/scripts/ci_build.sh
diff --git a/tools/scripts/ci_lint.sh b/tools/scripts/ci_lint.sh
new file mode 100755
index 000000000..75b3fb6a0
--- /dev/null
+++ b/tools/scripts/ci_lint.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+  paths=$(git diff --diff-filter ACMR --name-only $TRAVIS_BRANCH -- test/)
+
+  if [ "$paths" == "" ]; then
+    echo No test files added or modified. Exiting.
+    exit 0
+  fi
+
+  echo New or modified test files:
+  echo "$paths"
+
+else
+  paths="test/"
+fi
+
+./tools/lint/lint.py --whitelist lint.whitelist $paths
diff --git a/tools/scripts/deploy.sh b/tools/scripts/deploy.sh
index 44c6f968b..44c6f968b 100644..100755
--- a/tools/scripts/deploy.sh
+++ b/tools/scripts/deploy.sh