Enforce use of `throw` stmt in early error tests

Previously, test consumers were encouraged to insert a `throw` statement
as the first statement of tests for early errors. This recommendation
made tests harder to consume, and as an optional transformation,
consumers may have ignored it or simply been unaware it was made. By
explicitly including such a `throw` statement, the tests become more
literal, making them easier to consume and more transparent in their
expectations.

Document expectation for all tests for early errors to include an
explicit `throw` statement. Extend linting script to verify that
contributors are automatically notified of violations and to ensure that
future contributions satisfy this expectation.

10 files changed, 78 insertions, 15 deletions

diff --git a/tools/lint/lib/checks/frontmatter.py b/tools/lint/lib/checks/frontmatter.py
index a528f746a..b62f41935 100644
--- a/tools/lint/lib/checks/frontmatter.py
+++ b/tools/lint/lib/checks/frontmatter.py
@@ -29,14 +29,3 @@ class CheckFrontmatter(Check):
         unrecognized = fields - _VALID_FIELDS
         if len(unrecognized) > 0:
             return 'Unrecognized fields: %s' % ', '.join(list(unrecognized))
-
-        if 'negative' in meta:
-            negative = meta['negative']
-            if not isinstance(negative, dict):
-                return '"negative" must be a dictionary with fields "type" and "phase"'
-
-            if not 'type' in negative:
-                return '"negative" must specify a "type" field'
-
-            if not 'phase' in negative:
-                return '"negative" must specify a "phase" field'
diff --git a/tools/lint/lib/checks/negative.py b/tools/lint/lib/checks/negative.py
new file mode 100644
index 000000000..b1ae3e4fb
--- /dev/null
+++ b/tools/lint/lib/checks/negative.py
@@ -0,0 +1,27 @@
+import re
+from ..check import Check
+
+_THROW_STMT = re.compile(
+      r'^throw "Test262: This statement should not be evaluated\.";$',
+      re.MULTILINE)
+
+class CheckNegative(Check):
+    '''Ensure tests have the expected YAML-formatted metadata.'''
+    ID = 'NEGATIVE'
+
+    def run(self, name, meta, source):
+        if meta is None or meta.get('negative') is None:
+            return
+
+        negative = meta['negative']
+        if not isinstance(negative, dict):
+            return '"negative" must be a dictionary with fields "type" and "phase"'
+
+        if not 'type' in negative:
+            return '"negative" must specify a "type" field'
+
+        if not 'phase' in negative:
+            return '"negative" must specify a "phase" field'
+
+        if negative["phase"] == "early" and not _THROW_STMT.search(source):
+            return 'Negative tests of type "early" must include a `throw` statement'
diff --git a/tools/lint/lint.py b/tools/lint/lint.py
index 9cdbca474..09e22507a 100755
--- a/tools/lint/lint.py
+++ b/tools/lint/lint.py
@@ -9,6 +9,7 @@ from lib.collect_files import collect_files
 from lib.checks.features import CheckFeatures
 from lib.checks.frontmatter import CheckFrontmatter
 from lib.checks.license import CheckLicense
+from lib.checks.negative import CheckNegative
 from lib.eprint import eprint
 import lib.frontmatter
 import lib.whitelist
@@ -21,7 +22,10 @@ parser.add_argument('path',
         nargs='+',
         help='file name or directory of files to lint')
 
-checks = [CheckFrontmatter(), CheckFeatures('features.txt'), CheckLicense()]
+checks = [
+        CheckFrontmatter(), CheckFeatures('features.txt'), CheckLicense(),
+        CheckNegative()
+    ]
 
 def lint(file_names):
     errors = dict()
diff --git a/tools/lint/test/fixtures/negative_early_throw_bad_value.js b/tools/lint/test/fixtures/negative_early_throw_bad_value.js
new file mode 100644
index 000000000..22979c313
--- /dev/null
+++ b/tools/lint/test/fixtures/negative_early_throw_bad_value.js
@@ -0,0 +1,15 @@
+NEGATIVE
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  type: SyntaxError
+  phase: early
+---*/
+
+throw "Test262: This statement should not be evaluated!";
+
+!!!
diff --git a/tools/lint/test/fixtures/frontmatter_negative_valid.js b/tools/lint/test/fixtures/negative_early_throw_missing.js
index f1b5cc05d..44a9a22dd 100644
--- a/tools/lint/test/fixtures/frontmatter_negative_valid.js
+++ b/tools/lint/test/fixtures/negative_early_throw_missing.js
@@ -1,3 +1,4 @@
+NEGATIVE
 ^ expected errors | v input
 // Copyright (C) 2017 Mike Pennisi. All rights reserved.
 // This code is governed by the BSD license found in the LICENSE file.
diff --git a/tools/lint/test/fixtures/frontmatter_negative_missing_phase.js b/tools/lint/test/fixtures/negative_missing_phase.js
index 8315882ca..e830d4238 100644
--- a/tools/lint/test/fixtures/frontmatter_negative_missing_phase.js
+++ b/tools/lint/test/fixtures/negative_missing_phase.js
@@ -1,4 +1,4 @@
-FRONTMATTER
+NEGATIVE
 ^ expected errors | v input
 // Copyright (C) 2017 Mike Pennisi. All rights reserved.
 // This code is governed by the BSD license found in the LICENSE file.
diff --git a/tools/lint/test/fixtures/frontmatter_negative_missing_type.js b/tools/lint/test/fixtures/negative_missing_type.js
index 6e3ec6551..8642c9124 100644
--- a/tools/lint/test/fixtures/frontmatter_negative_missing_type.js
+++ b/tools/lint/test/fixtures/negative_missing_type.js
@@ -1,4 +1,4 @@
-FRONTMATTER
+NEGATIVE
 ^ expected errors | v input
 // Copyright (C) 2017 Mike Pennisi. All rights reserved.
 // This code is governed by the BSD license found in the LICENSE file.
diff --git a/tools/lint/test/fixtures/frontmatter_negative_string.js b/tools/lint/test/fixtures/negative_string.js
index b11d19a3e..706f773a5 100644
--- a/tools/lint/test/fixtures/frontmatter_negative_string.js
+++ b/tools/lint/test/fixtures/negative_string.js
@@ -1,4 +1,4 @@
-FRONTMATTER
+NEGATIVE
 ^ expected errors | v input
 // Copyright (C) 2017 Mike Pennisi. All rights reserved.
 // This code is governed by the BSD license found in the LICENSE file.
diff --git a/tools/lint/test/fixtures/negative_valid_early.js b/tools/lint/test/fixtures/negative_valid_early.js
new file mode 100644
index 000000000..5b6c7a261
--- /dev/null
+++ b/tools/lint/test/fixtures/negative_valid_early.js
@@ -0,0 +1,14 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  type: SyntaxError
+  phase: early
+---*/
+
+throw "Test262: This statement should not be evaluated.";
+
+!!!
diff --git a/tools/lint/test/fixtures/negative_valid_runtime.js b/tools/lint/test/fixtures/negative_valid_runtime.js
new file mode 100644
index 000000000..ed4b056fd
--- /dev/null
+++ b/tools/lint/test/fixtures/negative_valid_runtime.js
@@ -0,0 +1,13 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  type: ReferenceError
+  phase: runtime
+---*/
+
+x;
+let x;