Merge pull request #1488 from maciej-pawlowski-opera/issue-1227-2

 Fix parsing some special chars in depfiles (take 2)

4 files changed, 431 insertions, 510 deletions

diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc
index 7cee892..345fa15 100644
--- a/src/depfile_parser.cc
+++ b/src/depfile_parser.cc
@@ -1,4 +1,4 @@
-/* Generated by re2c 0.13.5 */
+/* Generated by re2c 0.16 */
 // Copyright 2011 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -53,7 +53,7 @@ bool DepfileParser::Parse(string* content, string* err) {
           0,   0,   0,   0,   0,   0,   0,   0, 
           0,   0,   0,   0,   0,   0,   0,   0, 
           0,   0,   0,   0,   0,   0,   0,   0, 
-          0, 128,   0,   0,   0,   0,   0,   0, 
+          0, 128,   0,   0,   0, 128,   0,   0, 
         128, 128,   0, 128, 128, 128, 128, 128, 
         128, 128, 128, 128, 128, 128, 128, 128, 
         128, 128, 128,   0,   0, 128,   0,   0, 
@@ -82,88 +82,37 @@ bool DepfileParser::Parse(string* content, string* err) {
         128, 128, 128, 128, 128, 128, 128, 128, 
         128, 128, 128, 128, 128, 128, 128, 128, 
       };
-
       yych = *in;
-      if (yych <= '=') {
-        if (yych <= '$') {
-          if (yych <= ' ') {
-            if (yych <= 0x00) goto yy7;
-            goto yy9;
-          } else {
-            if (yych <= '!') goto yy5;
-            if (yych <= '#') goto yy9;
-            goto yy4;
-          }
-        } else {
-          if (yych <= '*') {
-            if (yych <= '\'') goto yy9;
-            if (yych <= ')') goto yy5;
-            goto yy9;
-          } else {
-            if (yych <= ':') goto yy5;
-            if (yych <= '<') goto yy9;
-            goto yy5;
-          }
-        }
+      if (yybm[0+yych] & 128) {
+        goto yy6;
+      }
+      if (yych <= '$') {
+        if (yych <= 0x00) goto yy2;
+        if (yych <= '#') goto yy4;
+        goto yy9;
       } else {
-        if (yych <= '_') {
-          if (yych <= '[') {
-            if (yych <= '?') goto yy9;
-            if (yych <= 'Z') goto yy5;
-            goto yy9;
-          } else {
-            if (yych <= '\\') goto yy2;
-            if (yych <= '^') goto yy9;
-            goto yy5;
-          }
-        } else {
-          if (yych <= '|') {
-            if (yych <= '`') goto yy9;
-            if (yych <= '{') goto yy5;
-            goto yy9;
-          } else {
-            if (yych == 0x7F) goto yy9;
-            goto yy5;
-          }
-        }
+        if (yych == '\\') goto yy10;
+        goto yy4;
       }
 yy2:
       ++in;
-      if ((yych = *in) <= '"') {
-        if (yych <= '\f') {
-          if (yych <= 0x00) goto yy3;
-          if (yych != '\n') goto yy14;
-        } else {
-          if (yych <= '\r') goto yy3;
-          if (yych == ' ') goto yy16;
-          goto yy14;
-        }
-      } else {
-        if (yych <= 'Z') {
-          if (yych <= '#') goto yy16;
-          if (yych == '*') goto yy16;
-          goto yy14;
-        } else {
-          if (yych <= '\\') goto yy16;
-          if (yych == '|') goto yy16;
-          goto yy14;
-        }
+      {
+        break;
       }
-yy3:
+yy4:
+      ++in;
+yy5:
       {
         // For any other character (e.g. whitespace), swallow it here,
         // allowing the outer logic to loop around again.
         break;
       }
-yy4:
-      yych = *++in;
-      if (yych == '$') goto yy12;
-      goto yy3;
-yy5:
+yy6:
       ++in;
       yych = *in;
-      goto yy11;
-yy6:
+      if (yybm[0+yych] & 128) {
+        goto yy6;
+      }
       {
         // Got a span of plain text.
         int len = (int)(in - start);
@@ -173,30 +122,41 @@ yy6:
         out += len;
         continue;
       }
-yy7:
-      ++in;
-      {
-        break;
-      }
 yy9:
       yych = *++in;
-      goto yy3;
+      if (yych == '$') goto yy11;
+      goto yy5;
 yy10:
-      ++in;
-      yych = *in;
-yy11:
-      if (yybm[0+yych] & 128) {
-        goto yy10;
+      yych = *++in;
+      if (yych <= '"') {
+        if (yych <= '\f') {
+          if (yych <= 0x00) goto yy5;
+          if (yych == '\n') goto yy5;
+          goto yy13;
+        } else {
+          if (yych <= '\r') goto yy5;
+          if (yych == ' ') goto yy15;
+          goto yy13;
+        }
+      } else {
+        if (yych <= 'Z') {
+          if (yych <= '#') goto yy15;
+          if (yych == '*') goto yy15;
+          goto yy13;
+        } else {
+          if (yych <= ']') goto yy15;
+          if (yych == '|') goto yy15;
+          goto yy13;
+        }
       }
-      goto yy6;
-yy12:
+yy11:
       ++in;
       {
         // De-escape dollar character.
         *out++ = '$';
         continue;
       }
-yy14:
+yy13:
       ++in;
       {
         // Let backslash before other characters through verbatim.
@@ -204,7 +164,7 @@ yy14:
         *out++ = yych;
         continue;
       }
-yy16:
+yy15:
       ++in;
       {
         // De-escape backslashed character.
diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc
index 98c1621..464efda 100644
--- a/src/depfile_parser.in.cc
+++ b/src/depfile_parser.in.cc
@@ -55,7 +55,7 @@ bool DepfileParser::Parse(string* content, string* err) {
       re2c:indent:string = "  ";
 
       nul = "\000";
-      escape = [ \\#*[|];
+      escape = [ \\#*[|\]];
 
       '\\' escape {
         // De-escape backslashed character.
@@ -73,7 +73,7 @@ bool DepfileParser::Parse(string* content, string* err) {
         *out++ = yych;
         continue;
       }
-      [a-zA-Z0-9+,/_:.~()}{@=!\x80-\xFF-]+ {
+      [a-zA-Z0-9+,/_:.~()}{%@=!\x80-\xFF-]+ {
         // Got a span of plain text.
         int len = (int)(in - start);
         // Need to shift it over if we're overwriting backslashes.
diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc
index ee798f8..824073f 100644
--- a/src/depfile_parser_test.cc
+++ b/src/depfile_parser_test.cc
@@ -122,12 +122,13 @@ TEST_F(DepfileParserTest, SpecialChars) {
 "C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n"
 " en@quot.header~ t+t-x!=1 \n"
 " openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif\n"
-" Fu\303\244ball",
+" Fu\303\244ball\n"
+" a\\[1\\]b@2%c",
       &err));
   ASSERT_EQ("", err);
   EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h",
             parser_.out_.AsString());
-  ASSERT_EQ(4u, parser_.ins_.size());
+  ASSERT_EQ(5u, parser_.ins_.size());
   EXPECT_EQ("en@quot.header~",
             parser_.ins_[0].AsString());
   EXPECT_EQ("t+t-x!=1",
@@ -136,6 +137,8 @@ TEST_F(DepfileParserTest, SpecialChars) {
             parser_.ins_[2].AsString());
   EXPECT_EQ("Fu\303\244ball",
             parser_.ins_[3].AsString());
+  EXPECT_EQ("a[1]b@2%c",
+            parser_.ins_[4].AsString());
 }
 
 TEST_F(DepfileParserTest, UnifyMultipleOutputs) {
diff --git a/src/lexer.cc b/src/lexer.cc
index 3c6e70e..35ae97b 100644
--- a/src/lexer.cc
+++ b/src/lexer.cc
@@ -1,4 +1,4 @@
-/* Generated by re2c 0.13.5 */
+/* Generated by re2c 0.16 */
 // Copyright 2011 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -126,305 +126,325 @@ Lexer::Token Lexer::ReadToken() {
 	unsigned char yych;
 	unsigned int yyaccept = 0;
 	static const unsigned char yybm[] = {
-		  0,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,   0,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		192,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  96,  96,  64, 
-		 96,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  64,  64,  64,  64,  64,  64, 
-		 64,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  96,  64,  64,  64,  64,  96, 
-		 64,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  96,  96,  96,  96,  96,  96, 
-		 96,  96,  96,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
-		 64,  64,  64,  64,  64,  64,  64,  64, 
+		  0, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128,   0, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		160, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 192, 192, 128, 
+		192, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 128, 128, 128, 128, 128, 128, 
+		128, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 192, 128, 128, 128, 128, 192, 
+		128, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 192, 192, 192, 192, 192, 192, 
+		192, 192, 192, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
+		128, 128, 128, 128, 128, 128, 128, 128, 
 	};
-
 	yych = *p;
-	if (yych <= 'Z') {
-		if (yych <= '#') {
+	if (yybm[0+yych] & 32) {
+		goto yy9;
+	}
+	if (yych <= '^') {
+		if (yych <= ',') {
 			if (yych <= '\f') {
-				if (yych <= 0x00) goto yy23;
-				if (yych == '\n') goto yy7;
-				goto yy25;
+				if (yych <= 0x00) goto yy2;
+				if (yych == '\n') goto yy6;
+				goto yy4;
 			} else {
-				if (yych <= 0x1F) {
-					if (yych <= '\r') goto yy6;
-					goto yy25;
-				} else {
-					if (yych <= ' ') goto yy2;
-					if (yych <= '"') goto yy25;
-					goto yy4;
-				}
+				if (yych <= '\r') goto yy8;
+				if (yych == '#') goto yy12;
+				goto yy4;
 			}
 		} else {
-			if (yych <= '9') {
-				if (yych <= ',') goto yy25;
-				if (yych == '/') goto yy25;
-				goto yy22;
+			if (yych <= ':') {
+				if (yych == '/') goto yy4;
+				if (yych <= '9') goto yy13;
+				goto yy16;
 			} else {
-				if (yych <= '<') {
-					if (yych <= ':') goto yy16;
-					goto yy25;
+				if (yych <= '=') {
+					if (yych <= '<') goto yy4;
+					goto yy18;
 				} else {
-					if (yych <= '=') goto yy14;
-					if (yych <= '@') goto yy25;
-					goto yy22;
+					if (yych <= '@') goto yy4;
+					if (yych <= 'Z') goto yy13;
+					goto yy4;
 				}
 			}
 		}
 	} else {
 		if (yych <= 'i') {
-			if (yych <= 'a') {
-				if (yych == '_') goto yy22;
-				if (yych <= '`') goto yy25;
-				goto yy22;
+			if (yych <= 'b') {
+				if (yych == '`') goto yy4;
+				if (yych <= 'a') goto yy13;
+				goto yy20;
 			} else {
-				if (yych <= 'c') {
-					if (yych <= 'b') goto yy9;
-					goto yy22;
-				} else {
-					if (yych <= 'd') goto yy13;
-					if (yych <= 'h') goto yy22;
-					goto yy20;
-				}
+				if (yych == 'd') goto yy21;
+				if (yych <= 'h') goto yy13;
+				goto yy22;
 			}
 		} else {
 			if (yych <= 'r') {
-				if (yych == 'p') goto yy11;
-				if (yych <= 'q') goto yy22;
-				goto yy12;
+				if (yych == 'p') goto yy23;
+				if (yych <= 'q') goto yy13;
+				goto yy24;
 			} else {
 				if (yych <= 'z') {
-					if (yych <= 's') goto yy21;
-					goto yy22;
+					if (yych <= 's') goto yy25;
+					goto yy13;
 				} else {
-					if (yych == '|') goto yy18;
-					goto yy25;
+					if (yych == '|') goto yy26;
+					goto yy4;
 				}
 			}
 		}
 	}
 yy2:
-	yyaccept = 0;
-	yych = *(q = ++p);
-	goto yy73;
-yy3:
-	{ token = INDENT;   break; }
+	++p;
+	{ token = TEOF;     break; }
 yy4:
-	yyaccept = 1;
-	yych = *(q = ++p);
-	if (yych >= 0x01) goto yy68;
+	++p;
 yy5:
 	{ token = ERROR;    break; }
 yy6:
-	yych = *++p;
-	if (yych == '\n') goto yy65;
-	goto yy5;
-yy7:
 	++p;
-yy8:
 	{ token = NEWLINE;  break; }
+yy8:
+	yych = *++p;
+	if (yych == '\n') goto yy28;
+	goto yy5;
 yy9:
-	++p;
-	if ((yych = *p) == 'u') goto yy60;
-	goto yy27;
-yy10:
-	{ token = IDENT;    break; }
+	yyaccept = 0;
+	q = ++p;
+	yych = *p;
+	if (yybm[0+yych] & 32) {
+		goto yy9;
+	}
+	if (yych <= '\f') {
+		if (yych == '\n') goto yy6;
+	} else {
+		if (yych <= '\r') goto yy30;
+		if (yych == '#') goto yy32;
+	}
 yy11:
-	yych = *++p;
-	if (yych == 'o') goto yy56;
-	goto yy27;
+	{ token = INDENT;   break; }
 yy12:
-	yych = *++p;
-	if (yych == 'u') goto yy52;
-	goto yy27;
+	yyaccept = 1;
+	yych = *(q = ++p);
+	if (yych <= 0x00) goto yy5;
+	goto yy33;
 yy13:
-	yych = *++p;
-	if (yych == 'e') goto yy45;
-	goto yy27;
-yy14:
 	++p;
-	{ token = EQUALS;   break; }
+	yych = *p;
+yy14:
+	if (yybm[0+yych] & 64) {
+		goto yy13;
+	}
+	{ token = IDENT;    break; }
 yy16:
 	++p;
 	{ token = COLON;    break; }
 yy18:
 	++p;
-	if ((yych = *p) == '|') goto yy43;
-	{ token = PIPE;     break; }
+	{ token = EQUALS;   break; }
 yy20:
 	yych = *++p;
-	if (yych == 'n') goto yy36;
-	goto yy27;
+	if (yych == 'u') goto yy36;
+	goto yy14;
 yy21:
 	yych = *++p;
-	if (yych == 'u') goto yy28;
-	goto yy27;
+	if (yych == 'e') goto yy37;
+	goto yy14;
 yy22:
 	yych = *++p;
-	goto yy27;
+	if (yych == 'n') goto yy38;
+	goto yy14;
 yy23:
-	++p;
-	{ token = TEOF;     break; }
+	yych = *++p;
+	if (yych == 'o') goto yy39;
+	goto yy14;
+yy24:
+	yych = *++p;
+	if (yych == 'u') goto yy40;
+	goto yy14;
 yy25:
 	yych = *++p;
-	goto yy5;
+	if (yych == 'u') goto yy41;
+	goto yy14;
 yy26:
 	++p;
-	yych = *p;
-yy27:
-	if (yybm[0+yych] & 32) {
-		goto yy26;
-	}
-	goto yy10;
+	if ((yych = *p) == '|') goto yy42;
+	{ token = PIPE;     break; }
 yy28:
+	++p;
+	{ token = NEWLINE;  break; }
+yy30:
 	yych = *++p;
-	if (yych != 'b') goto yy27;
-	yych = *++p;
-	if (yych != 'n') goto yy27;
-	yych = *++p;
-	if (yych != 'i') goto yy27;
-	yych = *++p;
-	if (yych != 'n') goto yy27;
-	yych = *++p;
-	if (yych != 'j') goto yy27;
-	yych = *++p;
-	if (yych != 'a') goto yy27;
+	if (yych == '\n') goto yy28;
+yy31:
+	p = q;
+	if (yyaccept == 0) {
+		goto yy11;
+	} else {
+		goto yy5;
+	}
+yy32:
 	++p;
-	if (yybm[0+(yych = *p)] & 32) {
-		goto yy26;
+	yych = *p;
+yy33:
+	if (yybm[0+yych] & 128) {
+		goto yy32;
 	}
-	{ token = SUBNINJA; break; }
+	if (yych <= 0x00) goto yy31;
+	++p;
+	{ continue; }
 yy36:
 	yych = *++p;
-	if (yych != 'c') goto yy27;
+	if (yych == 'i') goto yy44;
+	goto yy14;
+yy37:
 	yych = *++p;
-	if (yych != 'l') goto yy27;
+	if (yych == 'f') goto yy45;
+	goto yy14;
+yy38:
 	yych = *++p;
-	if (yych != 'u') goto yy27;
+	if (yych == 'c') goto yy46;
+	goto yy14;
+yy39:
 	yych = *++p;
-	if (yych != 'd') goto yy27;
+	if (yych == 'o') goto yy47;
+	goto yy14;
+yy40:
 	yych = *++p;
-	if (yych != 'e') goto yy27;
-	++p;
-	if (yybm[0+(yych = *p)] & 32) {
-		goto yy26;
-	}
-	{ token = INCLUDE;  break; }
-yy43:
+	if (yych == 'l') goto yy48;
+	goto yy14;
+yy41:
+	yych = *++p;
+	if (yych == 'b') goto yy49;
+	goto yy14;
+yy42:
 	++p;
 	{ token = PIPE2;    break; }
+yy44:
+	yych = *++p;
+	if (yych == 'l') goto yy50;
+	goto yy14;
 yy45:
 	yych = *++p;
-	if (yych != 'f') goto yy27;
+	if (yych == 'a') goto yy51;
+	goto yy14;
+yy46:
 	yych = *++p;
-	if (yych != 'a') goto yy27;
+	if (yych == 'l') goto yy52;
+	goto yy14;
+yy47:
 	yych = *++p;
-	if (yych != 'u') goto yy27;
+	if (yych == 'l') goto yy53;
+	goto yy14;
+yy48:
 	yych = *++p;
-	if (yych != 'l') goto yy27;
+	if (yych == 'e') goto yy55;
+	goto yy14;
+yy49:
 	yych = *++p;
-	if (yych != 't') goto yy27;
-	++p;
-	if (yybm[0+(yych = *p)] & 32) {
-		goto yy26;
-	}
-	{ token = DEFAULT;  break; }
-yy52:
+	if (yych == 'n') goto yy57;
+	goto yy14;
+yy50:
 	yych = *++p;
-	if (yych != 'l') goto yy27;
+	if (yych == 'd') goto yy58;
+	goto yy14;
+yy51:
+	yych = *++p;
+	if (yych == 'u') goto yy60;
+	goto yy14;
+yy52:
 	yych = *++p;
-	if (yych != 'e') goto yy27;
+	if (yych == 'u') goto yy61;
+	goto yy14;
+yy53:
 	++p;
-	if (yybm[0+(yych = *p)] & 32) {
-		goto yy26;
+	if (yybm[0+(yych = *p)] & 64) {
+		goto yy13;
+	}
+	{ token = POOL;     break; }
+yy55:
+	++p;
+	if (yybm[0+(yych = *p)] & 64) {
+		goto yy13;
 	}
 	{ token = RULE;     break; }
-yy56:
-	yych = *++p;
-	if (yych != 'o') goto yy27;
+yy57:
 	yych = *++p;
-	if (yych != 'l') goto yy27;
+	if (yych == 'i') goto yy62;
+	goto yy14;
+yy58:
 	++p;
-	if (yybm[0+(yych = *p)] & 32) {
-		goto yy26;
+	if (yybm[0+(yych = *p)] & 64) {
+		goto yy13;
 	}
-	{ token = POOL;     break; }
+	{ token = BUILD;    break; }
 yy60:
 	yych = *++p;
-	if (yych != 'i') goto yy27;
+	if (yych == 'l') goto yy63;
+	goto yy14;
+yy61:
 	yych = *++p;
-	if (yych != 'l') goto yy27;
+	if (yych == 'd') goto yy64;
+	goto yy14;
+yy62:
 	yych = *++p;
-	if (yych != 'd') goto yy27;
-	++p;
-	if (yybm[0+(yych = *p)] & 32) {
-		goto yy26;
-	}
-	{ token = BUILD;    break; }
+	if (yych == 'n') goto yy65;
+	goto yy14;
+yy63:
+	yych = *++p;
+	if (yych == 't') goto yy66;
+	goto yy14;
+yy64:
+	yych = *++p;
+	if (yych == 'e') goto yy68;
+	goto yy14;
 yy65:
+	yych = *++p;
+	if (yych == 'j') goto yy70;
+	goto yy14;
+yy66:
 	++p;
-	{ token = NEWLINE;  break; }
-yy67:
-	++p;
-	yych = *p;
-yy68:
-	if (yybm[0+yych] & 64) {
-		goto yy67;
+	if (yybm[0+(yych = *p)] & 64) {
+		goto yy13;
 	}
-	if (yych >= 0x01) goto yy70;
-yy69:
-	p = q;
-	if (yyaccept <= 0) {
-		goto yy3;
-	} else {
-		goto yy5;
-	}
-yy70:
+	{ token = DEFAULT;  break; }
+yy68:
 	++p;
-	{ continue; }
-yy72:
-	yyaccept = 0;
-	q = ++p;
-	yych = *p;
-yy73:
-	if (yybm[0+yych] & 128) {
-		goto yy72;
-	}
-	if (yych <= '\f') {
-		if (yych != '\n') goto yy3;
-	} else {
-		if (yych <= '\r') goto yy75;
-		if (yych == '#') goto yy67;
-		goto yy3;
+	if (yybm[0+(yych = *p)] & 64) {
+		goto yy13;
 	}
+	{ token = INCLUDE;  break; }
+yy70:
 	yych = *++p;
-	goto yy8;
-yy75:
+	if (yych != 'a') goto yy14;
 	++p;
-	if ((yych = *p) == '\n') goto yy65;
-	goto yy69;
+	if (yybm[0+(yych = *p)] & 64) {
+		goto yy13;
+	}
+	{ token = SUBNINJA; break; }
 }
 
   }
@@ -487,49 +507,42 @@ void Lexer::EatWhitespace() {
 		  0,   0,   0,   0,   0,   0,   0,   0, 
 	};
 	yych = *p;
-	if (yych <= ' ') {
-		if (yych <= 0x00) goto yy82;
-		if (yych <= 0x1F) goto yy84;
-	} else {
-		if (yych == '$') goto yy80;
-		goto yy84;
+	if (yybm[0+yych] & 128) {
+		goto yy79;
 	}
+	if (yych <= 0x00) goto yy75;
+	if (yych == '$') goto yy82;
+	goto yy77;
+yy75:
 	++p;
-	yych = *p;
-	goto yy92;
-yy79:
-	{ continue; }
-yy80:
-	yych = *(q = ++p);
-	if (yych == '\n') goto yy85;
-	if (yych == '\r') goto yy87;
-yy81:
 	{ break; }
-yy82:
+yy77:
 	++p;
+yy78:
 	{ break; }
-yy84:
-	yych = *++p;
-	goto yy81;
-yy85:
+yy79:
 	++p;
+	yych = *p;
+	if (yybm[0+yych] & 128) {
+		goto yy79;
+	}
 	{ continue; }
-yy87:
+yy82:
+	yych = *(q = ++p);
+	if (yych == '\n') goto yy83;
+	if (yych == '\r') goto yy85;
+	goto yy78;
+yy83:
+	++p;
+	{ continue; }
+yy85:
 	yych = *++p;
-	if (yych == '\n') goto yy89;
+	if (yych == '\n') goto yy87;
 	p = q;
-	goto yy81;
-yy89:
+	goto yy78;
+yy87:
 	++p;
 	{ continue; }
-yy91:
-	++p;
-	yych = *p;
-yy92:
-	if (yybm[0+yych] & 128) {
-		goto yy91;
-	}
-	goto yy79;
 }
 
   }
@@ -578,45 +591,24 @@ bool Lexer::ReadIdent(string* out) {
 		  0,   0,   0,   0,   0,   0,   0,   0, 
 	};
 	yych = *p;
-	if (yych <= '@') {
-		if (yych <= '.') {
-			if (yych <= ',') goto yy97;
-		} else {
-			if (yych <= '/') goto yy97;
-			if (yych >= ':') goto yy97;
-		}
-	} else {
-		if (yych <= '_') {
-			if (yych <= 'Z') goto yy95;
-			if (yych <= '^') goto yy97;
-		} else {
-			if (yych <= '`') goto yy97;
-			if (yych >= '{') goto yy97;
-		}
+	if (yybm[0+yych] & 128) {
+		goto yy93;
 	}
-yy95:
-	++p;
-	yych = *p;
-	goto yy100;
-yy96:
-	{
-      out->assign(start, p - start);
-      break;
-    }
-yy97:
 	++p;
 	{
       last_token_ = start;
       return false;
     }
-yy99:
+yy93:
 	++p;
 	yych = *p;
-yy100:
 	if (yybm[0+yych] & 128) {
-		goto yy99;
+		goto yy93;
 	}
-	goto yy96;
+	{
+      out->assign(start, p - start);
+      break;
+    }
 }
 
   }
@@ -636,72 +628,69 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
 {
 	unsigned char yych;
 	static const unsigned char yybm[] = {
-		  0, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128,   0, 128, 128,   0, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		 16, 128, 128, 128,   0, 128, 128, 128, 
-		128, 128, 128, 128, 128, 224, 160, 128, 
-		224, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224,   0, 128, 128, 128, 128, 128, 
-		128, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224, 224, 128, 128, 128, 128, 224, 
-		128, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224, 224, 224, 224, 224, 224, 224, 
-		224, 224, 224, 128,   0, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
+		  0,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,   0,  16,  16,   0,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 32,  16,  16,  16,   0,  16,  16,  16, 
+		 16,  16,  16,  16,  16, 208, 144,  16, 
+		208, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208,   0,  16,  16,  16,  16,  16, 
+		 16, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208, 208,  16,  16,  16,  16, 208, 
+		 16, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208, 208, 208, 208, 208, 208, 208, 
+		208, 208, 208,  16,   0,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
+		 16,  16,  16,  16,  16,  16,  16,  16, 
 	};
 	yych = *p;
-	if (yych <= ' ') {
-		if (yych <= '\n') {
-			if (yych <= 0x00) goto yy110;
-			if (yych >= '\n') goto yy107;
-		} else {
-			if (yych == '\r') goto yy105;
-			if (yych >= ' ') goto yy107;
-		}
+	if (yybm[0+yych] & 16) {
+		goto yy100;
+	}
+	if (yych <= '\r') {
+		if (yych <= 0x00) goto yy98;
+		if (yych <= '\n') goto yy103;
+		goto yy105;
 	} else {
-		if (yych <= '9') {
-			if (yych == '$') goto yy109;
-		} else {
-			if (yych <= ':') goto yy107;
-			if (yych == '|') goto yy107;
-		}
+		if (yych <= ' ') goto yy103;
+		if (yych <= '$') goto yy107;
+		goto yy103;
 	}
+yy98:
 	++p;
-	yych = *p;
-	goto yy140;
-yy104:
 	{
-      eval->AddText(StringPiece(start, p - start));
-      continue;
+      last_token_ = start;
+      return Error("unexpected EOF", err);
     }
-yy105:
+yy100:
 	++p;
-	if ((yych = *p) == '\n') goto yy137;
+	yych = *p;
+	if (yybm[0+yych] & 16) {
+		goto yy100;
+	}
 	{
-      last_token_ = start;
-      return Error(DescribeLastError(), err);
+      eval->AddText(StringPiece(start, p - start));
+      continue;
     }
-yy107:
+yy103:
 	++p;
 	{
       if (path) {
@@ -714,152 +703,121 @@ yy107:
         continue;
       }
     }
-yy109:
+yy105:
+	++p;
+	if ((yych = *p) == '\n') goto yy108;
+	{
+      last_token_ = start;
+      return Error(DescribeLastError(), err);
+    }
+yy107:
 	yych = *++p;
-	if (yych <= '-') {
-		if (yych <= 0x1F) {
-			if (yych <= '\n') {
-				if (yych <= '\t') goto yy112;
-				goto yy124;
-			} else {
-				if (yych == '\r') goto yy114;
-				goto yy112;
-			}
+	if (yybm[0+yych] & 64) {
+		goto yy120;
+	}
+	if (yych <= ' ') {
+		if (yych <= '\f') {
+			if (yych == '\n') goto yy112;
+			goto yy110;
 		} else {
-			if (yych <= '#') {
-				if (yych <= ' ') goto yy115;
-				goto yy112;
-			} else {
-				if (yych <= '$') goto yy117;
-				if (yych <= ',') goto yy112;
-				goto yy119;
-			}
+			if (yych <= '\r') goto yy115;
+			if (yych <= 0x1F) goto yy110;
+			goto yy116;
 		}
 	} else {
-		if (yych <= 'Z') {
-			if (yych <= '9') {
-				if (yych <= '/') goto yy112;
-				goto yy119;
-			} else {
-				if (yych <= ':') goto yy121;
-				if (yych <= '@') goto yy112;
-				goto yy119;
-			}
+		if (yych <= '/') {
+			if (yych == '$') goto yy118;
+			goto yy110;
 		} else {
-			if (yych <= '`') {
-				if (yych == '_') goto yy119;
-				goto yy112;
-			} else {
-				if (yych <= 'z') goto yy119;
-				if (yych <= '{') goto yy123;
-				goto yy112;
-			}
+			if (yych <= ':') goto yy123;
+			if (yych <= '`') goto yy110;
+			if (yych <= '{') goto yy125;
+			goto yy110;
 		}
 	}
+yy108:
+	++p;
+	{
+      if (path)
+        p = start;
+      break;
+    }
 yy110:
 	++p;
+yy111:
 	{
       last_token_ = start;
-      return Error("unexpected EOF", err);
+      return Error("bad $-escape (literal $ must be written as $$)", err);
     }
 yy112:
 	++p;
-yy113:
+	yych = *p;
+	if (yybm[0+yych] & 32) {
+		goto yy112;
+	}
 	{
-      last_token_ = start;
-      return Error("bad $-escape (literal $ must be written as $$)", err);
+      continue;
     }
-yy114:
-	yych = *++p;
-	if (yych == '\n') goto yy134;
-	goto yy113;
 yy115:
+	yych = *++p;
+	if (yych == '\n') goto yy126;
+	goto yy111;
+yy116:
 	++p;
 	{
       eval->AddText(StringPiece(" ", 1));
       continue;
     }
-yy117:
+yy118:
 	++p;
 	{
       eval->AddText(StringPiece("$", 1));
       continue;
     }
-yy119:
+yy120:
 	++p;
 	yych = *p;
-	goto yy133;
-yy120:
+	if (yybm[0+yych] & 64) {
+		goto yy120;
+	}
 	{
       eval->AddSpecial(StringPiece(start + 1, p - start - 1));
       continue;
     }
-yy121:
+yy123:
 	++p;
 	{
       eval->AddText(StringPiece(":", 1));
       continue;
     }
-yy123:
+yy125:
 	yych = *(q = ++p);
-	if (yybm[0+yych] & 32) {
-		goto yy127;
+	if (yybm[0+yych] & 128) {
+		goto yy129;
 	}
-	goto yy113;
-yy124:
+	goto yy111;
+yy126:
 	++p;
 	yych = *p;
-	if (yybm[0+yych] & 16) {
-		goto yy124;
-	}
+	if (yych == ' ') goto yy126;
 	{
       continue;
     }
-yy127:
+yy129:
 	++p;
 	yych = *p;
-	if (yybm[0+yych] & 32) {
-		goto yy127;
+	if (yybm[0+yych] & 128) {
+		goto yy129;
 	}
-	if (yych == '}') goto yy130;
+	if (yych == '}') goto yy132;
 	p = q;
-	goto yy113;
-yy130:
-	++p;
-	{
-      eval->AddSpecial(StringPiece(start + 2, p - start - 3));
-      continue;
-    }
+	goto yy111;
 yy132:
 	++p;
-	yych = *p;
-yy133:
-	if (yybm[0+yych] & 64) {
-		goto yy132;
-	}
-	goto yy120;
-yy134:
-	++p;
-	yych = *p;
-	if (yych == ' ') goto yy134;
 	{
+      eval->AddSpecial(StringPiece(start + 2, p - start - 3));
       continue;
     }
-yy137:
-	++p;
-	{
-      if (path)
-        p = start;
-      break;
-    }
-yy139:
-	++p;
-	yych = *p;
-yy140:
-	if (yybm[0+yych] & 128) {
-		goto yy139;
-	}
-	goto yy104;
 }
 
   }