Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1156)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: src/lexer/lexer_py.re

Issue 66533002: Experimental parser: better generation (Closed) Base URL: https://v8.googlecode.com/svn/branches/experimental/parser
Patch Set: Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | tools/lexer_generator/generator.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: src/lexer/lexer_py.re
diff --git a/src/lexer/lexer_py.re b/src/lexer/lexer_py.re
index 26e3f21065d71b4b5653be28855f0aec0c3af86e..1a59620152d7f0f66006858cacf879eb57aa8a43 100644
--- a/src/lexer/lexer_py.re
+++ b/src/lexer/lexer_py.re
@@ -101,16 +101,63 @@ number { PUSH_TOKEN(NUMBER); }
line_terminator+ { PUSH_LINE_TERMINATOR(); }
whitespace <<continue>>
-"\"" <<DoubleQuoteString>> # TODO mark these transitions as ignoring this character
+"\"" <<DoubleQuoteString>>
"'" <<SingleQuoteString>>
-identifier_start <<Identifier>> # TODO merge identifier dfa...
+# all keywords
+"break" { PUSH_TOKEN(BREAK); } <<break>>
+"case" { PUSH_TOKEN(CASE); } <<break>>
+"catch" { PUSH_TOKEN(CATCH); } <<break>>
+"class" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"const" { PUSH_TOKEN(CONST); } <<break>>
+"continue" { PUSH_TOKEN(CONTINUE); } <<break>>
+"debugger" { PUSH_TOKEN(DEBUGGER); } <<break>>
+"default" { PUSH_TOKEN(DEFAULT); } <<break>>
+"delete" { PUSH_TOKEN(DELETE); } <<break>>
+"do" { PUSH_TOKEN(DO); } <<break>>
+"else" { PUSH_TOKEN(ELSE); } <<break>>
+"enum" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"export" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"extends" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"false" { PUSH_TOKEN(FALSE_LITERAL); } <<break>>
+"finally" { PUSH_TOKEN(FINALLY); } <<break>>
+"for" { PUSH_TOKEN(FOR); } <<break>>
+"function" { PUSH_TOKEN(FUNCTION); } <<break>>
+"if" { PUSH_TOKEN(IF); } <<break>>
+"implements" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"import" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"in" { PUSH_TOKEN(IN); } <<break>>
+"instanceof" { PUSH_TOKEN(INSTANCEOF); } <<break>>
+"interface" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"let" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"new" { PUSH_TOKEN(NEW); } <<break>>
+"null" { PUSH_TOKEN(NULL_LITERAL); } <<break>>
+"package" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"private" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"protected" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"public" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"return" { PUSH_TOKEN(RETURN); } <<break>>
+"static" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"super" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"switch" { PUSH_TOKEN(SWITCH); } <<break>>
+"this" { PUSH_TOKEN(THIS); } <<break>>
+"throw" { PUSH_TOKEN(THROW); } <<break>>
+"true" { PUSH_TOKEN(TRUE_LITERAL); } <<break>>
+"try" { PUSH_TOKEN(TRY); } <<break>>
+"typeof" { PUSH_TOKEN(TYPEOF); } <<break>>
+"var" { PUSH_TOKEN(VAR); } <<break>>
+"void" { PUSH_TOKEN(VOID); } <<break>>
+"while" { PUSH_TOKEN(WHILE); } <<break>>
+"with" { PUSH_TOKEN(WITH); } <<break>>
+"yield" { PUSH_TOKEN(YIELD); } <<break>>
+
+identifier_start <<Identifier>>
/\\u[0-9a-fA-F]{4}/ {
if (V8_LIKELY(ValidIdentifierStart())) {
JUMP(Identifier);
}
PUSH_TOKEN(ILLEGAL);
-}
+} <<Identifier>>
eof <<terminate>>
default { PUSH_TOKEN(ILLEGAL); }
@@ -121,7 +168,7 @@ default { PUSH_TOKEN(ILLEGAL); }
"\"" { PUSH_TOKEN(STRING); } <<break>>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
-/\n\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
+/\n|\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
eof <<terminate_illegal>>
default <<continue>>
@@ -131,7 +178,7 @@ default <<continue>>
"'" { PUSH_TOKEN(STRING); } <<break>>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
-/\n\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
+/\n|\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
eof <<terminate_illegal>>
default <<continue>>
@@ -161,51 +208,3 @@ default <<continue>>
line_terminator+ { PUSH_LINE_TERMINATOR(); }
eof <<terminate>>
default <<continue>>
-
-<default>
-# all keywords
-"break" { PUSH_TOKEN(BREAK); } <<break>>
-"case" { PUSH_TOKEN(CASE); } <<break>>
-"catch" { PUSH_TOKEN(CATCH); } <<break>>
-"class" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"const" { PUSH_TOKEN(CONST); } <<break>>
-"continue" { PUSH_TOKEN(CONTINUE); } <<break>>
-"debugger" { PUSH_TOKEN(DEBUGGER); } <<break>>
-"default" { PUSH_TOKEN(DEFAULT); } <<break>>
-"delete" { PUSH_TOKEN(DELETE); } <<break>>
-"do" { PUSH_TOKEN(DO); } <<break>>
-"else" { PUSH_TOKEN(ELSE); } <<break>>
-"enum" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"export" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"extends" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"false" { PUSH_TOKEN(FALSE_LITERAL); } <<break>>
-"finally" { PUSH_TOKEN(FINALLY); } <<break>>
-"for" { PUSH_TOKEN(FOR); } <<break>>
-"function" { PUSH_TOKEN(FUNCTION); } <<break>>
-"if" { PUSH_TOKEN(IF); } <<break>>
-"implements" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"import" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"in" { PUSH_TOKEN(IN); } <<break>>
-"instanceof" { PUSH_TOKEN(INSTANCEOF); } <<break>>
-"interface" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"let" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"new" { PUSH_TOKEN(NEW); } <<break>>
-"null" { PUSH_TOKEN(NULL_LITERAL); } <<break>>
-"package" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"private" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"protected" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"public" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"return" { PUSH_TOKEN(RETURN); } <<break>>
-"static" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"super" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"switch" { PUSH_TOKEN(SWITCH); } <<break>>
-"this" { PUSH_TOKEN(THIS); } <<break>>
-"throw" { PUSH_TOKEN(THROW); } <<break>>
-"true" { PUSH_TOKEN(TRUE_LITERAL); } <<break>>
-"try" { PUSH_TOKEN(TRY); } <<break>>
-"typeof" { PUSH_TOKEN(TYPEOF); } <<break>>
-"var" { PUSH_TOKEN(VAR); } <<break>>
-"void" { PUSH_TOKEN(VOID); } <<break>>
-"while" { PUSH_TOKEN(WHILE); } <<break>>
-"with" { PUSH_TOKEN(WITH); } <<break>>
-"yield" { PUSH_TOKEN(YIELD); } <<break>>
« no previous file with comments | « no previous file | tools/lexer_generator/generator.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698