Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(165)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: src/lexer/lexer_py.re

Issue 63773003: Experimental parser: add push_token syntax (Closed) Base URL: https://v8.googlecode.com/svn/branches/experimental/parser
Patch Set: Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | tools/lexer_generator/code_generator.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: src/lexer/lexer_py.re
diff --git a/src/lexer/lexer_py.re b/src/lexer/lexer_py.re
index df0be312cd405f77ecd403bc653c3ccf5ce5761c..679cbd8e3819843c35f34b4808c2a7e8f8b02ca4 100644
--- a/src/lexer/lexer_py.re
+++ b/src/lexer/lexer_py.re
@@ -36,21 +36,21 @@ maybe_exponent = ([eE] [\-+]? digit+)?;
number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("." digit*)? maybe_exponent));
<default>
-"|=" { PUSH_TOKEN(Token::ASSIGN_BIT_OR); }
-"^=" { PUSH_TOKEN(Token::ASSIGN_BIT_XOR); }
-"&=" { PUSH_TOKEN(Token::ASSIGN_BIT_AND); }
-"+=" { PUSH_TOKEN(Token::ASSIGN_ADD); }
-"-=" { PUSH_TOKEN(Token::ASSIGN_SUB); }
-"*=" { PUSH_TOKEN(Token::ASSIGN_MUL); }
-"/=" { PUSH_TOKEN(Token::ASSIGN_DIV); }
-"%=" { PUSH_TOKEN(Token::ASSIGN_MOD); }
-
-"===" { PUSH_TOKEN(Token::EQ_STRICT); }
-"==" { PUSH_TOKEN(Token::EQ); }
-"=" { PUSH_TOKEN(Token::ASSIGN); }
-"!==" { PUSH_TOKEN(Token::NE_STRICT); }
-"!=" { PUSH_TOKEN(Token::NE); }
-"!" { PUSH_TOKEN(Token::NOT); }
+"|=" push_token(ASSIGN_BIT_OR)
+"^=" push_token(ASSIGN_BIT_XOR)
+"&=" push_token(ASSIGN_BIT_AND)
+"+=" push_token(ASSIGN_ADD)
+"-=" push_token(ASSIGN_SUB)
+"*=" push_token(ASSIGN_MUL)
+"/=" push_token(ASSIGN_DIV)
+"%=" push_token(ASSIGN_MOD)
+
+"===" push_token(EQ_STRICT)
+"==" push_token(EQ)
+"=" push_token(ASSIGN)
+"!==" push_token(NE_STRICT)
+"!=" push_token(NE)
+"!" push_token(NOT)
"//" <<SingleLineComment>>
"/*" <<MultiLineComment>>
@@ -58,46 +58,46 @@ number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("." digit*)
#whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kConditionSingleLineComment); goto yyc_SingleLineComment; } else { --cursor_; send(Token::DEC); start_ = cursor_; goto yyc_Normal; } }
-">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); }
-">>>" { PUSH_TOKEN(Token::SHR); }
-"<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); }
-">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); }
-"<=" { PUSH_TOKEN(Token::LTE); }
-">=" { PUSH_TOKEN(Token::GTE); }
-"<<" { PUSH_TOKEN(Token::SHL); }
-">>" { PUSH_TOKEN(Token::SAR); }
-"<" { PUSH_TOKEN(Token::LT); }
-">" { PUSH_TOKEN(Token::GT); }
-
-number { PUSH_TOKEN(Token::NUMBER); }
-# number identifier_char { PUSH_TOKEN(Token::ILLEGAL); }
-
-"(" { PUSH_TOKEN(Token::LPAREN); }
-")" { PUSH_TOKEN(Token::RPAREN); }
-"[" { PUSH_TOKEN(Token::LBRACK); }
-"]" { PUSH_TOKEN(Token::RBRACK); }
-"{" { PUSH_TOKEN(Token::LBRACE); }
-"}" { PUSH_TOKEN(Token::RBRACE); }
-":" { PUSH_TOKEN(Token::COLON); }
-";" { PUSH_TOKEN(Token::SEMICOLON); }
-"." { PUSH_TOKEN(Token::PERIOD); }
-"?" { PUSH_TOKEN(Token::CONDITIONAL); }
-"++" { PUSH_TOKEN(Token::INC); }
-"--" { PUSH_TOKEN(Token::DEC); }
-
-"||" { PUSH_TOKEN(Token::OR); }
-"&&" { PUSH_TOKEN(Token::AND); }
-
-"|" { PUSH_TOKEN(Token::BIT_OR); }
-"^" { PUSH_TOKEN(Token::BIT_XOR); }
-"&" { PUSH_TOKEN(Token::BIT_AND); }
-"+" { PUSH_TOKEN(Token::ADD); }
-"-" { PUSH_TOKEN(Token::SUB); }
-"*" { PUSH_TOKEN(Token::MUL); }
-"/" { PUSH_TOKEN(Token::DIV); }
-"%" { PUSH_TOKEN(Token::MOD); }
-"~" { PUSH_TOKEN(Token::BIT_NOT); }
-"," { PUSH_TOKEN(Token::COMMA); }
+">>>=" push_token(ASSIGN_SHR)
+">>>" push_token(SHR)
+"<<=" push_token(ASSIGN_SHL)
+">>=" push_token(ASSIGN_SAR)
+"<=" push_token(LTE)
+">=" push_token(GTE)
+"<<" push_token(SHL)
+">>" push_token(SAR)
+"<" push_token(LT)
+">" push_token(GT)
+
+number push_token(NUMBER)
+# number identifier_char push_token(ILLEGAL)
+
+"(" push_token(LPAREN)
+")" push_token(RPAREN)
+"[" push_token(LBRACK)
+"]" push_token(RBRACK)
+"{" push_token(LBRACE)
+"}" push_token(RBRACE)
+":" push_token(COLON)
+";" push_token(SEMICOLON)
+"." push_token(PERIOD)
+"?" push_token(CONDITIONAL)
+"++" push_token(INC)
+"--" push_token(DEC)
+
+"||" push_token(OR)
+"&&" push_token(AND)
+
+"|" push_token(BIT_OR)
+"^" push_token(BIT_XOR)
+"&" push_token(BIT_AND)
+"+" push_token(ADD)
+"-" push_token(SUB)
+"*" push_token(MUL)
+"/" push_token(DIV)
+"%" push_token(MOD)
+"~" push_token(BIT_NOT)
+"," push_token(COMMA)
line_terminator+ { PUSH_LINE_TERMINATOR(); }
whitespace { SKIP(); } # TODO implement skip
@@ -106,53 +106,53 @@ whitespace { SKIP(); } # TODO implement skip
"'" <<SingleQuoteString>>
# all keywords
-"break" { PUSH_TOKEN(Token::BREAK); }
-"case" { PUSH_TOKEN(Token::CASE); }
-"catch" { PUSH_TOKEN(Token::CATCH); }
-"class" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"const" { PUSH_TOKEN(Token::CONST); }
-"continue" { PUSH_TOKEN(Token::CONTINUE); }
-"debugger" { PUSH_TOKEN(Token::DEBUGGER); }
-"default" { PUSH_TOKEN(Token::DEFAULT); }
-"delete" { PUSH_TOKEN(Token::DELETE); }
-"do" { PUSH_TOKEN(Token::DO); }
-"else" { PUSH_TOKEN(Token::ELSE); }
-"enum" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"export" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"extends" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"false" { PUSH_TOKEN(Token::FALSE_LITERAL); }
-"finally" { PUSH_TOKEN(Token::FINALLY); }
-"for" { PUSH_TOKEN(Token::FOR); }
-"function" { PUSH_TOKEN(Token::FUNCTION); }
-"if" { PUSH_TOKEN(Token::IF); }
-"implements" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"import" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"in" { PUSH_TOKEN(Token::IN); }
-"instanceof" { PUSH_TOKEN(Token::INSTANCEOF); }
-"interface" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"let" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"new" { PUSH_TOKEN(Token::NEW); }
-"null" { PUSH_TOKEN(Token::NULL_LITERAL); }
-"package" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"private" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"protected" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"public" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"return" { PUSH_TOKEN(Token::RETURN); }
-"static" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"super" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"switch" { PUSH_TOKEN(Token::SWITCH); }
-"this" { PUSH_TOKEN(Token::THIS); }
-"throw" { PUSH_TOKEN(Token::THROW); }
-"true" { PUSH_TOKEN(Token::TRUE_LITERAL); }
-"try" { PUSH_TOKEN(Token::TRY); }
-"typeof" { PUSH_TOKEN(Token::TYPEOF); }
-"var" { PUSH_TOKEN(Token::VAR); }
-"void" { PUSH_TOKEN(Token::VOID); }
-"while" { PUSH_TOKEN(Token::WHILE); }
-"with" { PUSH_TOKEN(Token::WITH); }
-"yield" { PUSH_TOKEN(Token::YIELD); }
-
-identifier_start { PUSH_TOKEN(Token::IDENTIFIER); } <<Identifier>>
+"break" push_token(BREAK)
+"case" push_token(CASE)
+"catch" push_token(CATCH)
+"class" push_token(FUTURE_RESERVED_WORD)
+"const" push_token(CONST)
+"continue" push_token(CONTINUE)
+"debugger" push_token(DEBUGGER)
+"default" push_token(DEFAULT)
+"delete" push_token(DELETE)
+"do" push_token(DO)
+"else" push_token(ELSE)
+"enum" push_token(FUTURE_RESERVED_WORD)
+"export" push_token(FUTURE_RESERVED_WORD)
+"extends" push_token(FUTURE_RESERVED_WORD)
+"false" push_token(FALSE_LITERAL)
+"finally" push_token(FINALLY)
+"for" push_token(FOR)
+"function" push_token(FUNCTION)
+"if" push_token(IF)
+"implements" push_token(FUTURE_STRICT_RESERVED_WORD)
+"import" push_token(FUTURE_RESERVED_WORD)
+"in" push_token(IN)
+"instanceof" push_token(INSTANCEOF)
+"interface" push_token(FUTURE_STRICT_RESERVED_WORD)
+"let" push_token(FUTURE_STRICT_RESERVED_WORD)
+"new" push_token(NEW)
+"null" push_token(NULL_LITERAL)
+"package" push_token(FUTURE_STRICT_RESERVED_WORD)
+"private" push_token(FUTURE_STRICT_RESERVED_WORD)
+"protected" push_token(FUTURE_STRICT_RESERVED_WORD)
+"public" push_token(FUTURE_STRICT_RESERVED_WORD)
+"return" push_token(RETURN)
+"static" push_token(FUTURE_STRICT_RESERVED_WORD)
+"super" push_token(FUTURE_RESERVED_WORD)
+"switch" push_token(SWITCH)
+"this" push_token(THIS)
+"throw" push_token(THROW)
+"true" push_token(TRUE_LITERAL)
+"try" push_token(TRY)
+"typeof" push_token(TYPEOF)
+"var" push_token(VAR)
+"void" push_token(VOID)
+"while" push_token(WHILE)
+"with" push_token(WITH)
+"yield" push_token(YIELD)
+
+identifier_start push_token(IDENTIFIER) <<Identifier>>
/\\u[0-9a-fA-F]{4}/ {
if (V8_UNLIKELY(!ValidIdentifierStart())) {
PUSH_TOKEN(Token::ILLEGAL);
@@ -160,14 +160,14 @@ identifier_start { PUSH_TOKEN(Token::IDENTIFIER); } <<Identifier>>
} <<Identifier>>
eof { PUSH_TOKEN(Token::EOS); return 0; }
-default_action { PUSH_TOKEN(Token::ILLEGAL); }
+default_action push_token(ILLEGAL)
<DoubleQuoteString>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
/\\./ <<continue>>
-/\n|\r/ { PUSH_TOKEN(Token::ILLEGAL); }
-"\"" { PUSH_TOKEN(Token::STRING); }
+/\n|\r/ push_token(ILLEGAL)
+"\"" push_token(STRING)
eof <<terminate_illegal>>
catch_all <<continue>>
@@ -175,8 +175,8 @@ catch_all <<continue>>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
/\\./ <<continue>>
-/\n|\r/ { PUSH_TOKEN(Token::ILLEGAL); }
-"'" { PUSH_TOKEN(Token::STRING); }
+/\n|\r/ push_token(ILLEGAL)
+"'" push_token(STRING)
eof <<terminate_illegal>>
catch_all <<continue>>
@@ -187,7 +187,7 @@ identifier_char <<continue>>
PUSH_TOKEN(Token::ILLEGAL);
}
} <<continue>>
-default_action { PUSH_TOKEN(Token::IDENTIFIER); }
+default_action push_token(IDENTIFIER)
<SingleLineComment>
line_terminator { PUSH_LINE_TERMINATOR(); }
« no previous file with comments | « no previous file | tools/lexer_generator/code_generator.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698