Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(742)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: src/lexer/lexer_py.re

Issue 59403010: Experimental parser: easier to read rules and default rule (Closed) Base URL: https://v8.googlecode.com/svn/branches/experimental/parser
Patch Set: Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | tools/lexer_generator/dfa.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2013 the V8 project authors. All rights reserved. 1 # Copyright 2013 the V8 project authors. All rights reserved.
2 # Redistribution and use in source and binary forms, with or without 2 # Redistribution and use in source and binary forms, with or without
3 # modification, are permitted provided that the following conditions are 3 # modification, are permitted provided that the following conditions are
4 # met: 4 # met:
5 # 5 #
6 # * Redistributions of source code must retain the above copyright 6 # * Redistributions of source code must retain the above copyright
7 # notice, this list of conditions and the following disclaimer. 7 # notice, this list of conditions and the following disclaimer.
8 # * Redistributions in binary form must reproduce the above 8 # * Redistributions in binary form must reproduce the above
9 # copyright notice, this list of conditions and the following 9 # copyright notice, this list of conditions and the following
10 # disclaimer in the documentation and/or other materials provided 10 # disclaimer in the documentation and/or other materials provided
11 # with the distribution. 11 # with the distribution.
12 # * Neither the name of Google Inc. nor the names of its 12 # * Neither the name of Google Inc. nor the names of its
13 # contributors may be used to endorse or promote products derived 13 # contributors may be used to endorse or promote products derived
14 # from this software without specific prior written permission. 14 # from this software without specific prior written permission.
15 # 15 #
16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 27
28 whitespace_char = [ \t\v\f\r:ws:\240]; 28 whitespace_char = [ \t\v\f\r:ws:\240];
29 whitespace = whitespace_char+; 29 whitespace = whitespace_char+;
30 identifier_start = [$_a-zA-Z:lit:]; 30 identifier_start = [$_a-zA-Z:lit:]; # TODO add relevant latin1 char codes
31 identifier_char = [$_a-zA-Z0-9:lit:]; 31 identifier_char = [0-9:identifier_start:];
32 not_identifier_char = [^:identifier_char:]; 32 line_terminator = [\n\r];
33 line_terminator = [\n\r]+;
34 digit = [0-9]; 33 digit = [0-9];
35 hex_digit = [0-9a-fA-F]; 34 hex_digit = [0-9a-fA-F];
36 maybe_exponent = ("e" [\-+]? digit+)?; 35 maybe_exponent = ("e" [\-+]? digit+)?;
37 number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("." digit*) ? maybe_exponent)); 36 number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("." digit*) ? maybe_exponent));
38 37
39 <Normal> "break" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::BREAK); } 38 <default>
40 <Normal> "case" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CASE); } 39 "|=" { PUSH_TOKEN(ASSIGN_BIT_OR); }
41 <Normal> "catch" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CATCH); } 40 "^=" { PUSH_TOKEN(ASSIGN_BIT_XOR); }
42 <Normal> "class" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); } 41 "&=" { PUSH_TOKEN(ASSIGN_BIT_AND); }
43 <Normal> "const" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CONST); } 42 "+=" { PUSH_TOKEN(ASSIGN_ADD); }
44 <Normal> "continue" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CONTINUE ); } 43 "-=" { PUSH_TOKEN(ASSIGN_SUB); }
45 <Normal> "debugger" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DEBUGGER ); } 44 "*=" { PUSH_TOKEN(ASSIGN_MUL); }
46 <Normal> "default" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DEFAULT) ; } 45 "/=" { PUSH_TOKEN(ASSIGN_DIV); }
47 <Normal> "delete" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DELETE); } 46 "%=" { PUSH_TOKEN(ASSIGN_MOD); }
48 <Normal> "do" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DO); }
49 <Normal> "else" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::ELSE); }
50 <Normal> "enum" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
51 <Normal> "export" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
52 <Normal> "extends" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
53 <Normal> "false" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FALSE_LI TERAL); }
54 <Normal> "finally" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FINALLY) ; }
55 <Normal> "for" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FOR); }
56 <Normal> "function" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUNCTION ); }
57 <Normal> "if" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::IF); }
58 <Normal> "implements" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
59 <Normal> "import" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
60 <Normal> "in" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::IN); }
61 <Normal> "instanceof" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::INSTANCE OF); }
62 <Normal> "interface" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
63 <Normal> "let" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
64 <Normal> "new" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NEW); }
65 <Normal> "null" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NULL_LIT ERAL); }
66 <Normal> "package" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
67 <Normal> "private" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
68 <Normal> "protected" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
69 <Normal> "public" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
70 <Normal> "return" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::RETURN); }
71 <Normal> "static" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
72 <Normal> "super" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
73 <Normal> "switch" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::SWITCH); }
74 <Normal> "this" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::THIS); }
75 <Normal> "throw" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::THROW); }
76 <Normal> "true" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TRUE_LIT ERAL); }
77 <Normal> "try" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TRY); }
78 <Normal> "typeof" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TYPEOF); }
79 <Normal> "var" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::VAR); }
80 <Normal> "void" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::VOID); }
81 <Normal> "while" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::WHILE); }
82 <Normal> "with" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::WITH); }
83 <Normal> "yield" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::YIELD); }
84 47
85 <Normal> "|=" { PUSH_TOKEN(Token::ASSIGN_BIT_OR); } 48 "===" { PUSH_TOKEN(EQ_STRICT); }
86 <Normal> "^=" { PUSH_TOKEN(Token::ASSIGN_BIT_XOR); } 49 "==" { PUSH_TOKEN(EQ); }
87 <Normal> "&=" { PUSH_TOKEN(Token::ASSIGN_BIT_AND); } 50 "=" { PUSH_TOKEN(ASSIGN); }
88 <Normal> "+=" { PUSH_TOKEN(Token::ASSIGN_ADD); } 51 "!==" { PUSH_TOKEN(NE_STRICT); }
89 <Normal> "-=" { PUSH_TOKEN(Token::ASSIGN_SUB); } 52 "!=" { PUSH_TOKEN(NE); }
90 <Normal> "*=" { PUSH_TOKEN(Token::ASSIGN_MUL); } 53 "!" { PUSH_TOKEN(NOT); }
91 <Normal> "/=" { PUSH_TOKEN(Token::ASSIGN_DIV); }
92 <Normal> "%=" { PUSH_TOKEN(Token::ASSIGN_MOD); }
93 54
94 <Normal> "===" { PUSH_TOKEN(Token::EQ_STRICT); } 55 "//" <<SingleLineComment>> # TODO save offset?
95 <Normal> "==" { PUSH_TOKEN(Token::EQ); } 56 "/*" <<MultiLineComment>>
96 <Normal> "=" { PUSH_TOKEN(Token::ASSIGN); } 57 "<!--" <<HtmlComment>>
97 <Normal> "!==" { PUSH_TOKEN(Token::NE_STRICT); } 58 #whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kCondition SingleLineComment); goto yyc_SingleLineComment; } else { --cursor_; send(Token:: DEC); start_ = cursor_; goto yyc_Normal; } }
98 <Normal> "!=" { PUSH_TOKEN(Token::NE); }
99 <Normal> "!" { PUSH_TOKEN(Token::NOT); }
100 59
101 <Normal> "//" :=> SingleLineComment 60 ">>>=" { PUSH_TOKEN(ASSIGN_SHR); }
102 <Normal> whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kC onditionSingleLineComment); goto yyc_SingleLineComment; } else { --cursor_; send (Token::DEC); start_ = cursor_; goto yyc_Normal; } } 61 ">>>" { PUSH_TOKEN(SHR); }
103 <Normal> "/*" :=> MultiLineComment 62 "<<=" { PUSH_TOKEN(ASSIGN_SHL); }
104 <Normal> "<!--" :=> HtmlComment 63 ">>=" { PUSH_TOKEN(ASSIGN_SAR); }
64 "<=" { PUSH_TOKEN(LTE); }
65 ">=" { PUSH_TOKEN(GTE); }
66 "<<" { PUSH_TOKEN(SHL); }
67 ">>" { PUSH_TOKEN(SAR); }
68 "<" { PUSH_TOKEN(LT); }
69 ">" { PUSH_TOKEN(GT); }
105 70
106 <Normal> ">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); } 71 number { PUSH_TOKEN(NUMBER); }
107 <Normal> ">>>" { PUSH_TOKEN(Token::SHR); } 72 # number identifier_char { PUSH_TOKEN(ILLEGAL); }
108 <Normal> "<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); }
109 <Normal> ">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); }
110 <Normal> "<=" { PUSH_TOKEN(Token::LTE); }
111 <Normal> ">=" { PUSH_TOKEN(Token::GTE); }
112 <Normal> "<<" { PUSH_TOKEN(Token::SHL); }
113 <Normal> ">>" { PUSH_TOKEN(Token::SAR); }
114 <Normal> "<" { PUSH_TOKEN(Token::LT); }
115 <Normal> ">" { PUSH_TOKEN(Token::GT); }
116 73
117 <Normal> number not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); } 74 "(" { PUSH_TOKEN(LPAREN); }
118 <Normal> number identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } 75 ")" { PUSH_TOKEN(RPAREN); }
76 "[" { PUSH_TOKEN(LBRACK); }
77 "]" { PUSH_TOKEN(RBRACK); }
78 "{" { PUSH_TOKEN(LBRACE); }
79 "}" { PUSH_TOKEN(RBRACE); }
80 ":" { PUSH_TOKEN(COLON); }
81 ";" { PUSH_TOKEN(SEMICOLON); }
82 "." { PUSH_TOKEN(PERIOD); }
83 "?" { PUSH_TOKEN(CONDITIONAL); }
84 "++" { PUSH_TOKEN(INC); }
85 "--" { PUSH_TOKEN(DEC); }
119 86
120 <Normal> "(" { PUSH_TOKEN(Token::LPAREN); } 87 "||" { PUSH_TOKEN(OR); }
121 <Normal> ")" { PUSH_TOKEN(Token::RPAREN); } 88 "&&" { PUSH_TOKEN(AND); }
122 <Normal> "[" { PUSH_TOKEN(Token::LBRACK); }
123 <Normal> "]" { PUSH_TOKEN(Token::RBRACK); }
124 <Normal> "{" { PUSH_TOKEN(Token::LBRACE); }
125 <Normal> "}" { PUSH_TOKEN(Token::RBRACE); }
126 <Normal> ":" { PUSH_TOKEN(Token::COLON); }
127 <Normal> ";" { PUSH_TOKEN(Token::SEMICOLON); }
128 <Normal> "." { PUSH_TOKEN(Token::PERIOD); }
129 <Normal> "?" { PUSH_TOKEN(Token::CONDITIONAL); }
130 <Normal> "++" { PUSH_TOKEN(Token::INC); }
131 <Normal> "--" { PUSH_TOKEN(Token::DEC); }
132 89
133 <Normal> "||" { PUSH_TOKEN(Token::OR); } 90 "|" { PUSH_TOKEN(BIT_OR); }
134 <Normal> "&&" { PUSH_TOKEN(Token::AND); } 91 "^" { PUSH_TOKEN(BIT_XOR); }
92 "&" { PUSH_TOKEN(BIT_AND); }
93 "+" { PUSH_TOKEN(ADD); }
94 "-" { PUSH_TOKEN(SUB); }
95 "*" { PUSH_TOKEN(MUL); }
96 "/" { PUSH_TOKEN(DIV); }
97 "%" { PUSH_TOKEN(MOD); }
98 "~" { PUSH_TOKEN(BIT_NOT); }
99 "," { PUSH_TOKEN(COMMA); }
135 100
136 <Normal> "|" { PUSH_TOKEN(Token::BIT_OR); } 101 line_terminator+ { PUSH_LINE_TERMINATOR(); }
137 <Normal> "^" { PUSH_TOKEN(Token::BIT_XOR); } 102 whitespace <<continue>>
138 <Normal> "&" { PUSH_TOKEN(Token::BIT_AND); }
139 <Normal> "+" { PUSH_TOKEN(Token::ADD); }
140 <Normal> "-" { PUSH_TOKEN(Token::SUB); }
141 <Normal> "*" { PUSH_TOKEN(Token::MUL); }
142 <Normal> "/" { PUSH_TOKEN(Token::DIV); }
143 <Normal> "%" { PUSH_TOKEN(Token::MOD); }
144 <Normal> "~" { PUSH_TOKEN(Token::BIT_NOT); }
145 <Normal> "," { PUSH_TOKEN(Token::COMMA); }
146 103
147 <Normal> line_terminator { PUSH_LINE_TERMINATOR(); } 104 "\"" <<DoubleQuoteString>> # TODO mark these transitions as ignoring t his character
148 <Normal> whitespace { SKIP(); } 105 "'" <<SingleQuoteString>>
149 106
150 <Normal> "\"" :=> DoubleQuoteString 107 identifier_start <<Identifier>> # TODO merge identifier dfa...
151 <Normal> "'" :=> SingleQuoteString 108 /\\u[0-9a-fA-F]{4}/ {
109 if (V8_LIKELY(ValidIdentifierStart())) {
110 JUMP(Identifier);
111 }
112 PUSH_TOKEN(ILLEGAL);
113 }
152 114
153 <Normal> identifier_start :=> Identifier 115 eof <<terminate>>
154 <Normal> /\\u[0-9a-fA-F]{4}/ { if (ValidIdentifierStart()) { YYSETCONDITION(kCon ditionIdentifier); goto yyc_Identifier; } send(Token::ILLEGAL); start_ = cursor_ ; goto yyc_Normal; } 116 default { PUSH_TOKEN(ILLEGAL); }
155 <Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
156 117
157 <Normal> eof { PUSH_EOF_AND_RETURN();} 118 <DoubleQuoteString>
158 <Normal> any { PUSH_TOKEN(Token::ILLEGAL); } 119 "\\" <<continue>>
120 "\\\"" <<continue>>
121 "\"" { PUSH_TOKEN(STRING); } <<break>>
122 /\\\n\r?/ <<continue>>
123 /\\\r\n?/ <<continue>>
124 /\n\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
125 eof <<terminate_illegal>>
126 default <<continue>>
159 127
160 <DoubleQuoteString> "\\\\" { goto yyc_DoubleQuoteString; } 128 <SingleQuoteString>
161 <DoubleQuoteString> "\\\"" { goto yyc_DoubleQuoteString; } 129 "\\" <<continue>>
162 <DoubleQuoteString> "\"" { PUSH_TOKEN(Token::STRING);} 130 "\\'" <<continue>>
163 <DoubleQuoteString> /\\\n\r?/ { goto yyc_DoubleQuoteString; } 131 "'" { PUSH_TOKEN(STRING); } <<break>>
164 <DoubleQuoteString> /\\\r\n?/ { goto yyc_DoubleQuoteString; } 132 /\\\n\r?/ <<continue>>
165 <DoubleQuoteString> /\n/ => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } 133 /\\\r\n?/ <<continue>>
166 <DoubleQuoteString> /\r/ => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } 134 /\n\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
167 <DoubleQuoteString> eof { TERMINATE_ILLEGAL(); } 135 eof <<terminate_illegal>>
168 <DoubleQuoteString> any { goto yyc_DoubleQuoteString; } 136 default <<continue>>
169 137
170 <SingleQuoteString> "\\" { goto yyc_SingleQuoteString; } 138 <Identifier>
171 <SingleQuoteString> "\\'" { goto yyc_SingleQuoteString; } 139 identifier_char+ <<continue>>
172 <SingleQuoteString> "'" { PUSH_TOKEN(Token::STRING); } 140 /\\u[0-9a-fA-F]{4}/ {
173 <SingleQuoteString> /\\\n\r?/ { goto yyc_SingleQuoteString; } 141 if (V8_UNLIKELY(!ValidIdentifierStart())) {
174 <SingleQuoteString> /\\\r\n?/ { goto yyc_SingleQuoteString; } 142 PUSH_TOKEN(ILLEGAL);
175 <SingleQuoteString> /\n/ => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } 143 JUMP(Normal);
176 <SingleQuoteString> /\r/ => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } 144 }
177 <SingleQuoteString> eof { TERMINATE_ILLEGAL(); } 145 }
178 <SingleQuoteString> any { goto yyc_SingleQuoteString; } 146 default { PUSH_TOKEN(IDENTIFIER); } <<break>>
179 147
180 <Identifier> identifier_char+ { goto yyc_Identifier; } 148 <SingleLineComment>
181 <Identifier> /\\u[0-9a-fA-F]{4}/ { if (ValidIdentifierPart()) { goto yyc_Identif ier; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_; goto yyc_Normal; } 149 line_terminator { PUSH_LINE_TERMINATOR(); } <<break>>
182 <Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); } 150 eof <<terminate>>
183 <Identifier> any { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); } 151 default <<continue>>
184 152
185 <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();} 153 <MultiLineComment>
186 <SingleLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EO S); } 154 "*/" <<break>>
187 <SingleLineComment> any { goto yyc_SingleLineComment; } 155 line_terminator+ { PUSH_LINE_TERMINATOR(); }
156 eof <<terminate>>
157 default <<continue>>
188 158
189 <MultiLineComment> "*/" { PUSH_LINE_TERMINATOR();} 159 <HtmlComment>
190 <MultiLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); } 160 "-->" <<break>>
191 <MultiLineComment> any { goto yyc_MultiLineComment; } 161 line_terminator+ { PUSH_LINE_TERMINATOR(); }
162 eof <<terminate>>
163 default <<continue>>
192 164
193 <HtmlComment> "-->" { PUSH_LINE_TERMINATOR();} 165 <default>
194 <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();} 166 # all keywords
195 <HtmlComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); } 167 "break" { PUSH_TOKEN(BREAK); } <<break>>
196 <HtmlComment> any { goto yyc_HtmlComment; } 168 "case" { PUSH_TOKEN(CASE); } <<break>>
169 "catch" { PUSH_TOKEN(CATCH); } <<break>>
170 "class" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
171 "const" { PUSH_TOKEN(CONST); } <<break>>
172 "continue" { PUSH_TOKEN(CONTINUE); } <<break>>
173 "debugger" { PUSH_TOKEN(DEBUGGER); } <<break>>
174 "default" { PUSH_TOKEN(DEFAULT); } <<break>>
175 "delete" { PUSH_TOKEN(DELETE); } <<break>>
176 "do" { PUSH_TOKEN(DO); } <<break>>
177 "else" { PUSH_TOKEN(ELSE); } <<break>>
178 "enum" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
179 "export" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
180 "extends" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
181 "false" { PUSH_TOKEN(FALSE_LITERAL); } <<break>>
182 "finally" { PUSH_TOKEN(FINALLY); } <<break>>
183 "for" { PUSH_TOKEN(FOR); } <<break>>
184 "function" { PUSH_TOKEN(FUNCTION); } <<break>>
185 "if" { PUSH_TOKEN(IF); } <<break>>
186 "implements" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
187 "import" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
188 "in" { PUSH_TOKEN(IN); } <<break>>
189 "instanceof" { PUSH_TOKEN(INSTANCEOF); } <<break>>
190 "interface" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
191 "let" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
192 "new" { PUSH_TOKEN(NEW); } <<break>>
193 "null" { PUSH_TOKEN(NULL_LITERAL); } <<break>>
194 "package" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
195 "private" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
196 "protected" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
197 "public" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
198 "return" { PUSH_TOKEN(RETURN); } <<break>>
199 "static" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
200 "super" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
201 "switch" { PUSH_TOKEN(SWITCH); } <<break>>
202 "this" { PUSH_TOKEN(THIS); } <<break>>
203 "throw" { PUSH_TOKEN(THROW); } <<break>>
204 "true" { PUSH_TOKEN(TRUE_LITERAL); } <<break>>
205 "try" { PUSH_TOKEN(TRY); } <<break>>
206 "typeof" { PUSH_TOKEN(TYPEOF); } <<break>>
207 "var" { PUSH_TOKEN(VAR); } <<break>>
208 "void" { PUSH_TOKEN(VOID); } <<break>>
209 "while" { PUSH_TOKEN(WHILE); } <<break>>
210 "with" { PUSH_TOKEN(WITH); } <<break>>
211 "yield" { PUSH_TOKEN(YIELD); } <<break>>
OLDNEW
« no previous file with comments | « no previous file | tools/lexer_generator/dfa.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698