Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(233)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: src/lexer/lexer_py.re

Issue 59643002: Experimental parser: simplified rule lexer and parser (Closed) Base URL: https://v8.googlecode.com/svn/branches/experimental/parser
Patch Set: Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | tools/lexer_generator/generator.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # Copyright 2013 the V8 project authors. All rights reserved.
2 # Redistribution and use in source and binary forms, with or without
3 # modification, are permitted provided that the following conditions are
4 # met:
5 #
6 # * Redistributions of source code must retain the above copyright
7 # notice, this list of conditions and the following disclaimer.
8 # * Redistributions in binary form must reproduce the above
9 # copyright notice, this list of conditions and the following
10 # disclaimer in the documentation and/or other materials provided
11 # with the distribution.
12 # * Neither the name of Google Inc. nor the names of its
13 # contributors may be used to endorse or promote products derived
14 # from this software without specific prior written permission.
15 #
16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28 whitespace_char = [ \t\v\f\r\240:ws:];
29 whitespace = whitespace_char+;
30 identifier_start = [$_a-zA-Z:lit:];
31 identifier_char = [$_a-zA-Z0-9:lit:];
32 not_identifier_char = [^:identifier_char:];
33 line_terminator = [\n\r]+;
34 digit = [0-9];
35 hex_digit = [0-9a-fA-F];
36 maybe_exponent = ("e" [-+]? digit+)?;
37 number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("." digit*) ? maybe_exponent));
38
39 <Normal> "break" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::BREAK); }
40 <Normal> "case" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CASE); }
41 <Normal> "catch" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CATCH); }
42 <Normal> "class" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
43 <Normal> "const" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CONST); }
44 <Normal> "continue" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CONTINUE ); }
45 <Normal> "debugger" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DEBUGGER ); }
46 <Normal> "default" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DEFAULT) ; }
47 <Normal> "delete" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DELETE); }
48 <Normal> "do" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DO); }
49 <Normal> "else" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::ELSE); }
50 <Normal> "enum" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
51 <Normal> "export" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
52 <Normal> "extends" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
53 <Normal> "false" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FALSE_LI TERAL); }
54 <Normal> "finally" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FINALLY) ; }
55 <Normal> "for" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FOR); }
56 <Normal> "function" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUNCTION ); }
57 <Normal> "if" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::IF); }
58 <Normal> "implements" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
59 <Normal> "import" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
60 <Normal> "in" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::IN); }
61 <Normal> "instanceof" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::INSTANCE OF); }
62 <Normal> "interface" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
63 <Normal> "let" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
64 <Normal> "new" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NEW); }
65 <Normal> "null" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NULL_LIT ERAL); }
66 <Normal> "package" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
67 <Normal> "private" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
68 <Normal> "protected" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
69 <Normal> "public" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
70 <Normal> "return" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::RETURN); }
71 <Normal> "static" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_S TRICT_RESERVED_WORD); }
72 <Normal> "super" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_R ESERVED_WORD); }
73 <Normal> "switch" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::SWITCH); }
74 <Normal> "this" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::THIS); }
75 <Normal> "throw" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::THROW); }
76 <Normal> "true" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TRUE_LIT ERAL); }
77 <Normal> "try" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TRY); }
78 <Normal> "typeof" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TYPEOF); }
79 <Normal> "var" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::VAR); }
80 <Normal> "void" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::VOID); }
81 <Normal> "while" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::WHILE); }
82 <Normal> "with" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::WITH); }
83 <Normal> "yield" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::YIELD); }
84
85 <Normal> "|=" { PUSH_TOKEN(Token::ASSIGN_BIT_OR); }
86 <Normal> "^=" { PUSH_TOKEN(Token::ASSIGN_BIT_XOR); }
87 <Normal> "&=" { PUSH_TOKEN(Token::ASSIGN_BIT_AND); }
88 <Normal> "+=" { PUSH_TOKEN(Token::ASSIGN_ADD); }
89 <Normal> "-=" { PUSH_TOKEN(Token::ASSIGN_SUB); }
90 <Normal> "*=" { PUSH_TOKEN(Token::ASSIGN_MUL); }
91 <Normal> "/=" { PUSH_TOKEN(Token::ASSIGN_DIV); }
92 <Normal> "%=" { PUSH_TOKEN(Token::ASSIGN_MOD); }
93
94 <Normal> "===" { PUSH_TOKEN(Token::EQ_STRICT); }
95 <Normal> "==" { PUSH_TOKEN(Token::EQ); }
96 <Normal> "=" { PUSH_TOKEN(Token::ASSIGN); }
97 <Normal> "!==" { PUSH_TOKEN(Token::NE_STRICT); }
98 <Normal> "!=" { PUSH_TOKEN(Token::NE); }
99 <Normal> "!" { PUSH_TOKEN(Token::NOT); }
100
101 <Normal> "//" :=> SingleLineComment
102 <Normal> whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kC onditionSingleLineComment); goto yyc_SingleLineComment; } else { --cursor_; send (Token::DEC); start_ = cursor_; goto yyc_Normal; } }
103 <Normal> "/*" :=> MultiLineComment
104 <Normal> "<!--" :=> HtmlComment
105
106 <Normal> ">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); }
107 <Normal> ">>>" { PUSH_TOKEN(Token::SHR); }
108 <Normal> "<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); }
109 <Normal> ">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); }
110 <Normal> "<=" { PUSH_TOKEN(Token::LTE); }
111 <Normal> ">=" { PUSH_TOKEN(Token::GTE); }
112 <Normal> "<<" { PUSH_TOKEN(Token::SHL); }
113 <Normal> ">>" { PUSH_TOKEN(Token::SAR); }
114 <Normal> "<" { PUSH_TOKEN(Token::LT); }
115 <Normal> ">" { PUSH_TOKEN(Token::GT); }
116
117 <Normal> number not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
118 <Normal> number identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
119
120 <Normal> "(" { PUSH_TOKEN(Token::LPAREN); }
121 <Normal> ")" { PUSH_TOKEN(Token::RPAREN); }
122 <Normal> "[" { PUSH_TOKEN(Token::LBRACK); }
123 <Normal> "]" { PUSH_TOKEN(Token::RBRACK); }
124 <Normal> "{" { PUSH_TOKEN(Token::LBRACE); }
125 <Normal> "}" { PUSH_TOKEN(Token::RBRACE); }
126 <Normal> ":" { PUSH_TOKEN(Token::COLON); }
127 <Normal> ";" { PUSH_TOKEN(Token::SEMICOLON); }
128 <Normal> "." { PUSH_TOKEN(Token::PERIOD); }
129 <Normal> "?" { PUSH_TOKEN(Token::CONDITIONAL); }
130 <Normal> "++" { PUSH_TOKEN(Token::INC); }
131 <Normal> "--" { PUSH_TOKEN(Token::DEC); }
132
133 <Normal> "||" { PUSH_TOKEN(Token::OR); }
134 <Normal> "&&" { PUSH_TOKEN(Token::AND); }
135
136 <Normal> "|" { PUSH_TOKEN(Token::BIT_OR); }
137 <Normal> "^" { PUSH_TOKEN(Token::BIT_XOR); }
138 <Normal> "&" { PUSH_TOKEN(Token::BIT_AND); }
139 <Normal> "+" { PUSH_TOKEN(Token::ADD); }
140 <Normal> "-" { PUSH_TOKEN(Token::SUB); }
141 <Normal> "*" { PUSH_TOKEN(Token::MUL); }
142 <Normal> "/" { PUSH_TOKEN(Token::DIV); }
143 <Normal> "%" { PUSH_TOKEN(Token::MOD); }
144 <Normal> "~" { PUSH_TOKEN(Token::BIT_NOT); }
145 <Normal> "," { PUSH_TOKEN(Token::COMMA); }
146
147 <Normal> line_terminator { PUSH_LINE_TERMINATOR(); }
148 <Normal> whitespace { SKIP(); }
149
150 <Normal> "\"" :=> DoubleQuoteString
151 <Normal> "'" :=> SingleQuoteString
152
153 <Normal> identifier_start :=> Identifier
154 <Normal> "\\u[0-9a-fA-F]{4}" { if (ValidIdentifierStart()) { YYSETCONDITION(kCon ditionIdentifier); goto yyc_Identifier; } send(Token::ILLEGAL); start_ = cursor_ ; goto yyc_Normal; }
155 <Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
156
157 <Normal> eof { PUSH_EOF_AND_RETURN();}
158 <Normal> any { PUSH_TOKEN(Token::ILLEGAL); }
159
160 <DoubleQuoteString> "\\\\" { goto yyc_DoubleQuoteString; }
161 <DoubleQuoteString> "\\\"" { goto yyc_DoubleQuoteString; }
162 <DoubleQuoteString> "\"" { PUSH_TOKEN(Token::STRING);}
163 <DoubleQuoteString> "\\\n\r?" { goto yyc_DoubleQuoteString; }
164 <DoubleQuoteString> "\\\r\n?" { goto yyc_DoubleQuoteString; }
165 <DoubleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
166 <DoubleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
167 <DoubleQuoteString> eof { TERMINATE_ILLEGAL(); }
168 <DoubleQuoteString> any { goto yyc_DoubleQuoteString; }
169
170 <SingleQuoteString> "\\\\" { goto yyc_SingleQuoteString; }
171 <SingleQuoteString> "\\'" { goto yyc_SingleQuoteString; }
172 <SingleQuoteString> "'" { PUSH_TOKEN(Token::STRING);}
173 <SingleQuoteString> "\\\n\r?" { goto yyc_SingleQuoteString; }
174 <SingleQuoteString> "\\\r\n?" { goto yyc_SingleQuoteString; }
175 <SingleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
176 <SingleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
177 <SingleQuoteString> eof { TERMINATE_ILLEGAL(); }
178 <SingleQuoteString> any { goto yyc_SingleQuoteString; }
179
180 <Identifier> identifier_char+ { goto yyc_Identifier; }
181 <Identifier> "\\u[0-9a-fA-F]{4}" { if (ValidIdentifierPart()) { goto yyc_Identif ier; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_; goto yyc_Normal; }
182 <Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
183 <Identifier> any { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
184
185 <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
186 <SingleLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EO S); }
187 <SingleLineComment> any { goto yyc_SingleLineComment; }
188
189 <MultiLineComment> "*//" { PUSH_LINE_TERMINATOR();}
190 <MultiLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); }
191 <MultiLineComment> any { goto yyc_MultiLineComment; }
192
193 <HtmlComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); }
194 <HtmlComment> "-->" { PUSH_LINE_TERMINATOR();}
195 <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();}
196 <HtmlComment> any { goto yyc_HtmlComment; }
OLDNEW
« no previous file with comments | « no previous file | tools/lexer_generator/generator.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698