| OLD | NEW |
|---|
| (Empty) |
| 1 // Portions of this code based on re2c: | |
| 2 // (re2c/examples/push.re) | |
| 3 // Copyright 2013 the V8 project authors. All rights reserved. | |
| 4 // Redistribution and use in source and binary forms, with or without | |
| 5 // modification, are permitted provided that the following conditions are | |
| 6 // met: | |
| 7 // | |
| 8 // * Redistributions of source code must retain the above copyright | |
| 9 // notice, this list of conditions and the following disclaimer. | |
| 10 // * Redistributions in binary form must reproduce the above | |
| 11 // copyright notice, this list of conditions and the following | |
| 12 // disclaimer in the documentation and/or other materials provided | |
| 13 // with the distribution. | |
| 14 // * Neither the name of Google Inc. nor the names of its | |
| 15 // contributors may be used to endorse or promote products derived | |
| 16 // from this software without specific prior written permission. | |
| 17 // | |
| 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 29 | |
| 30 #include "lexer.h" | |
| 31 | |
| 32 #include <stdio.h> | |
| 33 #include <stdlib.h> | |
| 34 #include <string.h> | |
| 35 | |
| 36 // FIXME: some of this is probably not needed. | |
| 37 #include "allocation.h" | |
| 38 #include "ast.h" | |
| 39 #include "preparse-data-format.h" | |
| 40 #include "preparse-data.h" | |
| 41 #include "scopes.h" | |
| 42 #include "preparser.h" | |
| 43 #include "api.h" | |
| 44 #include "ast.h" | |
| 45 #include "bootstrapper.h" | |
| 46 #include "char-predicates-inl.h" | |
| 47 #include "codegen.h" | |
| 48 #include "compiler.h" | |
| 49 #include "func-name-inferrer.h" | |
| 50 #include "messages.h" | |
| 51 #include "parser.h" | |
| 52 #include "platform.h" | |
| 53 #include "preparser.h" | |
| 54 #include "runtime.h" | |
| 55 #include "scanner-character-streams.h" | |
| 56 #include "scopeinfo.h" | |
| 57 #include "string-stream.h" | |
| 58 | |
| 59 #include "experimental-scanner.h" | |
| 60 | |
| 61 // TODO: | |
| 62 // - Run-time lexing modifications: harmony number literals, keywords depending | |
| 63 // on harmony_modules, harmony_scoping | |
| 64 // - Escaping the string literals (like the baseline does) | |
| 65 // - Error recovery after illegal tokens. | |
| 66 | |
| 67 enum Condition { | |
| 68 kConditionNormal, | |
| 69 kConditionDoubleQuoteString, | |
| 70 kConditionSingleQuoteString, | |
| 71 kConditionIdentifier, | |
| 72 kConditionIdentifierIllegal, | |
| 73 kConditionSingleLineComment, | |
| 74 kConditionMultiLineComment, | |
| 75 kConditionHtmlComment | |
| 76 }; | |
| 77 | |
| 78 using namespace v8::internal; | |
| 79 | |
| 80 #define PUSH_TOKEN(T) { send(T); SKIP(); } | |
| 81 #define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); } | |
| 82 #define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;} | |
| 83 #define PUSH_LINE_TERMINATOR() { just_seen_line_terminator_ = true; SKIP(); } | |
| 84 #define TERMINATE_ILLEGAL() { send(Token::ILLEGAL); send(Token::EOS); return 1;
} | |
| 85 | |
| 86 PushScanner::PushScanner(ExperimentalScanner* sink, UnicodeCache* unicode_cache) | |
| 87 : unicode_cache_(unicode_cache), | |
| 88 eof_(false), | |
| 89 state_(-1), | |
| 90 condition_(kConditionNormal), | |
| 91 limit_(NULL), | |
| 92 start_(NULL), | |
| 93 cursor_(NULL), | |
| 94 marker_(NULL), | |
| 95 real_start_(0), | |
| 96 buffer_(NULL), | |
| 97 buffer_end_(NULL), | |
| 98 yych(0), | |
| 99 yyaccept(0), | |
| 100 just_seen_line_terminator_(true), | |
| 101 sink_(sink) { | |
| 102 | |
| 103 } | |
| 104 | |
| 105 PushScanner::~PushScanner() { | |
| 106 } | |
| 107 | |
| 108 | |
| 109 uc32 PushScanner::ScanHexNumber(int length) { | |
| 110 // We have seen \uXXXX, let's see what it is. | |
| 111 // FIXME: we never end up in here if only a subset of the 4 chars are valid | |
| 112 // hex digits -> handle the case where they're not. | |
| 113 uc32 x = 0; | |
| 114 for (YYCTYPE* s = cursor_ - length; s != cursor_; ++s) { | |
| 115 int d = HexValue(*s); | |
| 116 if (d < 0) { | |
| 117 return -1; | |
| 118 } | |
| 119 x = x * 16 + d; | |
| 120 } | |
| 121 return x; | |
| 122 } | |
| 123 | |
| 124 | |
| 125 bool PushScanner::ValidIdentifierPart() { | |
| 126 return unicode_cache_->IsIdentifierPart(ScanHexNumber(4)); | |
| 127 } | |
| 128 | |
| 129 bool PushScanner::ValidIdentifierStart() { | |
| 130 return unicode_cache_->IsIdentifierStart(ScanHexNumber(4)); | |
| 131 } | |
| 132 | |
| 133 void PushScanner::send(Token::Value token) { | |
| 134 int beg = (start_ - buffer_) + real_start_; | |
| 135 int end = (cursor_ - buffer_) + real_start_; | |
| 136 if (FLAG_trace_lexer) { | |
| 137 printf("got %s at (%d, %d): ", Token::Name(token), beg, end); | |
| 138 for (YYCTYPE* s = start_; s != cursor_; s++) printf("%c", (char)*s); | |
| 139 printf(".\n"); | |
| 140 } | |
| 141 just_seen_line_terminator_ = false; | |
| 142 sink_->Record(token, beg, end); | |
| 143 } | |
| 144 | |
| 145 uint32_t PushScanner::push(const void *input, int input_size) { | |
| 146 if (FLAG_trace_lexer) { | |
| 147 printf( | |
| 148 "scanner is receiving a new data batch of length %d\n" | |
| 149 "scanner continues with saved state_ = %d\n", | |
| 150 input_size, | |
| 151 state_); | |
| 152 } | |
| 153 | |
| 154 // Data source is signaling end of file when batch size | |
| 155 // is less than max_fill. This is slightly annoying because | |
| 156 // max_fill is a value that can only be known after re2c does | |
| 157 // its thing. Practically though, max_fill is never bigger than | |
| 158 // the longest keyword, so given our grammar, 32 is a safe bet. | |
| 159 | |
| 160 YYCTYPE null[64]; | |
| 161 const int max_fill = 32; | |
| 162 if (input_size < max_fill) { // FIXME: do something about this!!! | |
| 163 eof_ = true; | |
| 164 input = null; | |
| 165 input_size = sizeof(null); | |
| 166 memset(null, 0, sizeof(null)); | |
| 167 } | |
| 168 | |
| 169 | |
| 170 // When we get here, we have a partially | |
| 171 // consumed buffer_ which is in the following state_: | |
| 172 // last valid char last valid buffer_ spot | |
| 173 // v v | |
| 174 // +-------------------+-------------+---------------+-------------+---------
-------------+ | |
| 175 // ^ ^ ^ ^ ^ ^ | |
| 176 // buffer_ start_ marker_ cursor_ limit_ buffer_end_ | |
| 177 // | |
| 178 // We need to stretch the buffer_ and concatenate the new chunk of input to i
t | |
| 179 | |
| 180 size_t used = limit_ - buffer_; | |
| 181 size_t needed = used + input_size; | |
| 182 size_t allocated = buffer_end_ - buffer_; | |
| 183 if (allocated < needed) { | |
| 184 size_t limit__offset = limit_ - buffer_; | |
| 185 size_t start_offset = start_ - buffer_; | |
| 186 size_t marker__offset = marker_ - buffer_; | |
| 187 size_t cursor__offset = cursor_ - buffer_; | |
| 188 | |
| 189 buffer_ = (YYCTYPE*)realloc(buffer_, needed); | |
| 190 buffer_end_ = needed + buffer_; | |
| 191 | |
| 192 marker_ = marker__offset + buffer_; | |
| 193 cursor_ = cursor__offset + buffer_; | |
| 194 start_ = buffer_ + start_offset; | |
| 195 limit_ = limit__offset + buffer_; | |
| 196 } | |
| 197 memcpy(limit_, input, input_size); | |
| 198 limit_ += input_size; | |
| 199 | |
| 200 #define SKIP() { start_ = cursor_; YYSETCONDITION(kConditionNorma
l); goto yyc_Normal; } | |
| 201 #define YYFILL(n) { goto fill; } | |
| 202 | |
| 203 #define YYGETSTATE() state_ | |
| 204 #define YYSETSTATE(x) { state_ = (x); } | |
| 205 | |
| 206 #define YYGETCONDITION() condition_ | |
| 207 #define YYSETCONDITION(x) { condition_ = (x); } | |
| 208 #define YYDEBUG(state, current) {printf("%d: |%c| (%d)\n", state, current, (int)
(current));} | |
| 209 | |
| 210 start_: | |
| 211 if (FLAG_trace_lexer) { | |
| 212 printf("Starting a round; state_: %d, condition_: %d\n", state_, condition_)
; | |
| 213 } | |
| 214 | |
| 215 /*!re2c | |
| 216 re2c:indent:top = 1; | |
| 217 re2c:yych:conversion = 0; | |
| 218 re2c:condenumprefix = kCondition; | |
| 219 re2c:define:YYCONDTYPE = Condition; | |
| 220 re2c:define:YYCURSOR = cursor_; | |
| 221 re2c:define:YYLIMIT = limit_; | |
| 222 re2c:define:YYMARKER = marker_; | |
| 223 | |
| 224 eof = "\000"; | |
| 225 any = [\000-\377]; | |
| 226 whitespace_char = [ \t\v\f\r\240]; | |
| 227 whitespace = whitespace_char+; | |
| 228 identifier_start = [$_a-zA-Z\300-\377]; | |
| 229 identifier_char = [$_a-zA-Z0-9\300-\377]; | |
| 230 not_identifier_char = any\identifier_char\[\\]; | |
| 231 line_terminator = [\n\r]+; | |
| 232 digit = [0-9]; | |
| 233 hex_digit = [0-9a-fA-F]; | |
| 234 maybe_exponent = ('e' [-+]? digit+)?; | |
| 235 number = ('0x' hex_digit+) | ("." digit+ maybe_exponent) | (digit+ ("." digi
t*)? maybe_exponent); | |
| 236 | |
| 237 <Normal> "break" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::BREA
K); } | |
| 238 <Normal> "case" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CASE
); } | |
| 239 <Normal> "catch" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CATC
H); } | |
| 240 <Normal> "class" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_RESERVED_WORD); } | |
| 241 <Normal> "const" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CONS
T); } | |
| 242 <Normal> "continue" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::CONT
INUE); } | |
| 243 <Normal> "debugger" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DEBU
GGER); } | |
| 244 <Normal> "default" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DEFA
ULT); } | |
| 245 <Normal> "delete" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DELE
TE); } | |
| 246 <Normal> "do" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::DO);
} | |
| 247 <Normal> "else" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::ELSE
); } | |
| 248 <Normal> "enum" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_RESERVED_WORD); } | |
| 249 <Normal> "export" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_RESERVED_WORD); } | |
| 250 <Normal> "extends" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_RESERVED_WORD); } | |
| 251 <Normal> "false" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FALS
E_LITERAL); } | |
| 252 <Normal> "finally" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FINA
LLY); } | |
| 253 <Normal> "for" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FOR)
; } | |
| 254 <Normal> "function" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUNC
TION); } | |
| 255 <Normal> "if" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::IF);
} | |
| 256 <Normal> "implements" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 257 <Normal> "import" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_RESERVED_WORD); } | |
| 258 <Normal> "in" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::IN);
} | |
| 259 <Normal> "instanceof" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::INST
ANCEOF); } | |
| 260 <Normal> "interface" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 261 <Normal> "let" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 262 <Normal> "new" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NEW)
; } | |
| 263 <Normal> "null" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NULL
_LITERAL); } | |
| 264 <Normal> "package" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 265 <Normal> "private" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 266 <Normal> "protected" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 267 <Normal> "public" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 268 <Normal> "return" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::RETU
RN); } | |
| 269 <Normal> "static" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_STRICT_RESERVED_WORD); } | |
| 270 <Normal> "super" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::FUTU
RE_RESERVED_WORD); } | |
| 271 <Normal> "switch" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::SWIT
CH); } | |
| 272 <Normal> "this" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::THIS
); } | |
| 273 <Normal> "throw" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::THRO
W); } | |
| 274 <Normal> "true" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TRUE
_LITERAL); } | |
| 275 <Normal> "try" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TRY)
; } | |
| 276 <Normal> "typeof" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::TYPE
OF); } | |
| 277 <Normal> "var" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::VAR)
; } | |
| 278 <Normal> "void" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::VOID
); } | |
| 279 <Normal> "while" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::WHIL
E); } | |
| 280 <Normal> "with" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::WITH
); } | |
| 281 <Normal> "yield" not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::YIEL
D); } | |
| 282 | |
| 283 <Normal> "|=" { PUSH_TOKEN(Token::ASSIGN_BIT_OR); } | |
| 284 <Normal> "^=" { PUSH_TOKEN(Token::ASSIGN_BIT_XOR); } | |
| 285 <Normal> "&=" { PUSH_TOKEN(Token::ASSIGN_BIT_AND); } | |
| 286 <Normal> "+=" { PUSH_TOKEN(Token::ASSIGN_ADD); } | |
| 287 <Normal> "-=" { PUSH_TOKEN(Token::ASSIGN_SUB); } | |
| 288 <Normal> "*=" { PUSH_TOKEN(Token::ASSIGN_MUL); } | |
| 289 <Normal> "/=" { PUSH_TOKEN(Token::ASSIGN_DIV); } | |
| 290 <Normal> "%=" { PUSH_TOKEN(Token::ASSIGN_MOD); } | |
| 291 | |
| 292 <Normal> "===" { PUSH_TOKEN(Token::EQ_STRICT); } | |
| 293 <Normal> "==" { PUSH_TOKEN(Token::EQ); } | |
| 294 <Normal> "=" { PUSH_TOKEN(Token::ASSIGN); } | |
| 295 <Normal> "!==" { PUSH_TOKEN(Token::NE_STRICT); } | |
| 296 <Normal> "!=" { PUSH_TOKEN(Token::NE); } | |
| 297 <Normal> "!" { PUSH_TOKEN(Token::NOT); } | |
| 298 | |
| 299 <Normal> "//" :=> SingleLineComment | |
| 300 <Normal> whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITIO
N(kConditionSingleLineComment); goto yyc_SingleLineComment; } else { --cursor_;
send(Token::DEC); start_ = cursor_; goto yyc_Normal; } } | |
| 301 <Normal> "/*" :=> MultiLineComment | |
| 302 <Normal> "<!--" :=> HtmlComment | |
| 303 | |
| 304 <Normal> ">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); } | |
| 305 <Normal> ">>>" { PUSH_TOKEN(Token::SHR); } | |
| 306 <Normal> "<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); } | |
| 307 <Normal> ">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); } | |
| 308 <Normal> "<=" { PUSH_TOKEN(Token::LTE); } | |
| 309 <Normal> ">=" { PUSH_TOKEN(Token::GTE); } | |
| 310 <Normal> "<<" { PUSH_TOKEN(Token::SHL); } | |
| 311 <Normal> ">>" { PUSH_TOKEN(Token::SAR); } | |
| 312 <Normal> "<" { PUSH_TOKEN(Token::LT); } | |
| 313 <Normal> ">" { PUSH_TOKEN(Token::GT); } | |
| 314 | |
| 315 <Normal> number not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); } | |
| 316 <Normal> number any { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } | |
| 317 | |
| 318 <Normal> "(" { PUSH_TOKEN(Token::LPAREN); } | |
| 319 <Normal> ")" { PUSH_TOKEN(Token::RPAREN); } | |
| 320 <Normal> "[" { PUSH_TOKEN(Token::LBRACK); } | |
| 321 <Normal> "]" { PUSH_TOKEN(Token::RBRACK); } | |
| 322 <Normal> "{" { PUSH_TOKEN(Token::LBRACE); } | |
| 323 <Normal> "}" { PUSH_TOKEN(Token::RBRACE); } | |
| 324 <Normal> ":" { PUSH_TOKEN(Token::COLON); } | |
| 325 <Normal> ";" { PUSH_TOKEN(Token::SEMICOLON); } | |
| 326 <Normal> "." { PUSH_TOKEN(Token::PERIOD); } | |
| 327 <Normal> "?" { PUSH_TOKEN(Token::CONDITIONAL); } | |
| 328 <Normal> "++" { PUSH_TOKEN(Token::INC); } | |
| 329 <Normal> "--" { PUSH_TOKEN(Token::DEC); } | |
| 330 | |
| 331 <Normal> "||" { PUSH_TOKEN(Token::OR); } | |
| 332 <Normal> "&&" { PUSH_TOKEN(Token::AND); } | |
| 333 | |
| 334 <Normal> "|" { PUSH_TOKEN(Token::BIT_OR); } | |
| 335 <Normal> "^" { PUSH_TOKEN(Token::BIT_XOR); } | |
| 336 <Normal> "&" { PUSH_TOKEN(Token::BIT_AND); } | |
| 337 <Normal> "+" { PUSH_TOKEN(Token::ADD); } | |
| 338 <Normal> "-" { PUSH_TOKEN(Token::SUB); } | |
| 339 <Normal> "*" { PUSH_TOKEN(Token::MUL); } | |
| 340 <Normal> "/" { PUSH_TOKEN(Token::DIV); } | |
| 341 <Normal> "%" { PUSH_TOKEN(Token::MOD); } | |
| 342 <Normal> "~" { PUSH_TOKEN(Token::BIT_NOT); } | |
| 343 <Normal> "," { PUSH_TOKEN(Token::COMMA); } | |
| 344 | |
| 345 <Normal> line_terminator { PUSH_LINE_TERMINATOR(); } | |
| 346 <Normal> whitespace { SKIP(); } | |
| 347 | |
| 348 <Normal> ["] :=> DoubleQuoteString | |
| 349 <Normal> ['] :=> SingleQuoteString | |
| 350 | |
| 351 <Normal> identifier_start :=> Identifier | |
| 352 <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION
(kConditionIdentifier); goto yyc_Identifier; } send(Token::ILLEGAL); start_ = cu
rsor_; goto yyc_Normal; } | |
| 353 <Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); } | |
| 354 | |
| 355 <Normal> eof { PUSH_EOF_AND_RETURN();} | |
| 356 <Normal> any { PUSH_TOKEN(Token::ILLEGAL); } | |
| 357 | |
| 358 <DoubleQuoteString> "\\\\" { goto yyc_DoubleQuoteString; } | |
| 359 <DoubleQuoteString> "\\\"" { goto yyc_DoubleQuoteString; } | |
| 360 <DoubleQuoteString> '"' { PUSH_TOKEN(Token::STRING);} | |
| 361 <DoubleQuoteString> "\\" "\n" "\r"? { goto yyc_DoubleQuoteString; } | |
| 362 <DoubleQuoteString> "\\" "\r" "\n"? { goto yyc_DoubleQuoteString; } | |
| 363 <DoubleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | |
| 364 <DoubleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | |
| 365 <DoubleQuoteString> eof { TERMINATE_ILLEGAL(); } | |
| 366 <DoubleQuoteString> any { goto yyc_DoubleQuoteString; } | |
| 367 | |
| 368 <SingleQuoteString> "\\\\" { goto yyc_SingleQuoteString; } | |
| 369 <SingleQuoteString> "\\'" { goto yyc_SingleQuoteString; } | |
| 370 <SingleQuoteString> "'" { PUSH_TOKEN(Token::STRING);} | |
| 371 <SingleQuoteString> "\\" "\n" "\r"? { goto yyc_SingleQuoteString; } | |
| 372 <SingleQuoteString> "\\" "\r" "\n"? { goto yyc_SingleQuoteString; } | |
| 373 <SingleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | |
| 374 <SingleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | |
| 375 <SingleQuoteString> eof { TERMINATE_ILLEGAL(); } | |
| 376 <SingleQuoteString> any { goto yyc_SingleQuoteString; } | |
| 377 | |
| 378 <Identifier> identifier_char+ { goto yyc_Identifier; } | |
| 379 <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) { goto yyc_Id
entifier; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cur
sor_; goto yyc_Normal; } | |
| 380 <Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); } | |
| 381 <Identifier> any { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); } | |
| 382 | |
| 383 <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();} | |
| 384 <SingleLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token
::EOS); } | |
| 385 <SingleLineComment> any { goto yyc_SingleLineComment; } | |
| 386 | |
| 387 <MultiLineComment> [*][//] { PUSH_LINE_TERMINATOR();} | |
| 388 <MultiLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS);
} | |
| 389 <MultiLineComment> any { goto yyc_MultiLineComment; } | |
| 390 | |
| 391 <HtmlComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); } | |
| 392 <HtmlComment> "-->" { PUSH_LINE_TERMINATOR();} | |
| 393 <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();} | |
| 394 <HtmlComment> any { goto yyc_HtmlComment; } | |
| 395 */ | |
| 396 | |
| 397 fill: | |
| 398 int unfinished_size = cursor_ - start_; | |
| 399 if (FLAG_trace_lexer) { | |
| 400 printf( | |
| 401 "scanner needs a refill. Exiting for now with:\n" | |
| 402 " saved fill state_ = %d\n" | |
| 403 " unfinished token size = %d\n", | |
| 404 state_, | |
| 405 unfinished_size); | |
| 406 if (0 < unfinished_size && start_ < limit_) { | |
| 407 printf(" unfinished token is: "); | |
| 408 fwrite(start_, 1, cursor_ - start_, stdout); | |
| 409 putchar('\n'); | |
| 410 } | |
| 411 putchar('\n'); | |
| 412 } | |
| 413 | |
| 414 if (eof_) goto start_; | |
| 415 | |
| 416 // Once we get here, we can get rid of | |
| 417 // everything before start_ and after limit_. | |
| 418 | |
| 419 if (buffer_ < start_) { | |
| 420 size_t start_offset = start_ - buffer_; | |
| 421 memmove(buffer_, start_, limit_ - start_); | |
| 422 marker_ -= start_offset; | |
| 423 cursor_ -= start_offset; | |
| 424 limit_ -= start_offset; | |
| 425 start_ -= start_offset; | |
| 426 real_start_ += start_offset; | |
| 427 } | |
| 428 return 0; | |
| 429 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 78463002:
Experimental scanner: Remove the re2c version for real. (Closed)
Base URL: https://v8.googlecode.com/svn/branches/experimental/parser