| OLD | NEW |
| 1 // Portions of this code based on re2c: | 1 // Portions of this code based on re2c: |
| 2 // (re2c/examples/push.re) | 2 // (re2c/examples/push.re) |
| 3 // Copyright 2013 the V8 project authors. All rights reserved. | 3 // Copyright 2013 the V8 project authors. All rights reserved. |
| 4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
| 6 // met: | 6 // met: |
| 7 // | 7 // |
| 8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
| (...skipping 191 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 202 buffer_end_ = needed + buffer_; | 202 buffer_end_ = needed + buffer_; |
| 203 | 203 |
| 204 marker_ = marker__offset + buffer_; | 204 marker_ = marker__offset + buffer_; |
| 205 cursor_ = cursor__offset + buffer_; | 205 cursor_ = cursor__offset + buffer_; |
| 206 start_ = buffer_ + start_offset; | 206 start_ = buffer_ + start_offset; |
| 207 limit_ = limit__offset + buffer_; | 207 limit_ = limit__offset + buffer_; |
| 208 } | 208 } |
| 209 memcpy(limit_, input, input_size); | 209 memcpy(limit_, input, input_size); |
| 210 limit_ += input_size; | 210 limit_ += input_size; |
| 211 | 211 |
| 212 #define SKIP() { start_ = cursor_; YYSETCONDITION(kConditionNorma
l); goto yy0; } | 212 #define SKIP() { start_ = cursor_; YYSETCONDITION(kConditionNorma
l); goto yyc_Normal; } |
| 213 #define YYFILL(n) { goto fill; } | 213 #define YYFILL(n) { goto fill; } |
| 214 | 214 |
| 215 #define YYGETSTATE() state_ | 215 #define YYGETSTATE() state_ |
| 216 #define YYSETSTATE(x) { state_ = (x); } | 216 #define YYSETSTATE(x) { state_ = (x); } |
| 217 | 217 |
| 218 #define YYGETCONDITION() condition_ | 218 #define YYGETCONDITION() condition_ |
| 219 #define YYSETCONDITION(x) { condition_ = (x); } | 219 #define YYSETCONDITION(x) { condition_ = (x); } |
| 220 | 220 |
| 221 start_: | 221 start_: |
| 222 if (FLAG_trace_lexer) { | 222 if (FLAG_trace_lexer) { |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 301 <Normal> "%=" { PUSH_TOKEN(Token::ASSIGN_MOD); } | 301 <Normal> "%=" { PUSH_TOKEN(Token::ASSIGN_MOD); } |
| 302 | 302 |
| 303 <Normal> "===" { PUSH_TOKEN(Token::EQ_STRICT); } | 303 <Normal> "===" { PUSH_TOKEN(Token::EQ_STRICT); } |
| 304 <Normal> "==" { PUSH_TOKEN(Token::EQ); } | 304 <Normal> "==" { PUSH_TOKEN(Token::EQ); } |
| 305 <Normal> "=" { PUSH_TOKEN(Token::ASSIGN); } | 305 <Normal> "=" { PUSH_TOKEN(Token::ASSIGN); } |
| 306 <Normal> "!==" { PUSH_TOKEN(Token::NE_STRICT); } | 306 <Normal> "!==" { PUSH_TOKEN(Token::NE_STRICT); } |
| 307 <Normal> "!=" { PUSH_TOKEN(Token::NE); } | 307 <Normal> "!=" { PUSH_TOKEN(Token::NE); } |
| 308 <Normal> "!" { PUSH_TOKEN(Token::NOT); } | 308 <Normal> "!" { PUSH_TOKEN(Token::NOT); } |
| 309 | 309 |
| 310 <Normal> "//" :=> SingleLineComment | 310 <Normal> "//" :=> SingleLineComment |
| 311 <Normal> whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITIO
N(kConditionSingleLineComment); goto yy0; } else { --cursor_; send(Token::DEC);
start_ = cursor_; goto yy0; } } | 311 <Normal> whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITIO
N(kConditionSingleLineComment); goto yyc_SingleLineComment; } else { --cursor_;
send(Token::DEC); start_ = cursor_; goto yyc_Normal; } } |
| 312 <Normal> "/*" :=> MultiLineComment | 312 <Normal> "/*" :=> MultiLineComment |
| 313 <Normal> "<!--" :=> HtmlComment | 313 <Normal> "<!--" :=> HtmlComment |
| 314 | 314 |
| 315 <Normal> ">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); } | 315 <Normal> ">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); } |
| 316 <Normal> ">>>" { PUSH_TOKEN(Token::SHR); } | 316 <Normal> ">>>" { PUSH_TOKEN(Token::SHR); } |
| 317 <Normal> "<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); } | 317 <Normal> "<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); } |
| 318 <Normal> ">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); } | 318 <Normal> ">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); } |
| 319 <Normal> "<=" { PUSH_TOKEN(Token::LTE); } | 319 <Normal> "<=" { PUSH_TOKEN(Token::LTE); } |
| 320 <Normal> ">=" { PUSH_TOKEN(Token::GTE); } | 320 <Normal> ">=" { PUSH_TOKEN(Token::GTE); } |
| 321 <Normal> "<<" { PUSH_TOKEN(Token::SHL); } | 321 <Normal> "<<" { PUSH_TOKEN(Token::SHL); } |
| 322 <Normal> ">>" { PUSH_TOKEN(Token::SAR); } | 322 <Normal> ">>" { PUSH_TOKEN(Token::SAR); } |
| 323 <Normal> "<" { PUSH_TOKEN(Token::LT); } | 323 <Normal> "<" { PUSH_TOKEN(Token::LT); } |
| 324 <Normal> ">" { PUSH_TOKEN(Token::GT); } | 324 <Normal> ">" { PUSH_TOKEN(Token::GT); } |
| 325 | 325 |
| 326 <Normal> number not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); } | 326 <Normal> number not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); } |
| 327 <Normal> number "\\u" [0-9a-fA-F]{4} { cursor_ -= 6; send(Token::ILLEGAL); s
tart_ = cursor_; cursor_ += 6; if (ValidIdentifierStart()) { YYSETCONDITION(kCon
ditionIdentifier); goto yy0; } else { PUSH_TOKEN(Token::ILLEGAL); } } | 327 <Normal> number "\\u" [0-9a-fA-F]{4} { cursor_ -= 6; send(Token::ILLEGAL); s
tart_ = cursor_; cursor_ += 6; if (ValidIdentifierStart()) { YYSETCONDITION(kCon
ditionIdentifier); goto yyc_Identifier; } else { PUSH_TOKEN(Token::ILLEGAL); } } |
| 328 <Normal> number any { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } | 328 <Normal> number any { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); } |
| 329 | 329 |
| 330 <Normal> "(" { PUSH_TOKEN(Token::LPAREN); } | 330 <Normal> "(" { PUSH_TOKEN(Token::LPAREN); } |
| 331 <Normal> ")" { PUSH_TOKEN(Token::RPAREN); } | 331 <Normal> ")" { PUSH_TOKEN(Token::RPAREN); } |
| 332 <Normal> "[" { PUSH_TOKEN(Token::LBRACK); } | 332 <Normal> "[" { PUSH_TOKEN(Token::LBRACK); } |
| 333 <Normal> "]" { PUSH_TOKEN(Token::RBRACK); } | 333 <Normal> "]" { PUSH_TOKEN(Token::RBRACK); } |
| 334 <Normal> "{" { PUSH_TOKEN(Token::LBRACE); } | 334 <Normal> "{" { PUSH_TOKEN(Token::LBRACE); } |
| 335 <Normal> "}" { PUSH_TOKEN(Token::RBRACE); } | 335 <Normal> "}" { PUSH_TOKEN(Token::RBRACE); } |
| 336 <Normal> ":" { PUSH_TOKEN(Token::COLON); } | 336 <Normal> ":" { PUSH_TOKEN(Token::COLON); } |
| 337 <Normal> ";" { PUSH_TOKEN(Token::SEMICOLON); } | 337 <Normal> ";" { PUSH_TOKEN(Token::SEMICOLON); } |
| (...skipping 16 matching lines...) Expand all Loading... |
| 354 <Normal> "~" { PUSH_TOKEN(Token::BIT_NOT); } | 354 <Normal> "~" { PUSH_TOKEN(Token::BIT_NOT); } |
| 355 <Normal> "," { PUSH_TOKEN(Token::COMMA); } | 355 <Normal> "," { PUSH_TOKEN(Token::COMMA); } |
| 356 | 356 |
| 357 <Normal> line_terminator { PUSH_LINE_TERMINATOR(); } | 357 <Normal> line_terminator { PUSH_LINE_TERMINATOR(); } |
| 358 <Normal> whitespace { SKIP(); } | 358 <Normal> whitespace { SKIP(); } |
| 359 | 359 |
| 360 <Normal> ["] :=> DoubleQuoteString | 360 <Normal> ["] :=> DoubleQuoteString |
| 361 <Normal> ['] :=> SingleQuoteString | 361 <Normal> ['] :=> SingleQuoteString |
| 362 | 362 |
| 363 <Normal> identifier_start :=> Identifier | 363 <Normal> identifier_start :=> Identifier |
| 364 <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION
(kConditionIdentifier); goto yy0; } send(Token::ILLEGAL); start_ = cursor_; goto
yy0; } | 364 <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION
(kConditionIdentifier); goto yyc_Identifier; } send(Token::ILLEGAL); start_ = cu
rsor_; goto yyc_Normal; } |
| 365 <Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); } | 365 <Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); } |
| 366 | 366 |
| 367 <Normal> eof { PUSH_EOF_AND_RETURN();} | 367 <Normal> eof { PUSH_EOF_AND_RETURN();} |
| 368 <Normal> any { PUSH_TOKEN(Token::ILLEGAL); } | 368 <Normal> any { PUSH_TOKEN(Token::ILLEGAL); } |
| 369 | 369 |
| 370 <DoubleQuoteString> "\\\\" { goto yy0; } | 370 <DoubleQuoteString> "\\\\" { goto yyc_DoubleQuoteString; } |
| 371 <DoubleQuoteString> "\\\"" { goto yy0; } | 371 <DoubleQuoteString> "\\\"" { goto yyc_DoubleQuoteString; } |
| 372 <DoubleQuoteString> '"' { PUSH_TOKEN(Token::STRING);} | 372 <DoubleQuoteString> '"' { PUSH_TOKEN(Token::STRING);} |
| 373 <DoubleQuoteString> "\\" "\n" "\r"? { goto yy0; } | 373 <DoubleQuoteString> "\\" "\n" "\r"? { goto yyc_DoubleQuoteString; } |
| 374 <DoubleQuoteString> "\\" "\r" "\n"? { goto yy0; } | 374 <DoubleQuoteString> "\\" "\r" "\n"? { goto yyc_DoubleQuoteString; } |
| 375 <DoubleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | 375 <DoubleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } |
| 376 <DoubleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | 376 <DoubleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } |
| 377 <DoubleQuoteString> eof { TERMINATE_ILLEGAL(); } | 377 <DoubleQuoteString> eof { TERMINATE_ILLEGAL(); } |
| 378 <DoubleQuoteString> any { goto yy0; } | 378 <DoubleQuoteString> any { goto yyc_DoubleQuoteString; } |
| 379 | 379 |
| 380 <SingleQuoteString> "\\\\" { goto yy0; } | 380 <SingleQuoteString> "\\\\" { goto yyc_SingleQuoteString; } |
| 381 <SingleQuoteString> "\\'" { goto yy0; } | 381 <SingleQuoteString> "\\'" { goto yyc_SingleQuoteString; } |
| 382 <SingleQuoteString> "'" { PUSH_TOKEN(Token::STRING);} | 382 <SingleQuoteString> "'" { PUSH_TOKEN(Token::STRING);} |
| 383 <SingleQuoteString> "\\" "\n" "\r"? { goto yy0; } | 383 <SingleQuoteString> "\\" "\n" "\r"? { goto yyc_SingleQuoteString; } |
| 384 <SingleQuoteString> "\\" "\r" "\n"? { goto yy0; } | 384 <SingleQuoteString> "\\" "\r" "\n"? { goto yyc_SingleQuoteString; } |
| 385 <SingleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | 385 <SingleQuoteString> "\n" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } |
| 386 <SingleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | 386 <SingleQuoteString> "\r" => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } |
| 387 <SingleQuoteString> eof { TERMINATE_ILLEGAL(); } | 387 <SingleQuoteString> eof { TERMINATE_ILLEGAL(); } |
| 388 <SingleQuoteString> any { goto yy0; } | 388 <SingleQuoteString> any { goto yyc_SingleQuoteString; } |
| 389 | 389 |
| 390 <Identifier> identifier_char+ { goto yy0; } | 390 <Identifier> identifier_char+ { goto yyc_Identifier; } |
| 391 <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) { YYSETCONDIT
ION(kConditionIdentifier); goto yy0; } YYSETCONDITION(kConditionNormal); send(To
ken::ILLEGAL); start_ = cursor_; goto yy0; } | 391 <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) { goto yyc_Id
entifier; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cur
sor_; goto yyc_Normal; } |
| 392 <Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); } | 392 <Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); } |
| 393 <Identifier> any { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); } | 393 <Identifier> any { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); } |
| 394 | 394 |
| 395 <IdentifierIllegal> identifier_start { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | 395 <IdentifierIllegal> identifier_start { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } |
| 396 <IdentifierIllegal> identifier_char\identifier_start { marker_ = cursor_; g
oto yy0; } | 396 <IdentifierIllegal> identifier_char\identifier_start { marker_ = cursor_; g
oto yyc_IdentifierIllegal; } |
| 397 <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { cur
sor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } marker_ = cursor_; PUSH_TOKEN(Token::IL
LEGAL); } | 397 <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { cur
sor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } marker_ = cursor_; PUSH_TOKEN(Token::IL
LEGAL); } |
| 398 <IdentifierIllegal> "\\"+ { marker_ = cursor_; goto yy0; } | 398 <IdentifierIllegal> "\\"+ { marker_ = cursor_; goto yyc_Identifi
erIllegal; } |
| 399 <IdentifierIllegal> number not_identifier_char { YYCTYPE* temp = cursor_; cu
rsor_ = marker_; send(Token::ILLEGAL); cursor_ = temp; YYSETCONDITION(kCondition
Normal); PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); } | 399 <IdentifierIllegal> number not_identifier_char { YYCTYPE* temp = cursor_; cu
rsor_ = marker_; send(Token::ILLEGAL); cursor_ = temp; YYSETCONDITION(kCondition
Normal); PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); } |
| 400 <IdentifierIllegal> number "\\u" [0-9a-fA-F]{4} { YYCTYPE* temp = cursor_; c
ursor_ = marker_; send(Token::ILLEGAL); cursor_ = temp; send(Token::ILLEGAL); st
art_ = cursor_; goto yy0; } | 400 <IdentifierIllegal> number "\\u" [0-9a-fA-F]{4} { YYCTYPE* temp = cursor_; c
ursor_ = marker_; send(Token::ILLEGAL); cursor_ = temp; send(Token::ILLEGAL); st
art_ = cursor_; goto yyc_IdentifierIllegal; } |
| 401 <IdentifierIllegal> any { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } | 401 <IdentifierIllegal> any { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL)
; } |
| 402 | 402 |
| 403 <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();} | 403 <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();} |
| 404 <SingleLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token
::EOS); } | 404 <SingleLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token
::EOS); } |
| 405 <SingleLineComment> any { goto yy0; } | 405 <SingleLineComment> any { goto yyc_SingleLineComment; } |
| 406 | 406 |
| 407 <MultiLineComment> [*][//] { PUSH_LINE_TERMINATOR();} | 407 <MultiLineComment> [*][//] { PUSH_LINE_TERMINATOR();} |
| 408 <MultiLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS);
} | 408 <MultiLineComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS);
} |
| 409 <MultiLineComment> any { goto yy0; } | 409 <MultiLineComment> any { goto yyc_MultiLineComment; } |
| 410 | 410 |
| 411 <HtmlComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); } | 411 <HtmlComment> eof { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); } |
| 412 <HtmlComment> "-->" { PUSH_LINE_TERMINATOR();} | 412 <HtmlComment> "-->" { PUSH_LINE_TERMINATOR();} |
| 413 <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();} | 413 <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();} |
| 414 <HtmlComment> any { goto yy0; } | 414 <HtmlComment> any { goto yyc_HtmlComment; } |
| 415 */ | 415 */ |
| 416 | 416 |
| 417 fill: | 417 fill: |
| 418 int unfinished_size = cursor_ - start_; | 418 int unfinished_size = cursor_ - start_; |
| 419 if (FLAG_trace_lexer) { | 419 if (FLAG_trace_lexer) { |
| 420 printf( | 420 printf( |
| 421 "scanner needs a refill. Exiting for now with:\n" | 421 "scanner needs a refill. Exiting for now with:\n" |
| 422 " saved fill state_ = %d\n" | 422 " saved fill state_ = %d\n" |
| 423 " unfinished token size = %d\n", | 423 " unfinished token size = %d\n", |
| 424 state_, | 424 state_, |
| (...skipping 15 matching lines...) Expand all Loading... |
| 440 size_t start_offset = start_ - buffer_; | 440 size_t start_offset = start_ - buffer_; |
| 441 memmove(buffer_, start_, limit_ - start_); | 441 memmove(buffer_, start_, limit_ - start_); |
| 442 marker_ -= start_offset; | 442 marker_ -= start_offset; |
| 443 cursor_ -= start_offset; | 443 cursor_ -= start_offset; |
| 444 limit_ -= start_offset; | 444 limit_ -= start_offset; |
| 445 start_ -= start_offset; | 445 start_ -= start_offset; |
| 446 real_start_ += start_offset; | 446 real_start_ += start_offset; |
| 447 } | 447 } |
| 448 return 0; | 448 return 0; |
| 449 } | 449 } |
| OLD | NEW |