Experimental parser: implement skip

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 82 matching lines @@
 "&"           push_token(BIT_AND)
 "+"           push_token(ADD)
 "-"           push_token(SUB)
 "*"           push_token(MUL)
 "/"           push_token(DIV)
 "%"           push_token(MOD)
 "~"           push_token(BIT_NOT)
 ","           push_token(COMMA)
 
 line_terminator+  { PUSH_LINE_TERMINATOR(); }
-whitespace     { SKIP(); } # TODO implement skip
+whitespace     <<skip>>
 
 "\""           <<DoubleQuoteString>>
 "'"            <<SingleQuoteString>>
 
 # all keywords
 "break"       push_token(BREAK)
 "case"        push_token(CASE)
 "catch"       push_token(CATCH)
 "class"       push_token(FUTURE_RESERVED_WORD)
 "const"       push_token(CONST)
@@ skipping 38 matching lines @@
 "with"        push_token(WITH)
 "yield"       push_token(YIELD)
 
 identifier_start push_token(IDENTIFIER) <<Identifier>>
 /\\u[0-9a-fA-F]{4}/ {
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
     PUSH_TOKEN(Token::ILLEGAL);
   }
 } <<Identifier>>
 
-eof             { PUSH_TOKEN(Token::EOS); return 0; }
+eof             <<terminate>>
 default_action  push_token(ILLEGAL)
 
 <DoubleQuoteString>
 /\\\n\r?/ <<continue>>
 /\\\r\n?/ <<continue>>
 /\\./     <<continue>>
 /\n|\r/   push_token(ILLEGAL)
 "\""      push_token(STRING)
 eof       <<terminate_illegal>>
 catch_all <<continue>>
 
 <SingleQuoteString>
 /\\\n\r?/ <<continue>>
 /\\\r\n?/ <<continue>>
 /\\./     <<continue>>
 /\n|\r/   push_token(ILLEGAL)
 "'"       push_token(STRING)
 eof       <<terminate_illegal>>
 catch_all <<continue>>
 
 <Identifier>
-identifier_char    <<continue>>
+identifier_char push_token(IDENTIFIER) <<continue>>
 /\\u[0-9a-fA-F]{4}/ {
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
     PUSH_TOKEN(Token::ILLEGAL);
   }
 } <<continue>>
-default_action push_token(IDENTIFIER)
 
 <SingleLineComment>
 line_terminator  { PUSH_LINE_TERMINATOR(); }
-eof              <<terminate>>
 catch_all <<continue>>
 
 <MultiLineComment>
-"*/"             { SKIP(); goto code_start;}
+"*/"             <<skip>>
 /\*[^\057]/      <<continue>>
-# need to force action
-line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
-eof              <<terminate>>
+line_terminator { PUSH_LINE_TERMINATOR(); } <<continue>>
 catch_all <<continue>>
 
 <HtmlComment>
-"-->"            { SKIP(); }
+"-->"            <<skip>>
 /--./            <<continue>>
 /-./             <<continue>>
-# need to force action
-line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
-eof              <<terminate>>
+line_terminator { PUSH_LINE_TERMINATOR(); } <<continue>>
 catch_all <<continue>>