Experimental parser: cleanup code requiring calls to the unicode cache

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 50 matching lines @@
 "="           <|push_token(ASSIGN)|>
 "!=="         <|push_token(NE_STRICT)|>
 "!="          <|push_token(NE)|>
 "!"           <|push_token(NOT)|>
 
 "//"          <||SingleLineComment>
 "/*"          <set_marker(2)||MultiLineComment>
 "<!--"        <||SingleLineComment>
 
 "<!-"        <|{
-  cursor_ -= 2;
-  yych = *(cursor_);
+  BACKWARD(2);
   PUSH_TOKEN(Token::LT);
 }|>
 
 "<!"        <|{
-  cursor_ -= 1;
-  yych = *(cursor_);
+  BACKWARD(1);
   PUSH_TOKEN(Token::LT);
 }|>
 
 
 "-->" <{
   if (!just_seen_line_terminator_) {
-    yych = *(--cursor_);
+    BACKWARD(1);
     PUSH_TOKEN(Token::DEC);
   }
 }||SingleLineComment>
 
 ">>>="        <|push_token(ASSIGN_SHR)|>
 ">>>"         <|push_token(SHR)|>
 "<<="         <|push_token(ASSIGN_SHL)|>
 ">>="         <|push_token(ASSIGN_SAR)|>
 "<="          <|push_token(LTE)|>
 ">="          <|push_token(GTE)|>
@@ skipping 31 matching lines @@
 "&"           <|push_token(BIT_AND)|>
 "+"           <|push_token(ADD)|>
 "-"           <|push_token(SUB)|>
 "*"           <|push_token(MUL)|>
 "/"           <|push_token(DIV)|>
 "%"           <|push_token(MOD)|>
 "~"           <|push_token(BIT_NOT)|>
 ","           <|push_token(COMMA)|>
 
 line_terminator+                     <|push_line_terminator|>
-/[:whitespace::byte_order_mark:]+/   <|skip|>
+/[:whitespace:]+/   <|skip|>
 
 "\""           <set_marker(1)||DoubleQuoteString>
 "'"            <set_marker(1)||SingleQuoteString>
 
 # all keywords
 "break"       <|push_token(BREAK)|>
 "case"        <|push_token(CASE)|>
 "catch"       <|push_token(CATCH)|>
 "class"       <|push_token(FUTURE_RESERVED_WORD)|>
 "const"       <|push_token(CONST)|>
@@ skipping 84 matching lines @@
 eos <|skip_and_terminate|>
 catch_all <||continue>
 
 <<MultiLineComment>>
 /\*+\//             <|skip|>
 # TODO find a way to generate the below rule
 /\*+[^\/*]/       <||continue>
 line_terminator  <push_line_terminator||continue>
 eos <|terminate_illegal|>
 catch_all        <||continue>