Experimental parser: cleanup cursing

tools/lexer_generator/code_generator.jinja

Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja b/tools/lexer_generator/code_generator.jinja
index 37b0870fa7949edf55d99794a30ca18f6c2c4c2b..77ae4742fb92f54972df27ebe2e4acddf2669e5b 100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -74,12 +74,12 @@
   {% elif type == 'set_marker' %}
     marker_ = cursor_ - {{value}};
   {% elif type == 'set_has_escapes' %}
-     next_.has_escapes = true;
+    next_.has_escapes = true;
   {% elif type == 'octal_inside_string' %}
-     last_octal_end_ = cursor_;
-     next_.has_escapes = true;
+    last_octal_end_ = cursor_;
+    next_.has_escapes = true;
   {% elif type == 'line_terminator_in_multiline_comment' %}
-     has_multiline_comment_before_next_ = true;
+    has_multiline_comment_before_next_ = true;
   {% elif type == 'check_escaped_identifier_start' %}
     if (!ValidIdentifierStart()) goto default_action;
     next_.has_escapes = true;
@@ -99,22 +99,20 @@
 {#- match actions must all explicitly jump or return -#}
 {% macro dispatch_match_action(type, value) -%}
   {% if type == 'terminate' %}
-    DO_EOS();
-    return;
+    {{dispatch_match_action('backtrack', ('1', 'EOS'))}}
   {% elif type == 'terminate_illegal' %}
     start_ = marker_;
-    BACKWARD(1);
-    DO_TOKEN(Token::ILLEGAL);
-    return;
+    {{dispatch_match_action('backtrack', ('1', 'ILLEGAL'))}}
   {% elif type == 'skip' %}
-    SKIP();
+    RESET_START();
     goto state_entry_0;
   {% elif type == 'skip_and_terminate' %}
-    SKIP();
+    RESET_START();
     --start_;
     {{dispatch_match_action('terminate', None)}}
   {% elif type == 'line_terminator' %}
-    DO_LINE_TERMINATOR();
+    RESET_START();
+    has_line_terminator_before_next_ = true;
     goto state_entry_0;
   {% elif type == 'token' %}
     DO_TOKEN(Token::{{value}})
@@ -128,7 +126,9 @@
     DO_TOKEN(stored_token)
     return;
   {% elif type == 'do_token_and_go_forward' %}
-    DO_TOKEN_AND_GO_FORWARD(Token::{{value}})
+    DO_TOKEN(Token::{{value}});
+    FORWARD();
+    RESET_START();
     return;
   {% elif type == 'harmony_token' %}
     if (harmony_{{value[0]}}_) {
@@ -282,47 +282,30 @@
 
 #include "lexer/experimental-scanner.h"
 
-#define PREPARE_TOKEN() {             \
-  next_.beg_pos = start_ - buffer_;   \
-  next_.end_pos = cursor_ - buffer_;  \
+#define RESET_START() {               \
   start_ = cursor_;                   \
 }
 
 #define DO_TOKEN(T) {                 \
-  PREPARE_TOKEN();                    \
-  next_.token = T;                    \
-}
-
-#define DO_TOKEN_AND_GO_FORWARD(T) {  \
-  PREPARE_TOKEN();                    \
-  FORWARD();                          \
+  next_.beg_pos = start_ - buffer_;   \
+  next_.end_pos = cursor_ - buffer_;  \
   next_.token = T;                    \
-  start_ = cursor_;                   \
-}
-
-#define DO_EOS() {                    \
-  cursor_ -= 1;                       \
-  DO_TOKEN(Token::EOS);               \
-}
-
-#define DO_LINE_TERMINATOR(s) {            \
-  start_ = cursor_;                        \
-  has_line_terminator_before_next_ = true; \
+  RESET_START();                      \
 }
 
 #define FORWARD() {                               \
-  if (++cursor_ >= buffer_end_) primary_char = 0; \
-  else primary_char = *(cursor_);                 \
+  cursor_++;                                      \
+  READ_CURSOR();                                  \
 }
 
 #define BACKWARD(n) {                             \
   cursor_ -= n;                                   \
-  if (cursor_ >= buffer_end_) primary_char = 0;   \
-  else primary_char = *(cursor_);                 \
+  READ_CURSOR();                                  \
 }
 
-#define SKIP() {                      \
-  start_ = cursor_;                   \
+#define READ_CURSOR() {                           \
+  if (cursor_ >= buffer_end_) primary_char = 0;   \
+  else primary_char = *(cursor_);                 \
 }
 
 #ifdef DEBUG
@@ -335,12 +318,12 @@ namespace v8 {
 namespace internal {
 template<>
 void ExperimentalScanner<{{char_type}}>::Scan() {
-  next_.has_escapes = false;
+
   // Setup environment.
+  next_.has_escapes = false;
   Token::Value stored_token;
   {{char_type}} primary_char;
-  if (cursor_ >= buffer_end_) primary_char = 0;
-  else primary_char = *(cursor_);
+  READ_CURSOR();
 
 {# first node is start node #}
 {% for dfa_state in dfa_states -%}