Experimental parser: better eos handling

tools/lexer_generator/code_generator.jinja

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 219 matching lines @@
     {{ dispatch_entry_action(entry_action[0], entry_action[1]) }}
   {%- endif %}
 
   {{ write_label('after_entry_code', node_number) }}
 
   {%- if debug_print %}
     fprintf(stderr, "char at hand is %c (%d)\n", primary_char, primary_char);
   {% endif -%}
 
   {%- macro do_transition(jump_id) -%}
-    {%- set transition_state_id = jump_table[jump_id][0] -%}
-    {%- set inline_transition = jump_table[jump_id][1] == 'inline' %}
     FORWARD();
-    {%- if inline_transition %}
-      {{ do_dfa_state(transition_state_id) }}
-    {% else %}
-      {{ jump(jump_id) }}
-    {% endif %}
+    if (cursor_ < buffer_end_) {
+      {%- set transition_state_id = jump_table[jump_id][0] -%}
+      {%- set inline_transition = jump_table[jump_id][1] == 'inline' %}
+      {%- if inline_transition %}
+        {{ do_dfa_state(transition_state_id) }}
+      {% else %}
+        {{ jump(jump_id) }}
+      {% endif %}
+    }
+    {% if 'eos' in state['unique_transitions'] -%}
+      {{ jump(state['unique_transitions']['eos']) }} // eos handler
+    {%- else -%}
+      BACKWARD(1);
+      {{ jump(state['jump_before_match']) }} // no eos handler
+    {%- endif %}
   {%- endmacro -%}
 
   {%- if state['switch_transitions'] -%}
     switch(primary_char) {
     {%- for ranges, jump_id in state['switch_transitions'] %}
       {%- for r in ranges -%}
         {%- for key in range(r[0], r[1] + 1) -%}
           case {{key}}:
         {% endfor %}
       {%- endfor -%}
@@ skipping 19 matching lines @@
       next_.is_onebyte = false;
       {{long_char_create()}}
       {%- for key, jump_id in state['long_char_transitions'] %}
         if ({{do_key(key)}}) { // long_char transition
           {{ do_transition(jump_id) }}
         }
       {% endfor -%}
     }
   {%- endif-%}
 
+  {{ write_label('before_match', node_number) }}
+
   {%- set match_action = state.match_action -%}
-
   {%- if match_action %}
     {{ dispatch_match_action(match_action[0], match_action[1]) }}
     CRASH();
   {% else %}
     goto default_action;
   {%- endif %}
 
 {%- endmacro %}
 
 #include "lexer/experimental-scanner.h"
@@ skipping 11 matching lines @@
 
 #define FORWARD() {                               \
   cursor_++;                                      \
 }
 
 #define BACKWARD(n) {                             \
   cursor_ -= n;                                   \
 }
 
 #define READ_CURSOR() {                           \
-  if (cursor_ >= buffer_end_) primary_char = 0;   \
-  else primary_char = *(cursor_);                 \
+  ASSERT(cursor_ < buffer_end_);                  \
+  primary_char = *(cursor_);                      \
 }
 
 #ifdef DEBUG
 #define CRASH() { ((void(*)())0)(); }
 #else
 #define CRASH() { }
 #endif
 
 namespace v8 {
 namespace internal {
@@ skipping 26 matching lines @@
     // force use of stored_token
     stored_token = Token::ILLEGAL;
     // force use of marker
     marker = NULL;
     // force use of state_entry_0
     goto state_entry_0;
   }
 }
 } }