Experimental lexer generator: comment fixes.

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 46 matching lines @@
     return 'PUSH_TOKEN(Token::EOS); return 0;'
 
   @staticmethod
   def __terminate_illegal_code(value):
     assert value == None
     return 'PUSH_TOKEN(Token::ILLEGAL); return 1;'
 
   @staticmethod
   def __skip_code(value):
     assert value == None
-    return 'SKIP(); goto code_start;'
+    return 'SKIP();'
 
   @staticmethod
   def __push_line_terminator_code(value):
     assert value == None
     return 'PUSH_LINE_TERMINATOR();'
 
   @staticmethod
   def __push_token_code(value):
     assert value != None
     return 'PUSH_TOKEN(Token::%s);' % value
 
   @staticmethod
   def __code_code(value):
     assert value != None
     return '%s\n' % value
 
+  @staticmethod
+  def __skip_and_terminate_code(value):
+    return 'SKIP(); --start_; ' + CodeGenerator.__terminate_code(value)
+
   def __init__(self, dfa, default_action):
     self.__dfa = dfa
     self.__start_node_number = dfa.start_state().node_number()
     self.__default_action = default_action
     # make this better
     self.__action_code_map = {
       "terminate" : self.__terminate_code,
       "terminate_illegal" : self.__terminate_illegal_code,
       "push_token" : self.__push_token_code,
       "push_line_terminator" : self.__push_line_terminator_code,
       "skip" : self.__skip_code,
       "code" : self.__code_code,
+      "skip_and_terminate" : self.__skip_and_terminate_code,
     }
 
   def __dfa_state_to_code(self, state):
     # FIXME: add different check types (if, switch, lookup table)
     # FIXME: add action + break / continue
     # FIXME: add default action
     code = ''
     if self.__start_node_number == state.node_number():
       code += '''
 code_start:
@@ skipping 43 matching lines @@
     def f(state, code):
       code += self.__dfa_state_to_code(state)
       return code
     code = dfa.visit_all_states(f, code)
 
     default_action_code = ''
     assert(default_action and default_action.match_action())
     action = default_action.match_action()
     default_action_code = self.__action_code_map[action[0]](action[1])
     code += '''
-  CHECK(false);
+  CHECK(false); goto code_start;
 default_action:
   //fprintf(stderr, "default action\\n");
   %s
   FORWARD();
   goto code_%s;
   return 0;
 }
 }
 }
 ''' % (default_action_code, self.__start_node_number)
     return code
 
   @staticmethod
   def rule_processor_to_code(rule_processor, use_mdfa):
     if use_mdfa:
       dfa = rule_processor.default_automata().minimal_dfa()
     else:
       dfa = rule_processor.default_automata().dfa()
     return CodeGenerator(dfa, rule_processor.default_action).__process()