Experimental parser: add catch all rule

tools/lexer_generator/generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 73 matching lines @@
       body = "\n".join(scripts) + (load_outer_template % "\n".join(loads))
     return file_template % body
 
   def process_rules(self, parser_state):
     rule_map = {}
     builder = NfaBuilder()
     builder.set_character_classes(parser_state.character_classes)
     assert 'default' in parser_state.rules
     def process(k, v):
       graphs = []
+      continues = 0
       for (graph, (precedence, code, transition)) in v['regex']:
         default_code = v['default_action']
         action = code if code else default_code
         if action:
           graph = NfaBuilder.add_action(graph, (precedence, action))
-        if not transition:
+        if not transition or transition == 'break':
           pass
         elif transition == 'continue':
           assert not k == 'default'
+          continues += 1
           graph = NfaBuilder.add_continue(graph)
-        elif transition == 'break':
-          assert code
-          graph = NfaBuilder.add_break(graph)
         elif (transition == 'terminate' or
               transition == 'terminate_illegal'):
           assert not code
           graph = NfaBuilder.add_action(graph, (-1, transition))
         else:
           assert k == 'default'
           subgraph_modifier = '*' if code else None
           graph = NfaBuilder.join_subgraph(
             graph, transition, rule_map[transition], subgraph_modifier)
         graphs.append(graph)
+      if continues == len(graphs):
+        graphs.append(NfaBuilder.epsilon())
+      if v['catch_all']:
+        assert v['catch_all'] == 'continue'
+        graphs.append(NfaBuilder.add_continue(NfaBuilder.catch_all()))
       graph = NfaBuilder.or_graphs(graphs)
       rule_map[k] = graph
     # process first the subgraphs, then the default graph
     for k, v in parser_state.rules.items():
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
     # build the automata
     for rule_name, graph in rule_map.items():
       nfa = builder.nfa(graph)
@@ skipping 39 matching lines @@
     with open(code_file, 'w') as f:
       f.write(code)
       print "wrote code to %s" % code_file
 
   input_file = args.input
   if input_file:
     with open(input_file, 'r') as f:
       input_text = f.read() + '\0'
     for t in generator.lex(input_text):
       print t