Experimental parser: implement skip

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 34 matching lines @@
     self.transitions = set()
 
   def parse(self, string):
     return RuleParser.parse(string, self)
 
 class RuleParser:
 
   tokens = RuleLexer.tokens
   __rule_precedence_counter = 0
   __keyword_transitions = set([
-      'continue', 'break', 'terminate', 'terminate_illegal'])
+      'continue', 'break', 'terminate', 'terminate_illegal', 'skip'])
 
   def __init__(self):
     self.__state = None
 
   def p_statements(self, p):
     'statements : aliases rules'
 
   def p_aliases(self, p):
     '''aliases : alias_rule aliases
                | empty'''
@@ skipping 38 matching lines @@
                        | DEFAULT_ACTION code_or_token empty
                        | CATCH_ALL empty action'''
     transition = p[3]
     if transition and not transition in self.__keyword_transitions:
       assert not transition == 'default'
       self.__state.transitions.add(transition)
     RuleParser.__rule_precedence_counter += 1
     rules = self.__state.rules[self.__state.current_state]
     code = p[2]
     if p[1] == 'default_action':
+      assert self.__state.current_state == 'default'
       assert not rules['default_action']
       rules['default_action'] = code
     elif p[1] == 'catch_all':
       assert not rules['catch_all']
       rules['catch_all'] = transition
     else:
       rule = (p[1], RuleParser.__rule_precedence_counter, code, transition)
       rules['regex'].append(rule)
 
   def p_code_or_token(self, p):
@@ skipping 157 matching lines @@
           (code_type, code_value) = code if code else default_code
           action = Action(code_type, code_value, precedence)
           graph = NfaBuilder.add_action(graph, action)
         if not transition or transition == 'break':
           pass
         elif transition == 'continue':
           assert not k == 'default'
           continues += 1
           graph = NfaBuilder.add_continue(graph)
         elif (transition == 'terminate' or
-              transition == 'terminate_illegal'):
+              transition == 'terminate_illegal' or
+              transition == 'skip'):
           assert not code
           graph = NfaBuilder.add_action(graph, Action(transition, None, -1))
         else:
           assert k == 'default'
           subgraph_modifier = '*' if code else None
           graph = NfaBuilder.join_subgraph(
             graph, transition, rule_map[transition], subgraph_modifier)
         graphs.append(graph)
       if continues == len(graphs):
         graphs.append(NfaBuilder.epsilon())
       if v['catch_all']:
         assert v['catch_all'] == 'continue'
         graphs.append(NfaBuilder.add_continue(NfaBuilder.catch_all()))
       graph = NfaBuilder.or_graphs(graphs)
       rule_map[k] = graph
     # process first the subgraphs, then the default graph
     for k, v in parser_state.rules.items():
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
     # build the automata
     for rule_name, graph in rule_map.items():
       self.__automata[rule_name] = RuleProcessor.Automata(builder, graph)
 
     default_action = parser_state.rules['default']['default_action']
     self.default_action = Action(default_action[0], default_action[1]) if default_action else None