Experimental parser: always apply default transitions

tools/lexer_generator/generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 48 matching lines @@
 %s
     </script>
 '''
 
 load_template = '''      draw('%s', '%s');'''
 
 load_outer_template = '''    <script>
 %s
     </script>'''
 
-def generate_html(rule_processor, minimize_default):
+def generate_html(rule_processor, minimize_default, merge):
   scripts = []
   loads = []
   for i, (name, automata) in enumerate(list(rule_processor.automata_iter())):
     (nfa, dfa) = (automata.nfa(), automata.dfa())
     mdfa = None
     if name != 'default' or minimize_default:
       mdfa = automata.minimal_dfa()
     (nfa_i, dfa_i, mdfa_i) = ("nfa_%d" % i, "dfa_%d" % i, "mdfa_%d" % i)
     scripts.append(script_template % (nfa_i, automaton_to_dot(nfa)))
     loads.append(load_template % ("nfa [%s]" % name, nfa_i))
-    scripts.append(script_template % (dfa_i, automaton_to_dot(dfa)))
+    scripts.append(script_template % (dfa_i, automaton_to_dot(dfa, merge)))
     loads.append(load_template % ("dfa [%s]" % name, dfa_i))
     if mdfa and mdfa.node_count() != dfa.node_count():
-      scripts.append(script_template % (mdfa_i, automaton_to_dot(mdfa)))
+      scripts.append(script_template % (mdfa_i, automaton_to_dot(mdfa, merge)))
       loads.append(load_template % ("mdfa [%s]" % name, mdfa_i))
   body = "\n".join(scripts) + (load_outer_template % "\n".join(loads))
   return file_template % body
 
 def generate_rule_tree_html(rule_processor):
   scripts = []
   loads = []
   mapper = lambda x : map_characters(rule_processor.encoding(), x)
   for i, (name, alias) in enumerate(list(rule_processor.alias_iter())):
     alias_i = "alias_%d" % i
@@ skipping 27 matching lines @@
   s = StringIO.StringIO()
   sortby = 'cumulative'
   ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
   ps.print_stats()
   print s.getvalue()
 
 if __name__ == '__main__':
 
   parser = argparse.ArgumentParser()
   parser.add_argument('--html')
+  parser.add_argument('--no-merge-html', action='store_true')
   parser.add_argument('--re', default='src/lexer/lexer_py.re')
   parser.add_argument('--input')
   parser.add_argument('--code')
   parser.add_argument('--encoding', default='latin1')
   parser.add_argument('--no-optimize-default', action='store_true')
   parser.add_argument('--no-minimize-default', action='store_true')
   parser.add_argument('--no-verify-default', action='store_true')
   parser.add_argument('--no-inline', action='store_true')
   parser.add_argument('--verbose', action='store_true')
   parser.add_argument('--debug-code', action='store_true')
@@ skipping 21 matching lines @@
       DfaMinimizer.set_verify(False)
     dfa = rule_processor.default_automata().dfa()
     mdfa = rule_processor.default_automata().minimal_dfa()
     if verbose:
       print "nodes reduced from %s to %s" % (
         dfa.node_count(), mdfa.node_count())
     DfaMinimizer.set_verify(True)
 
   html_file = args.html
   if html_file:
-    html = generate_html(rule_processor, minimize_default)
+    html = generate_html(
+      rule_processor, minimize_default, not args.no_merge_html)
     with open(args.html, 'w') as f:
       f.write(html)
       if verbose:
         print "wrote html to %s" % html_file
 
   rule_html_file = args.rule_html
   if rule_html_file:
     html = generate_rule_tree_html(rule_processor)
     with open(rule_html_file, 'w') as f:
       f.write(html)
@@ skipping 13 matching lines @@
       if verbose:
         print "wrote code to %s" % code_file
 
   input_file = args.input
   if input_file:
     with open(input_file, 'r') as f:
       lex(rule_processor, f.read())
 
   if args.profile:
     stop_profiling(profiler)