Experimental lexer generator: random refactoring + comments.

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 130 matching lines @@
     assert state['inline'] == None
     inline = False
     if not state['transitions']:
       inline = True
     # TODO add in 1 or 2 element if blocks
     state['inline'] = inline
     return count + 1 if inline else count
 
   @staticmethod
   def __split_transitions(split_count, state):
+    '''Goes through the transitions for 'state' and decides which of them should
+    use the if statement and which should use the switch statement.'''
     assert not state['switch_transitions']
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
     for (t, r) in state['disjoint_keys']:
       if t == 'CLASS':
         class_keys += 1
       elif t == 'LATIN_1':
         distinct_keys += r[1] - r[0] + 1
         ranges += 1
       else:
         raise Exception()
@@ skipping 30 matching lines @@
     def f((key, original_node_number)):
       return (key, id_map[original_node_number]['node_number'])
     for state in dfa_states:
       state['transitions'] = map(f, state['transitions'])
     assert id_map[self.__start_node_number]['node_number'] == 0
     assert len(dfa_states) == self.__dfa.node_count()
     # set nodes to inline
     if self.__inline:
       inlined = reduce(CodeGenerator.__set_inline, dfa_states, 0)
       if self.__log:
-        print "inlined %s" % inlined
+        print "%s states inlined" % inlined
     elif self.__log:
       print "no inlining"
     # split transitions
     if self.__switching:
       switched = reduce(CodeGenerator.__split_transitions, dfa_states, 0)
       if self.__log:
-        print "switched states %s" % inlined
+        print "%s states use switch (instead of if)" % switched
     elif self.__log:
       print "no switching"
     # store states
     self.__dfa_states = dfa_states
 
   def process(self):
 
     self.__canonicalize_traversal()
 
     dfa_states = self.__dfa_states
 
     default_action = self.__default_action
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states)