Experimental parser: cleanup after adding encodings

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 64 matching lines @@
     if c:
       return c
     # if left['original_node_number'] != right['original_node_number']:
     #   # TODO fix
     #   print "noncanonical node ordering %d %d" % (left['original_node_number'],
     #                                               right['original_node_number'])
     return cmp(left['original_node_number'], right['original_node_number'])
 
   @staticmethod
   def __range_cmp(left, right):
-    if left[0] == 'LATIN_1':
-      if right[0] == 'LATIN_1':
+    if left[0] == 'PRIMARY_RANGE':
+      if right[0] == 'PRIMARY_RANGE':
         return cmp(left[1], right[1])
       assert right[0] == 'CLASS'
       return -1
     assert left[0] == 'CLASS'
-    if right[0] == 'LATIN_1':
+    if right[0] == 'PRIMARY_RANGE':
       return 1
     # TODO store numeric values and cmp
     return cmp(left[1], right[1])
 
   @staticmethod
   def __transform_state(encoding, state):
     # action data
     action = state.action()
     entry_action = None if not action else action.entry_action()
     match_action = None if not action else action.match_action()
     # generate ordered transitions
     transitions = map(lambda (k, v) : (k, v.node_number()),
                       state.transitions().items())
     def cmp(left, right):
       return TransitionKey.canonical_compare(left[0], right[0])
     transitions = sorted(transitions, cmp)
     # map transition keys to disjoint ranges
     disjoint_keys = {'value' : []}
     def f((key, state)):
       ranges = list(key.range_iter(encoding))
       disjoint_keys['value'] += ranges
       return (ranges, state)
     transitions = map(f, transitions)
     disjoint_keys = sorted(disjoint_keys['value'], CodeGenerator.__range_cmp)
     # dictionary object representing state
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
     for (t, r) in disjoint_keys:
       if t == 'CLASS':
         class_keys += 1
-      elif t == 'LATIN_1':
+      elif t == 'PRIMARY_RANGE':
         distinct_keys += r[1] - r[0] + 1
         ranges += 1
       else:
         raise Exception()
     return {
       'node_number' : state.node_number(),
       'original_node_number' : state.node_number(),
       'transitions' : transitions,
       'switch_transitions' : [],
       'deferred_transitions' : [],
@@ skipping 107 matching lines @@
     default_action = self.__default_action
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
-    encoding = self.__dfa.encoding().name()
+    encoding = self.__dfa.encoding()
     char_types = {'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'}
-    char_type = char_types[encoding]
+    char_type = char_types[encoding.name()]
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
-      encoding = encoding,
-      char_type = char_type)
+      encoding = encoding.name(),
+      char_type = char_type,
+      upper_bound = encoding.primary_range()[1])