Experimental parser: better eos handling

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 457 matching lines @@
     elif last[1] < upper_bound:
       inverted.append((last[1] + 1, upper_bound))
     inverted += list(classes)
     return inverted
 
 class Latin1Encoding(KeyEncoding):
 
   def __init__(self):
     super(Latin1Encoding, self).__init__(
       'latin1',
-      (1, 255),
-      ['eos', 'zero'])
+      (0, 255),
+      [])
     self.add_predefined_range(
       'whitespace', [(9, 9), (11, 12), (32, 32), (133, 133), (160, 160)])
     self.add_predefined_range(
       'letter', [
         (65, 90), (97, 122), (170, 170), (181, 181),
         (186, 186), (192, 214), (216, 246), (248, 255)])
     self.add_predefined_range('line_terminator', [(10, 10), (13, 13)])
     self.add_predefined_range(
       'identifier_part_not_letter', [(48, 57), (95, 95)])
 
 class Utf16Encoding(KeyEncoding):
 
   def __init__(self):
     super(Utf16Encoding, self).__init__(
       'utf16',
-      (1, 255),
-      ['eos', 'zero', 'byte_order_mark',
+      (0, 255),
+      ['byte_order_mark',
        'non_primary_whitespace',
        'non_primary_letter',
        'non_primary_identifier_part_not_letter',
        'non_primary_line_terminator',
        'non_primary_everything_else'])
     self.add_predefined_range(
       'whitespace',
       [(9, 9), (11, 12), (32, 32), (133, 133), (160, 160),
        self.class_range('byte_order_mark'),
        self.class_range('non_primary_whitespace')])
     self.add_predefined_range(
       'letter', [
         (65, 90), (97, 122), (170, 170), (181, 181),
         (186, 186), (192, 214), (216, 246), (248, 255),
         self.class_range('non_primary_letter')])
     self.add_predefined_range(
       'line_terminator',
       [(10, 10), (13, 13), self.class_range('non_primary_line_terminator')])
     self.add_predefined_range(
       'identifier_part_not_letter',
       [(48, 57), (95, 95),
        self.class_range('non_primary_identifier_part_not_letter')])
 
 class Utf8Encoding(KeyEncoding):
 
   def __init__(self):
     super(Utf8Encoding, self).__init__(
       'utf8',
-      (1, 127),
-      ['eos', 'zero', 'byte_order_mark',
+      (0, 127),
+      ['byte_order_mark',
        'non_primary_whitespace',
        'non_primary_letter',
        'non_primary_identifier_part_not_letter',
        'non_primary_line_terminator',
        'non_primary_everything_else'])
     self.add_predefined_range(
       'whitespace',
       [(9, 9), (11, 12), (32, 32),
         self.class_range('byte_order_mark'),
         self.class_range('non_primary_whitespace')])
     self.add_predefined_range(
       'letter', [(65, 90), (97, 122), self.class_range('non_primary_letter')])
     self.add_predefined_range(
       'line_terminator',
       [(10, 10), (13, 13), self.class_range('non_primary_line_terminator')])
     self.add_predefined_range(
       'identifier_part_not_letter',
       [(48, 57), (95, 95),
        self.class_range('non_primary_identifier_part_not_letter')])