Experimental parser: KeyEncoding class added

tools/lexer_generator/lexer_test.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 15 matching lines @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import unittest
 from automaton import Action
 from rule_parser import RuleProcessor
 
 class LexerTestCase(unittest.TestCase):
 
   def __verify_action_stream(self, rules, string, expected):
     expected = map(lambda (action, s) : (Action(None, (action, None)), s), expected)
-    automata = RuleProcessor.parse(rules).default_automata()
+    automata = RuleProcessor.parse(rules, 'latin1').default_automata()
     for automaton in [automata.nfa(), automata.dfa(), automata.minimal_dfa()]:
         for i, (action, start, stop) in enumerate(automaton.lex(string)):
           self.assertEquals(expected[i][0], action)
           self.assertEquals(expected[i][1], string[start : stop])
 
   @staticmethod
   def __terminate():
     return (Action(None, ('terminate', None)), '\0')
 
   def test_simple(self):
@@ skipping 48 matching lines @@
     rules = '''
     <<default>>
     /[a-z]+/ <|ID|>
     "key" <|KEYWORD|>'''
 
     # The keyword is not recognized because of the rule preference order (ID
     # is preferred over KEYWORD).
     self.__verify_action_stream(rules, 'ke', [('ID', 'ke')])
     self.__verify_action_stream(rules, 'key', [('ID', 'key')])
     self.__verify_action_stream(rules, 'keys', [('ID', 'keys')])