Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/transition_key_test.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 33 matching lines @@
   def test_hash(self):
     for (left, right) in self.__equal_pairs:
       self.assertEqual(hash(left), hash(right))
 
   def test_classes(self):
     # class regex, should match, should not match
     data = [
       ("1-2", "12", "ab"),
       ("a-zA-Z", "abyzABYZ" , "123"),
       ("a-zA-Z0g" , "abyzABYZ0" , "123"),
-      ("a-z:ws::lit:" , "abc" , "123"),
+      ("a-z:whitespace::letter:" , "abc" , "123"),
     ]
     classes = {}
     for (string, match, no_match) in data:
       for invert in [False, True]:
         if invert:
           regex = "[^%s]" % string
           token = "NOT_CLASS"
         else:
           regex = "[%s]" % string
           token = "CLASS"
@@ skipping 14 matching lines @@
     self.assertTrue(key.matches_char('A'))
 
 
   def test_disjoint_keys(self):
     key1 = TransitionKey([(1, 10)])
     key2 = TransitionKey([(5, 15)])
     disjoint_set = TransitionKey.disjoint_keys(set([key1, key2]))
     self.assertTrue(TransitionKey([(1, 4)]) in disjoint_set)
     self.assertTrue(TransitionKey([(5, 10)]) in disjoint_set)
     self.assertTrue(TransitionKey([(11, 15)]) in disjoint_set)