Experimental parser: add utf8tolatin1 conversion

tools/lexer_generator/test/run_lexing_tests.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 87 matching lines @@
 
   @staticmethod
   def crashed(data):
     return data['process'].returncode != 0
 
   @staticmethod
   def buffer_contents(data):
     data['buffer'].append(data['process'].stdout.read())
     return ''.join(data['buffer'])
 
+  @staticmethod
+  def analyse_diff(left_data, right_data):
+    left = left_data.split("\n");
+    right = right_data.split("\n");
+    for i in range(min(len(left), len(right))):
+      if left[i] != right[i]:
+        message = "differ at token %d" % i
+        for j in range(i-4, i-1):
+          if j >= 0:
+            message += "\n\n%s\n%s" % (left[j], right[j])
+        message += "\n\n%s\n%s\n" % (left[i], right[i])
+        logging.info(message)
+        return
+    if len(right) > len(left):
+      logging.info("right longer")
+      return
+    if len(left) > len(right):
+      logging.info("left longer")
+      return
+
   def compare_results(self, left, right):
     f = left['file']
     assert f == right['file']
     logging.info('checking results for %s' % f)
     if self.crashed(left) or self.crashed(right):
       print "%s failed" % f
       return
     if left['path'] == self.right_path:
       left, right = right, left
     left_data = self.buffer_contents(left)
     right_data = self.buffer_contents(right)
     if left_data != right_data:
-      # TODO(dcarney): analyse differences
+      self.analyse_diff(left_data, right_data)
       print "%s failed" % f
       return
     print "%s succeeded" % f
 
   def process_complete_processes(self):
     complete_processes = self.complete_processes
     complete_ids = []
     for i, data in complete_processes.iteritems():
       if not self.right_path:
         assert not i in complete_ids
@@ skipping 46 matching lines @@
 
 if __name__ == '__main__':
 
   parser = argparse.ArgumentParser()
   parser.add_argument('-l', '--left-path')
   parser.add_argument('-r', '--right-path', default='')
   parser.add_argument('-i', '--input-files-path', default='')
   parser.add_argument('-f', '--single-file', default='')
   parser.add_argument('-p', '--parallel-process-count', default=1, type=int)
   parser.add_argument('-e', '--encoding',
-    choices=['latin1', 'utf8', 'utf8to16', 'utf16'], default='utf8')
+    choices=['latin1', 'utf8', 'utf16', 'utf8to16', 'utf8tolatin1'],
+    default='utf8')
   parser.add_argument('--use-harmony', action='store_true')
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
   if args.verbose:
     logging.basicConfig(level=logging.INFO)
 
   files = []
   if args.input_files_path:
     with open(args.input_files_path, 'r') as f:
       files = [filename for filename in f.read().split('\n') if filename]
   if args.single_file:
     files.append(args.single_file)
   assert files
 
   process_runner = ProcessRunner(files, args)
   process_runner.run()