Experimental parser: add utf8tolatin1 conversion

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 35 matching lines @@
 #include "lexer/lexer.h"
 
 using namespace v8::internal;
 
 
 enum Encoding {
   LATIN1,
   UTF8,
   UTF16,
   UTF8TO16,  // Convert stream via scanner input stream
-  UTF8TO16_PRECONVERT  // Convert stream during file read
+  UTF8TOLATIN1,  // Convert stream via scanner input stream
 };
 
 
 struct LexerShellSettings {
   Encoding encoding;
   bool print_tokens;
   bool print_tokens_for_compare;
   bool break_after_illegal;
   bool eos_test;
   int repeat;
   bool harmony_numeric_literals;
   bool harmony_modules;
   bool harmony_scoping;
   LexerShellSettings()
       : encoding(LATIN1),
         print_tokens(false),
         print_tokens_for_compare(false),
         break_after_illegal(false),
         eos_test(false),
         repeat(1),
         harmony_numeric_literals(false),
         harmony_modules(false),
         harmony_scoping(false) {}
 };
 
 
 static uint16_t* ConvertUtf8ToUtf16(const uint16_t* const data_in,
-                                    unsigned* length) {
+                                    unsigned* length,
+                                    bool* is_one_byte) {
   const unsigned file_size = *length;
   const uint8_t* char_data = reinterpret_cast<const uint8_t*>(data_in);
   const uint32_t kMaxUtf16Character = 0xffff;
   // Get utf8 length.
   unsigned utf16_chars = 0;
+  *is_one_byte = true;
   {
     unsigned position = 0;
     while (position < file_size) {
       uint32_t c = char_data[position];
       if (c <= unibrow::Utf8::kMaxOneByteChar) {
         position++;
       } else {
+        *is_one_byte = false;
         c =  unibrow::Utf8::CalculateValue(char_data + position,
                                            file_size - position,
                                            &position);
       }
       if (c > kMaxUtf16Character) {
         utf16_chars += 2;
       } else {
         utf16_chars += 1;
       }
     }
@@ skipping 16 matching lines @@
       data[i++] = unibrow::Utf16::TrailSurrogate(c);
     } else {
       data[i++] = static_cast<uc16>(c);
     }
   }
   *length = 2 * utf16_chars;
   return data;
 }
 
 
+static uint16_t* ConvertUtf16ToLatin1(const uint16_t* const data_in,
+                                      unsigned* length) {
+  const unsigned size = *length / 2 + *length % 2;
+  uint16_t* data = new uint16_t[size];
+  uint8_t* char_data = reinterpret_cast<uint8_t*>(data);
+  CopyChars(char_data, data_in, size);
+  *length = size;
+  return data;
+}
+
+
 static uint16_t* Repeat(int repeat,
                         const uint16_t* const data_in,
                         unsigned* length) {
   const unsigned file_size = *length;
   unsigned size = file_size * repeat;
   uint16_t* data = new uint16_t[size / 2 + size % 2];
   uint8_t* char_data = reinterpret_cast<uint8_t*>(data);
   for (int i = 0; i < repeat; i++) {
     memcpy(&char_data[i * file_size], data_in, file_size);
   }
@@ skipping 16 matching lines @@
     i += fread(&char_data[i], 1, file_size - i, file);
   }
   fclose(file);
   *length = file_size;
   return data;
 }
 
 
 static uint16_t* ReadFile(const char* name,
                           const LexerShellSettings& settings,
-                          unsigned* length) {
+                          unsigned* length,
+                          Encoding* output_encoding) {
   uint16_t* data = ReadFile(name, length);
   CHECK_GE(*length, 0);
   if (*length == 0) return data;
 
-  if (settings.encoding == UTF8TO16_PRECONVERT) {
-    uint16_t* new_data = ConvertUtf8ToUtf16(data, length);
+  *output_encoding = settings.encoding;
+
+  if (settings.encoding == UTF8TO16 ||
+      settings.encoding == UTF8TOLATIN1) {
+    bool is_one_byte;
+    uint16_t* new_data = ConvertUtf8ToUtf16(data, length, &is_one_byte);
+    if (settings.encoding == UTF8TOLATIN1 && is_one_byte) {
+      *output_encoding = LATIN1;
+    } else {
+      *output_encoding = UTF16;
+    }
     delete data;
     data = new_data;
   }
+
+  if (settings.encoding == UTF8TOLATIN1 && *output_encoding == LATIN1) {
+    uint16_t* new_data = ConvertUtf16ToLatin1(data, length);
+    delete data;
+    data = new_data;
+  }
 
   if (settings.repeat > 1) {
     uint16_t* new_data = Repeat(settings.repeat, data, length);
     delete data;
     data = new_data;
   }
 
   return data;
 }
 
@@ skipping 69 matching lines @@
   }
 
  private:
   DISALLOW_COPY_AND_ASSIGN(TokenWithLocation);
 };
 
 
 static TimeDelta RunLexer(const uint16_t* source,
                           const uint8_t* source_end,
                           Isolate* isolate,
+                          Encoding output_encoding,
                           const LexerShellSettings& settings) {
   SmartPointer<Utf16CharacterStream> stream;
   const uint8_t* one_byte_source = reinterpret_cast<const uint8_t*>(source);
   int bytes = source_end - one_byte_source;
-  switch (settings.encoding) {
-    case UTF8TO16:
+  switch (output_encoding) {
     case UTF8:
       stream.Reset(new Utf8ToUtf16CharacterStream(one_byte_source, bytes));
       break;
-    case UTF8TO16_PRECONVERT:
     case UTF16: {
       CHECK_EQ(0, bytes % 2);
       Handle<String> result = isolate->factory()->NewStringFromTwoByte(
           Vector<const uint16_t>(source, bytes / 2));
       stream.Reset(
           new GenericStringUtf16CharacterStream(result, 0, result->length()));
       break;
     }
     case LATIN1: {
       Handle<String> result = isolate->factory()->NewStringFromOneByte(
           Vector<const uint8_t>(one_byte_source, bytes));
       stream.Reset(
           new GenericStringUtf16CharacterStream(result, 0, result->length()));
       break;
     }
+    case UTF8TO16:
+    case UTF8TOLATIN1:
+      CHECK(false);
   }
   Scanner scanner(isolate->unicode_cache());
   scanner.SetHarmonyNumericLiterals(settings.harmony_numeric_literals);
   scanner.SetHarmonyModules(settings.harmony_modules);
   scanner.SetHarmonyScoping(settings.harmony_scoping);
   ElapsedTimer timer;
   std::vector<TokenWithLocation*> tokens;
   timer.Start();
   scanner.Initialize(stream.get());
   Token::Value token;
@@ skipping 30 matching lines @@
     const LexerShellSettings& settings,
     int truncate_by,
     bool* can_truncate) {
   if (settings.print_tokens && !settings.print_tokens_for_compare) {
     printf("Processing file %s, truncating by %d bytes\n", fname, truncate_by);
   }
   HandleScope handle_scope(isolate);
   TimeDelta time;
   {
     unsigned length_in_bytes;
-    const uint16_t* buffer = ReadFile(fname, settings, &length_in_bytes);
+    Encoding output_encoding;
+    const uint16_t* buffer =
+        ReadFile(fname, settings, &length_in_bytes, &output_encoding);
     const uint8_t* char_data = reinterpret_cast<const uint8_t*>(buffer);
     const uint8_t* buffer_end = &char_data[length_in_bytes];
     if (truncate_by > buffer_end - char_data) {
       *can_truncate = false;
     } else {
       buffer_end -= truncate_by;
-      time = RunLexer(buffer, buffer_end, isolate, settings);
+      time = RunLexer(buffer, buffer_end, isolate, output_encoding, settings);
     }
     delete[] buffer;
   }
 
   return time;
 }
 
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   std::vector<std::string> fnames;
   LexerShellSettings settings;
   for (int i = 0; i < argc; ++i) {
     if (strcmp(argv[i], "--latin1") == 0) {
       settings.encoding = LATIN1;
     } else if (strcmp(argv[i], "--utf8") == 0) {
       settings.encoding = UTF8;
     } else if (strcmp(argv[i], "--utf16") == 0) {
       settings.encoding = UTF16;
     } else if (strcmp(argv[i], "--utf8to16") == 0) {
 #ifdef V8_USE_GENERATED_LEXER
-      settings.encoding = UTF8TO16_PRECONVERT;
+      settings.encoding = UTF8TO16;
 #else
-      settings.encoding = UTF8TO16;
+      settings.encoding = UTF8;
+#endif
+    } else if (strcmp(argv[i], "--utf8tolatin1") == 0) {
+#ifdef V8_USE_GENERATED_LEXER
+      settings.encoding = UTF8TOLATIN1;
+#else
+      settings.encoding = UTF8;
 #endif
     } else if (strcmp(argv[i], "--print-tokens") == 0) {
       settings.print_tokens = true;
     } else if (strcmp(argv[i], "--print-tokens-for-compare") == 0) {
       settings.print_tokens = true;
       settings.print_tokens_for_compare = true;
     } else if (strcmp(argv[i], "--no-baseline") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--no-experimental") == 0) {
       // Ignore.
@@ skipping 38 matching lines @@
         ++truncate_by;
       } while (can_truncate);
     }
     if (!settings.print_tokens_for_compare) {
       printf("RunTime: %.f ms\n", total_time);
     }
   }
   v8::V8::Dispose();
   return 0;
 }