Experimental parser: proper utf16 conversion

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 34 matching lines @@
 #include "scanner.h"
 #include "lexer/lexer.h"
 
 using namespace v8::internal;
 
 
 enum Encoding {
   LATIN1,
   UTF8,
   UTF16,
-  UTF8TO16  // Read as UTF8, convert to UTF16 before giving it to the lexers.
+  UTF8TO16,  // Convert stream via scanner input stream
+  UTF8TO16_PRECONVERT  // Convert stream during file read
 };
 
 
 struct LexerShellSettings {
   Encoding encoding;
   bool print_tokens;
   bool break_after_illegal;
   bool eos_test;
   int repeat;
   bool harmony_numeric_literals;
   bool harmony_modules;
   bool harmony_scoping;
   LexerShellSettings()
       : encoding(LATIN1),
         print_tokens(false),
         break_after_illegal(false),
         eos_test(false),
         repeat(1),
         harmony_numeric_literals(false),
         harmony_modules(false),
         harmony_scoping(false) {}
 };
 
 
-static uint16_t* ReadFile(const char* name, const uint8_t** end,
-                          const LexerShellSettings& settings) {
-  FILE* file = fopen(name, "rb");
-  CHECK(file != NULL);
-
-  fseek(file, 0, SEEK_END);
-  unsigned file_size = ftell(file);
-  rewind(file);
-
-  uint16_t* two_byte_data = new uint16_t[file_size / 2 + file_size % 2];
-
-  uint8_t* char_data = reinterpret_cast<uint8_t*>(two_byte_data);
-  for (unsigned i = 0; i < file_size;) {
-    i += fread(&char_data[i], 1, file_size - i, file);
-  }
-  fclose(file);
-
-  if (settings.encoding == UTF8TO16) {
-    const uint32_t kMaxUtf16Character = 0xffff;
-    // Get utf8 length.
-    unsigned utf16_chars = 0;
-    {
-      unsigned position = 0;
-      while (position < file_size) {
-        uint32_t c = char_data[position];
-        if (c <= unibrow::Utf8::kMaxOneByteChar) {
-          position++;
-        } else {
-          c =  unibrow::Utf8::CalculateValue(char_data + position,
-                                             file_size - position,
-                                             &position);
-        }
-        if (c > kMaxUtf16Character) {
-          utf16_chars += 2;
-        } else {
-          utf16_chars += 1;
-        }
-      }
-    }
-    // Write new buffer out.
-    uint16_t* data = new uint16_t[utf16_chars];
+static uint16_t* ConvertUtf8ToUtf16(const uint16_t* const data_in,
+                                    unsigned* length) {
+  const unsigned file_size = *length;
+  const uint8_t* char_data = reinterpret_cast<const uint8_t*>(data_in);
+  const uint32_t kMaxUtf16Character = 0xffff;
+  // Get utf8 length.
+  unsigned utf16_chars = 0;
+  {
     unsigned position = 0;
-    unsigned i = 0;
     while (position < file_size) {
       uint32_t c = char_data[position];
       if (c <= unibrow::Utf8::kMaxOneByteChar) {
         position++;
       } else {
         c =  unibrow::Utf8::CalculateValue(char_data + position,
                                            file_size - position,
                                            &position);
       }
       if (c > kMaxUtf16Character) {
-        data[i++] = unibrow::Utf16::LeadSurrogate(c);
-        data[i++] = unibrow::Utf16::TrailSurrogate(c);
+        utf16_chars += 2;
       } else {
-        data[i++] = static_cast<uc16>(c);
+        utf16_chars += 1;
       }
     }
-    // Swap buffers.
-    delete two_byte_data;
-    file_size = utf16_chars * 2;
-    two_byte_data = data;
-    char_data = reinterpret_cast<uint8_t*>(two_byte_data);
   }
-
-  // Duplicate buffer if necessary.
-  if (settings.repeat > 1) {
-    unsigned size = file_size * settings.repeat;
-    uint16_t* data = new uint16_t[size / 2 + size % 2];
-    char_data = reinterpret_cast<uint8_t*>(two_byte_data);
-    for (int i = 0; i < settings.repeat; i++) {
-      memcpy(&char_data[i * file_size], two_byte_data, file_size);
+  // Write new buffer out.
+  uint16_t* data = new uint16_t[utf16_chars];
+  unsigned position = 0;
+  unsigned i = 0;
+  while (position < file_size) {
+    uint32_t c = char_data[position];
+    if (c <= unibrow::Utf8::kMaxOneByteChar) {
+      position++;
+    } else {
+      c =  unibrow::Utf8::CalculateValue(char_data + position,
+                                         file_size - position,
+                                         &position);
     }
-    delete two_byte_data;
-    file_size = size;
-    two_byte_data = data;
+    if (c > kMaxUtf16Character) {
+      data[i++] = unibrow::Utf16::LeadSurrogate(c);
+      data[i++] = unibrow::Utf16::TrailSurrogate(c);
+    } else {
+      data[i++] = static_cast<uc16>(c);
+    }
   }
-
-  *end = &char_data[file_size];
-  return two_byte_data;
+  *length = 2 * utf16_chars;
+  return data;
 }
 
 
+static uint16_t* Repeat(int repeat,
+                        const uint16_t* const data_in,
+                        unsigned* length) {
+  const unsigned file_size = *length;
+  unsigned size = file_size * repeat;
+  uint16_t* data = new uint16_t[size / 2 + size % 2];
+  uint8_t* char_data = reinterpret_cast<uint8_t*>(data);
+  for (int i = 0; i < repeat; i++) {
+    memcpy(&char_data[i * file_size], data_in, file_size);
+  }
+  *length = size;
+  return data;
+}
+
+
+static uint16_t* ReadFile(const char* name, unsigned* length) {
+  FILE* file = fopen(name, "rb");
+  CHECK(file != NULL);
+  // Get file size.
+  fseek(file, 0, SEEK_END);
+  unsigned file_size = ftell(file);
+  rewind(file);
+  // Read file contents.
+  uint16_t* data = new uint16_t[file_size / 2 + file_size % 2];
+  uint8_t* char_data = reinterpret_cast<uint8_t*>(data);
+  for (unsigned i = 0; i < file_size;) {
+    i += fread(&char_data[i], 1, file_size - i, file);
+  }
+  fclose(file);
+  *length = file_size;
+  return data;
+}
+
+
+static uint16_t* ReadFile(const char* name,
+                          const LexerShellSettings& settings,
+                          unsigned* length) {
+  uint16_t* data = ReadFile(name, length);
+  CHECK_GE(*length, 0);
+  if (*length == 0) return data;
+
+  if (settings.encoding == UTF8TO16_PRECONVERT) {
+    uint16_t* new_data = ConvertUtf8ToUtf16(data, length);
+    delete data;
+    data = new_data;
+  }
+
+  if (settings.repeat > 1) {
+    uint16_t* new_data = Repeat(settings.repeat, data, length);
+    delete data;
+    data = new_data;
+  }
+
+  return data;
+}
+
+
 struct TokenWithLocation {
   Token::Value value;
   size_t beg;
   size_t end;
   std::vector<int> literal;
   bool is_ascii;
   // The location of the latest octal position when the token was seen.
   int octal_beg;
   int octal_end;
   TokenWithLocation() :
@@ skipping 62 matching lines @@
 
 static TimeDelta RunLexer(const uint16_t* source,
                           const uint8_t* source_end,
                           Isolate* isolate,
                           std::vector<TokenWithLocation>* tokens,
                           const LexerShellSettings& settings) {
   SmartPointer<Utf16CharacterStream> stream;
   const uint8_t* one_byte_source = reinterpret_cast<const uint8_t*>(source);
   int bytes = source_end - one_byte_source;
   switch (settings.encoding) {
+    case UTF8TO16:
     case UTF8:
       stream.Reset(new Utf8ToUtf16CharacterStream(one_byte_source, bytes));
       break;
-    case UTF8TO16:
+    case UTF8TO16_PRECONVERT:
     case UTF16: {
       CHECK_EQ(0, bytes % 2);
       Handle<String> result = isolate->factory()->NewStringFromTwoByte(
           Vector<const uint16_t>(source, bytes / 2));
       stream.Reset(
           new GenericStringUtf16CharacterStream(result, 0, result->length()));
       break;
     }
     case LATIN1: {
       Handle<String> result = isolate->factory()->NewStringFromOneByte(
@@ skipping 33 matching lines @@
     const LexerShellSettings& settings,
     int truncate_by,
     bool* can_truncate) {
   if (settings.print_tokens) {
     printf("Processing file %s, truncating by %d bytes\n", fname, truncate_by);
   }
   HandleScope handle_scope(isolate);
   std::vector<TokenWithLocation> tokens;
   TimeDelta time;
   {
-    const uint8_t* buffer_end = 0;
-    const uint16_t* buffer = ReadFile(fname, &buffer_end, settings);
-    if (truncate_by > buffer_end - reinterpret_cast<const uint8_t*>(buffer)) {
+    unsigned length_in_bytes;
+    const uint16_t* buffer = ReadFile(fname, settings, &length_in_bytes);
+    const uint8_t* char_data = reinterpret_cast<const uint8_t*>(buffer);
+    const uint8_t* buffer_end = &char_data[length_in_bytes];
+    if (truncate_by > buffer_end - char_data) {
       *can_truncate = false;
     } else {
       buffer_end -= truncate_by;
       time = RunLexer(buffer, buffer_end, isolate, &tokens, settings);
     }
     delete[] buffer;
   }
   if (settings.print_tokens) {
     printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
     for (size_t i = 0; i < tokens.size(); ++i) {
@@ skipping 14 matching lines @@
   std::vector<std::string> fnames;
   LexerShellSettings settings;
   for (int i = 0; i < argc; ++i) {
     if (strcmp(argv[i], "--latin1") == 0) {
       settings.encoding = LATIN1;
     } else if (strcmp(argv[i], "--utf8") == 0) {
       settings.encoding = UTF8;
     } else if (strcmp(argv[i], "--utf16") == 0) {
       settings.encoding = UTF16;
     } else if (strcmp(argv[i], "--utf8to16") == 0) {
+#ifdef V8_USE_GENERATED_LEXER
+      settings.encoding = UTF8TO16_PRECONVERT;
+#else
       settings.encoding = UTF8TO16;
+#endif
     } else if (strcmp(argv[i], "--print-tokens") == 0) {
       settings.print_tokens = true;
     } else if (strcmp(argv[i], "--no-baseline") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--no-experimental") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--no-check") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--break-after-illegal") == 0) {
       settings.break_after_illegal = true;
@@ skipping 32 matching lines @@
                                   &can_truncate);
         total_time += t.InMillisecondsF();
         ++truncate_by;
       } while (can_truncate);
     }
     printf("RunTime: %.f ms\n", total_time);
   }
   v8::V8::Dispose();
   return 0;
 }