Experimental parser: merge to r19637

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 27 matching lines @@
 #include "ast.h"
 #include "char-predicates-inl.h"
 #include "messages.h"
 #include "platform.h"
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 #include "scanner.h"
 
-#include "experimental-scanner.h"
-
 using namespace v8::internal;
 
 byte* ReadFile(const char* name, const byte** end, int repeat,
                bool convert_to_utf16) {
   FILE* file = fopen(name, "rb");
   if (file == NULL) return NULL;
 
   fseek(file, 0, SEEK_END);
   int file_size = ftell(file);
   rewind(file);
@@ skipping 12 matching lines @@
   }
   *end = &chars[size];
 
   if (!convert_to_utf16) return chars;
 
   // Length of new_chars is not strictly accurate, but should be enough.
   uint16_t* new_chars = new uint16_t[size];
   {
     Utf8ToUtf16CharacterStream stream(chars, size);
     uint16_t* cursor = new_chars;
-    uc32 c;
+    // uc32 c;
     // The 32-bit char type is probably only so that we can have -1 as a return
     // value. If the char is not -1, it should fit into 16 bits.
-    while ((c = stream.Advance()) != -1) {
-      *cursor++ = c;
-    }
+    CHECK(false);
+    // while ((c = stream.Advance()) != -1) {
+    //   *cursor++ = c;
+    // }
     *end = reinterpret_cast<byte*>(cursor);
   }
   delete[] chars;
   return reinterpret_cast<byte*>(new_chars);
 }
 
 
 enum Encoding {
   LATIN1,
   UTF8,
@@ skipping 31 matching lines @@
 };
 
 class BaselineScanner {
  public:
   BaselineScanner(const byte* source,
                   const byte* source_end,
                   Isolate* isolate,
                   ElapsedTimer* timer,
                   const LexerShellSettings& settings)
       : source_(source), stream_(NULL) {
-    unicode_cache_ = new UnicodeCache();
-    scanner_ = new Scanner(unicode_cache_);
+    scanner_ = new Scanner(isolate->unicode_cache());
     scanner_->SetHarmonyNumericLiterals(settings.harmony_numeric_literals);
     scanner_->SetHarmonyModules(settings.harmony_modules);
     scanner_->SetHarmonyScoping(settings.harmony_scoping);
     switch (settings.encoding) {
       case UTF8:
       case UTF8TO16:
         stream_ = new Utf8ToUtf16CharacterStream(source_, source_end - source_);
         break;
       case UTF16: {
         Handle<String> result = isolate->factory()->NewStringFromTwoByte(
@@ skipping 20 matching lines @@
     delete scanner_;
     delete stream_;
     delete unicode_cache_;
   }
 
   Scanner* scanner_;
 
  private:
   UnicodeCache* unicode_cache_;
   const byte* source_;
-  BufferedUtf16CharacterStream* stream_;
+  Utf16CharacterStream* stream_;
 };
 
 
 struct TokenWithLocation {
   Token::Value value;
   size_t beg;
   size_t end;
   std::vector<int> literal;
   bool is_ascii;
   // The location of the latest octal position when the token was seen.
@@ skipping 83 matching lines @@
         scanner.scanner_->literal_ascii_string();
       } else {
         scanner.scanner_->literal_utf16_string();
       }
     }
   } while (token != Token::EOS);
   return timer.Elapsed();
 }
 
 
-template<typename Char>
-TimeDelta RunExperimentalScanner(Handle<String> source,
-                                 Isolate* isolate,
-                                 std::vector<TokenWithLocation>* tokens,
-                                 LexerShellSettings settings) {
-  ElapsedTimer timer;
-  ExperimentalScanner<Char> scanner(source, isolate);
-  scanner.SetHarmonyNumericLiterals(settings.harmony_numeric_literals);
-  scanner.SetHarmonyModules(settings.harmony_modules);
-  scanner.SetHarmonyScoping(settings.harmony_scoping);
-
-  timer.Start();
-  scanner.Init();
-  Token::Value token;
-  do {
-    token = scanner.Next();
-    if (settings.dump_tokens) {
-      tokens->push_back(GetTokenWithLocation(&scanner, token));
-    } else if (HasLiteral(token)) {
-      if (scanner.is_literal_ascii()) {
-        scanner.literal_ascii_string();
-      } else {
-        scanner.literal_utf16_string();
-      }
-    }
-  } while (token != Token::EOS);
-  return timer.Elapsed();
-}
-
-
 void PrintTokens(const char* name,
                  const std::vector<TokenWithLocation>& tokens) {
   printf("No of tokens: %d\n",
          static_cast<int>(tokens.size()));
   printf("%s:\n", name);
   for (size_t i = 0; i < tokens.size(); ++i) {
     tokens[i].Print("=>");
   }
 }
 
@@ skipping 18 matching lines @@
     const byte* buffer = ReadFile(fname, &buffer_end, settings.repeat, false);
     if (truncate_by > buffer_end - buffer) {
       *can_truncate = false;
     } else {
       buffer_end -= truncate_by;
       baseline_time = RunBaselineScanner(
           buffer, buffer_end, isolate, &baseline_tokens, settings);
     }
     delete[] buffer;
   }
-  if (run_experimental) {
-    Handle<String> source;
-    const byte* buffer_end = 0;
-    const byte* buffer = ReadFile(fname, &buffer_end, settings.repeat,
-                                  settings.encoding == UTF8TO16);
-    if (truncate_by > buffer_end - buffer) {
-      *can_truncate = false;
-    } else {
-      buffer_end -= truncate_by;
-      switch (settings.encoding) {
-        case UTF8:
-        case LATIN1:
-          source = isolate->factory()->NewStringFromAscii(
-              Vector<const char>(reinterpret_cast<const char*>(buffer),
-                                 buffer_end - buffer));
-          experimental_time = RunExperimentalScanner<uint8_t>(
-              source, isolate, &experimental_tokens, settings);
-          break;
-        case UTF16:
-        case UTF8TO16: {
-          const uc16* buffer_16 = reinterpret_cast<const uc16*>(buffer);
-          const uc16* buffer_end_16 = reinterpret_cast<const uc16*>(buffer_end);
-          source = isolate->factory()->NewStringFromTwoByte(
-              Vector<const uc16>(buffer_16, buffer_end_16 - buffer_16));
-          // If the string was just an expaneded one byte string, V8 detects it
-          // and doesn't store it as two byte.
-          if (!source->IsTwoByteRepresentation()) {
-            experimental_time = RunExperimentalScanner<uint8_t>(
-              source, isolate, &experimental_tokens, settings);
-          } else {
-            experimental_time = RunExperimentalScanner<uint16_t>(
-              source, isolate, &experimental_tokens, settings);
-          }
-          break;
-        }
-        default:
-          printf("Encoding not supported by the experimental scanner\n");
-          exit(1);
-          break;
-      }
-    }
-    delete[] buffer;
-  }
   if (print_tokens && !run_experimental) {
     PrintTokens("Baseline", baseline_tokens);
   }
   if (print_tokens && !run_baseline) {
     PrintTokens("Experimental", experimental_tokens);
   }
   if ((print_tokens || settings.check_tokens) &&
        run_baseline && run_experimental) {
     if (print_tokens) {
       printf("No of tokens in Baseline:     %d\n",
@@ skipping 39 matching lines @@
       settings.encoding = LATIN1;
     } else if (strcmp(argv[i], "--utf8") == 0) {
       settings.encoding = UTF8;
     } else if (strcmp(argv[i], "--utf16") == 0) {
       settings.encoding = UTF16;
     } else if (strcmp(argv[i], "--utf8to16") == 0) {
       settings.encoding = UTF8TO16;
     } else if (strcmp(argv[i], "--print-tokens") == 0) {
       settings.print_tokens = true;
     } else if (strcmp(argv[i], "--no-baseline") == 0) {
-      settings.run_baseline = false;
     } else if (strcmp(argv[i], "--no-experimental") == 0) {
-      settings.run_experimental = false;
     } else if (strcmp(argv[i], "--no-check") == 0) {
       settings.check_tokens = false;
     } else if (strcmp(argv[i], "--break-after-illegal") == 0) {
       settings.break_after_illegal = true;
     } else if (strcmp(argv[i], "--use-harmony") == 0) {
       settings.harmony_numeric_literals = true;
       settings.harmony_modules = true;
       settings.harmony_scoping = true;
     } else if (strncmp(argv[i], "--benchmark=", 12) == 0) {
       benchmark = std::string(argv[i]).substr(12);
@@ skipping 43 matching lines @@
       if (settings.run_experimental) {
         if (benchmark.empty()) benchmark = "Experimental";
         printf("%s(RunTime): %.f ms\n", benchmark.c_str(),
                experimental_total);
       }
     }
   }
   v8::V8::Dispose();
   return 0;
 }