Experimental scanner += API which takes Handle<String>.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 193 matching lines @@
     end = scanner.scanner_->location().end_pos;
     if (dump_tokens) {
       tokens->push_back(TokenWithLocation(token, beg, end));
     }
   } while (token != Token::EOS);
   return timer.Elapsed();
 }
 
 
 template<typename YYCTYPE>
-TimeDelta RunExperimentalScanner(const char* fname,
+TimeDelta RunExperimentalScanner(Handle<String> source,
                                  Isolate* isolate,
                                  Encoding encoding,
                                  bool dump_tokens,
                                  std::vector<TokenWithLocation>* tokens,
                                  int repeat,
                                  HarmonySettings harmony_settings) {
   ElapsedTimer timer;
-  byte* buffer_end = 0;
-  YYCTYPE* buffer = reinterpret_cast<YYCTYPE*>(
-      ReadFile(fname, &buffer_end, repeat, encoding == UTF8TO16));
-
   timer.Start();
-  ExperimentalScanner<YYCTYPE> scanner(
-      buffer, reinterpret_cast<YYCTYPE*>(buffer_end), isolate);
+  ExperimentalScanner<YYCTYPE> scanner(source, isolate);
   scanner.SetHarmonyNumericLiterals(harmony_settings.numeric_literals);
   scanner.SetHarmonyModules(harmony_settings.modules);
   scanner.SetHarmonyScoping(harmony_settings.scoping);
 
   Token::Value token;
   int beg, end;
   do {
     token = scanner.Next();
     beg = scanner.location().beg_pos;
     end = scanner.location().end_pos;
@@ skipping 32 matching lines @@
   }
   HandleScope handle_scope(isolate);
   std::vector<TokenWithLocation> baseline_tokens, experimental_tokens;
   TimeDelta baseline_time, experimental_time;
   if (run_baseline) {
     baseline_time = RunBaselineScanner(
         fname, isolate, encoding, print_tokens || check_tokens,
         &baseline_tokens, repeat, harmony_settings);
   }
   if (run_experimental) {
+    Handle<String> source;
+    byte* buffer_end = 0;
+    const byte* buffer = ReadFile(fname, &buffer_end, repeat,
+                                  encoding == UTF8TO16);
     switch (encoding) {
       case UTF8:
-        experimental_time = RunExperimentalScanner<int8_t>(
-            fname, isolate, encoding, print_tokens || check_tokens,
-            &experimental_tokens, repeat, harmony_settings);
-        break;
       case LATIN1:
+        source = isolate->factory()->NewStringFromAscii(
+            Vector<const char>(reinterpret_cast<const char*>(buffer),
+                               buffer_end - buffer));
         experimental_time = RunExperimentalScanner<uint8_t>(
-            fname, isolate, encoding, print_tokens || check_tokens,
+            source, isolate, encoding, print_tokens || check_tokens,
             &experimental_tokens, repeat, harmony_settings);
         break;
       case UTF16:
+      case UTF8TO16: {
+        const uc16* buffer_16 = reinterpret_cast<const uc16*>(buffer);
+        const uc16* buffer_end_16 = reinterpret_cast<const uc16*>(buffer_end);
+        source = isolate->factory()->NewStringFromTwoByte(
+            Vector<const uc16>(buffer_16, buffer_end_16 - buffer_16));
+        // If the string was just an expaneded one byte string, V8 detects it
+        // and doesn't store it as two byte.
+        CHECK(source->IsTwoByteRepresentation());
         experimental_time = RunExperimentalScanner<uint16_t>(
-            fname, isolate, encoding, print_tokens || check_tokens,
+            source, isolate, encoding, print_tokens || check_tokens,
             &experimental_tokens, repeat, harmony_settings);
         break;
-      case UTF8TO16:
-        experimental_time = RunExperimentalScanner<uint16_t>(
-            fname, isolate, encoding, print_tokens || check_tokens,
-            &experimental_tokens, repeat, harmony_settings);
-        break;
+      }
       default:
         printf("Encoding not supported by the experimental scanner\n");
         exit(1);
         break;
     }
   }
   if (print_tokens && !run_experimental) {
     PrintTokens("Baseline", baseline_tokens);
   }
   if (print_tokens && !run_baseline) {
@@ skipping 79 matching lines @@
     }
   }
   v8::Isolate* isolate = v8::Isolate::GetCurrent();
   {
     v8::HandleScope handle_scope(isolate);
     v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
     v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);
     ASSERT(!context.IsEmpty());
     {
       v8::Context::Scope scope(context);
-      Isolate* isolate = Isolate::Current();
+      Isolate* internal_isolate = Isolate::Current();
       double baseline_total = 0, experimental_total = 0;
       for (size_t i = 0; i < fnames.size(); i++) {
         std::pair<TimeDelta, TimeDelta> times;
         check_tokens = check_tokens && run_baseline && run_experimental;
-        times = ProcessFile(fnames[i].c_str(), encoding, isolate, run_baseline,
-                            run_experimental, print_tokens, check_tokens,
-                            break_after_illegal, repeat,
+        times = ProcessFile(fnames[i].c_str(),
+                            encoding,
+                            internal_isolate,
+                            run_baseline,
+                            run_experimental,
+                            print_tokens,
+                            check_tokens,
+                            break_after_illegal,
+                            repeat,
                             harmony_settings);
         baseline_total += times.first.InMillisecondsF();
         experimental_total += times.second.InMillisecondsF();
       }
       if (run_baseline) {
         printf("Baseline%s(RunTime): %.f ms\n", benchmark.c_str(),
                baseline_total);
       }
       if (run_experimental) {
         if (benchmark.empty()) benchmark = "Experimental";
         printf("%s(RunTime): %.f ms\n", benchmark.c_str(),
                experimental_total);
       }
     }
   }
   v8::V8::Dispose();
   return 0;
 }