Experimental lexer: fix internalization and allocation of literals.

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 238 matching lines @@
 }
 
 
 Handle<String> Parser::LookupSymbol(int symbol_id) {
   // Length of symbol cache is the number of identified symbols.
   // If we are larger than that, or negative, it's not a cached symbol.
   // This might also happen if there is no preparser symbol data, even
   // if there is some preparser data.
   if (static_cast<unsigned>(symbol_id)
       >= static_cast<unsigned>(symbol_cache_.length())) {
-    if (scanner().is_literal_ascii()) {
-      return isolate()->factory()->InternalizeOneByteString(
-          Vector<const uint8_t>::cast(scanner().literal_ascii_string()));
-    } else {
-      return isolate()->factory()->InternalizeTwoByteString(
-          scanner().literal_utf16_string());
-    }
+    return scanner().GetLiteralSymbol();
   }
   return LookupCachedSymbol(symbol_id);
 }
 
 
 Handle<String> Parser::LookupCachedSymbol(int symbol_id) {
   // Make sure the cache is large enough to hold the symbol identifier.
   if (symbol_cache_.length() <= symbol_id) {
     // Increase length to index + 1.
     symbol_cache_.AddBlock(Handle<String>::null(),
                            symbol_id + 1 - symbol_cache_.length(), zone());
   }
   Handle<String> result = symbol_cache_.at(symbol_id);
   if (result.is_null()) {
-    if (scanner().is_literal_ascii()) {
-      result = isolate()->factory()->InternalizeOneByteString(
-          Vector<const uint8_t>::cast(scanner().literal_ascii_string()));
-    } else {
-      result = isolate()->factory()->InternalizeTwoByteString(
-          scanner().literal_utf16_string());
-    }
+    result = scanner().GetLiteralSymbol();
     symbol_cache_.at(symbol_id) = result;
     return result;
   }
   isolate()->counters()->total_preparse_symbols_skipped()->Increment();
   return result;
 }
 
 
 FunctionEntry ScriptDataImpl::GetFunctionEntry(int start) {
   // The current pre-data entry must be a FunctionEntry with the given
@@ skipping 272 matching lines @@
   HistogramTimerScope timer_scope(isolate()->counters()->parse(), true);
   Handle<String> source(String::cast(script_->source()));
   isolate()->counters()->total_parse_size()->Increment(source->length());
   ElapsedTimer timer;
   if (FLAG_trace_parse) {
     timer.Start();
   }
   fni_ = new(zone()) FuncNameInferrer(isolate(), zone());
 
   // Initialize parser state.
-  FlattenString(source);
+  source = FlattenGetString(source);
   FunctionLiteral* result;
   if (source->IsTwoByteRepresentation()) {
     delete reusable_preparser_;
     delete scanner_;
     scanner_ = new ExperimentalScanner<uint16_t>(source, isolate());
   } else {
     delete reusable_preparser_;
     delete scanner_;
     scanner_ = new ExperimentalScanner<uint8_t>(source, isolate());
   }
@@ skipping 110 matching lines @@
 
 FunctionLiteral* Parser::ParseLazy() {
   HistogramTimerScope timer_scope(isolate()->counters()->parse_lazy());
   Handle<String> source(String::cast(script_->source()));
   isolate()->counters()->total_parse_size()->Increment(source->length());
   ElapsedTimer timer;
   if (FLAG_trace_parse) {
     timer.Start();
   }
   // Initialize parser state.
-  FlattenString(source);
   Handle<SharedFunctionInfo> shared_info = info()->shared_info();
   FunctionLiteral* result = ParseLazy(
       source, shared_info->start_position(), shared_info->end_position());
   if (FLAG_trace_parse && result != NULL) {
     double ms = timer.Elapsed().InMillisecondsF();
     SmartArrayPointer<char> name_chars = result->debug_name()->ToCString();
     PrintF("[parsing function: %s - took %0.3f ms]\n", name_chars.get(), ms);
   }
   return result;
 }
 
 
 FunctionLiteral* Parser::ParseLazy(Handle<String> source, int start, int end) {
+  source = FlattenGetString(source);
   delete reusable_preparser_;
   delete scanner_;
   if (source->IsTwoByteRepresentation()) {
     scanner_ = new ExperimentalScanner<uint16_t>(source, isolate());
   } else {
     scanner_ = new ExperimentalScanner<uint8_t>(source, isolate());
   }
   SetScannerFlags();
   // We don't need to Init() if we immediately SeekForward.
   scanner_->SeekForward(start);
@@ skipping 4873 matching lines @@
   return result;
 }
 
 
 // Create a ScannerBase for the preparser to use as input, and preparse the
 // source.
 ScriptDataImpl* PreParserApi::PreParse(Isolate* isolate,
                                        Handle<String> source) {
   CompleteParserRecorder recorder;
   HistogramTimerScope timer(isolate->counters()->pre_parse());
+  source = FlattenGetString(source);
   ScannerBase* scanner = NULL;
   if (source->IsTwoByteRepresentation()) {
     scanner = new ExperimentalScanner<uint16_t>(source, isolate);
   } else {
     scanner = new ExperimentalScanner<uint8_t>(source, isolate);
   }
   intptr_t stack_limit = isolate->stack_guard()->real_climit();
   PreParser preparser(scanner, &recorder, stack_limit);
   preparser.set_allow_lazy(true);
   preparser.set_allow_generators(FLAG_harmony_generators);
@@ skipping 72 matching lines @@
 }
 
 
 void Parser::SetScannerFlags() {
   scanner_->SetHarmonyScoping(allow_harmony_scoping_);
   scanner_->SetHarmonyModules(allow_harmony_modules_);
   scanner_->SetHarmonyNumericLiterals(allow_harmony_numeric_literals_);
 }
 
 } }  // namespace v8::internal