Add separate scanner only intended for preparsing.

src/parser.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 19 matching lines @@
 #include "api.h"
 #include "ast.h"
 #include "bootstrapper.h"
 #include "codegen.h"
 #include "compiler.h"
 #include "func-name-inferrer.h"
 #include "messages.h"
 #include "parser.h"
 #include "platform.h"
 #include "preparser.h"
+#include "prescanner.h"
 #include "runtime.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 
 #include "ast-inl.h"
 #include "jump-target-inl.h"
 
 namespace v8 {
 namespace internal {
 
@@ skipping 4580 matching lines @@
     if (data >= symbol_data_end_) return -1;
     input = *data;
     result = (result << 7) | (input & 0x7f);
     data++;
   }
   *source = data;
   return result;
 }
 
 
+static ScriptDataImpl* DoPreParse(UTF16Buffer* stream,
+                                  bool allow_lazy,
+                                  PartialParserRecorder* recorder) {
+  typedef preparser::Scanner<UTF16Buffer, UTF8Buffer> PreScanner;
+  PreScanner scanner;
+  scanner.Initialize(stream);
+  preparser::PreParser<PreScanner, PartialParserRecorder> preparser;
+  if (!preparser.PreParseProgram(&scanner, recorder, allow_lazy)) {
+    Top::StackOverflow();
+    return NULL;
+  }
+
+  // Extract the accumulated data from the recorder as a single
+  // contiguous vector that we are responsible for disposing.
+  Vector<unsigned> store = recorder->ExtractData();
+  return new ScriptDataImpl(store);
+}
+
+
+// Create an UTF16Buffer for the preparser to use as input,
+// and preparse the source.
+static ScriptDataImpl* DoPreParse(Handle<String> source,
+                                  unibrow::CharacterStream* stream,
+                                  bool allow_lazy,
+                                  PartialParserRecorder* recorder) {
+  if (source.is_null()) {
+    CharacterStreamUTF16Buffer buffer;
+    int length = stream->Length();
+    buffer.Initialize(source, stream, 0, length);
+    return DoPreParse(&buffer, allow_lazy, recorder);
+  } else if (source->IsExternalAsciiString()) {
+    ExternalStringUTF16Buffer<ExternalAsciiString, char> buffer;
+    int length = source->length();
+    buffer.Initialize(Handle<ExternalAsciiString>::cast(source), 0, length);
+    return DoPreParse(&buffer, allow_lazy, recorder);
+  } else if (source->IsExternalTwoByteString()) {
+    ExternalStringUTF16Buffer<ExternalTwoByteString, uint16_t> buffer;
+    int length = source->length();
+    buffer.Initialize(Handle<ExternalTwoByteString>::cast(source), 0, length);
+    return DoPreParse(&buffer, allow_lazy, recorder);
+  } else {
+    CharacterStreamUTF16Buffer buffer;
+    SafeStringInputBuffer input;
+    input.Reset(0, source.location());
+    int length = source->length();
+    buffer.Initialize(source, &input, 0, length);
+    return DoPreParse(&buffer, allow_lazy, recorder);
+  }
+}
+
+
 // Preparse, but only collect data that is immediately useful,
 // even if the preparser data is only used once.
 ScriptDataImpl* ParserApi::PartialPreParse(Handle<String> source,
                                            unibrow::CharacterStream* stream,
                                            v8::Extension* extension) {
   Handle<Script> no_script;
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   if (!allow_lazy) {
     // Partial preparsing is only about lazily compiled functions.
     // If we don't allow lazy compilation, the log data will be empty.
     return NULL;
   }
-  preparser::PreParser<Scanner, PartialParserRecorder> parser;
-  Scanner scanner;
-  scanner.Initialize(source, stream, JAVASCRIPT);
   PartialParserRecorder recorder;
-  if (!parser.PreParseProgram(&scanner, &recorder, allow_lazy)) {
-    Top::StackOverflow();
-    return NULL;
-  }
 
-  // Extract the accumulated data from the recorder as a single
-  // contiguous vector that we are responsible for disposing.
-  Vector<unsigned> store = recorder.ExtractData();
-  return new ScriptDataImpl(store);
+  return DoPreParse(source, stream, allow_lazy, &recorder);
 }
 
 
 ScriptDataImpl* ParserApi::PreParse(Handle<String> source,
                                     unibrow::CharacterStream* stream,
                                     v8::Extension* extension) {
   Handle<Script> no_script;
-  preparser::PreParser<Scanner, CompleteParserRecorder> parser;
-  Scanner scanner;
-  scanner.Initialize(source, stream, JAVASCRIPT);
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   CompleteParserRecorder recorder;
-  if (!parser.PreParseProgram(&scanner, &recorder, allow_lazy)) {
-    Top::StackOverflow();
-    return NULL;
-  }
-  // Extract the accumulated data from the recorder as a single
-  // contiguous vector that we are responsible for disposing.
-  Vector<unsigned> store = recorder.ExtractData();
-  return new ScriptDataImpl(store);
+  return DoPreParse(source, stream, allow_lazy, &recorder);
 }
 
 
 bool RegExpParser::ParseRegExp(FlatStringReader* input,
                                bool multiline,
                                RegExpCompileData* result) {
   ASSERT(result != NULL);
   RegExpParser parser(input, &result->error, multiline);
   RegExpTree* tree = parser.ParsePattern();
   if (parser.failed()) {
@@ skipping 39 matching lines @@
       Handle<String> source = Handle<String>(String::cast(script->source()));
       result = parser.ParseProgram(source, info->is_global());
     }
   }
 
   info->SetFunction(result);
   return (result != NULL);
 }
 
 } }  // namespace v8::internal