Merge preparser Scanner with main JavaScript scanner.

src/parser.cc

Index: src/parser.cc
diff --git a/src/parser.cc b/src/parser.cc
index c454099ade7f191a896e780bff7b80eba174af37..9b681a13b937a3517c81cf34613d99ec25e03017 100644
--- a/src/parser.cc
+++ b/src/parser.cc
@@ -37,7 +37,6 @@
 #include "parser.h"
 #include "platform.h"
 #include "preparser.h"
-#include "prescanner.h"
 #include "runtime.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
@@ -4637,12 +4636,15 @@ int ScriptDataImpl::ReadNumber(byte** source) {
 }
 
 
-static ScriptDataImpl* DoPreParse(UTF16Buffer* stream,
+// Create a Scannerfor the preparser to use as input, and preparse the source.

[Søren Thygesen Gjesse, 2010/11/19 08:17:21]

Scannerfor -> Scanner for

[Lasse Reichstein, 2010/11/19 08:51:24]

Done.

+static ScriptDataImpl* DoPreParse(Handle<String> source,
+                                  unibrow::CharacterStream* stream,
                                   bool allow_lazy,
-                                  PartialParserRecorder* recorder) {
-  preparser::Scanner scanner;
-  scanner.Initialize(stream);
-  preparser::PreParser<preparser::Scanner, PartialParserRecorder> preparser;
+                                  PartialParserRecorder* recorder,
+                                  int kLiteralFlags) {

[Søren Thygesen Gjesse, 2010/11/19 08:17:21]

kLiteralFlags -> literal_flags.

[Søren Thygesen Gjesse, 2010/11/19 08:17:21]

Instead of passing flags could the PartialParserRecorder have a method for
requesting this - or does that not make any sense?

[Lasse Reichstein, 2010/11/19 08:51:24]

Done.

[Lasse Reichstein, 2010/11/19 08:51:24]

I'm afraid it doesn't make any sense :)
The PartialParserRecorder is not the one deciding which literals to record in
the scanner. The PartialParserRecorder has some needs for literals (the
CompleteParserRecorder needs strings and identifiers recorded in order to make
preparser data for symbols, the PartialParserRecorder doesn't), but the scanner
is also used by the real parser which doesn't have a PartialParserRecorder.

I still don't *like* this approach, and I'll try to find a better one where we
don't have to make a test for each character that is scanned.
And perhaps without using utf-8 as intermediate storage.

+  V8JavaScriptScanner scanner;
+  scanner.Initialize(source, stream, kLiteralFlags);
+  preparser::PreParser<JavaScriptScanner, PartialParserRecorder> preparser;
   if (!preparser.PreParseProgram(&scanner, recorder, allow_lazy)) {
     Top::StackOverflow();
     return NULL;
@@ -4655,44 +4657,11 @@ static ScriptDataImpl* DoPreParse(UTF16Buffer* stream,
 }
 
 
-// Create an UTF16Buffer for the preparser to use as input,
-// and preparse the source.
-static ScriptDataImpl* DoPreParse(Handle<String> source,
-                                  unibrow::CharacterStream* stream,
-                                  bool allow_lazy,
-                                  PartialParserRecorder* recorder) {
-  if (source.is_null()) {
-    CharacterStreamUTF16Buffer buffer;
-    int length = stream->Length();
-    buffer.Initialize(source, stream, 0, length);
-    return DoPreParse(&buffer, allow_lazy, recorder);
-  } else if (source->IsExternalAsciiString()) {
-    ExternalStringUTF16Buffer<ExternalAsciiString, char> buffer;
-    int length = source->length();
-    buffer.Initialize(Handle<ExternalAsciiString>::cast(source), 0, length);
-    return DoPreParse(&buffer, allow_lazy, recorder);
-  } else if (source->IsExternalTwoByteString()) {
-    ExternalStringUTF16Buffer<ExternalTwoByteString, uint16_t> buffer;
-    int length = source->length();
-    buffer.Initialize(Handle<ExternalTwoByteString>::cast(source), 0, length);
-    return DoPreParse(&buffer, allow_lazy, recorder);
-  } else {
-    CharacterStreamUTF16Buffer buffer;
-    SafeStringInputBuffer input;
-    input.Reset(0, source.location());
-    int length = source->length();
-    buffer.Initialize(source, &input, 0, length);
-    return DoPreParse(&buffer, allow_lazy, recorder);
-  }
-}
-
-
 // Preparse, but only collect data that is immediately useful,
 // even if the preparser data is only used once.
 ScriptDataImpl* ParserApi::PartialPreParse(Handle<String> source,
                                            unibrow::CharacterStream* stream,
                                            v8::Extension* extension) {
-  Handle<Script> no_script;
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   if (!allow_lazy) {
     // Partial preparsing is only about lazily compiled functions.
@@ -4701,7 +4670,8 @@ ScriptDataImpl* ParserApi::PartialPreParse(Handle<String> source,
   }
   PartialParserRecorder recorder;
 
-  return DoPreParse(source, stream, allow_lazy, &recorder);
+  const int kNoLiteralsFlag = 0;

[Søren Thygesen Gjesse, 2010/11/19 08:17:21]

Shouldn't this constant be part of the enum in the JavaScriptScanner?

[Lasse Reichstein, 2010/11/19 08:51:24]

I was split about that. Should the LiteralType be the type of elements in a set,
or the type of the set itself. I went for the former, so the full and empty set
are defined separatly.  

Let's add the full and empty set of literal types to the type.

+  return DoPreParse(source, stream, allow_lazy, &recorder, kNoLiteralsFlag);
 }
 
 
@@ -4711,7 +4681,10 @@ ScriptDataImpl* ParserApi::PreParse(Handle<String> source,
   Handle<Script> no_script;
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   CompleteParserRecorder recorder;
-  return DoPreParse(source, stream, allow_lazy, &recorder);
+  int kPreParseLiteralsFlags =
+      JavaScriptScanner::kLiteralString | JavaScriptScanner::kLiteralIdentifier;
+  return DoPreParse(source, stream, allow_lazy,
+                    &recorder, kPreParseLiteralsFlags);
 }