[Isolates] Merge from bleeding_edge to isolates, revisions 6100-6300.

src/parser.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 306 matching lines @@
     parent_(*variable) {
   *variable = this;
 }
 
 
 TemporaryScope::~TemporaryScope() {
   *variable_ = parent_;
 }
 
 
-Handle<String> Parser::LookupSymbol(int symbol_id,
-                                    Vector<const char> string) {
+Handle<String> Parser::LookupSymbol(int symbol_id) {
   // Length of symbol cache is the number of identified symbols.
   // If we are larger than that, or negative, it's not a cached symbol.
   // This might also happen if there is no preparser symbol data, even
   // if there is some preparser data.
   if (static_cast<unsigned>(symbol_id)
       >= static_cast<unsigned>(symbol_cache_.length())) {
-    return isolate()->factory()->LookupSymbol(string);
+    if (scanner().is_literal_ascii()) {
+      return isolate()->factory()->LookupAsciiSymbol(
+          scanner().literal_ascii_string());
+    } else {
+      return isolate()->factory()->LookupTwoByteSymbol(
+          scanner().literal_uc16_string());
+    }
   }
-  return LookupCachedSymbol(symbol_id, string);
+  return LookupCachedSymbol(symbol_id);
 }
 
 
-Handle<String> Parser::LookupCachedSymbol(int symbol_id,
-                                          Vector<const char> string) {
+Handle<String> Parser::LookupCachedSymbol(int symbol_id) {
   // Make sure the cache is large enough to hold the symbol identifier.
   if (symbol_cache_.length() <= symbol_id) {
     // Increase length to index + 1.
     symbol_cache_.AddBlock(Handle<String>::null(),
                            symbol_id + 1 - symbol_cache_.length());
   }
   Handle<String> result = symbol_cache_.at(symbol_id);
   if (result.is_null()) {
-    result = isolate()->factory()->LookupSymbol(string);
+    if (scanner().is_literal_ascii()) {
+      result = isolate()->factory()->LookupAsciiSymbol(
+          scanner().literal_ascii_string());
+    } else {
+      result = isolate()->factory()->LookupTwoByteSymbol(
+          scanner().literal_uc16_string());
+    }
     symbol_cache_.at(symbol_id) = result;
     return result;
   }
   COUNTERS->total_preparse_symbols_skipped()->Increment();
   return result;
 }
 
 
 FunctionEntry ScriptDataImpl::GetFunctionEntry(int start) {
   // The current pre-data entry must be a FunctionEntry with the given
@@ skipping 248 matching lines @@
   fni_ = new FuncNameInferrer();
 
   // Initialize parser state.
   source->TryFlatten();
   if (source->IsExternalTwoByteString()) {
     // Notice that the stream is destroyed at the end of the branch block.
     // The last line of the blocks can't be moved outside, even though they're
     // identical calls.
     ExternalTwoByteStringUC16CharacterStream stream(
         Handle<ExternalTwoByteString>::cast(source), 0, source->length());
-    scanner_.Initialize(&stream, JavaScriptScanner::kAllLiterals);
+    scanner_.Initialize(&stream);
     return DoParseProgram(source, in_global_context, &zone_scope);
   } else {
     GenericStringUC16CharacterStream stream(source, 0, source->length());
-    scanner_.Initialize(&stream, JavaScriptScanner::kAllLiterals);
+    scanner_.Initialize(&stream);
     return DoParseProgram(source, in_global_context, &zone_scope);
   }
 }
 
 
 FunctionLiteral* Parser::DoParseProgram(Handle<String> source,
                                         bool in_global_context,
                                         ZoneScope* zone_scope) {
   ASSERT(target_stack_ == NULL);
   if (pre_data_ != NULL) pre_data_->Initialize();
@@ skipping 65 matching lines @@
                                             info->end_position());
     FunctionLiteral* result = ParseLazy(info, &stream, &zone_scope);
     return result;
   }
 }
 
 
 FunctionLiteral* Parser::ParseLazy(Handle<SharedFunctionInfo> info,
                                    UC16CharacterStream* source,
                                    ZoneScope* zone_scope) {
-  scanner_.Initialize(source, JavaScriptScanner::kAllLiterals);
+  scanner_.Initialize(source);
   ASSERT(target_stack_ == NULL);
 
   Handle<String> name(String::cast(info->name()));
   fni_ = new FuncNameInferrer();
   fni_->PushEnclosingName(name);
 
   mode_ = PARSE_EAGERLY;
 
   // Place holder for the result.
   FunctionLiteral* result = NULL;
@@ skipping 31 matching lines @@
   }
   return result;
 }
 
 
 Handle<String> Parser::GetSymbol(bool* ok) {
   int symbol_id = -1;
   if (pre_data() != NULL) {
     symbol_id = pre_data()->GetSymbolIdentifier();
   }
-  return LookupSymbol(symbol_id, scanner().literal());
+  return LookupSymbol(symbol_id);
 }
 
 
 void Parser::ReportMessage(const char* type, Vector<const char*> args) {
   Scanner::Location source_location = scanner().location();
   ReportMessageAt(source_location, type, args);
 }
 
 
 void Parser::ReportMessageAt(Scanner::Location source_location,
@@ skipping 1547 matching lines @@
             uint32_t shift = DoubleToInt32(y_val) & 0x1f;
             int value = ArithmeticShiftRight(DoubleToInt32(x_val), shift);
             x = NewNumberLiteral(value);
             continue;
           }
           default:
             break;
         }
       }
 
-      // Convert constant divisions to multiplications for speed.
-      if (op == Token::DIV &&
-          y && y->AsLiteral() && y->AsLiteral()->handle()->IsNumber()) {
-        double y_val = y->AsLiteral()->handle()->Number();
-        int64_t y_int = static_cast<int64_t>(y_val);
-        // There are rounding issues with this optimization, but they don't
-        // apply if the number to be divided with has a reciprocal that can be
-        // precisely represented as a floating point number.  This is the case
-        // if the number is an integer power of 2.  Negative integer powers of
-        // 2 work too, but for -2, -1, 1 and 2 we don't do the strength
-        // reduction because the inlined optimistic idiv has a reasonable
-        // chance of succeeding by producing a Smi answer with no remainder.
-        if (static_cast<double>(y_int) == y_val &&
-            (IsPowerOf2(y_int) || IsPowerOf2(-y_int)) &&
-            (y_int > 2 || y_int < -2)) {
-          y = NewNumberLiteral(1 / y_val);
-          op = Token::MUL;
-        }
-      }
-
       // For now we distinguish between comparisons and other binary
       // operations.  (We could combine the two and get rid of this
       // code and AST node eventually.)
       if (Token::IsCompareOp(op)) {
         // We have a comparison.
         Token::Value cmp = op;
         switch (op) {
           case Token::NE: cmp = Token::EQ; break;
           case Token::NE_STRICT: cmp = Token::EQ_STRICT; break;
           default: break;
@@ skipping 361 matching lines @@
 
     case Token::IDENTIFIER: {
       Handle<String> name = ParseIdentifier(CHECK_OK);
       if (fni_ != NULL) fni_->PushVariableName(name);
       result = top_scope_->NewUnresolved(name, inside_with());
       break;
     }
 
     case Token::NUMBER: {
       Consume(Token::NUMBER);
-      double value =
-        StringToDouble(scanner().literal(), ALLOW_HEX | ALLOW_OCTALS);
+      ASSERT(scanner().is_literal_ascii());
+      double value = StringToDouble(scanner().literal_ascii_string(),
+                                    ALLOW_HEX | ALLOW_OCTALS);
       result = NewNumberLiteral(value);
       break;
     }
 
     case Token::STRING: {
       Consume(Token::STRING);
       Handle<String> symbol = GetSymbol(CHECK_OK);
       result = new Literal(symbol);
       if (fni_ != NULL) fni_->PushLiteralName(symbol);
       break;
@@ skipping 251 matching lines @@
   *depth = depth_acc;
 }
 
 
 ObjectLiteral::Property* Parser::ParseObjectLiteralGetSet(bool is_getter,
                                                           bool* ok) {
   // Special handling of getter and setter syntax:
   // { ... , get foo() { ... }, ... , set foo(v) { ... v ... } , ... }
   // We have already read the "get" or "set" keyword.
   Token::Value next = Next();
-  // TODO(820): Allow NUMBER and STRING as well (and handle array indices).
-  if (next == Token::IDENTIFIER || Token::IsKeyword(next)) {
-    Handle<String> name = GetSymbol(CHECK_OK);
+  bool is_keyword = Token::IsKeyword(next);
+  if (next == Token::IDENTIFIER || next == Token::NUMBER ||
+      next == Token::STRING || is_keyword) {
+    Handle<String> name;
+    if (is_keyword) {
+      name = isolate_->factory()->LookupAsciiSymbol(Token::String(next));
+    } else {
+      name = GetSymbol(CHECK_OK);
+    }
     FunctionLiteral* value =
         ParseFunctionLiteral(name,
                              RelocInfo::kNoPosition,
                              DECLARATION,
                              CHECK_OK);
+    // Allow any number of parameters for compatiabilty with JSC.
+    // Specification only allows zero parameters for get and one for set.
     ObjectLiteral::Property* property =
         new ObjectLiteral::Property(is_getter, value);
     return property;
   } else {
     ReportUnexpectedToken(next);
     *ok = false;
     return NULL;
   }
 }
 
@@ skipping 50 matching lines @@
         uint32_t index;
         if (!string.is_null() && string->AsArrayIndex(&index)) {
           key = NewNumberLiteral(index);
           break;
         }
         key = new Literal(string);
         break;
       }
       case Token::NUMBER: {
         Consume(Token::NUMBER);
-        double value =
-          StringToDouble(scanner().literal(), ALLOW_HEX | ALLOW_OCTALS);
+        ASSERT(scanner().is_literal_ascii());
+        double value = StringToDouble(scanner().literal_ascii_string(),
+                                      ALLOW_HEX | ALLOW_OCTALS);
         key = NewNumberLiteral(value);
         break;
       }
       default:
         if (Token::IsKeyword(next)) {
           Consume(next);
           Handle<String> string = GetSymbol(CHECK_OK);
           key = new Literal(string);
         } else {
           // Unexpected token.
@@ skipping 49 matching lines @@
 Expression* Parser::ParseRegExpLiteral(bool seen_equal, bool* ok) {
   if (!scanner().ScanRegExpPattern(seen_equal)) {
     Next();
     ReportMessage("unterminated_regexp", Vector<const char*>::empty());
     *ok = false;
     return NULL;
   }
 
   int literal_index = temp_scope_->NextMaterializedLiteralIndex();
 
-  Handle<String> js_pattern =
-      isolate_->factory()->NewStringFromUtf8(scanner().next_literal(), TENURED);
+  Handle<String> js_pattern = NextLiteralString(TENURED);
   scanner().ScanRegExpFlags();
-  Handle<String> js_flags =
-      isolate_->factory()->NewStringFromUtf8(scanner().next_literal(), TENURED);
+  Handle<String> js_flags = NextLiteralString(TENURED);
   Next();
 
   return new RegExpLiteral(js_pattern, js_flags, literal_index);
 }
 
 
 ZoneList<Expression*>* Parser::ParseArguments(bool* ok) {
   // Arguments ::
   //   '(' (AssignmentExpression)*[','] ')'
 
@@ skipping 263 matching lines @@
 
 // This function reads an identifier and determines whether or not it
 // is 'get' or 'set'.  The reason for not using ParseIdentifier and
 // checking on the output is that this involves heap allocation which
 // we can't do during preparsing.
 Handle<String> Parser::ParseIdentifierOrGetOrSet(bool* is_get,
                                                  bool* is_set,
                                                  bool* ok) {
   Expect(Token::IDENTIFIER, ok);
   if (!*ok) return Handle<String>();
-  if (scanner().literal_length() == 3) {
-    const char* token = scanner().literal_string();
-    *is_get = strcmp(token, "get") == 0;
-    *is_set = !*is_get && strcmp(token, "set") == 0;
+  if (scanner().is_literal_ascii() && scanner().literal_length() == 3) {
+    const char* token = scanner().literal_ascii_string().start();
+    *is_get = strncmp(token, "get", 3) == 0;
+    *is_set = !*is_get && strncmp(token, "set", 3) == 0;
   }
   return GetSymbol(ok);
 }
 
 
 // ----------------------------------------------------------------------------
 // Parser support
 
 
 bool Parser::TargetStackContainsLabel(Handle<String> label) {
@@ skipping 161 matching lines @@
   }
   return result;
 }
 
 
 Handle<String> JsonParser::GetString() {
   int literal_length = scanner_.literal_length();
   if (literal_length == 0) {
     return isolate()->factory()->empty_string();
   }
-  const char* literal_string = scanner_.literal_string();
-  Vector<const char> literal(literal_string, literal_length);
-  return isolate()->factory()->NewStringFromUtf8(literal);
+  if (scanner_.is_literal_ascii()) {
+    return isolate()->factory()->NewStringFromAscii(
+        scanner_.literal_ascii_string());
+  } else {
+    return isolate()->factory()->NewStringFromTwoByte(
+        scanner_.literal_uc16_string());
+  }
 }
 
 
 // Parse any JSON value.
 Handle<Object> JsonParser::ParseJsonValue() {
   Token::Value token = scanner_.Next();
   switch (token) {
     case Token::STRING: {
       return GetString();
     }
     case Token::NUMBER: {
-      double value = StringToDouble(scanner_.literal(),
+      ASSERT(scanner_.is_literal_ascii());
+      double value = StringToDouble(scanner_.literal_ascii_string(),
                                     NO_FLAGS,  // Hex, octal or trailing junk.
                                     OS::nan_value());
       return isolate()->factory()->NewNumber(value);
     }
     case Token::FALSE_LITERAL:
       return isolate()->factory()->false_value();
     case Token::TRUE_LITERAL:
       return isolate()->factory()->true_value();
     case Token::NULL_LITERAL:
       return isolate()->factory()->null_value();
@@ skipping 25 matching lines @@
         return ReportUnexpectedToken();
       }
       Handle<String> key = GetString();
       if (scanner_.Next() != Token::COLON) {
         return ReportUnexpectedToken();
       }
       Handle<Object> value = ParseJsonValue();
       if (value.is_null()) return Handle<Object>::null();
       uint32_t index;
       if (key->AsArrayIndex(&index)) {
-        SetElement(json_object, index, value);
+        SetOwnElement(json_object, index, value);
       } else {
-        SetProperty(json_object, key, value, NONE);
+        SetLocalPropertyIgnoreAttributes(json_object, key, value, NONE);
       }
     } while (scanner_.Next() == Token::COMMA);
     if (scanner_.current_token() != Token::RBRACE) {
       return ReportUnexpectedToken();
     }
   }
   return json_object;
 }
 
 
@@ skipping 340 matching lines @@
         break;
       case 't':
         Advance(2);
         builder->AddCharacter('\t');
         break;
       case 'v':
         Advance(2);
         builder->AddCharacter('\v');
         break;
       case 'c': {
-        Advance(2);
-        uc32 control = ParseControlLetterEscape();
-        builder->AddCharacter(control);
+        Advance();
+        uc32 controlLetter = Next();
+        // Special case if it is an ASCII letter.
+        // Convert lower case letters to uppercase.
+        uc32 letter = controlLetter & ~('a' ^ 'A');
+        if (letter < 'A' || 'Z' < letter) {
+          // controlLetter is not in range 'A'-'Z' or 'a'-'z'.
+          // This is outside the specification. We match JSC in
+          // reading the backslash as a literal character instead
+          // of as starting an escape.
+          builder->AddCharacter('\\');
+        } else {
+          Advance(2);
+          builder->AddCharacter(controlLetter & 0x1f);
+        }
         break;
       }
       case 'x': {
         Advance(2);
         uc32 value;
         if (ParseHexEscape(2, &value)) {
           builder->AddCharacter(value);
         } else {
           builder->AddCharacter('x');
         }
@@ skipping 230 matching lines @@
   } else {
     Reset(start);
     return false;
   }
   *min_out = min;
   *max_out = max;
   return true;
 }
 
 
-// Upper and lower case letters differ by one bit.
-STATIC_CHECK(('a' ^ 'A') == 0x20);
-
-uc32 RegExpParser::ParseControlLetterEscape() {
-  if (!has_more())
-    return 'c';
-  uc32 letter = current() & ~(0x20);  // Collapse upper and lower case letters.
-  if (letter < 'A' || 'Z' < letter) {
-    // Non-spec error-correction: "\c" followed by non-control letter is
-    // interpreted as an IdentityEscape of 'c'.
-    return 'c';
-  }
-  Advance();
-  return letter & 0x1f;  // Remainder modulo 32, per specification.
-}
-
-
 uc32 RegExpParser::ParseOctalLiteral() {
   ASSERT('0' <= current() && current() <= '7');
   // For compatibility with some other browsers (not all), we parse
   // up to three octal digits with a value below 256.
   uc32 value = current() - '0';
   Advance();
   if ('0' <= current() && current() <= '7') {
     value = value * 8 + current() - '0';
     Advance();
     if (value < 32 && '0' <= current() && current() <= '7') {
@@ skipping 45 matching lines @@
       return '\n';
     case 'r':
       Advance();
       return '\r';
     case 't':
       Advance();
       return '\t';
     case 'v':
       Advance();
       return '\v';
-    case 'c':
-      Advance();
-      return ParseControlLetterEscape();
+    case 'c': {
+      uc32 controlLetter = Next();
+      uc32 letter = controlLetter & ~('A' ^ 'a');
+      // For compatibility with JSC, inside a character class
+      // we also accept digits and underscore as control characters.
+      if ((controlLetter >= '0' && controlLetter <= '9') ||
+          controlLetter == '_' ||
+          (letter >= 'A' && letter <= 'Z')) {
+        Advance(2);
+        // Control letters mapped to ASCII control characters in the range
+        // 0x00-0x1f.
+        return controlLetter & 0x1f;
+      }
+      // We match JSC in reading the backslash as a literal
+      // character instead of as starting an escape.
+      return '\\';
+    }
     case '0': case '1': case '2': case '3': case '4': case '5':
     case '6': case '7':
       // For compatibility, we interpret a decimal escape that isn't
       // a back reference (and therefore either \0 or not valid according
       // to the specification) as a 1..3 digit octal character code.
       return ParseOctalLiteral();
     case 'x': {
       Advance();
       uc32 value;
       if (ParseHexEscape(2, &value)) {
@@ skipping 190 matching lines @@
     data++;
   }
   *source = data;
   return result;
 }
 
 
 // Create a Scanner for the preparser to use as input, and preparse the source.
 static ScriptDataImpl* DoPreParse(UC16CharacterStream* source,
                                   bool allow_lazy,
-                                  ParserRecorder* recorder,
-                                  int literal_flags) {
+                                  ParserRecorder* recorder) {
   Isolate* isolate = Isolate::Current();
   V8JavaScriptScanner scanner(isolate);
-  scanner.Initialize(source, literal_flags);
+  scanner.Initialize(source);
   intptr_t stack_limit = isolate->stack_guard()->real_climit();
   if (!preparser::PreParser::PreParseProgram(&scanner,
                                              recorder,
                                              allow_lazy,
                                              stack_limit)) {
     isolate->StackOverflow();
     return NULL;
   }
 
   // Extract the accumulated data from the recorder as a single
   // contiguous vector that we are responsible for disposing.
   Vector<unsigned> store = recorder->ExtractData();
   return new ScriptDataImpl(store);
 }
 
 
 // Preparse, but only collect data that is immediately useful,
 // even if the preparser data is only used once.
 ScriptDataImpl* ParserApi::PartialPreParse(UC16CharacterStream* source,
                                            v8::Extension* extension) {
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   if (!allow_lazy) {
     // Partial preparsing is only about lazily compiled functions.
     // If we don't allow lazy compilation, the log data will be empty.
     return NULL;
   }
   PartialParserRecorder recorder;
-  return DoPreParse(source, allow_lazy, &recorder,
-                    JavaScriptScanner::kNoLiterals);
+  return DoPreParse(source, allow_lazy, &recorder);
 }
 
 
 ScriptDataImpl* ParserApi::PreParse(UC16CharacterStream* source,
                                     v8::Extension* extension) {
   Handle<Script> no_script;
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   CompleteParserRecorder recorder;
-  int kPreParseLiteralsFlags =
-      JavaScriptScanner::kLiteralString | JavaScriptScanner::kLiteralIdentifier;
-  return DoPreParse(source, allow_lazy, &recorder, kPreParseLiteralsFlags);
+  return DoPreParse(source, allow_lazy, &recorder);
 }
 
 
 bool RegExpParser::ParseRegExp(FlatStringReader* input,
                                bool multiline,
                                RegExpCompileData* result) {
   ASSERT(result != NULL);
   RegExpParser parser(input, &result->error, multiline);
   RegExpTree* tree = parser.ParsePattern();
   if (parser.failed()) {
@@ skipping 40 matching lines @@
       Handle<String> source = Handle<String>(String::cast(script->source()));
       result = parser.ParseProgram(source, info->is_global());
     }
   }
 
   info->SetFunction(result);
   return (result != NULL);
 }
 
 } }  // namespace v8::internal