Experimental parser: allocate substrings

src/lexer/lexer.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 446 matching lines @@
 template <>
 const uint16_t* Lexer<uint16_t>::GetNewBufferBasedOnHandle()
     const {
   String::FlatContent content = source_handle_->GetFlatContent();
   return content.ToUC16Vector().start();
 }
 
 
 template<>
 const int8_t* Lexer<int8_t>::GetNewBufferBasedOnHandle() const {
-  String::FlatContent content = source_handle_->GetFlatContent();
-  return reinterpret_cast<const int8_t*>(content.ToOneByteVector().start());
+  UNREACHABLE();
+  return NULL;
 }
 
 
 template<typename Char>
 void Lexer<Char>::UpdateBufferBasedOnHandle() {
   // We get a raw pointer from the Handle, but we also update it every time
   // there is a GC, so it is safe.
   DisallowHeapAllocation no_gc;
   const Char* new_buffer = GetNewBufferBasedOnHandle();
   if (new_buffer != buffer_) {
@@ skipping 41 matching lines @@
       one_byte_string_.Dispose();
     }
     is_one_byte_string_owned_ = false;
     one_byte_string_ = buffer.one_byte_literal();
   } else {
     two_byte_string_ = buffer.two_byte_literal();
   }
 }
 
 
-Handle<String> LexerBase::AllocateNextLiteralString(Isolate* isolate,
-                                                  PretenureFlag tenured) {
-  if (is_next_literal_one_byte()) {
-    return isolate->factory()->NewStringFromOneByte(
-        Vector<const uint8_t>::cast(next_literal_one_byte_string()), tenured);
-  } else {
-    return isolate->factory()->NewStringFromTwoByte(
-          next_literal_two_byte_string(), tenured);
-  }
-}
-
-
-Handle<String> LexerBase::AllocateInternalizedString(Isolate* isolate) {
-  if (is_literal_one_byte()) {
-    return isolate->factory()->InternalizeOneByteString(
-        literal_one_byte_string());
-  } else {
-    return isolate->factory()->InternalizeTwoByteString(
-        literal_two_byte_string());
-  }
-}
-
-
 double LexerBase::DoubleValue() {
-  ASSERT(is_literal_one_byte());
   return StringToDouble(
       unicode_cache_, Vector<const char>::cast(literal_one_byte_string()),
       ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
 }
 
 
 int LexerBase::FindNumber(DuplicateFinder* finder, int value) {
   return finder->AddNumber(literal_one_byte_string(), value);
 }
 
 
 int LexerBase::FindSymbol(DuplicateFinder* finder, int value) {
   if (is_literal_one_byte()) {
     return finder->AddOneByteSymbol(literal_one_byte_string(), value);
   }
   return finder->AddTwoByteSymbol(literal_two_byte_string(), value);
 }
 
 
 void LexerBase::LogSymbol(ParserRecorder* log, int position) {
   if (is_literal_one_byte()) {
     log->LogOneByteSymbol(position, literal_one_byte_string());
   } else {
     log->LogTwoByteSymbol(position, literal_two_byte_string());
   }
 }
 
 
-static inline bool IsOneByte(const uint8_t* cursor, const uint8_t* end) {
-  return true;
-}
-
-
-static inline bool IsOneByte(const uint16_t* cursor, const uint16_t* end) {
-  uint16_t acc = 0;
-  while (cursor != end) {
-    acc |= *cursor++ >> 8;
-  }
-  return acc == 0;
-}
-
-
-static inline bool IsOneByte(const int8_t* cursor, const int8_t* end) {
-  int8_t acc = 0;
-  while (cursor != end) {
-    acc |= *cursor++ >> 7;
-  }
-  return acc == 0;
-}
-
-
 template<>
 template<>
 inline void Lexer<uint16_t>::SetLiteral<true>(const uint16_t* cursor,
-                                              const uint16_t* end,
                                               LiteralDesc* literal) {
   Vector<uint8_t> vector = Vector<uint8_t>::New(literal->length);
-  uint8_t* data = vector.start();
-  while (cursor < end) {
-    *data++ = *cursor++;
-  }
+  CopyChars(vector.start(), cursor, literal->length);
   literal->SetOneByteString(Vector<const uint8_t>::cast(vector), true);
 }
 
 
 template<>
 template<>
 inline void Lexer<uint16_t>::SetLiteral<false>(const uint16_t* start,
-                                        const uint16_t* end,
-                                        LiteralDesc* literal) {
+                                               LiteralDesc* literal) {
   literal->SetTwoByteString(Vector<const uint16_t>(start, literal->length));
 }
 
 
 template<>
 template<>
 inline void Lexer<uint8_t>::SetLiteral<true>(const uint8_t* start,
-                                             const uint8_t* end,
                                              LiteralDesc* literal) {
   literal->SetOneByteString(
       Vector<const uint8_t>(start, literal->length), false);
 }
 
 
 template<>
 template<>
 inline void Lexer<int8_t>::SetLiteral<true>(const int8_t* start,
-                                            const int8_t* end,
                                             LiteralDesc* literal) {
   const uint8_t* cast = reinterpret_cast<const uint8_t*>(start);
   literal->SetOneByteString(
       Vector<const uint8_t>(cast, literal->length), false);
 }
 
 
 template<class Char>
+static inline void GetStartAndEnd(const Char* buffer,
+                                  const LexerBase::TokenDesc& token,
+                                  const Char** start,
+                                  const Char** end) {
+  *start = buffer + token.beg_pos;
+  *end = buffer + token.end_pos;
+  if (token.token == Token::STRING) {
+    ++(*start);
+    --(*end);
+  }
+}
+
+
+template<class Char>
+static inline const Char* LiteralOffsetAndLength(
+    const Char* buffer,
+    const LexerBase::TokenDesc& token,
+    int* offset,
+    int* length) {
+  ASSERT(!Lexer<Char>::MustBeInBuffer(token));
+  const Char* start = NULL;
+  const Char* end = NULL;
+  GetStartAndEnd<Char>(buffer, token, &start, &end);
+  *offset = start - buffer;
+  *length = end - start;
+  return start;
+}
+
+
+template<class Char>
 bool Lexer<Char>::FillLiteral(const TokenDesc& token, LiteralDesc* literal) {
   literal->beg_pos = token.beg_pos;
-  const Char* start = buffer_ + token.beg_pos;
-  const Char* end = buffer_ + token.end_pos;
-  if (token.token == Token::STRING) {
-    ++start;
-    --end;
+  if (!MustBeInBuffer(token)) {
+    const Char* start = LiteralOffsetAndLength<Char>(buffer_,
+                                                     token,
+                                                     &literal->offset,
+                                                     &literal->length);
+    if (sizeof(Char) == 1) {
+      SetLiteral<true>(start, literal);
+    } else if (token.is_in_primary_range) {
+      SetLiteral<true>(start, literal);
+    } else {
+      SetLiteral<false>(start, literal);
+    }
+    return true;
   }
-  if (!token.has_escapes) {
-    bool is_one_byte = IsOneByte(start, end);
-    if (sizeof(Char) == 2 || is_one_byte) {
-      literal->offset = start - buffer_;
-      literal->length = end - start;
-      if (sizeof(Char) == 1) {
-        SetLiteral<true>(start, end, literal);
-      } else if (is_one_byte) {
-        SetLiteral<true>(start, end, literal);
-      } else {
-        SetLiteral<false>(start, end, literal);
-      }
-      return true;
-    }
-  }
-  return CopyToLiteralBuffer(start, end, token, literal);
+  return CopyToLiteralBuffer(token, literal);
 }
 
 
 template<class Char>
-bool Lexer<Char>::CopyToLiteralBuffer(const Char* start,
-                                      const Char* end,
-                                      const TokenDesc& token,
+bool Lexer<Char>::CopyToLiteralBuffer(const TokenDesc& token,
                                       LiteralDesc* literal) {
   literal->buffer.Reset();
+  const Char* start = NULL;
+  const Char* end = NULL;
+  GetStartAndEnd<Char>(buffer_, token, &start, &end);
   if (token.has_escapes) {
     for (const Char* cursor = start; cursor != end;) {
       if (*cursor != '\\') {
         literal->buffer.AddChar(*cursor++);
       } else if (token.token == Token::IDENTIFIER) {
         uc32 c;
         cursor = ScanIdentifierUnicodeEscape(cursor, end, &c);
         ASSERT(cursor != NULL);
         if (cursor == NULL) return false;
         literal->buffer.AddChar(c);
       } else {
         cursor = ScanEscape(unicode_cache_, cursor, end, &literal->buffer);
         ASSERT(cursor != NULL);
         if (cursor == NULL) return false;
       }
     }
   } else {
     // TODO(dcarney): This can only happen for utf8 strings
     // use a helper function.
     for (const Char* cursor = start; cursor != end;) {
         literal->buffer.AddChar(*cursor++);
     }
   }
   literal->SetStringFromLiteralBuffer();
   return true;
 }
 
 
 template<class Char>
-Handle<String> Lexer<Char>::InternalizeLiteral(
-    LiteralDesc* literal) {
-  // Factory* factory = isolate_->factory();
-  // if (literal->is_in_buffer) {
-  //   return literal->is_one_byte
-  //       ? factory->InternalizeOneByteString(
-  //           Vector<const uint8_t>::cast(literal->one_byte_string))
-  //       : factory->InternalizeTwoByteString(literal->two_byte_string);
-  // }
-  // if (sizeof(Char) == 1) {
-  //   SubStringKey<uint8_t> key(
-  //       source_handle_, literal->offset, literal->length);
-  //   return factory->InternalizeStringWithKey(&key);
-  // } else {
-  //   SubStringKey<uint16_t> key(
-  //       source_handle_, literal->offset, literal->length);
-  //   return factory->InternalizeStringWithKey(&key);
-  // }
-  CHECK(false);
-  return Handle<String>();
+Handle<String> Lexer<Char>::AllocateInternalizedString(
+    Isolate* isolate) {
+  Factory* factory = isolate->factory();
+  LiteralDesc* literal = current_literal_;
+  const TokenDesc& token = current_;
+  // TODO(dcarney): handle utf8 directly.
+  if (source_handle_.is_null()  || MustBeInBuffer(token)) {
+    EnsureLiteralIsValid(token, literal);
+    return literal->is_one_byte() ?
+        factory->InternalizeOneByteString(literal->one_byte_string()) :
+        factory->InternalizeTwoByteString(literal->two_byte_string());
+  }
+  int offset = 0, length = 0;
+  LiteralOffsetAndLength<Char>(buffer_, token, &offset, &length);
+  if (sizeof(Char) == 1) {
+    SubStringKey<uint8_t> key(source_handle_, offset, length);
+    return factory->InternalizeStringWithKey(&key);
+  } else {
+    SubStringKey<uint16_t> key(source_handle_, offset, length);
+    return factory->InternalizeStringWithKey(&key);
+  }
 }
 
 
-template<>
-Handle<String> Lexer<uint8_t>::AllocateLiteral(
-    LiteralDesc* literal, PretenureFlag pretenured) {
-  // Factory* factory = isolate_->factory();
-  // if (literal->is_in_buffer) {
-  //   return literal->is_one_byte
-  //       ? factory->NewStringFromOneByte(literal->one_byte_string, pretenured)
-  //       : factory->NewStringFromTwoByte(literal->two_byte_string, pretenured)
-  // }
-  // int from = literal->offset;
-  // int length = literal->length;
-  // // Save the offset and the length before allocating the string as it may
-  // // cause a GC, invalidate the literal, and move the source.
-  // Handle<String> result = factory->NewRawOneByteString(length, pretenured);
-  // uint8_t* chars = SeqOneByteString::cast(*result)->GetChars();
-  // String::WriteToFlat(*source_handle_, chars, from, from + length);
-  // return result;
-  CHECK(false);
-  return Handle<String>();
+template<class Char>
+Handle<String> Lexer<Char>::AllocateNextLiteralString(Isolate* isolate,
+                                                      PretenureFlag tenured) {
+  Factory* factory = isolate->factory();
+  LiteralDesc* literal = next_literal_;
+  const TokenDesc& token = next_;
+  // TODO(dcarney): handle utf8 directly.
+  if (source_handle_.is_null()  || MustBeInBuffer(token)) {
+    EnsureLiteralIsValid(token, literal);
+    return literal->is_one_byte() ?
+        factory->NewStringFromOneByte(literal->one_byte_string(), tenured) :
+        factory->NewStringFromTwoByte(literal->two_byte_string(), tenured);
+  }
+  int offset = 0, length = 0;
+  LiteralOffsetAndLength<Char>(buffer_, token, &offset, &length);
+  return factory->NewSubString(source_handle_, offset, offset + length);
 }
 
 
-template<>
-Handle<String> Lexer<uint16_t>::AllocateLiteral(
-    LiteralDesc* literal, PretenureFlag pretenured) {
-  // Factory* factory = isolate_->factory();
-  // if (literal->is_in_buffer) {
-  //   return literal->is_one_byte
-  //       ? factory->NewStringFromOneByte(literal->one_byte_string, pretenured)
-  //       : factory->NewStringFromTwoByte(literal->two_byte_string, pretenured)
-  // }
-  // // Save the offset and the length before allocating the string as it may
-  // // cause a GC, invalidate the literal, and move the source.
-  // int from = literal->offset;
-  // int length = literal->length;
-  // Handle<String> result = factory->NewRawTwoByteString(length, pretenured);
-  // uint16_t* chars = SeqTwoByteString::cast(*result)->GetChars();
-  // String::WriteToFlat(*source_handle_, chars, from, from + length);
-  // return result;
-  CHECK(false);
-  return Handle<String>();
-}
-
-
-template<>
-Handle<String> Lexer<int8_t>::AllocateLiteral(
-    LiteralDesc* literal, PretenureFlag pretenured) {
-  CHECK(false);
-  return Handle<String>();
-}
-
-
 template class Lexer<uint8_t>;
 template class Lexer<uint16_t>;
 template class Lexer<int8_t>;
 
 } }  // v8::internal