Use external byte arrays for token stream, this moves the token stream out of the isolate heap. For…

vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/bigint_operations.h"
@@ skipping 4765 matching lines @@
 RawArray* TokenStream::TokenObjects() const {
   return raw_ptr()->token_objects_;
 }
 
 
 void TokenStream::SetTokenObjects(const Array& value) const {
   StorePointer(&raw_ptr()->token_objects_, value.raw());
 }
 
 
-void TokenStream::SetLength(intptr_t value) const {
-  raw_ptr()->length_ = Smi::New(value);
+RawExternalUint8Array* TokenStream::GetStream() const {
+  return raw_ptr()->stream_;
+}
+
+
+void TokenStream::SetStream(const ExternalUint8Array& value) const {
+  StorePointer(&raw_ptr()->stream_, value.raw());
+}
+
+
+void TokenStream::DataFinalizer(void *peer) {
+  ASSERT(peer != NULL);
+  ::free(peer);

[cshapiro, 2012/09/05 20:33:50]

It is always safe to free NULL.  That is a no-op.

[siva, 2012/09/06 18:19:29]

I added the ASSERT more to ensure that this finalizer callback is not registered
for the case when we point
straight to the snapshot read only buffer.

In that case I setup the peer to be NULL and callback to NULL.

On 2012/09/05 20:33:50, cshapiro wrote:

 }
 
 
 RawString* TokenStream::PrivateKey() const {
   return raw_ptr()->private_key_;
 }
 
 
 void TokenStream::SetPrivateKey(const String& value) const {
   StorePointer(&raw_ptr()->private_key_, value.raw());
 }
 
 
 RawString* TokenStream::GenerateSource() const {
   Iterator iterator(*this, 0);
+  const ExternalUint8Array& data = ExternalUint8Array::Handle(GetStream());
   const GrowableObjectArray& literals =
-      GrowableObjectArray::Handle(GrowableObjectArray::New(Length()));
+      GrowableObjectArray::Handle(GrowableObjectArray::New(data.Length()));
   const String& private_key = String::Handle(PrivateKey());
   intptr_t private_len = private_key.Length();
 
   String& blank = String::Handle(String::New(" "));
   String& newline = String::Handle(String::New("\n"));
   String& two_newlines = String::Handle(String::New("\n\n"));
   String& double_quotes = String::Handle(String::New("\""));
   String& dollar = String::Handle(String::New("$"));
   String& two_spaces = String::Handle(String::New("  "));
 
@@ skipping 143 matching lines @@
   Token::Kind kind = iterator.CurrentTokenKind();
   while (index < src_pos && kind != Token::kEOS) {
     iterator.Advance();
     kind = iterator.CurrentTokenKind();
     index += 1;
   }
   return iterator.CurrentPosition();
 }
 
 
+RawTokenStream* TokenStream::New() {
+  ASSERT(Object::token_stream_class() != Class::null());
+  TokenStream& result = TokenStream::Handle();
+  {
+    RawObject* raw = Object::Allocate(TokenStream::kClassId,
+                                      TokenStream::InstanceSize(),
+                                      Heap::kOld);
+    NoGCScope no_gc;
+    result ^= raw;
+  }
+  return result.raw();
+}
+
+
 RawTokenStream* TokenStream::New(intptr_t len) {
-  ASSERT(Object::token_stream_class() != Class::null());
-  if (len < 0 || len > kMaxElements) {
-    // This should be caught before we reach here.
-    FATAL1("Fatal error in TokenStream::New: invalid len %ld\n", len);
-  }
-  TokenStream& result = TokenStream::Handle();
-  {
-    RawObject* raw = Object::Allocate(TokenStream::kClassId,
-                                      TokenStream::InstanceSize(len),
-                                      Heap::kOld);
-    NoGCScope no_gc;
-    result ^= raw;
-    result.SetLength(len);
-  }
+  uint8_t* data = reinterpret_cast<uint8_t*>(::malloc(len));
+  ASSERT(data != NULL);
+  const ExternalUint8Array& stream = ExternalUint8Array::Handle(
+      ExternalUint8Array::New(data, len, data, DataFinalizer, Heap::kOld));
+  const TokenStream& result = TokenStream::Handle(TokenStream::New());
+  result.SetStream(stream);
   return result.raw();
 }
 
 
 // Helper class for creation of compressed token stream data.
 class CompressedTokenStreamData : public ValueObject {
  public:
   CompressedTokenStreamData() :
       buffer_(NULL),
       stream_(&buffer_, Reallocate),
       token_objects_(GrowableObjectArray::Handle(
           GrowableObjectArray::New(kInitialTokenCount, Heap::kOld))),
       token_obj_(Object::Handle()),
       literal_token_(LiteralToken::Handle()),
       literal_str_(String::Handle()) {
     const String& empty_literal = String::Handle();
     token_objects_.Add(empty_literal);
   }
   ~CompressedTokenStreamData() {
-    free(buffer_);
   }
 
   // Add an IDENT token into the stream and the token objects array.
   void AddIdentToken(String* ident) {
     if (ident != NULL) {
       // If the IDENT token is already in the tokens object array use the
       // same index instead of duplicating it.
       intptr_t index = FindIdentIndex(ident);
       if (index == -1) {
         WriteIndex(token_objects_.Length());
@@ skipping 129 matching lines @@
       ASSERT(token.kind < Token::kNumTokens);
       data.AddSimpleToken(token.kind);
     }
   }
   if (FLAG_compiler_stats) {
     CompilerStats::num_tokens_total += len;
   }
   data.AddSimpleToken(Token::kEOS);  // End of stream.
 
   // Create and setup the token stream object.
-  const TokenStream& result = TokenStream::Handle(New(data.Length()));
+  const ExternalUint8Array& stream = ExternalUint8Array::Handle(
+      ExternalUint8Array::New(data.GetStream(),
+                              data.Length(),
+                              data.GetStream(),
+                              DataFinalizer,
+                              Heap::kOld));
+  const TokenStream& result = TokenStream::Handle(New());
   result.SetPrivateKey(private_key);
   {
     NoGCScope no_gc;
-    memmove(result.EntryAddr(0), data.GetStream(), data.Length());
+    result.SetStream(stream);
     const Array& tokens = Array::Handle(Array::MakeArray(data.TokenObjects()));
     result.SetTokenObjects(tokens);
   }
   return result.raw();
 }
 
 
 const char* TokenStream::ToCString() const {
   return "TokenStream";
 }
 
 
 TokenStream::Iterator::Iterator(const TokenStream& tokens, intptr_t token_pos)
     : tokens_(tokens),
+      data_(ExternalUint8Array::Handle(tokens.GetStream())),
+      stream_(reinterpret_cast<uint8_t*>(data_.GetPeer()), data_.Length()),
       token_objects_(Array::Handle(tokens.TokenObjects())),
       obj_(Object::Handle()),
       cur_token_pos_(token_pos),
-      stream_token_pos_(token_pos),
       cur_token_kind_(Token::kILLEGAL),
       cur_token_obj_index_(-1) {
   SetCurrentPosition(token_pos);
 }
 
 
 bool TokenStream::Iterator::IsValid() const {
   return !tokens_.IsNull();
 }
 
 
 Token::Kind TokenStream::Iterator::LookaheadTokenKind(intptr_t num_tokens) {
-  intptr_t saved_position = stream_token_pos_;
+  intptr_t saved_position = stream_.Position();
   Token::Kind kind = Token::kILLEGAL;
   intptr_t value = -1;
   intptr_t count = 0;
   while (count < num_tokens && value != Token::kEOS) {
     value = ReadToken();
     count += 1;
   }
   if (value < Token::kNumTokens) {
     kind = static_cast<Token::Kind>(value);
   } else {
     value = value - Token::kNumTokens;
     obj_ = token_objects_.At(value);
     if (obj_.IsLiteralToken()) {
       const LiteralToken& literal_token = LiteralToken::Cast(obj_);
       kind = literal_token.kind();
     } else {
       ASSERT(obj_.IsString());  // Must be an identifier.
       kind = Token::kIDENT;
     }
   }
-  stream_token_pos_ = saved_position;
+  stream_.SetPosition(saved_position);
   return kind;
 }
 
 
 intptr_t TokenStream::Iterator::CurrentPosition() const {
   return cur_token_pos_;
 }
 
 
 void TokenStream::Iterator::SetCurrentPosition(intptr_t value) {
-  stream_token_pos_ = value;
+  stream_.SetPosition(value);
   Advance();
 }
 
 
 void TokenStream::Iterator::Advance() {
-  cur_token_pos_ = stream_token_pos_;
+  cur_token_pos_ = stream_.Position();
   intptr_t value = ReadToken();
   if (value < Token::kNumTokens) {
     cur_token_kind_ = static_cast<Token::Kind>(value);
     cur_token_obj_index_ = -1;
     return;
   }
   cur_token_obj_index_ = value - Token::kNumTokens;
   obj_ = token_objects_.At(cur_token_obj_index_);
   if (obj_.IsLiteralToken()) {
     const LiteralToken& literal_token = LiteralToken::Cast(obj_);
@@ skipping 40 matching lines @@
     }
     return Symbols::New(Token::Str(kind));
   } else {
     ASSERT(obj.IsLiteralToken());  // Must be a literal token.
     const LiteralToken& literal_token = LiteralToken::Cast(obj);
     return literal_token.literal();
   }
 }
 
 
-intptr_t TokenStream::Iterator::ReadToken() {
-  uint8_t b = ReadByte();
-  if (b > kMaxUnsignedDataPerByte) {
-    return static_cast<intptr_t>(b) - kEndUnsignedByteMarker;
-  }
-  intptr_t value = 0;
-  uint8_t s = 0;
-  do {
-    value |= static_cast<intptr_t>(b) << s;
-    s += kDataBitsPerByte;
-    b = ReadByte();
-  } while (b <= kMaxUnsignedDataPerByte);
-  value |= ((static_cast<intptr_t>(b) - kEndUnsignedByteMarker) << s);
-  ASSERT((value >= 0) && (value <= kIntptrMax));
-  return value;
-}
-
-
-uint8_t TokenStream::Iterator::ReadByte() {
-  ASSERT(stream_token_pos_ < tokens_.Length());
-  return *(tokens_.EntryAddr(stream_token_pos_++));
-}
-
-
 bool Script::HasSource() const {
   return raw_ptr()->source_ != String::null();
 }
 
 
 RawString* Script::Source() const {
   String& source = String::Handle(raw_ptr()->source_);
   if (source.IsNull()) {
     const TokenStream& token_stream = TokenStream::Handle(tokens());
     return token_stream.GenerateSource();
@@ skipping 5870 matching lines @@
   }
   return result.raw();
 }
 
 
 const char* WeakProperty::ToCString() const {
   return "_WeakProperty";
 }
 
 }  // namespace dart