Use external byte arrays for token stream, this moves the token stream out of the isolate heap. For…

vm/object.cc

Index: vm/object.cc
===================================================================
--- vm/object.cc	(revision 11700)
+++ vm/object.cc	(working copy)
@@ -4783,11 +4783,22 @@
 }
 
 
-void TokenStream::SetLength(intptr_t value) const {
-  raw_ptr()->length_ = Smi::New(value);
+RawExternalUint8Array* TokenStream::GetStream() const {
+  return raw_ptr()->stream_;
 }
 
 
+void TokenStream::SetStream(const ExternalUint8Array& value) const {
+  StorePointer(&raw_ptr()->stream_, value.raw());
+}
+
+
+void TokenStream::DataFinalizer(void *peer) {
+  ASSERT(peer != NULL);
+  ::free(peer);

[cshapiro, 2012/09/05 20:33:50]

It is always safe to free NULL.  That is a no-op.

[siva, 2012/09/06 18:19:29]

I added the ASSERT more to ensure that this finalizer callback is not registered
for the case when we point
straight to the snapshot read only buffer.

In that case I setup the peer to be NULL and callback to NULL.

On 2012/09/05 20:33:50, cshapiro wrote:

+}
+
+
 RawString* TokenStream::PrivateKey() const {
   return raw_ptr()->private_key_;
 }
@@ -4800,8 +4811,9 @@
 
 RawString* TokenStream::GenerateSource() const {
   Iterator iterator(*this, 0);
+  const ExternalUint8Array& data = ExternalUint8Array::Handle(GetStream());
   const GrowableObjectArray& literals =
-      GrowableObjectArray::Handle(GrowableObjectArray::New(Length()));
+      GrowableObjectArray::Handle(GrowableObjectArray::New(data.Length()));
   const String& private_key = String::Handle(PrivateKey());
   intptr_t private_len = private_key.Length();
 
@@ -4965,25 +4977,31 @@
 }
 
 
-RawTokenStream* TokenStream::New(intptr_t len) {
+RawTokenStream* TokenStream::New() {
   ASSERT(Object::token_stream_class() != Class::null());
-  if (len < 0 || len > kMaxElements) {
-    // This should be caught before we reach here.
-    FATAL1("Fatal error in TokenStream::New: invalid len %ld\n", len);
-  }
   TokenStream& result = TokenStream::Handle();
   {
     RawObject* raw = Object::Allocate(TokenStream::kClassId,
-                                      TokenStream::InstanceSize(len),
+                                      TokenStream::InstanceSize(),
                                       Heap::kOld);
     NoGCScope no_gc;
     result ^= raw;
-    result.SetLength(len);
   }
   return result.raw();
 }
 
 
+RawTokenStream* TokenStream::New(intptr_t len) {
+  uint8_t* data = reinterpret_cast<uint8_t*>(::malloc(len));
+  ASSERT(data != NULL);
+  const ExternalUint8Array& stream = ExternalUint8Array::Handle(
+      ExternalUint8Array::New(data, len, data, DataFinalizer, Heap::kOld));
+  const TokenStream& result = TokenStream::Handle(TokenStream::New());
+  result.SetStream(stream);
+  return result.raw();
+}
+
+
 // Helper class for creation of compressed token stream data.
 class CompressedTokenStreamData : public ValueObject {
  public:
@@ -4999,7 +5017,6 @@
     token_objects_.Add(empty_literal);
   }
   ~CompressedTokenStreamData() {
-    free(buffer_);
   }
 
   // Add an IDENT token into the stream and the token objects array.
@@ -5149,11 +5166,17 @@
   data.AddSimpleToken(Token::kEOS);  // End of stream.
 
   // Create and setup the token stream object.
-  const TokenStream& result = TokenStream::Handle(New(data.Length()));
+  const ExternalUint8Array& stream = ExternalUint8Array::Handle(
+      ExternalUint8Array::New(data.GetStream(),
+                              data.Length(),
+                              data.GetStream(),
+                              DataFinalizer,
+                              Heap::kOld));
+  const TokenStream& result = TokenStream::Handle(New());
   result.SetPrivateKey(private_key);
   {
     NoGCScope no_gc;
-    memmove(result.EntryAddr(0), data.GetStream(), data.Length());
+    result.SetStream(stream);
     const Array& tokens = Array::Handle(Array::MakeArray(data.TokenObjects()));
     result.SetTokenObjects(tokens);
   }
@@ -5168,10 +5191,11 @@
 
 TokenStream::Iterator::Iterator(const TokenStream& tokens, intptr_t token_pos)
     : tokens_(tokens),
+      data_(ExternalUint8Array::Handle(tokens.GetStream())),
+      stream_(reinterpret_cast<uint8_t*>(data_.GetPeer()), data_.Length()),
       token_objects_(Array::Handle(tokens.TokenObjects())),
       obj_(Object::Handle()),
       cur_token_pos_(token_pos),
-      stream_token_pos_(token_pos),
       cur_token_kind_(Token::kILLEGAL),
       cur_token_obj_index_(-1) {
   SetCurrentPosition(token_pos);
@@ -5184,7 +5208,7 @@
 
 
 Token::Kind TokenStream::Iterator::LookaheadTokenKind(intptr_t num_tokens) {
-  intptr_t saved_position = stream_token_pos_;
+  intptr_t saved_position = stream_.Position();
   Token::Kind kind = Token::kILLEGAL;
   intptr_t value = -1;
   intptr_t count = 0;
@@ -5205,7 +5229,7 @@
       kind = Token::kIDENT;
     }
   }
-  stream_token_pos_ = saved_position;
+  stream_.SetPosition(saved_position);
   return kind;
 }
 
@@ -5216,13 +5240,13 @@
 
 
 void TokenStream::Iterator::SetCurrentPosition(intptr_t value) {
-  stream_token_pos_ = value;
+  stream_.SetPosition(value);
   Advance();
 }
 
 
 void TokenStream::Iterator::Advance() {
-  cur_token_pos_ = stream_token_pos_;
+  cur_token_pos_ = stream_.Position();
   intptr_t value = ReadToken();
   if (value < Token::kNumTokens) {
     cur_token_kind_ = static_cast<Token::Kind>(value);
@@ -5283,30 +5307,6 @@
 }
 
 
-intptr_t TokenStream::Iterator::ReadToken() {
-  uint8_t b = ReadByte();
-  if (b > kMaxUnsignedDataPerByte) {
-    return static_cast<intptr_t>(b) - kEndUnsignedByteMarker;
-  }
-  intptr_t value = 0;
-  uint8_t s = 0;
-  do {
-    value |= static_cast<intptr_t>(b) << s;
-    s += kDataBitsPerByte;
-    b = ReadByte();
-  } while (b <= kMaxUnsignedDataPerByte);
-  value |= ((static_cast<intptr_t>(b) - kEndUnsignedByteMarker) << s);
-  ASSERT((value >= 0) && (value <= kIntptrMax));
-  return value;
-}
-
-
-uint8_t TokenStream::Iterator::ReadByte() {
-  ASSERT(stream_token_pos_ < tokens_.Length());
-  return *(tokens_.EntryAddr(stream_token_pos_++));
-}
-
-
 bool Script::HasSource() const {
   return raw_ptr()->source_ != String::null();
 }