Changes to shrink the token stream representation from two words to one word.

vm/object.cc

Index: vm/object.cc
===================================================================
--- vm/object.cc	(revision 4579)
+++ vm/object.cc	(working copy)
@@ -68,6 +68,7 @@
     reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::function_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::field_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
+RawClass* Object::literal_token_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::token_stream_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::script_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::library_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
@@ -117,6 +118,8 @@
     return kFunctionClass;
   } else if (raw_class == field_class()) {
     return kFieldClass;
+  } else if (raw_class == literal_token_class()) {
+    return kLiteralTokenClass;
   } else if (raw_class == token_stream_class()) {
     return kTokenStreamClass;
   } else if (raw_class == script_class()) {
@@ -170,6 +173,7 @@
         return instantiated_type_arguments_class();
     case kFunctionClass: return function_class();
     case kFieldClass: return field_class();
+    case kLiteralTokenClass: return literal_token_class();
     case kTokenStreamClass: return token_stream_class();
     case kScriptClass: return script_class();
     case kLibraryClass: return library_class();
@@ -333,6 +337,9 @@
   cls = Class::New<Field>();
   field_class_ = cls.raw();
 
+  cls = Class::New<LiteralToken>();
+  literal_token_class_ = cls.raw();
+
   cls = Class::New<TokenStream>();
   token_stream_class_ = cls.raw();
 
@@ -3732,6 +3739,95 @@
 }
 
 
+void LiteralToken::set_literal(const String& literal) const {
+  StorePointer(&raw_ptr()->literal_, literal.raw());
+}
+
+
+void LiteralToken::set_value(const Object& value) const {
+  StorePointer(&raw_ptr()->value_, value.raw());
+}
+
+
+RawLiteralToken* LiteralToken::Canonicalize() const {
+  const String& literal_value = String::Handle(literal());
+  return NewCanonical(kind(), literal_value);
+}
+
+
+RawLiteralToken* LiteralToken::New() {
+  const Class& cls = Class::Handle(Object::literal_token_class());
+  RawObject* raw = Object::Allocate(cls,
+                                    LiteralToken::InstanceSize(),
+                                    Heap::kOld);
+  return reinterpret_cast<RawLiteralToken*>(raw);
+}
+
+
+RawLiteralToken* LiteralToken::New(Token::Kind kind, const String& literal) {
+  const LiteralToken& result = LiteralToken::Handle(LiteralToken::New());
+  result.set_kind(kind);
+  result.set_literal(literal);
+  if (kind == Token::kINTEGER) {
+    const Integer& value = Integer::Handle(Integer::New(literal));
+    result.set_value(value);
+  } else if (kind == Token::kDOUBLE) {
+    const Double& value = Double::Handle(Double::NewCanonical(literal));
+    result.set_value(value);
+  } else {
+    ASSERT(Token::NeedsLiteralToken(kind));
+    result.set_value(literal);
+  }
+  return result.raw();
+}
+
+
+RawLiteralToken* LiteralToken::NewCanonical(Token::Kind kind,
+                                            const String& literal) {
+  ObjectStore* object_store = Isolate::Current()->object_store();
+  Array& canonical_list =
+      Array::Handle(object_store->literal_tokens_canonical_list());
+  const intptr_t canonical_list_len =
+      canonical_list.IsNull() ? 0 : canonical_list.Length();
+
+  // Linear search to see whether this value is already present in the
+  // list of canonicalized literal tokens.
+  intptr_t index = 0;
+  LiteralToken& canonical_value = LiteralToken::Handle();
+  String& literal_value = String::Handle();
+  while (index < canonical_list_len) {

[hausner, 2012/02/25 00:41:57]

This linear search might become a bottleneck once we load large libraries. 

[siva, 2012/02/28 19:57:31]

I removed this canonicalization step, the literal tokens are not canonicalized.
I think we need to evaluate further as to whether there is any benefit from
canonicalizing these. 

On 2012/02/25 00:41:57, hausner wrote:

+    canonical_value ^= canonical_list.At(index);
+    if (canonical_value.IsNull()) {
+      break;
+    }
+    if ((canonical_value.kind() == kind)) {
+      literal_value = canonical_value.literal();
+      if (literal_value.Equals(literal, 0, literal.Length())) {

[hausner, 2012/02/25 00:41:57]

As discussed in person: no need to call Equals, you can do a pointer comparison.

[siva, 2012/02/28 19:57:31]

This code has been removed.

On 2012/02/25 00:41:57, hausner wrote:

+        return canonical_value.raw();
+      }
+    }
+    index++;
+  }
+  // The value needs to be added to the canonical list.
+  // Grow the list if it is full.
+  canonical_value = LiteralToken::New(kind, literal);
+  if (index >= canonical_list_len) {
+    const intptr_t new_length = canonical_list_len + 4;
+    canonical_list = Array::Grow(canonical_list, new_length, Heap::kOld);
+    object_store->set_literal_tokens_canonical_list(canonical_list);
+  }
+  canonical_list.SetAt(index, canonical_value);
+  canonical_value.SetCanonical();
+  return canonical_value.raw();
+}
+
+
+const char* LiteralToken::ToCString() const {
+  const String& token = String::Handle(literal());
+  return token.ToCString();
+}
+
+
 void TokenStream::SetLength(intptr_t value) const {
   raw_ptr()->length_ = Smi::New(value);
 }
@@ -3740,12 +3836,61 @@
 void TokenStream::SetTokenAt(intptr_t index,
                              Token::Kind kind,
                              const String& literal) {
-  *(SmiAddr(index, RawTokenStream::kKindEntry)) = Smi::New(kind);
-  StorePointer(EntryAddr(index, RawTokenStream::kLiteralEntry),
-               reinterpret_cast<RawObject*>(literal.raw()));
+  if (kind == Token::kIDENT) {
+    if (FLAG_compiler_stats) {
+      CompilerStats::num_ident_tokens_total += 1;
+    }
+    StorePointer(EntryAddr(index, RawTokenStream::kTokenEntry),
+                 reinterpret_cast<RawObject*>(literal.raw()));
+  } else if (Token::NeedsLiteralToken(kind)) {
+    if (FLAG_compiler_stats) {
+      CompilerStats::num_literal_tokens_total += 1;
+    }
+    StorePointer(
+        EntryAddr(index, RawTokenStream::kTokenEntry),
+        reinterpret_cast<RawObject*>(LiteralToken::NewCanonical(kind,
+                                                                literal)));
+  } else {
+    ASSERT(kind < Token::kNumTokens);
+    *(SmiAddr(index, RawTokenStream::kTokenEntry)) = Smi::New(kind);
+  }
 }
 
 
+void TokenStream::SetTokenAt(intptr_t index, const Object& token) {
+  StorePointer(EntryAddr(index, RawTokenStream::kTokenEntry), token.raw());
+}
+
+
+RawObject* TokenStream::TokenAt(intptr_t index) const {
+  return *EntryAddr(index, RawTokenStream::kTokenEntry);
+}
+
+
+RawString* TokenStream::LiteralAt(intptr_t index) const {
+  const Object& obj = Object::Handle(TokenAt(index));
+  if (obj.IsString()) {
+    return reinterpret_cast<RawString*>(obj.raw());
+  } else if (obj.IsSmi()) {
+    Token::Kind kind = static_cast<Token::Kind>(
+        Smi::Value(reinterpret_cast<RawSmi*>(obj.raw())));
+    ASSERT(kind < Token::kNumTokens);
+    if (Token::IsPseudoKeyword(kind) || Token::IsKeyword(kind)) {
+      ObjectStore* object_store = Isolate::Current()->object_store();
+      String& str = String::Handle();
+      str ^= object_store->GetKeywordSymbol(kind - Token::kFirstKeyword);
+      ASSERT(!str.IsNull());
+      return str.raw();
+    }
+    return String::NewSymbol(Token::Str(kind));
+  } else {
+    LiteralToken& token = LiteralToken::Handle();
+    token ^= obj.raw();  // Must be a literal token.
+    return token.literal();
+  }
+}
+
+
 RawTokenStream* TokenStream::New(intptr_t len) {
   const Class& token_stream_class = Class::Handle(Object::token_stream_class());
   TokenStream& result = TokenStream::Handle();
@@ -3766,12 +3911,13 @@
 
   TokenStream& result = TokenStream::Handle(New(len));
   // Copy the relevant data out of the scanner's token stream.
+  const String& empty_literal = String::Handle();
   for (intptr_t i = 0; i < len; i++) {
     Scanner::TokenDescriptor token = tokens[i];
     if (token.literal != NULL) {
       result.SetTokenAt(i, token.kind, *(token.literal));
     } else {
-      result.SetTokenAt(i, token.kind, String::Handle());
+      result.SetTokenAt(i, token.kind, empty_literal);
     }
   }
   return result.raw();