Changes to shrink the token stream representation from two words to one word.

vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/bigint_operations.h"
 #include "vm/bootstrap.h"
@@ skipping 50 matching lines @@
 RawClass* Object::type_parameter_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::instantiated_type_class_ =
     reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::abstract_type_arguments_class_ =
     reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::type_arguments_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::instantiated_type_arguments_class_ =
     reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::function_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::field_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
+RawClass* Object::literal_token_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::token_stream_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::script_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::library_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::library_prefix_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::code_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::instructions_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::pc_descriptors_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::var_descriptors_class_ =
     reinterpret_cast<RawClass*>(RAW_NULL);
 RawClass* Object::exception_handlers_class_ =
@@ skipping 29 matching lines @@
   } else if (raw_class == abstract_type_arguments_class()) {
     return kAbstractTypeArgumentsClass;
   } else if (raw_class == type_arguments_class()) {
     return kTypeArgumentsClass;
   } else if (raw_class == instantiated_type_arguments_class()) {
     return kInstantiatedTypeArgumentsClass;
   } else if (raw_class == function_class()) {
     return kFunctionClass;
   } else if (raw_class == field_class()) {
     return kFieldClass;
+  } else if (raw_class == literal_token_class()) {
+    return kLiteralTokenClass;
   } else if (raw_class == token_stream_class()) {
     return kTokenStreamClass;
   } else if (raw_class == script_class()) {
     return kScriptClass;
   } else if (raw_class == library_class()) {
     return kLibraryClass;
   } else if (raw_class == library_prefix_class()) {
     return kLibraryPrefixClass;
   } else if (raw_class == code_class()) {
     return kCodeClass;
@@ skipping 33 matching lines @@
     case kUnresolvedClassClass: return unresolved_class_class();
     case kTypeClass: return type_class();
     case kTypeParameterClass: return type_parameter_class();
     case kInstantiatedTypeClass: return instantiated_type_class();
     case kAbstractTypeArgumentsClass: return abstract_type_arguments_class();
     case kTypeArgumentsClass: return type_arguments_class();
     case kInstantiatedTypeArgumentsClass:
         return instantiated_type_arguments_class();
     case kFunctionClass: return function_class();
     case kFieldClass: return field_class();
+    case kLiteralTokenClass: return literal_token_class();
     case kTokenStreamClass: return token_stream_class();
     case kScriptClass: return script_class();
     case kLibraryClass: return library_class();
     case kLibraryPrefixClass: return library_prefix_class();
     case kCodeClass: return code_class();
     case kInstructionsClass: return instructions_class();
     case kPcDescriptorsClass: return pc_descriptors_class();
     case kLocalVarDescriptorsClass: return var_descriptors_class();
     case kExceptionHandlersClass: return exception_handlers_class();
     case kContextClass: return context_class();
@@ skipping 143 matching lines @@
 
   cls = Class::New<InstantiatedTypeArguments>();
   instantiated_type_arguments_class_ = cls.raw();
 
   cls = Class::New<Function>();
   function_class_ = cls.raw();
 
   cls = Class::New<Field>();
   field_class_ = cls.raw();
 
+  cls = Class::New<LiteralToken>();
+  literal_token_class_ = cls.raw();
+
   cls = Class::New<TokenStream>();
   token_stream_class_ = cls.raw();
 
   cls = Class::New<Script>();
   script_class_ = cls.raw();
 
   cls = Class::New<Library>();
   library_class_ = cls.raw();
 
   cls = Class::New<LibraryPrefix>();
@@ skipping 3379 matching lines @@
   const char* cls_name = String::Handle(cls.Name()).ToCString();
   intptr_t len =
       OS::SNPrint(NULL, 0, kFormat, cls_name, field_name, kF0, kF1) + 1;
   char* chars = reinterpret_cast<char*>(
       Isolate::Current()->current_zone()->Allocate(len));
   OS::SNPrint(chars, len, kFormat, cls_name, field_name, kF0, kF1);
   return chars;
 }
 
 
+void LiteralToken::set_literal(const String& literal) const {
+  StorePointer(&raw_ptr()->literal_, literal.raw());
+}
+
+
+void LiteralToken::set_value(const Object& value) const {
+  StorePointer(&raw_ptr()->value_, value.raw());
+}
+
+
+RawLiteralToken* LiteralToken::Canonicalize() const {
+  const String& literal_value = String::Handle(literal());
+  return NewCanonical(kind(), literal_value);
+}
+
+
+RawLiteralToken* LiteralToken::New() {
+  const Class& cls = Class::Handle(Object::literal_token_class());
+  RawObject* raw = Object::Allocate(cls,
+                                    LiteralToken::InstanceSize(),
+                                    Heap::kOld);
+  return reinterpret_cast<RawLiteralToken*>(raw);
+}
+
+
+RawLiteralToken* LiteralToken::New(Token::Kind kind, const String& literal) {
+  const LiteralToken& result = LiteralToken::Handle(LiteralToken::New());
+  result.set_kind(kind);
+  result.set_literal(literal);
+  if (kind == Token::kINTEGER) {
+    const Integer& value = Integer::Handle(Integer::New(literal));
+    result.set_value(value);
+  } else if (kind == Token::kDOUBLE) {
+    const Double& value = Double::Handle(Double::NewCanonical(literal));
+    result.set_value(value);
+  } else {
+    ASSERT(Token::NeedsLiteralToken(kind));
+    result.set_value(literal);
+  }
+  return result.raw();
+}
+
+
+RawLiteralToken* LiteralToken::NewCanonical(Token::Kind kind,
+                                            const String& literal) {
+  ObjectStore* object_store = Isolate::Current()->object_store();
+  Array& canonical_list =
+      Array::Handle(object_store->literal_tokens_canonical_list());
+  const intptr_t canonical_list_len =
+      canonical_list.IsNull() ? 0 : canonical_list.Length();
+
+  // Linear search to see whether this value is already present in the
+  // list of canonicalized literal tokens.
+  intptr_t index = 0;
+  LiteralToken& canonical_value = LiteralToken::Handle();
+  String& literal_value = String::Handle();
+  while (index < canonical_list_len) {

[hausner, 2012/02/25 00:41:57]

This linear search might become a bottleneck once we load large libraries. 

[siva, 2012/02/28 19:57:31]

I removed this canonicalization step, the literal tokens are not canonicalized.
I think we need to evaluate further as to whether there is any benefit from
canonicalizing these. 

On 2012/02/25 00:41:57, hausner wrote:

+    canonical_value ^= canonical_list.At(index);
+    if (canonical_value.IsNull()) {
+      break;
+    }
+    if ((canonical_value.kind() == kind)) {
+      literal_value = canonical_value.literal();
+      if (literal_value.Equals(literal, 0, literal.Length())) {

[hausner, 2012/02/25 00:41:57]

As discussed in person: no need to call Equals, you can do a pointer comparison.

[siva, 2012/02/28 19:57:31]

This code has been removed.

On 2012/02/25 00:41:57, hausner wrote:

+        return canonical_value.raw();
+      }
+    }
+    index++;
+  }
+  // The value needs to be added to the canonical list.
+  // Grow the list if it is full.
+  canonical_value = LiteralToken::New(kind, literal);
+  if (index >= canonical_list_len) {
+    const intptr_t new_length = canonical_list_len + 4;
+    canonical_list = Array::Grow(canonical_list, new_length, Heap::kOld);
+    object_store->set_literal_tokens_canonical_list(canonical_list);
+  }
+  canonical_list.SetAt(index, canonical_value);
+  canonical_value.SetCanonical();
+  return canonical_value.raw();
+}
+
+
+const char* LiteralToken::ToCString() const {
+  const String& token = String::Handle(literal());
+  return token.ToCString();
+}
+
+
 void TokenStream::SetLength(intptr_t value) const {
   raw_ptr()->length_ = Smi::New(value);
 }
 
 
 void TokenStream::SetTokenAt(intptr_t index,
                              Token::Kind kind,
                              const String& literal) {
-  *(SmiAddr(index, RawTokenStream::kKindEntry)) = Smi::New(kind);
-  StorePointer(EntryAddr(index, RawTokenStream::kLiteralEntry),
-               reinterpret_cast<RawObject*>(literal.raw()));
+  if (kind == Token::kIDENT) {
+    if (FLAG_compiler_stats) {
+      CompilerStats::num_ident_tokens_total += 1;
+    }
+    StorePointer(EntryAddr(index, RawTokenStream::kTokenEntry),
+                 reinterpret_cast<RawObject*>(literal.raw()));
+  } else if (Token::NeedsLiteralToken(kind)) {
+    if (FLAG_compiler_stats) {
+      CompilerStats::num_literal_tokens_total += 1;
+    }
+    StorePointer(
+        EntryAddr(index, RawTokenStream::kTokenEntry),
+        reinterpret_cast<RawObject*>(LiteralToken::NewCanonical(kind,
+                                                                literal)));
+  } else {
+    ASSERT(kind < Token::kNumTokens);
+    *(SmiAddr(index, RawTokenStream::kTokenEntry)) = Smi::New(kind);
+  }
+}
+
+
+void TokenStream::SetTokenAt(intptr_t index, const Object& token) {
+  StorePointer(EntryAddr(index, RawTokenStream::kTokenEntry), token.raw());
+}
+
+
+RawObject* TokenStream::TokenAt(intptr_t index) const {
+  return *EntryAddr(index, RawTokenStream::kTokenEntry);
+}
+
+
+RawString* TokenStream::LiteralAt(intptr_t index) const {
+  const Object& obj = Object::Handle(TokenAt(index));
+  if (obj.IsString()) {
+    return reinterpret_cast<RawString*>(obj.raw());
+  } else if (obj.IsSmi()) {
+    Token::Kind kind = static_cast<Token::Kind>(
+        Smi::Value(reinterpret_cast<RawSmi*>(obj.raw())));
+    ASSERT(kind < Token::kNumTokens);
+    if (Token::IsPseudoKeyword(kind) || Token::IsKeyword(kind)) {
+      ObjectStore* object_store = Isolate::Current()->object_store();
+      String& str = String::Handle();
+      str ^= object_store->GetKeywordSymbol(kind - Token::kFirstKeyword);
+      ASSERT(!str.IsNull());
+      return str.raw();
+    }
+    return String::NewSymbol(Token::Str(kind));
+  } else {
+    LiteralToken& token = LiteralToken::Handle();
+    token ^= obj.raw();  // Must be a literal token.
+    return token.literal();
+  }
 }
 
 
 RawTokenStream* TokenStream::New(intptr_t len) {
   const Class& token_stream_class = Class::Handle(Object::token_stream_class());
   TokenStream& result = TokenStream::Handle();
   {
     RawObject* raw = Object::Allocate(token_stream_class,
                                       TokenStream::InstanceSize(len),
                                       Heap::kOld);
     NoGCScope no_gc;
     result ^= raw;
     result.SetLength(len);
   }
   return result.raw();
 }
 
 
 RawTokenStream* TokenStream::New(const Scanner::GrowableTokenStream& tokens) {
   intptr_t len = tokens.length();
 
   TokenStream& result = TokenStream::Handle(New(len));
   // Copy the relevant data out of the scanner's token stream.
+  const String& empty_literal = String::Handle();
   for (intptr_t i = 0; i < len; i++) {
     Scanner::TokenDescriptor token = tokens[i];
     if (token.literal != NULL) {
       result.SetTokenAt(i, token.kind, *(token.literal));
     } else {
-      result.SetTokenAt(i, token.kind, String::Handle());
+      result.SetTokenAt(i, token.kind, empty_literal);
     }
   }
   return result.raw();
 }
 
 
 const char* TokenStream::ToCString() const {
   return "TokenStream";
 }
 
@@ skipping 4517 matching lines @@
   result.set_num_args_tested(num_args_tested);
   // Number of array elements in one test entry (num_args_tested + 1)
   intptr_t len = num_args_tested + 1;
   // IC data array must be null terminated (sentinel entry).
   Array& ic_data = Array::Handle(Array::New(len, Heap::kOld));
   result.set_ic_data(ic_data);
   return result.raw();
 }
 
 }  // namespace dart