Changes to shrink the token stream representation from two words to one word.

vm/scanner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scanner.h"
 
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
+#include "vm/object_store.h"
 #include "vm/thread.h"
 #include "vm/token.h"
 
 namespace dart {
 
 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens.");
 
 void Scanner::InitKeywordTable() {
   for (int i = 0; i < Token::numKeywords; i++) {
     Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
     keywords_[i].kind = token;
     keywords_[i].keyword_chars = Token::Str(token);
     keywords_[i].keyword_len = strlen(Token::Str(token));
-    keywords_[i].keyword_symbol = NULL;
   }
 }
 
 
 void Scanner::Reset() {
   lookahead_pos_ = 0;
   token_start_ = 0;
   c0_ = source_length_ == 0 ? '\0' : source_.CharAt(0);
   newline_seen_ = false;
   while (saved_context_ != NULL) {
@@ skipping 218 matching lines @@
   while (i < Token::numKeywords &&
          keywords_[i].keyword_chars[0] <= ident_char0) {
     if (keywords_[i].keyword_len == ident_length) {
       const char* keyword = keywords_[i].keyword_chars;
       int char_pos = 0;
       while ((char_pos < ident_length) &&
              (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) {
         char_pos++;
       }
       if (char_pos == ident_length) {
-        if (keywords_[i].keyword_symbol == NULL) {
-          keywords_[i].keyword_symbol = &String::ZoneHandle(
-              String::NewSymbol(source_, ident_pos, ident_length));
+        ObjectStore* object_store = Isolate::Current()->object_store();
+        String& symbol = String::ZoneHandle();

[hausner, 2012/02/25 00:41:57]

NB: for each keyword occurrence in the source text we now call Isolate::Current
and allocate a zone handle. Before, we only allocated one handle for each
keyword kind. Hopefully not a big deal. 

[siva, 2012/02/28 19:57:31]

I have redone this code a bit to address the concern you had about handle
creation, please take another look.

On 2012/02/25 00:41:57, hausner wrote:

[hausner, 2012/02/29 00:22:37]

Nice, thank you.

On 2012/02/28 19:57:31, asiva wrote:

+        symbol ^= object_store->GetKeywordSymbol(i);
+        if (symbol.IsNull()) {
+          symbol = String::NewSymbol(source_, ident_pos, ident_length);
+          object_store->SetKeywordSymbol(i, symbol);
         }
-        current_token_.literal = keywords_[i].keyword_symbol;
+        current_token_.literal = &symbol;
         current_token_.kind = keywords_[i].kind;
         return;
       }
     }
     i++;
   }
 
   // We did not read a keyword.
   current_token_.kind = Token::kIDENT;
   String& literal =
@@ skipping 619 matching lines @@
               "%c%"PRIxPTR, kPrivateKeySeparator, key_value);
   const String& result = String::Handle(String::New(private_key));
   return result.raw();
 }
 
 
 void Scanner::InitOnce() {
 }
 
 }  // namespace dart