Changes to shrink the token stream representation from two words to one word.

vm/scanner.cc

Index: vm/scanner.cc
===================================================================
--- vm/scanner.cc	(revision 4579)
+++ vm/scanner.cc	(working copy)
@@ -7,6 +7,7 @@
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
+#include "vm/object_store.h"
 #include "vm/thread.h"
 #include "vm/token.h"
 
@@ -20,7 +21,6 @@
     keywords_[i].kind = token;
     keywords_[i].keyword_chars = Token::Str(token);
     keywords_[i].keyword_len = strlen(Token::Str(token));
-    keywords_[i].keyword_symbol = NULL;
   }
 }
 
@@ -259,11 +259,14 @@
         char_pos++;
       }
       if (char_pos == ident_length) {
-        if (keywords_[i].keyword_symbol == NULL) {
-          keywords_[i].keyword_symbol = &String::ZoneHandle(
-              String::NewSymbol(source_, ident_pos, ident_length));
+        ObjectStore* object_store = Isolate::Current()->object_store();
+        String& symbol = String::ZoneHandle();

[hausner, 2012/02/25 00:41:57]

NB: for each keyword occurrence in the source text we now call Isolate::Current
and allocate a zone handle. Before, we only allocated one handle for each
keyword kind. Hopefully not a big deal. 

[siva, 2012/02/28 19:57:31]

I have redone this code a bit to address the concern you had about handle
creation, please take another look.

On 2012/02/25 00:41:57, hausner wrote:

[hausner, 2012/02/29 00:22:37]

Nice, thank you.

On 2012/02/28 19:57:31, asiva wrote:

+        symbol ^= object_store->GetKeywordSymbol(i);
+        if (symbol.IsNull()) {
+          symbol = String::NewSymbol(source_, ident_pos, ident_length);
+          object_store->SetKeywordSymbol(i, symbol);
         }
-        current_token_.literal = keywords_[i].keyword_symbol;
+        current_token_.literal = &symbol;
         current_token_.kind = keywords_[i].kind;
         return;
       }