Changes to shrink the token stream representation from two words to one word.

vm/raw_object.h

Index: vm/raw_object.h
===================================================================
--- vm/raw_object.h	(revision 4579)
+++ vm/raw_object.h	(working copy)
@@ -7,6 +7,7 @@
 
 #include "platform/assert.h"
 #include "vm/globals.h"
+#include "vm/token.h"
 #include "vm/snapshot.h"
 
 #include "include/dart_api.h"
@@ -26,6 +27,7 @@
     V(InstantiatedTypeArguments)                                               \
   V(Function)                                                                  \
   V(Field)                                                                     \
+  V(LiteralToken)                                                              \
   V(TokenStream)                                                               \
   V(Script)                                                                    \
   V(Library)                                                                   \
@@ -527,12 +529,28 @@
 };
 
 
+class RawLiteralToken : public RawObject {
+  RAW_HEAP_OBJECT_IMPLEMENTATION(LiteralToken);
+
+  RawObject** from() {
+    return reinterpret_cast<RawObject**>(&ptr()->literal_);
+  }
+  RawString* literal_;  // The actual token string.

[hausner, 2012/02/25 00:41:57]

// literal characters as they appear in source text.

[siva, 2012/02/28 19:57:31]

Done.

+  RawObject* value_;  // The actual object corresponding to the token.
+  RawObject** to() {
+    return reinterpret_cast<RawObject**>(&ptr()->value_);
+  }
+  Token::Kind kind_;  // The literal kind (string, integer, double).
+
+  friend class SnapshotReader;
+};
+
+
 class RawTokenStream : public RawObject {
   RAW_HEAP_OBJECT_IMPLEMENTATION(TokenStream);
 
   enum {
-    kKindEntry = 0,
-    kLiteralEntry,
+    kTokenEntry = 0,
     kNumberOfEntries
   };