Make the parser agnostic to the TokenStream implementation. This is the first step towards compacti…

vm/parser.h

Index: vm/parser.h
===================================================================
--- vm/parser.h	(revision 8936)
+++ vm/parser.h	(working copy)
@@ -105,6 +105,34 @@
 };
 
 
+// The class TokenStreamIterator encapsulates iteration over the TokenStream
+// object. The parser uses this to iterate over tokens while parsing a script
+// or a function.
+class TokenStreamIterator : ValueObject {
+ public:
+  TokenStreamIterator(const TokenStream& tokens, intptr_t token_index)
+      : tokens_(tokens), token_index_(token_index) { }
+
+  intptr_t Length() const { return tokens_.Length(); }

[hausner, 2012/06/22 00:02:27]

What is the "Length" of an iterator? Do you mean the index of the token at the
current token position? Or the number of tokens iterated over since the iterator
was initialized?

[siva, 2012/06/22 17:59:02]

Renamed Length() to NumberOfTokens() basically it is the number of tokens in
this stream.

On 2012/06/22 00:02:27, hausner wrote:

+  bool IsNull() const;
+
+  inline Token::Kind CurrentTokenKind() const;
+  Token::Kind LookaheadTokenKind(intptr_t num_tokens) const;
+
+  intptr_t CurrentIndex() const { return token_index_; }

[hausner, 2012/06/22 00:02:27]

I would not call it an index anymore. What you abstract is a position on the
stream, not a particular index. They happen to be the same currently, but will
be different once the tokens become variable length.

[siva, 2012/06/22 17:59:02]

Renamed this to CurrentPosition

On 2012/06/22 00:02:27, hausner wrote:

+  void SetCurrentIndex(intptr_t value) { token_index_ = value; }
+
+  void AdvanceToNextToken() { token_index_ += 1; }
+
+  RawObject* CurrentToken() const;
+  RawString* CurrentLiteral() const;
+
+ private:
+  const TokenStream& tokens_;
+  intptr_t token_index_;
+};
+
+
 class Parser : ValueObject {
  public:
   Parser(const Script& script, const Library& library);
@@ -170,6 +198,7 @@
         (script_.kind() == RawScript::kLibrary);
   }
 
+  intptr_t TokenIndex() const { return tokens_iterator_.CurrentIndex(); }
   inline Token::Kind CurrentToken();
   Token::Kind LookaheadToken(int num_tokens);
   String* CurrentLiteral() const;
@@ -182,7 +211,7 @@
   void ConsumeToken() {
     // Reset cache and advance the token.
     token_kind_ = Token::kILLEGAL;
-    token_index_++;
+    tokens_iterator_.AdvanceToNextToken();
     CompilerStats::num_tokens_consumed++;
   }
   void ConsumeRightAngleBracket();
@@ -500,9 +529,8 @@
 
 
   const Script& script_;
-  const TokenStream& tokens_;
-  intptr_t token_index_;
-  Token::Kind token_kind_;  // Cached token kind for the token_index_.
+  TokenStreamIterator tokens_iterator_;
+  Token::Kind token_kind_;  // Cached token kind for current token.
   Block* current_block_;
 
   // is_top_level_ is true if parsing the "top level" of a compilation unit,