Make the parser agnostic to the TokenStream implementation. This is the first step towards compacti…

vm/parser.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_PARSER_H_
 #define VM_PARSER_H_
 
 #include "include/dart_api.h"
 
 #include "vm/ast.h"
@@ skipping 87 matching lines @@
 
   int first_parameter_index_;
   int first_stack_local_index_;
   int copied_parameter_count_;
   int stack_local_count_;
 
   DISALLOW_COPY_AND_ASSIGN(ParsedFunction);
 };
 
 
+// The class TokenStreamIterator encapsulates iteration over the TokenStream
+// object. The parser uses this to iterate over tokens while parsing a script
+// or a function.
+class TokenStreamIterator : ValueObject {
+ public:
+  TokenStreamIterator(const TokenStream& tokens, intptr_t token_index)
+      : tokens_(tokens), token_index_(token_index) { }
+
+  intptr_t Length() const { return tokens_.Length(); }

[hausner, 2012/06/22 00:02:27]

What is the "Length" of an iterator? Do you mean the index of the token at the
current token position? Or the number of tokens iterated over since the iterator
was initialized?

[siva, 2012/06/22 17:59:02]

Renamed Length() to NumberOfTokens() basically it is the number of tokens in
this stream.

On 2012/06/22 00:02:27, hausner wrote:

+  bool IsNull() const;
+
+  inline Token::Kind CurrentTokenKind() const;
+  Token::Kind LookaheadTokenKind(intptr_t num_tokens) const;
+
+  intptr_t CurrentIndex() const { return token_index_; }

[hausner, 2012/06/22 00:02:27]

I would not call it an index anymore. What you abstract is a position on the
stream, not a particular index. They happen to be the same currently, but will
be different once the tokens become variable length.

[siva, 2012/06/22 17:59:02]

Renamed this to CurrentPosition

On 2012/06/22 00:02:27, hausner wrote:

+  void SetCurrentIndex(intptr_t value) { token_index_ = value; }
+
+  void AdvanceToNextToken() { token_index_ += 1; }
+
+  RawObject* CurrentToken() const;
+  RawString* CurrentLiteral() const;
+
+ private:
+  const TokenStream& tokens_;
+  intptr_t token_index_;
+};
+
+
 class Parser : ValueObject {
  public:
   Parser(const Script& script, const Library& library);
   Parser(const Script& script,
          const Function& function,
          intptr_t token_index);
 
   // Parse the top level of a whole script file and register declared classes
   // and interfaces in the given library.
   static void ParseCompilationUnit(const Library& library,
@@ skipping 45 matching lines @@
   // The class or interface being parsed.
   const Class& current_class() const;
   void set_current_class(const Class& value);
 
   // Parsing a library or a regular source script.
   bool is_library_source() const {
     return (script_.kind() == RawScript::kScript) ||
         (script_.kind() == RawScript::kLibrary);
   }
 
+  intptr_t TokenIndex() const { return tokens_iterator_.CurrentIndex(); }
   inline Token::Kind CurrentToken();
   Token::Kind LookaheadToken(int num_tokens);
   String* CurrentLiteral() const;
   RawDouble* CurrentDoubleLiteral() const;
   RawInteger* CurrentIntegerLiteral() const;
 
   // Sets parser to given token position in the stream.
   void SetPosition(intptr_t position);
 
   void ConsumeToken() {
     // Reset cache and advance the token.
     token_kind_ = Token::kILLEGAL;
-    token_index_++;
+    tokens_iterator_.AdvanceToNextToken();
     CompilerStats::num_tokens_consumed++;
   }
   void ConsumeRightAngleBracket();
   void ExpectToken(Token::Kind token_expected);
   void ExpectSemicolon();
   void UnexpectedToken();
   String* ExpectTypeIdentifier(const char* msg);
   String* ExpectIdentifier(const char* msg);
   bool IsLiteral(const char* literal);
 
@@ skipping 297 matching lines @@
   AstNode* InsertClosureCallNodes(AstNode* condition);
 
   ConstructorCallNode* CreateConstructorCallNode(
       intptr_t token_index,
       const AbstractTypeArguments& type_arguments,
       const Function& constructor,
       ArgumentListNode* arguments);
 
 
   const Script& script_;
-  const TokenStream& tokens_;
-  intptr_t token_index_;
-  Token::Kind token_kind_;  // Cached token kind for the token_index_.
+  TokenStreamIterator tokens_iterator_;
+  Token::Kind token_kind_;  // Cached token kind for current token.
   Block* current_block_;
 
   // is_top_level_ is true if parsing the "top level" of a compilation unit,
   // that is interface and class definitions.
   bool is_top_level_;
 
   // The member currently being parsed during "top level" parsing.
   MemberDesc* current_member_;
 
   // Parser mode to allow/disallow function literals. This is used in
@@ skipping 18 matching lines @@
 
   // Allocate temporary only once per function.
   LocalVariable* expression_temp_;
 
   DISALLOW_COPY_AND_ASSIGN(Parser);
 };
 
 }  // namespace dart
 
 #endif  // VM_PARSER_H_