Eliminate GrowableTokenStream

runtime/vm/scanner.h

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 //
 // Scanner class for the Dart language. The scanner reads source text
 // and produces a stream of tokens which is used by the parser.
 //
 
 #ifndef VM_SCANNER_H_
 #define VM_SCANNER_H_
 
 #include "vm/growable_array.h"
 #include "vm/token.h"
 
 namespace dart {
 
 // Forward declarations.
 class Array;
 class Library;
 class RawString;
 class ScanContext;
 class String;
 
 // A call to Scan() scans the source one token at at time.
-// The scanned token is returned by cur_token().
+// The scanned token is returned by current_token().
 // GetStream() scans the entire source text and returns a stream of tokens.
 class Scanner : ValueObject {
  public:
   typedef uint16_t (*CharAtFunc)(const String& str, intptr_t index);
 
   // SourcePosition describes a text location in user friendly
   // terms of line number and column.
   struct SourcePosition {
     int line;
     int column;
   };
 
   // TokenDesc defines the kind of a token and its location in
   // the source text.
   struct TokenDescriptor {
     Token::Kind kind;
     int offset;               // Offset in source string.
     SourcePosition position;  // Text position in source.
     const String* literal;    // Identifier, number or string literal.
   };
 
-  typedef ZoneGrowableArray<TokenDescriptor> GrowableTokenStream;
+  class TokenCollector : public ValueObject {
+   public:
+    TokenCollector() { }
+    virtual ~TokenCollector() { }
+    virtual void AddToken(const TokenDescriptor& token);
+   private:
+    DISALLOW_COPY_AND_ASSIGN(TokenCollector);
+  };
 
   // Initializes scanner to scan string source.
   Scanner(const String& source, const String& private_key);
   ~Scanner();
 
   // Scans one token at a time.
   void Scan();
 
+  // Scans the entire source and collects tokens in the provided collector.
+  void ScanAll(TokenCollector* collector);
+
   // Scans to specified token position.
   // Use CurrentPosition() to extract line and column number.
   void ScanTo(intptr_t token_index);
 
-  // Scans entire source and returns a stream of tokens.
-  // Should be called only once.
-  const GrowableTokenStream& GetStream();
-
   // Info about most recently recognized token.
   const TokenDescriptor& current_token() const { return current_token_; }
 
   // Was there a line break before the current token?
   bool NewlineBeforeToken() const { return newline_seen_; }
 
   // Source code line number and column of current token.
   const SourcePosition& CurrentPosition() const {
     return current_token_.position;
   }
 
   static void InitOnce();
 
   // Return true if str is an identifier.
   bool IsIdent(const String& str);
 
-  // Does the token stream contain a valid literal. This is used to implement
-  // the Dart methods int.parse and double.parse.
-  static bool IsValidLiteral(const Scanner::GrowableTokenStream& tokens,
-                             Token::Kind literal_kind,
+  // Does the token stream contain a valid integer literal.
+  static bool IsValidInteger(const String& str,
                              bool* is_positive,
                              const String** value);
 
  private:
   friend class ScanContext;
 
   static const int kNumLowercaseChars = 26;
 
   struct KeywordTable {
     Token::Kind kind;
@@ skipping 14 matching lines @@
   // Recognizes token 'kind' and reads next character in input.
   void Recognize(Token::Kind kind) {
     ReadChar();
     current_token_.kind = kind;
   }
 
   int32_t LookaheadChar(int how_many);
 
   void ErrorMsg(const char* msg);
 
-  // Scans entire source into a given stream of tokens.
-  void ScanAll(GrowableTokenStream* token_stream);
-
   // These functions return true if the given character is a letter,
   // a decimal digit, a hexadecimal digit, etc.
   static bool IsLetter(int32_t c);
   static bool IsDecimalDigit(int32_t c);
   static bool IsNumberStart(int32_t);
   static bool IsHexDigit(int32_t c);
   static bool IsIdentStartChar(int32_t c);
   static bool IsIdentChar(int32_t c);
 
   // Skips up to next non-whitespace character.
@@ skipping 44 matching lines @@
   // Reads a number literal.
   void ScanNumber(bool dec_point_seen);
 
   void ScanScriptTag();
 
   CharAtFunc CallCharAt() const { return char_at_func_; }
 
   Thread* thread() const { return thread_; }
   Zone* zone() const { return zone_; }
 
-  static void PrintTokens(const GrowableTokenStream& ts);
-
   TokenDescriptor current_token_;  // Current token.
   TokenDescriptor newline_token_;  // Newline token.
   TokenDescriptor empty_string_token_;  // Token for "".
   const String& source_;           // The source text being tokenized.
   intptr_t source_length_;         // The length of the source text.
   intptr_t lookahead_pos_;         // Position of lookahead character
                                    // within source_.
   intptr_t token_start_;           // Begin of current token in src_.
   int32_t c0_;                     // Lookahead character.
   bool newline_seen_;              // Newline before current token.
@@ skipping 16 matching lines @@
   Zone* zone_;
 
   static KeywordTable keywords_[Token::kNumKeywords];
   static int keywords_char_offset_[kNumLowercaseChars];
 };
 
 
 }  // namespace dart
 
 #endif  // VM_SCANNER_H_