Experimentar scanner: Unify the API to Scanner API some more.

src/lexer/experimental-scanner.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 66 matching lines @@
     int end_pos;
   };
 
   explicit ExperimentalScanner(
       YYCTYPE* source,
       YYCTYPE* source_end,
       Isolate* isolate);
 
   ~ExperimentalScanner();
 
+  // Returns the next token and advances input.
   Token::Value Next() {
+    has_line_terminator_before_next_ = false;
     current_ = next_;
     Scan();  // will fill in next_.
     return current_.token;
   }
 
+  // Returns the current token again.
+  Token::Value current_token() { return current_.token; }
+
+  // Returns the location information for the current token
+  // (the token last returned by Next()).
   Location location() {
     return Location(current_.beg_pos, current_.end_pos);
   }
 
+  // One token look-ahead (past the token returned by Next()).
+  Token::Value peek() const { return next_.token; }
+
+  Location peek_location() const { return next_.location; }
+
+  UnicodeCache* unicode_cache() { return unicode_cache_; }
+
+  bool HarmonyScoping() const {
+    return harmony_scoping_;
+  }
+  void SetHarmonyScoping(bool scoping) {
+    harmony_scoping_ = scoping;
+  }
+  bool HarmonyModules() const {
+    return harmony_modules_;
+  }
+  void SetHarmonyModules(bool modules) {
+    harmony_modules_ = modules;
+  }
+  bool HarmonyNumericLiterals() const {
+    return harmony_numeric_literals_;
+  }
   void SetHarmonyNumericLiterals(bool numeric_literals) {
     harmony_numeric_literals_ = numeric_literals;
   }
 
-  void SetHarmonyModules(bool modules) {
-    harmony_modules_ = modules;
-  }
-
-  void SetHarmonyScoping(bool scoping) {
-    harmony_scoping_ = scoping;
+  // Returns true if there was a line terminator before the peek'ed token,
+  // possibly inside a multi-line comment.
+  bool HasAnyLineTerminatorBeforeNext() const {
+    return has_line_terminator_before_next_;
+    // FIXME: do we need to distinguish between newlines inside and outside
+    // multiline comments? Atm doesn't look like we need to.
   }
 
  private:
   struct TokenDesc {
     Token::Value token;
     int beg_pos;
     int end_pos;
     LiteralBuffer* literal_chars;
   };
 
   void Scan();
 
   bool ValidIdentifierStart();
   bool ValidIdentifierPart();
   uc32 ScanHexNumber(int length);
 
   UnicodeCache* unicode_cache_;
 
   YYCTYPE* buffer_;
   YYCTYPE* buffer_end_;
   YYCTYPE* start_;
   YYCTYPE* cursor_;
   YYCTYPE* marker_;
-  bool just_seen_line_terminator_;
+  bool has_line_terminator_before_next_;
 
   YYCTYPE yych;
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
   bool harmony_numeric_literals_;
   bool harmony_modules_;
   bool harmony_scoping_;
 };
 
 const byte* ReadFile(const char* name, Isolate* isolate, int* size, int repeat);
 
 template<typename YYCTYPE>
 ExperimentalScanner<YYCTYPE>::ExperimentalScanner(
     YYCTYPE* source,
     YYCTYPE* source_end,
     Isolate* isolate)
     : unicode_cache_(isolate->unicode_cache()),
-      just_seen_line_terminator_(true),
+      has_line_terminator_before_next_(true),
       harmony_numeric_literals_(false),
       harmony_modules_(false),
       harmony_scoping_(false) {
   buffer_ = source;
   buffer_end_ = source_end;
   start_ = buffer_;
   cursor_ = buffer_;
   marker_ = buffer_;
   Scan();
 }
@@ skipping 29 matching lines @@
 
 
 template<typename YYCTYPE>
 bool ExperimentalScanner<YYCTYPE>::ValidIdentifierStart() {
   return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
 }
 
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H