Add literal handling to experimental scanner.

src/lexer/experimental-scanner.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 18 matching lines @@
 #define V8_LEXER_EXPERIMENTAL_SCANNER_H
 
 #include <set>
 
 #include "compiler.h"
 #include "isolate.h"
 #include "scanner.h"  // UnicodeCache.
 #include "token.h"
 #include "utils.h"
 #include "v8stdint.h"
+#include "char-predicates-inl.h"
 
 namespace v8 {
 namespace internal {
 
 class UnicodeCache;
 
 // Base class for scanners for different encodings. The meat is the pure virtual
 // Scan() which each of them specializes.
 class ScannerBase {
  public:
   struct Location {
     Location(int b, int e) : beg_pos(b), end_pos(e) { }
     Location() : beg_pos(0), end_pos(0) { }
 
     bool IsValid() const {
       return beg_pos >= 0 && end_pos >= beg_pos;
     }
 
     static Location invalid() { return Location(-1, -1); }
 
     int beg_pos;
     int end_pos;
   };
 
   explicit ScannerBase(Isolate* isolate)
     : isolate_(isolate),
       unicode_cache_(isolate->unicode_cache()),
       has_line_terminator_before_next_(true),
+      current_literal_(&literals_[0]),
+      next_literal_(&literals_[1]),
+      octal_pos_(Location::invalid()),
       harmony_numeric_literals_(false),
       harmony_modules_(false),
       harmony_scoping_(false) {
     if (!scanners_) {
       scanners_ = new std::set<ScannerBase*>();
       isolate->heap()->AddGCEpilogueCallback(&ScannerBase::UpdateBuffersAfterGC,
                                              kGCTypeAll, false);
     }
     scanners_->insert(this);
   }
 
   virtual ~ScannerBase() {
     scanners_->erase(this);
     if (scanners_->empty()) {
       isolate_->heap()->RemoveGCEpilogueCallback(
           &ScannerBase::UpdateBuffersAfterGC);
       delete scanners_;
       scanners_ = NULL;
     }
   }
 
   // Returns the next token and advances input.
   Token::Value Next() {
     has_line_terminator_before_next_ = false;
     current_ = next_;
+    std::swap(current_literal_, next_literal_);
     Scan();  // Virtual! Will fill in next_.
     return current_.token;
   }
 
   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
 
   // Returns the location information for the current token
   // (the token last returned by Next()).
   Location location() {
@@ skipping 29 matching lines @@
   }
 
   // Returns true if there was a line terminator before the peek'ed token,
   // possibly inside a multi-line comment.
   bool HasAnyLineTerminatorBeforeNext() const {
     return has_line_terminator_before_next_;
     // FIXME: do we need to distinguish between newlines inside and outside
     // multiline comments? Atm doesn't look like we need to.
   }
 
-  // FIXME: implement these
   Vector<const char> literal_ascii_string() {
-    return Vector<const char>();  // FIXME
+    if (!current_literal_->Valid(current_.beg_pos)) {
+      FillLiteral(current_, current_literal_);
+    }
+    return current_literal_->ascii_string;
   }
+
   Vector<const uc16> literal_utf16_string() {
-    return Vector<const uc16>();  // FIXME
+    if (!current_literal_->Valid(current_.beg_pos)) {
+      FillLiteral(current_, current_literal_);
+    }
+    return current_literal_->utf16_string;
   }
+
+  int literal_length() {
+    if (!current_literal_->Valid(current_.beg_pos)) {
+      FillLiteral(current_, current_literal_);
+    }
+    return current_literal_->length;
+  }
+
   bool is_literal_ascii() {
-    return true;  // FIXME
+    if (!current_literal_->Valid(current_.beg_pos)) {
+      FillLiteral(current_, current_literal_);
+    }
+    return current_literal_->is_ascii;
   }
+
   bool is_literal_contextual_keyword(Vector<const char> keyword) {
-    return false;  // FIXME
+    if (!is_literal_ascii()) return false;
+    Vector<const char> literal = literal_ascii_string();
+    return literal.length() == keyword.length() &&
+        (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
-  int literal_length() const {
-    return 0;  // FIXME
-  }
+
   bool literal_contains_escapes() const {
-    return false;  // FIXME
+    return current_.has_escapes;
   }
 
   Vector<const char> next_literal_ascii_string() {
-    return Vector<const char>();  // FIXME
-  }
-  Vector<const uc16> next_literal_utf16_string() {
-    return Vector<const uc16>();  // FIXME
-  }
-  bool is_next_literal_ascii() {
-    return true;  // FIXME
-  }
-  bool is_next_contextual_keyword(Vector<const char> keyword) {
-    return false;  // FIXME
-  }
-  int next_literal_length() const {
-    return 0;  // FIXME
+    if (!next_literal_->Valid(next_.beg_pos)) {
+      FillLiteral(next_, next_literal_);
+    }
+    return next_literal_->ascii_string;
   }
 
-  uc32 ScanOctalEscape(uc32 c, int length) { return 0; }  // FIXME
+  Vector<const uc16> next_literal_utf16_string() {
+    if (!next_literal_->Valid(next_.beg_pos)) {
+      FillLiteral(next_, next_literal_);
+    }
+    return next_literal_->utf16_string;
+  }
 
-  Location octal_position() const {
-    return Location(0, 0);  // FIXME
+  int next_literal_length() {
+    if (!next_literal_->Valid(next_.beg_pos)) {
+      FillLiteral(next_, next_literal_);
+    }
+    return next_literal_->length;
   }
-  void clear_octal_position() { }  // FIXME
+
+  bool is_next_literal_ascii() {
+    if (!next_literal_->Valid(next_.beg_pos)) {
+      FillLiteral(next_, next_literal_);
+    }
+    return next_literal_->is_ascii;
+  }
+
+  bool is_next_contextual_keyword(Vector<const char> keyword) {
+    if (!is_next_literal_ascii()) return false;
+    Vector<const char> literal = next_literal_ascii_string();
+    return literal.length() == keyword.length() &&
+        (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
+  }
+
+  // Returns the location of the last seen octal literal.
+  Location octal_position() const { return octal_pos_; }
+  void clear_octal_position() { octal_pos_ = Location::invalid(); }
 
   // Seek forward to the given position. This operation works for simple cases
   // such as seeking forward until simple delimiter tokens, which is what it is
   // used for. After this call, we will have the token at the given position as
   // the "next" token. The "current" token will be invalid. FIXME: for utf-8,
   // we need to decide if pos is counted in characters or in bytes.
   virtual void SeekForward(int pos) = 0;
+  virtual void SetEnd(int pos) = 0;
 
   // Scans the input as a regular expression pattern, previous character(s) must
   // be /(=). Returns true if a pattern is scanned. FIXME: this won't work for
   // utf-8 newlines.
   virtual bool ScanRegExpPattern(bool seen_equal) = 0;
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   virtual bool ScanRegExpFlags() = 0;
 
  protected:
   struct TokenDesc {
     Token::Value token;
     int beg_pos;
     int end_pos;
     bool has_escapes;
   };
 
+  struct LiteralDesc {
+    int beg_pos;
+    bool is_ascii;
+    int length;
+    Vector<const char> ascii_string;
+    Vector<const uc16> utf16_string;
+    LiteralBuffer buffer;
+    bool Valid(int pos) { return beg_pos == pos; }
+  };
+
   virtual void Scan() = 0;
   virtual void SetBufferBasedOnHandle() = 0;
 
   static void UpdateBuffersAfterGC(v8::Isolate*, GCType, GCCallbackFlags);
+  virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal) = 0;
 
   Isolate* isolate_;
   UnicodeCache* unicode_cache_;
 
   bool has_line_terminator_before_next_;
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
+  LiteralDesc* current_literal_;
+  LiteralDesc* next_literal_;
+  LiteralDesc literals_[2];
+
+  Location octal_pos_;
+
   bool harmony_numeric_literals_;
   bool harmony_modules_;
   bool harmony_scoping_;
 
  private:
   static std::set<ScannerBase*>* scanners_;
 };
 
 
 template<typename Char>
 class ExperimentalScanner : public ScannerBase {
  public:
   explicit ExperimentalScanner(
       Handle<String> source,
       Isolate* isolate)
       : ScannerBase(isolate),
         source_handle_(source),
         buffer_(NULL),
         buffer_end_(NULL),
         start_(NULL),
         cursor_(NULL),
         marker_(NULL) {
     ASSERT(source->IsFlat());
     SetBufferBasedOnHandle();
     Scan();
   }
 
   virtual ~ExperimentalScanner() { }
 
+ protected:
   virtual void Scan();
   virtual void SeekForward(int pos);
+  virtual void SetEnd(int pos);
   virtual bool ScanRegExpPattern(bool seen_equal);
   virtual bool ScanRegExpFlags();
 
   virtual void SetBufferBasedOnHandle() {
     // We get a raw pointer from the Handle, but we also update it every time
     // there is a GC, so it is safe.
     DisallowHeapAllocation no_gc;
     const Char* new_buffer = GetNewBufferBasedOnHandle();
     if (new_buffer != buffer_) {
       int start_offset = start_ - buffer_;
       int cursor_offset = cursor_ - buffer_;
       int marker_offset = marker_ - buffer_;
       buffer_ = new_buffer;
       buffer_end_ = buffer_ + source_handle_->length();
       start_ = buffer_ + start_offset;
       cursor_ = buffer_ + cursor_offset;
       marker_ = buffer_ + marker_offset;
     }
   }
 
   const Char* GetNewBufferBasedOnHandle() const;
 
+  virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal);
+
  private:
   bool ValidIdentifierPart() {
       return unicode_cache_->IsIdentifierPart(ScanHexNumber(4));
   }
 
   bool ValidIdentifierStart() {
     return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
   }
 
   uc32 ScanHexNumber(int length);
   bool ScanLiteralUnicodeEscape();
 
+  const Char* ScanHexNumber(const Char* start,
+                            const Char* end,
+                            uc32* result);
+  const Char* ScanOctalEscape(const Char* start,
+                              const Char* end,
+                              uc32* result);
+  const Char* ScanIdentifierUnicodeEscape(const Char* start,
+                                          const Char* end,
+                                          uc32* result);
+  const Char* ScanEscape(const Char* start,
+                         const Char* end,
+                         LiteralBuffer* literal);
+
   Handle<String> source_handle_;
   const Char* buffer_;
   const Char* buffer_end_;
   const Char* start_;
   const Char* cursor_;
   const Char* marker_;
 };
 
 
 template<typename Char>
 void ExperimentalScanner<Char>::SeekForward(int pos) {
   cursor_ = buffer_ + pos;
   start_ = cursor_;
   marker_ = cursor_;
   has_line_terminator_before_next_ = false;
   Scan();  // Fills in next_.
 }
 
 
 template<typename Char>
+void ExperimentalScanner<Char>::SetEnd(int pos) {
+  buffer_end_ = buffer_ + pos;
+}
+
+
+template<typename Char>
 bool ExperimentalScanner<Char>::ScanRegExpPattern(bool seen_equal) {
   // Scan: ('/' | '/=') RegularExpressionBody '/' RegularExpressionFlags
   bool in_character_class = false;
 
   // Previous token is either '/' or '/=', in the second case, the
   // pattern starts at =.
   next_.beg_pos = (cursor_ - buffer_) - (seen_equal ? 2 : 1);
   next_.end_pos = (cursor_ - buffer_) - (seen_equal ? 1 : 0);
 
   // Scan regular expression body: According to ECMA-262, 3rd, 7.8.5,
@@ skipping 38 matching lines @@
       if (++cursor_ >= buffer_end_) break;
     } else {
       if (!ScanLiteralUnicodeEscape()) break;
       if (++cursor_ >= buffer_end_) break;
     }
   }
   next_.end_pos = cursor_ - buffer_ - 1;
   return true;
 }
 
+
 template<typename Char>
 uc32 ExperimentalScanner<Char>::ScanHexNumber(int length) {
   // We have seen \uXXXX, let's see what it is.
   uc32 x = 0;
   for (const Char* s = cursor_ - length; s != cursor_; ++s) {
     int d = HexValue(*s);
     if (d < 0) {
       return -1;
     }
     x = x * 16 + d;
   }
   return x;
 }
 
+
+template<typename Char>
+const Char* ExperimentalScanner<Char>::ScanHexNumber(
+    const Char* cursor, const Char* end, uc32* result) {
+  uc32 x = 0;
+  for ( ; cursor < end; ++cursor) {
+    int d = HexValue(*cursor);
+    if (d < 0) {
+      *result = -1;
+      return NULL;
+    }
+    x = x * 16 + d;
+  }
+  *result = x;
+  return cursor;
+}
+
+
+// Octal escapes of the forms '\0xx' and '\xxx' are not a part of
+// ECMA-262. Other JS VMs support them.
+template<typename Char>
+const Char* ExperimentalScanner<Char>::ScanOctalEscape(
+    const Char* start, const Char* end, uc32* result) {
+  uc32 x = *result - '0';
+  const Char* cursor;
+  for (cursor = start; cursor < end; cursor++) {
+    int d = *cursor - '0';
+    if (d < 0 || d > 7) break;
+    int nx = x * 8 + d;
+    if (nx >= 256) break;
+    x = nx;
+  }
+  // Anything except '\0' is an octal escape sequence, illegal in strict mode.
+  // Remember the position of octal escape sequences so that an error
+  // can be reported later (in strict mode).
+  // We don't report the error immediately, because the octal escape can
+  // occur before the "use strict" directive.
+  if (*result != '0' || cursor > start) {
+    octal_pos_ = Location(start - 1 - buffer_, cursor - 1 - buffer_);
+  }
+  *result = x;
+  return cursor;
+}
+
+
 template<typename Char>
 bool ExperimentalScanner<Char>::ScanLiteralUnicodeEscape() {
   ASSERT(cursor_ < buffer_end_);
   Char primary_char = *(cursor_);
   ASSERT(primary_char == '\\');
   if (++cursor_ >= buffer_end_) return false;
   primary_char = *(cursor_);
   int i = 1;
   if (primary_char == 'u') {
     i++;
     while (i < 6) {
       if (++cursor_ >= buffer_end_) return false;
       primary_char = *(cursor_);
       if (!IsHexDigit(primary_char)) break;
       i++;
     }
   }
   return i == 6;
 }
 
 
+template<typename Char>
+const Char* ExperimentalScanner<Char>::ScanIdentifierUnicodeEscape(
+    const Char* cursor, const Char* end, uc32* result) {
+  ASSERT(*cursor == '\\');
+  if (++cursor >= end) return NULL;
+  if (*cursor != 'u') return NULL;
+  ++cursor;
+  if (cursor + 4 > end) return NULL;
+  cursor = ScanHexNumber(cursor, cursor + 4, result);
+  return cursor;
+}
+
+
+template<typename Char>
+const Char* ExperimentalScanner<Char>::ScanEscape(
+    const Char* cursor, const Char* end, LiteralBuffer* literal) {
+  ASSERT(*cursor == '\\');
+  if (++cursor >= end) return NULL;
+  uc32 c = *cursor;
+  if (++cursor > end) return NULL;
+  // Skip escaped newlines.
+  if (unicode_cache_->IsLineTerminator(c)) {
+    uc32 peek = *cursor;
+    // Allow CR+LF newlines in multiline string literals.
+    if (IsCarriageReturn(c) && IsLineFeed(peek)) cursor++;
+    // Allow LF+CR newlines in multiline string literals.
+    if (IsLineFeed(c) && IsCarriageReturn(peek)) cursor++;
+    return cursor;
+  }
+
+  switch (c) {
+    case '\'':  // fall through
+    case '"' :  // fall through
+    case '\\': break;
+    case 'b' : c = '\b'; break;
+    case 'f' : c = '\f'; break;
+    case 'n' : c = '\n'; break;
+    case 'r' : c = '\r'; break;
+    case 't' : c = '\t'; break;
+    case 'u' : {
+      if (end > cursor + 4) return NULL;
+      cursor = ScanHexNumber(cursor, cursor + 4, &c);
+      if (cursor == NULL) return NULL;
+      break;
+    }
+    case 'v' : c = '\v'; break;
+    case 'x' : {
+      if (end > cursor + 2) return NULL ;
+      cursor = ScanHexNumber(cursor, cursor + 2, &c);
+      if (cursor == NULL) return NULL;
+      break;
+    }
+    case '0' :  // fall through
+    case '1' :  // fall through
+    case '2' :  // fall through
+    case '3' :  // fall through
+    case '4' :  // fall through
+    case '5' :  // fall through
+    case '6' :  // fall through
+    case '7' :
+      if (end > cursor + 2) end = cursor + 2;
+      cursor = ScanOctalEscape(cursor, end, &c); break;
+  }
+
+  // According to ECMA-262, section 7.8.4, characters not covered by the
+  // above cases should be illegal, but they are commonly handled as
+  // non-escaped characters by JS VMs.
+  literal->AddChar(c);
+  return cursor;
+}
+
+
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H