Experimental parser: make utf8 sort of work

src/lexer/lexer.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 39 matching lines @@
   LexerSet lexers_;
 };
 
 
 class LexerBase {
  public:
   struct Location {
     Location(int b, int e) : beg_pos(b), end_pos(e) { }
     Location() : beg_pos(0), end_pos(0) { }
 
-    bool IsValid() const {
-      return beg_pos >= 0 && end_pos >= beg_pos;
-    }
-
+    bool IsValid() const { return beg_pos >= 0 && end_pos >= beg_pos; }
     static Location invalid() { return Location(-1, -1); }
 
     int beg_pos;
     int end_pos;
   };
 
   explicit LexerBase(UnicodeCache* unicode_cache);
 
   virtual ~LexerBase();
 
@@ skipping 39 matching lines @@
 
   // Returns true if there was a line terminator before the peek'ed token,
   // possibly inside a multi-line comment.
   bool HasAnyLineTerminatorBeforeNext() const {
     return has_line_terminator_before_next_ ||
            has_multiline_comment_before_next_;
   }
 
   Vector<const uint8_t> literal_one_byte_string() {
     EnsureCurrentLiteralIsValid();
-    return current_literal_->one_byte_string;
+    return current_literal_->one_byte_string();
   }
 
   Vector<const uint16_t> literal_two_byte_string() {
     EnsureCurrentLiteralIsValid();
-    return current_literal_->two_byte_string;
+    return current_literal_->two_byte_string();
   }
 
   int literal_length() {
     EnsureCurrentLiteralIsValid();
     return current_literal_->length;
   }
 
   bool is_literal_one_byte() {
     EnsureCurrentLiteralIsValid();
-    return current_literal_->is_one_byte;
+    return current_literal_->is_one_byte();
   }
 
   bool is_literal_contextual_keyword(Vector<const uint8_t> keyword) {
     if (!is_literal_one_byte()) return false;
     Vector<const uint8_t> literal = literal_one_byte_string();
     return literal.length() == keyword.length() &&
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
   bool literal_contains_escapes() const {
     return current_.has_escapes;
   }
 
   Vector<const uint8_t> next_literal_one_byte_string() {
     EnsureNextLiteralIsValid();
-    return next_literal_->one_byte_string;
+    return next_literal_->one_byte_string();
   }
 
   Vector<const uint16_t> next_literal_two_byte_string() {
     EnsureNextLiteralIsValid();
-    return next_literal_->two_byte_string;
+    return next_literal_->two_byte_string();
   }
 
   int next_literal_length() {
     EnsureNextLiteralIsValid();
     return next_literal_->length;
   }
 
   bool is_next_literal_one_byte() {
     EnsureNextLiteralIsValid();
-    return next_literal_->is_one_byte;
+    return next_literal_->is_one_byte();
   }
 
   bool is_next_contextual_keyword(Vector<const uint8_t> keyword) {
     if (!is_next_literal_one_byte()) return false;
     Vector<const uint8_t> literal = next_literal_one_byte_string();
     return literal.length() == keyword.length() &&
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
   bool HarmonyScoping() const {
@@ skipping 15 matching lines @@
   bool HarmonyNumericLiterals() const {
     return harmony_numeric_literals_;
   }
 
   void SetHarmonyNumericLiterals(bool numeric_literals) {
     harmony_numeric_literals_ = numeric_literals;
   }
 
   UnicodeCache* unicode_cache() { return unicode_cache_; }
 
+  class LiteralDesc {
+   public:
+    LiteralDesc()
+        : beg_pos(-1),
+          offset(0),
+          length(0),
+          is_one_byte_(false),
+          is_in_buffer_(false),
+          is_one_byte_string_owned_(false)  // TODO(dcarney): move to buffer
+    { }
+
+    ~LiteralDesc() {
+      if (is_one_byte_string_owned_) {
+        one_byte_string_.Dispose();
+      }
+    }
+
+    inline bool is_one_byte() { return is_one_byte_; }
+    inline Vector<const uint8_t> one_byte_string() {
+      ASSERT(is_one_byte_);
+      return one_byte_string_;
+    }
+    inline  Vector<const uint16_t> two_byte_string() {
+      ASSERT(!is_one_byte_);
+      return two_byte_string_;
+    }
+
+    inline bool Valid(int pos) { return beg_pos == pos; }
+    inline void Invalidate() { if (is_in_buffer_) beg_pos = -1; }
+
+    // TODO(dcarney): make private as well.
+    int beg_pos;
+    int offset;
+    int length;
+    LiteralBuffer buffer;
+
+    void SetOneByteString(Vector<const uint8_t> string, bool owned);
+    void SetTwoByteString(Vector<const uint16_t> string);
+    void SetStringFromLiteralBuffer();
+
+   private:
+    bool is_one_byte_;
+    bool is_in_buffer_;
+    bool is_one_byte_string_owned_;
+    Vector<const uint8_t> one_byte_string_;
+    Vector<const uint16_t> two_byte_string_;
+
+    DISALLOW_COPY_AND_ASSIGN(LiteralDesc);
+  };
+
  protected:
   struct TokenDesc {
-    Token::Value token;
     int beg_pos;
     int end_pos;
+    Token::Value token;
     bool has_escapes;
     bool is_onebyte;
   };
 
-  struct LiteralDesc {
-    int beg_pos;
-    bool is_one_byte;
-    bool is_in_buffer;
-    int offset;
-    int length;
-    Vector<const uint8_t> one_byte_string;
-    Vector<const uint16_t> two_byte_string;
-    LiteralBuffer buffer;
-    LiteralDesc() : beg_pos(-1), is_one_byte(false), is_in_buffer(false),
-                    offset(0), length(0) { }
-    bool Valid(int pos) { return beg_pos == pos; }
-  };
-
   virtual void Scan() = 0;
-
   virtual void UpdateBufferBasedOnHandle() = 0;
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal) = 0;
   virtual Handle<String> InternalizeLiteral(LiteralDesc* literal) = 0;
   virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
                                          PretenureFlag tenured) = 0;
 
-  void ResetLiterals() {
-    if (!current_literal_->is_in_buffer) current_literal_->beg_pos = -1;
-    if (!next_literal_->is_in_buffer) next_literal_->beg_pos = -1;
-  }
-
   void EnsureCurrentLiteralIsValid() {
     if (!current_literal_->Valid(current_.beg_pos)) {
       FillLiteral(current_, current_literal_);
     }
   }
 
   void EnsureNextLiteralIsValid() {
     if (!next_literal_->Valid(next_.beg_pos)) {
       FillLiteral(next_, next_literal_);
     }
   }
 
   UnicodeCache* unicode_cache_;
+  LiteralDesc* current_literal_;
+  LiteralDesc* next_literal_;
+  LiteralDesc literals_[2];
 
+  TokenDesc current_;  // desc for current token (as returned by Next())
+  TokenDesc next_;     // desc for next token (one token look-ahead)
+
+  // TODO(dcarney): encode flags in uint8_t
   bool has_line_terminator_before_next_;
   // Whether there is a multiline comment *with a line break* before the next
   // token.
   bool has_multiline_comment_before_next_;
-
-  TokenDesc current_;  // desc for current token (as returned by Next())
-  TokenDesc next_;     // desc for next token (one token look-ahead)
-
-  LiteralDesc* current_literal_;
-  LiteralDesc* next_literal_;
-  LiteralDesc literals_[2];
-
   bool harmony_numeric_literals_;
   bool harmony_modules_;
   bool harmony_scoping_;
 
   friend class Scanner;
   friend class LexerGCHandler;
 };
 
 
 template<typename Char>
 class Lexer : public LexerBase {
  public:
   Lexer(UnicodeCache* unicode_cache,
         Handle<String> source,
         int start_position,
         int end_position);
   Lexer(UnicodeCache* unicode_cache, const Char* source_ptr, int length);
   virtual ~Lexer();
 
   virtual void SeekForward(int pos);
   virtual bool ScanRegExpPattern(bool seen_equal);
   virtual bool ScanRegExpFlags();
   virtual Location octal_position() const;
   virtual void clear_octal_position() { last_octal_end_ = NULL; }
 
  protected:
   virtual void Scan();
 
+ private:
+  uc32 ScanHexNumber(int length);
+
+  bool ScanLiteralUnicodeEscape();
+
   const Char* GetNewBufferBasedOnHandle() const;
   virtual void UpdateBufferBasedOnHandle();
 
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal);
   virtual Handle<String> InternalizeLiteral(LiteralDesc* literal);
   virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
                                          PretenureFlag tenured);
 
- private:
-  uc32 ScanHexNumber(int length);
-
-  bool ScanLiteralUnicodeEscape();
-
-  const Char* ScanHexNumber(const Char* start,
-                            const Char* end,
-                            uc32* result);
-  const Char* ScanOctalEscape(const Char* start,
-                              const Char* end,
-                              uc32* result);
-  const Char* ScanIdentifierUnicodeEscape(const Char* start,
-                                          const Char* end,
-                                          uc32* result);
-  const Char* ScanEscape(const Char* start,
-                         const Char* end,
-                         LiteralBuffer* literal);
-
-  // Returns true if the literal of the token can be represented as a
-  // substring of the source.
-  bool IsSubstringOfSource(const TokenDesc& token);
+  // Helper function for FillLiteral.
+  template<bool is_one_byte>
+  static void SetLiteral(
+      const Char* start, const Char* end, LiteralDesc* literal);
 
   bool CopyToLiteralBuffer(const Char* start,
                            const Char* end,
                            const TokenDesc& token,
                            LiteralDesc* literal);
 
   // One of source_handle_ or source_ptr_ is set.
   // If source_ptr_ is set, isolate_ is 0 and no isolate accesses are allowed.
   Isolate* isolate_;
   const Handle<String> source_handle_;
   const Char* const source_ptr_;
-  const int start_position_;
   const int end_position_;
   // Stream variables.
   const Char* buffer_;
   const Char* buffer_end_;
   const Char* start_;
   const Char* cursor_;
   // Where we have seen the last octal number or an octal escape inside a
   // string. Used by octal_position().
   const Char* last_octal_end_;
 };
@@ skipping 126 matching lines @@
   bool harmony_scoping_;
 };
 
 
 #endif
 
 
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H