Allow identifier code points from supplementary multilingual planes.

src/scanner.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #ifndef V8_SCANNER_H_
 #define V8_SCANNER_H_
 
 #include "src/allocation.h"
@@ skipping 194 matching lines @@
   INLINE(void AddChar(uint32_t code_unit)) {
     if (position_ >= backing_store_.length()) ExpandBuffer();
     if (is_one_byte_) {
       if (code_unit <= unibrow::Latin1::kMaxChar) {
         backing_store_[position_] = static_cast<byte>(code_unit);
         position_ += kOneByteSize;
         return;
       }
       ConvertToTwoByte();
     }
-    DCHECK(code_unit < 0x10000u);
-    *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit;
-    position_ += kUC16Size;
+    if (code_unit <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
+      *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit;
+      position_ += kUC16Size;
+    } else {
+      *reinterpret_cast<uint16_t*>(&backing_store_[position_]) =
+          unibrow::Utf16::LeadSurrogate(code_unit);
+      position_ += kUC16Size;
+      *reinterpret_cast<uint16_t*>(&backing_store_[position_]) =
+          unibrow::Utf16::TrailSurrogate(code_unit);
+      position_ += kUC16Size;
+    }
   }
 
   bool is_one_byte() const { return is_one_byte_; }
 
   bool is_contextual_keyword(Vector<const char> keyword) const {
     return is_one_byte() && keyword.length() == position_ &&
         (memcmp(keyword.start(), backing_store_.start(), position_) == 0);
   }
 
   Vector<const uint16_t> two_byte_literal() const {
@@ skipping 284 matching lines @@
   inline void DropLiteral() {
     next_.literal_chars = NULL;
   }
 
   inline void AddLiteralCharAdvance() {
     AddLiteralChar(c0_);
     Advance();
   }
 
   // Low-level scanning support.
-  void Advance() { c0_ = source_->Advance(); }
+  void Advance() {
+    c0_ = source_->Advance();
+    if (unibrow::Utf16::IsLeadSurrogate(c0_)) {

[marja, 2015/01/29 15:39:38]

Looks like code load doesn't like adding this branch to Advance. For more info,
-> verwaest@.

+      uc32 c1 = source_->Advance();
+      if (!unibrow::Utf16::IsTrailSurrogate(c1)) {
+        source_->PushBack(c1);
+      } else {
+        c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
+      }
+    }
+  }
+
   void PushBack(uc32 ch) {
-    source_->PushBack(c0_);
+    if (ch > static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
+      source_->PushBack(unibrow::Utf16::TrailSurrogate(c0_));
+      source_->PushBack(unibrow::Utf16::LeadSurrogate(c0_));
+    } else {
+      source_->PushBack(c0_);
+    }
     c0_ = ch;
   }
 
   inline Token::Value Select(Token::Value tok) {
     Advance();
     return tok;
   }
 
   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
     Advance();
@@ skipping 119 matching lines @@
   bool harmony_modules_;
   // Whether we scan 0o777 and 0b111 as numbers.
   bool harmony_numeric_literals_;
   // Whether we scan 'class', 'extends', 'static' and 'super' as keywords.
   bool harmony_classes_;
 };
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_