Allow identifier code points from supplementary multilingual planes.

src/scanner.h

Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index aeadf9da23b411c966790f6e699ad04e32fcd9b9..d40c6266b729b7edc5a97b1e4057de42a93a0d04 100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -212,9 +212,17 @@ class LiteralBuffer {
       }
       ConvertToTwoByte();
     }
-    DCHECK(code_unit < 0x10000u);
-    *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit;
-    position_ += kUC16Size;
+    if (code_unit <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
+      *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit;
+      position_ += kUC16Size;
+    } else {
+      *reinterpret_cast<uint16_t*>(&backing_store_[position_]) =
+          unibrow::Utf16::LeadSurrogate(code_unit);
+      position_ += kUC16Size;
+      *reinterpret_cast<uint16_t*>(&backing_store_[position_]) =
+          unibrow::Utf16::TrailSurrogate(code_unit);
+      position_ += kUC16Size;
+    }
   }
 
   bool is_one_byte() const { return is_one_byte_; }
@@ -519,9 +527,25 @@ class Scanner {
   }
 
   // Low-level scanning support.
-  void Advance() { c0_ = source_->Advance(); }
+  void Advance() {
+    c0_ = source_->Advance();
+    if (unibrow::Utf16::IsLeadSurrogate(c0_)) {

[marja, 2015/01/29 15:39:38]

Looks like code load doesn't like adding this branch to Advance. For more info,
-> verwaest@.

+      uc32 c1 = source_->Advance();
+      if (!unibrow::Utf16::IsTrailSurrogate(c1)) {
+        source_->PushBack(c1);
+      } else {
+        c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
+      }
+    }
+  }
+
   void PushBack(uc32 ch) {
-    source_->PushBack(c0_);
+    if (ch > static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
+      source_->PushBack(unibrow::Utf16::TrailSurrogate(c0_));
+      source_->PushBack(unibrow::Utf16::LeadSurrogate(c0_));
+    } else {
+      source_->PushBack(c0_);
+    }
     c0_ = ch;
   }