Merged bleeding edge 746:795 into regexp2000.

src/scanner.cc

Index: src/scanner.cc
===================================================================
--- src/scanner.cc	(revision 796)
+++ src/scanner.cc	(working copy)
@@ -234,11 +234,25 @@
 }
 
 
+static inline bool IsByteOrderMark(uc32 c) {
+  // The Unicode value U+FFFE is guaranteed never to be assigned as a
+  // Unicode character; this implies that in a Unicode context the
+  // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
+  // character expressed in little-endian byte order (since it could
+  // not be a U+FFFE character expressed in big-endian byte
+  // order). Nevertheless, we check for it to be compatible with
+  // Spidermonkey.
+  return c == 0xFEFF || c == 0xFFFE;
+}
+
+
 void Scanner::SkipWhiteSpace(bool initial) {
   has_line_terminator_before_next_ = initial;
 
   while (true) {
-    while (kIsWhiteSpace.get(c0_)) {
+    // We treat byte-order marks (BOMs) as whitespace for better
+    // compatibility with Spidermonkey and other JavaScript engines.
+    while (kIsWhiteSpace.get(c0_) || IsByteOrderMark(c0_)) {
       // IsWhiteSpace() includes line terminators!
       if (kIsLineTerminator.get(c0_))
         // Ignore line terminators, but remember them. This is necessary