Fix inconsistencies wrt whitespaces.

src/scanner.h

Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index 3cefc833ac3323c536ca98015fdfbee1522060d5..9b5a3c5e0f78b47b62c79e4e75074b0f0b4db15c 100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -138,7 +138,10 @@ class UnicodeCache {
   bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); }
   bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); }
   bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); }
-  bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); }
+  bool IsWhiteSpace(unibrow::uchar c) {
+    // BYTE ORDER MARK is considered a white space in ECMA-262 5.1, 7.2.
+    return kIsWhiteSpace.get(c) || c == 0xFEFF;

[Michael Starzinger, 2014/02/07 10:02:39]

This is kind of brittle! Can we please move this special casing into a
char-predicate (e.g struct WhiteSpace) and use that predicate in the cache
instead of special-casing after the cache lookup?

+  }
 
  private:
   unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;