Handle BOMs in dart2js.

sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of scanner;
 
 /**
  * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens
  * that points to substrings.
  */
@@ skipping 51 matching lines @@
    * string text of the source file is decoded.
    *
    * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an
    * array whose last element is '0' to signal the end of the file. If this
    * is not the case, the entire array is copied before scanning.
    */
   Utf8BytesScanner(SourceFile file, {bool includeComments: false})
       : bytes = file.slowUtf8Bytes(),
         super(file, includeComments) {
     ensureZeroTermination();
+    // Skip a leading BOM.
+    if (_containsBomAt(0)) byteOffset += 3;
   }
 
   /**
    * Creates a new Utf8BytesScanner from a list of UTF-8 bytes.
    *
    * The last element of the list is expected to be '0' to signal the end of
    * the file. If this is not the case, the entire array is copied before
    * scanning.
    */
   Utf8BytesScanner.fromBytes(this.bytes, {bool includeComments: false})
       : super(null, includeComments) {
     ensureZeroTermination();
   }
 
   void ensureZeroTermination() {
     if (bytes.isEmpty || bytes[bytes.length - 1] != 0) {
       // TODO(lry): abort instead of copying the array, or warn?
       var newBytes =  new Uint8List(bytes.length + 1);
       for (int i = 0; i < bytes.length; i++) {
         newBytes[i] = bytes[i];
       }
       newBytes[bytes.length] = 0;
       bytes = newBytes;
     }
   }
 
+  bool _containsBomAt(int offset) {
+    const BOM_UTF8 = const [0xEF, 0xBB, 0xBF];
+
+    return offset + 3 < bytes.length &&
+        bytes[offset] == BOM_UTF8[0] &&
+        bytes[offset + 1] == BOM_UTF8[1] &&
+        bytes[offset + 2] == BOM_UTF8[2];
+  }
+
   int advance() => bytes[++byteOffset];
 
   int peek() => bytes[byteOffset + 1];
 
   /**
    * Returns the unicode code point starting at the byte offset [startOffset]
    * with the byte [nextByte]. If [advance] is true the current [byteOffset]
    * is advanced to the last byte of the code point.
    */
   int nextCodePoint(int startOffset, int nextByte, bool advance) {
     // The number of 1s in the first byte indicate the number of bytes, at
     // least 2.
     int numBytes = 2;
     int bit = 0x20;
     while ((nextByte & bit) != 0) {
       numBytes++;
       bit >>= 1;
     }
     int end = startOffset + numBytes;
     if (advance) {
       byteOffset = end - 1;
     }
     // TODO(lry): measurably slow, decode creates first a Utf8Decoder and a
     // _Utf8Decoder instance. Also the sublist is eagerly allocated.
     String codePoint = UTF8.decode(bytes.sublist(startOffset, end));
+    if (codePoint.length == 0) {
+      // The UTF-8 decoder discards leading BOM characters.
+      // TODO(floitsch): don't just assume that removed characters were the
+      // BOM.
+      assert(_containsBomAt(startOffset));
+      codePoint = new String.fromCharCode(UNICODE_BOM_CHARACTER_RUNE);
+    }
     if (codePoint.length == 1) {
       if (advance) {
         utf8Slack += (numBytes - 1);
         scanSlack = numBytes - 1;
         scanSlackOffset = byteOffset;
       }
       return codePoint.codeUnitAt(0);
     } else if (codePoint.length == 2) {
       if (advance) {
         utf8Slack += (numBytes - 2);
@@ skipping 53 matching lines @@
 
   void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly,
                             [int extraOffset = 0]) {
     tail.next = new StringToken.fromUtf8Bytes(
         info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart);
     tail = tail.next;
   }
 
   bool atEndOfFile() => byteOffset >= bytes.length - 1;
 }