Port parser and scanner fixes from rasta branch.

pkg/dart_scanner/lib/src/utf8_bytes_scanner.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart_scanner.utf8_bytes_scanner;
 
 import 'dart:convert' show
     UNICODE_BOM_CHARACTER_RUNE,
     UTF8;
 
+import '../dart_scanner.dart' show
+    unicodeReplacementCharacter;
+
 import 'precedence.dart' show
     PrecedenceInfo;
 
 import 'token.dart' show
     StringToken,
     Token;
 
 import 'array_based_scanner.dart' show
     ArrayBasedScanner;
 
@@ skipping 77 matching lines @@
         bytes[offset + 1] == BOM_UTF8[1] &&
         bytes[offset + 2] == BOM_UTF8[2];
   }
 
   int advance() => bytes[++byteOffset];
 
   int peek() => bytes[byteOffset + 1];
 
   /**
    * Returns the unicode code point starting at the byte offset [startOffset]
    * with the byte [nextByte]. If [advance] is true the current [byteOffset]

[Johnni Winther, 2017/01/30 09:04:38]

Remove doc about [advance].

[ahe, 2017/01/30 13:26:22]

Done.

    * is advanced to the last byte of the code point.
    */
-  int nextCodePoint(int startOffset, int nextByte, bool advance) {
-    // The number of 1s in the first byte indicate the number of bytes, at
-    // least 2.
-    int numBytes = 2;
-    int bit = 0x20;
-    while ((nextByte & bit) != 0) {
+  int nextCodePoint(int startOffset, int nextByte) {
+    int expectedHighBytes;
+    if (nextByte < 0xC2) {
+      expectedHighBytes = 1; // Bad code unit.
+    } else if (nextByte < 0xE0) {
+      expectedHighBytes = 2;
+    } else if (nextByte < 0xF0) {
+      expectedHighBytes = 3;
+    } else if (nextByte < 0xF5) {
+      expectedHighBytes = 4;
+    } else {
+      expectedHighBytes = 1; // Bad code unit.
+    }
+    int numBytes = 0;
+    for (int i = 0; i < expectedHighBytes; i++) {
+      if (bytes[byteOffset + i] < 0x80) {
+        break;
+      }
       numBytes++;
-      bit >>= 1;
     }
     int end = startOffset + numBytes;
-    if (advance) {
-      byteOffset = end - 1;
+    byteOffset = end - 1;
+    if (expectedHighBytes == 1 || numBytes != expectedHighBytes) {
+      return unicodeReplacementCharacter;
     }
     // TODO(lry): measurably slow, decode creates first a Utf8Decoder and a
     // _Utf8Decoder instance. Also the sublist is eagerly allocated.
-    String codePoint = UTF8.decode(bytes.sublist(startOffset, end));
+    String codePoint =
+        UTF8.decode(bytes.sublist(startOffset, end), allowMalformed: true);
     if (codePoint.length == 0) {
       // The UTF-8 decoder discards leading BOM characters.
       // TODO(floitsch): don't just assume that removed characters were the
       // BOM.
       assert(containsBomAt(startOffset));
       codePoint = new String.fromCharCode(UNICODE_BOM_CHARACTER_RUNE);
     }
     if (codePoint.length == 1) {
-      if (advance) {
-        utf8Slack += (numBytes - 1);
-        scanSlack = numBytes - 1;
-        scanSlackOffset = byteOffset;
-      }
+      utf8Slack += (numBytes - 1);
+      scanSlack = numBytes - 1;
+      scanSlackOffset = byteOffset;
       return codePoint.codeUnitAt(0);
     } else if (codePoint.length == 2) {
-      if (advance) {
-        utf8Slack += (numBytes - 2);
-        scanSlack = numBytes - 1;
-        scanSlackOffset = byteOffset;
-        stringOffsetSlackOffset = byteOffset;
-      }
+      utf8Slack += (numBytes - 2);
+      scanSlack = numBytes - 1;
+      scanSlackOffset = byteOffset;
+      stringOffsetSlackOffset = byteOffset;
       // In case of a surrogate pair, return a single code point.
       return codePoint.runes.single;
     } else {
-      throw "Invalid UTF-8 byte sequence: ${bytes.sublist(startOffset, end)}";
+      return unicodeReplacementCharacter;
     }
   }
 
   int lastUnicodeOffset = -1;
   int currentAsUnicode(int next) {
     if (next < 128) return next;
     // Check if currentAsUnicode was already invoked.
     if (byteOffset == lastUnicodeOffset) return next;
-    int res = nextCodePoint(byteOffset, next, true);
+    int res = nextCodePoint(byteOffset, next);
     lastUnicodeOffset = byteOffset;
     return res;
   }
 
   void handleUnicode(int startScanOffset) {
     int end = byteOffset;
     // TODO(lry): this measurably slows down the scanner for files with unicode.
-    String s = UTF8.decode(bytes.sublist(startScanOffset, end));
+    String s =
+        UTF8.decode(bytes.sublist(startScanOffset, end), allowMalformed: true);
     utf8Slack += (end - startScanOffset) - s.length;
   }
 
   /**
    * This field remembers the byte offset of the last character decoded with
    * [nextCodePoint] that used two code units in UTF-16.
    *
    * [nextCodePoint] returns a single code point for each unicode character,
    * even if it needs two code units in UTF-16.
    *
@@ skipping 18 matching lines @@
 
   void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly,
       [int extraOffset = 0]) {
     tail.next = new StringToken.fromUtf8Bytes(
         info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart);
     tail = tail.next;
   }
 
   bool atEndOfFile() => byteOffset >= bytes.length - 1;
 }