Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(556)

Unified Diff: sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart

Issue 137343002: Handle BOMs in dart2js. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Address comments. Created 6 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « sdk/lib/_internal/compiler/implementation/scanner/scannerlib.dart ('k') | tests/language/language.status » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart
diff --git a/sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart b/sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart
index 62722ea6f7d0357831bf497801eca8244806f53f..c2adeb41672d726ff10b7d4a3a8c3f31a7cc0189 100644
--- a/sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart
+++ b/sdk/lib/_internal/compiler/implementation/scanner/utf8_bytes_scanner.dart
@@ -69,6 +69,8 @@ class Utf8BytesScanner extends ArrayBasedScanner {
: bytes = file.slowUtf8Bytes(),
super(file, includeComments) {
ensureZeroTermination();
+ // Skip a leading BOM.
+ if (_containsBomAt(0)) byteOffset += 3;
}
/**
@@ -95,6 +97,15 @@ class Utf8BytesScanner extends ArrayBasedScanner {
}
}
+ bool _containsBomAt(int offset) {
+ const BOM_UTF8 = const [0xEF, 0xBB, 0xBF];
+
+ return offset + 3 < bytes.length &&
+ bytes[offset] == BOM_UTF8[0] &&
+ bytes[offset + 1] == BOM_UTF8[1] &&
+ bytes[offset + 2] == BOM_UTF8[2];
+ }
+
int advance() => bytes[++byteOffset];
int peek() => bytes[byteOffset + 1];
@@ -120,6 +131,13 @@ class Utf8BytesScanner extends ArrayBasedScanner {
// TODO(lry): measurably slow, decode creates first a Utf8Decoder and a
// _Utf8Decoder instance. Also the sublist is eagerly allocated.
String codePoint = UTF8.decode(bytes.sublist(startOffset, end));
+ if (codePoint.length == 0) {
+ // The UTF-8 decoder discards leading BOM characters.
+ // TODO(floitsch): don't just assume that removed characters were the
+ // BOM.
+ assert(_containsBomAt(startOffset));
+ codePoint = new String.fromCharCode(UNICODE_BOM_CHARACTER_RUNE);
+ }
if (codePoint.length == 1) {
if (advance) {
utf8Slack += (numBytes - 1);
« no previous file with comments | « sdk/lib/_internal/compiler/implementation/scanner/scannerlib.dart ('k') | tests/language/language.status » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698