Scanner for UTF-8 byte arrays

sdk/lib/_internal/compiler/implementation/source_file_provider.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library source_file_provider;
 
 import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
 import '../compiler.dart' as api show Diagnostic;
 import 'dart2js.dart' show AbortLeg;
 import 'colors.dart' as colors;
 import 'source_file.dart';
 import 'filenames.dart';
 import 'util/uri_extras.dart';
+import 'dart:typed_data';
 
-String readAll(String filename) {
+List<int> readAll(String filename) {
   var file = (new File(filename)).openSync();
   var length = file.lengthSync();
-  var buffer = new List<int>(length);
+  // +1 to have a 0 terminated list, see [Scanner].
+  var buffer = new Uint8List(length + 1);
   var bytes = file.readIntoSync(buffer, 0, length);
   file.closeSync();
-  return UTF8.decode(buffer);
+  return buffer;
 }
 
 class SourceFileProvider {
   bool isWindows = (Platform.operatingSystem == 'windows');
   Uri cwd = currentDirectory;
   Map<String, SourceFile> sourceFiles = <String, SourceFile>{};
   int dartCharactersRead = 0;
 
   Future<String> readStringFromUri(Uri resourceUri) {
+    return readFromUri(resourceUri).then(UTF8.decode);
+  }
+
+  Future<List<int>> readFromUri(Uri resourceUri) {
     if (resourceUri.scheme != 'file') {
       throw new ArgumentError("Unknown scheme in uri '$resourceUri'");
     }
-    String source;
+    List<int> source;
     try {
       source = readAll(uriPathToNative(resourceUri.path));
     } on FileException catch (ex) {
       return new Future.error(
           "Error reading '${relativize(cwd, resourceUri, isWindows)}' "
           "(${ex.osError})");
     }
     dartCharactersRead += source.length;
-    sourceFiles[resourceUri.toString()] = new SourceFile(
+    sourceFiles[resourceUri.toString()] = new Utf8BytesSourceFile(
         relativize(cwd, resourceUri, isWindows), source);
     return new Future.value(source);
   }
 
-  Future<String> call(Uri resourceUri) => readStringFromUri(resourceUri);
+  Future<List<int>> call(Uri resourceUri) => readFromUri(resourceUri);
 }
 
 class FormattingDiagnosticHandler {
   final SourceFileProvider provider;
   bool showWarnings = true;
   bool showHints = true;
   bool verbose = false;
   bool isAborting = false;
   bool enableColors = false;
   bool throwOnError = false;
@@ skipping 70 matching lines @@
     if (fatal && throwOnError) {
       isAborting = true;
       throw new AbortLeg(message);
     }
   }
 
   void call(Uri uri, int begin, int end, String message, api.Diagnostic kind) {
     return diagnosticHandler(uri, begin, end, message, kind);
   }
 }