Scanner for UTF-8 byte arrays

sdk/lib/_internal/compiler/implementation/source_file_provider.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library source_file_provider;
 
 import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
 import '../compiler.dart' as api show Diagnostic;
 import 'dart2js.dart' show AbortLeg;
 import 'colors.dart' as colors;
 import 'source_file.dart';
 import 'filenames.dart';
 import 'util/uri_extras.dart';
+import 'dart:typed_data';
 
-String readAll(String filename) {
+List<int> readAll(String filename) {
   var file = (new File(filename)).openSync();
   var length = file.lengthSync();
-  var buffer = new List<int>(length);
+  // +1 to have a 0 terminated list, see [Scanner].
+  var buffer = new Uint8List(length + 1);
   var bytes = file.readIntoSync(buffer, 0, length);
   file.closeSync();
-  return UTF8.decode(buffer);
+  return buffer;
 }
 
-class SourceFileProvider {
+abstract class SourceFileProvider {
   bool isWindows = (Platform.operatingSystem == 'windows');
   Uri cwd = currentDirectory;
   Map<String, SourceFile> sourceFiles = <String, SourceFile>{};
   int dartCharactersRead = 0;
 
   Future<String> readStringFromUri(Uri resourceUri) {
+    return readUtf8BytesFromUri(resourceUri).then(UTF8.decode);
+  }
+
+  Future<List<int>> readUtf8BytesFromUri(Uri resourceUri) {
     if (resourceUri.scheme != 'file') {
       throw new ArgumentError("Unknown scheme in uri '$resourceUri'");
     }
-    String source;
+    List<int> source;
     try {
       source = readAll(uriPathToNative(resourceUri.path));
     } on FileException catch (ex) {
       return new Future.error(
           "Error reading '${relativize(cwd, resourceUri, isWindows)}' "
           "(${ex.osError})");
     }
     dartCharactersRead += source.length;
-    sourceFiles[resourceUri.toString()] = new SourceFile(
+    sourceFiles[resourceUri.toString()] = new Utf8BytesSourceFile(
         relativize(cwd, resourceUri, isWindows), source);
     return new Future.value(source);
   }
 
-  Future<String> call(Uri resourceUri) => readStringFromUri(resourceUri);
+  Future/*<List<int> | String>*/ call(Uri resourceUri);
+}
+
+class CompilerSourceFileProvider extends SourceFileProvider {
+  Future<List<int>> call(Uri resourceUri) => readUtf8BytesFromUri(resourceUri);
 }
 
 class FormattingDiagnosticHandler {
   final SourceFileProvider provider;
   bool showWarnings = true;
   bool showHints = true;
   bool verbose = false;
   bool isAborting = false;
   bool enableColors = false;
   bool throwOnError = false;
   api.Diagnostic lastKind = null;
 
   final int FATAL = api.Diagnostic.CRASH.ordinal | api.Diagnostic.ERROR.ordinal;
   final int INFO =
       api.Diagnostic.INFO.ordinal | api.Diagnostic.VERBOSE_INFO.ordinal;
 
   FormattingDiagnosticHandler([SourceFileProvider provider])
       : this.provider =
-          (provider == null) ? new SourceFileProvider() : provider;
+          (provider == null) ? new CompilerSourceFileProvider() : provider;
 
   void info(var message, [api.Diagnostic kind = api.Diagnostic.VERBOSE_INFO]) {
     if (!verbose && kind == api.Diagnostic.VERBOSE_INFO) return;
     if (enableColors) {
       print('${colors.green("info:")} $message');
     } else {
       print('info: $message');
     }
   }
 
@@ skipping 51 matching lines @@
     if (fatal && throwOnError) {
       isAborting = true;
       throw new AbortLeg(message);
     }
   }
 
   void call(Uri uri, int begin, int end, String message, api.Diagnostic kind) {
     return diagnosticHandler(uri, begin, end, message, kind);
   }
 }