Scanner for UTF-8 byte arrays

sdk/lib/_internal/compiler/implementation/scanner/scanner_task.dart

Index: sdk/lib/_internal/compiler/implementation/scanner/scanner_task.dart
diff --git a/sdk/lib/_internal/compiler/implementation/scanner/scanner_task.dart b/sdk/lib/_internal/compiler/implementation/scanner/scanner_task.dart
index 765f49eade5d0892511758a4085baeed8869b6f1..869e20e369c7e37ac00ea2d81cd225c5efb2c282 100644
--- a/sdk/lib/_internal/compiler/implementation/scanner/scanner_task.dart
+++ b/sdk/lib/_internal/compiler/implementation/scanner/scanner_task.dart
@@ -28,7 +28,7 @@ class ScannerTask extends CompilerTask {
 
   void scanElements(CompilationUnitElement compilationUnit) {
     Script script = compilationUnit.script;
-    Token tokens = new StringScanner(script.text,
+    Token tokens = new Scanner(script.file,
         includeComments: compiler.preserveComments).tokenize();
     if (compiler.preserveComments) {
       tokens = compiler.processAndStripComments(tokens);
@@ -36,9 +36,17 @@ class ScannerTask extends CompilerTask {
     compiler.dietParser.dietParse(compilationUnit, tokens);
   }
 
+  /**
+   * Returns the tokens for the [source].
+   *
+   * The [StringScanner] implementation works on strings that end with a '0'
+   * value ('\x00'). If [source] does not with '0', the string is copied before

[ngeoffray, 2013/10/18 10:19:37]

not *end* with

[lukas, 2013/10/24 16:48:36]

Done.

+   * scanning.
+   */
   Token tokenize(String source) {
     return measure(() {
-      return new StringScanner(source, includeComments: false).tokenize();
+      return new StringScanner.fromString(source, includeComments: false)
+          .tokenize();
     });
   }
 }