Fixed utf8 encoding/decoding issues in the testing scripts

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
  */
 library test_runner;
 
 import "dart:async";
 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow
 // CommandOutput.exitCode in subclasses of CommandOutput.
 import "dart:io" as io;
 import "dart:isolate";
 import "dart:uri";
+import "http_server.dart" as http_server;
 import "status_file_parser.dart";
 import "test_progress.dart";
 import "test_suite.dart";
-import "http_server.dart" as http_server;
+import "utils.dart";
 
 const int NO_TIMEOUT = 0;
 const int SLOW_TIMEOUT_MULTIPLIER = 4;
 
 typedef void TestCaseEvent(TestCase testCase);
 typedef void ExitCodeEvent(int exitCode);
 typedef void EnqueueMoreWork(ProcessQueue queue);
 
 
 /**
@@ skipping 584 matching lines @@
     }
     return _browserTestFailure;
   }
 
   bool get _failedBecauseOfMissingXDisplay {
     // Browser case:
     // If the browser test failed, it may have been because DumpRenderTree
     // and the virtual framebuffer X server didn't hook up, or DRT crashed with
     // a core dump. Sometimes DRT crashes after it has set the stdout to PASS,
     // so we have to do this check first.
-    var stderrLines = new String.fromCharCodes(super.stderr).split("\n");
+    var stderrLines = decodeUtf8(super.stderr).split("\n");
     for (String line in stderrLines) {
       // TODO(kustermann,ricow): Issue: 7564
       // This seems to happen quite frequently, we need to figure out why.
       if (line.contains('Gtk-WARNING **: cannot open display') ||
           line.contains('Failed to run command. return code=1')) {
         // If we get the X server error, or DRT crashes with a core dump, retry
         // the test.
         if ((testCase as BrowserTestCase).numRetries > 0) {
           requestRetry = true;
         }
@@ skipping 54 matching lines @@
                                    stdout.length);
       }
     }
     return true;
   }
 
   bool get _browserTestFailure {
     // Browser tests fail unless stdout contains
     // 'Content-Type: text/plain' followed by 'PASS'.
     bool has_content_type = false;
-    var stdoutLines = new String.fromCharCodes(super.stdout).split("\n");
+    var stdoutLines = decodeUtf8(super.stdout).split("\n");
     for (String line in stdoutLines) {
       switch (line) {
         case 'Content-Type: text/plain':
           has_content_type = true;
           break;
         case 'PASS':
           if (has_content_type) {
             if (exitCode != 0) {
               print("Warning: All tests passed, but exitCode != 0 "
                     "(${testCase.displayName})");
@@ skipping 52 matching lines @@
     return failResult;
   }
 
   bool _didFail() {
     if (hasCrashed) return false;
 
     List<String> errors = [];
     List<String> staticWarnings = [];
 
     // Read the returned list of errors and stuff them away.
-    var stderrLines = new String.fromCharCodes(super.stderr).split("\n");
+    var stderrLines = decodeUtf8(super.stderr).split("\n");
     for (String line in stderrLines) {
       if (line.length == 0) continue;
       List<String> fields = splitMachineError(line);
       if (fields[ERROR_LEVEL] == 'ERROR') {
         errors.add(fields[FORMATTED_ERROR]);
       } else if (fields[ERROR_LEVEL] == 'WARNING') {
         // We only care about testing Static type warnings
         // ignore all others
         if (fields[ERROR_TYPE] == 'STATIC_TYPE') {
           staticWarnings.add(fields[FORMATTED_ERROR]);
@@ skipping 155 matching lines @@
     var command = lastCommandOutput.command;
 
     if (timeoutTimer != null) {
       timeoutTimer.cancel();
     }
     if (lastCommandOutput.unexpectedOutput
         && testCase.configuration['verbose'] != null
         && testCase.configuration['verbose']) {
       print(testCase.displayName);
 
-      print(new String.fromCharCodes(lastCommandOutput.stderr));
+      print(decodeUtf8(lastCommandOutput.stderr));
       if (!lastCommandOutput.command.isPixelTest) {
-        print(new String.fromCharCodes(lastCommandOutput.stdout));
+        print(decodeUtf8(lastCommandOutput.stdout));
       } else {
         print("DRT pixel test failed! stdout is not printed because it "
               "contains binary data!");
       }
       print('');
       if (notifications.length > 0) {
         print("Notifications:");
         for (var line in notifications) {
           print(notifications);
         }
@@ skipping 347 matching lines @@
       // Otherwise, process output and call _reportResult() when done.
       var line = stream.readLine();
       while (line != null) {
         if (line.startsWith('>>> TEST')) {
           _status = line;
         } else if (line.startsWith('>>> BATCH START')) {
           // ignore
         } else if (line.startsWith('>>> ')) {
           throw new Exception('Unexpected command from dartc batch runner.');
         } else {
-          buffer.addAll("$line\n".charCodes);
+          buffer.addAll(decodeUtf8(line));
+          buffer.addAll("\n".charCodes);
         }
         line = stream.readLine();
       }
       if (_status != null) {
         _timer.cancel();
         _stdoutDone();
       }
     }
     stream.onLine = onLineHandler;
   }
 
   void _readStderr(io.StringInputStream stream, List<int> buffer) {
     var ignoreStreams = _ignoreStreams;  // Capture this mutable object.
     void onLineHandler() {
       if (ignoreStreams.value) {
         while (stream.readLine() != null) {
           // Do nothing.
         }
         return;
       }
       // Otherwise, process output and call _reportResult() when done.
       var line = stream.readLine();
       while (line != null) {
         if (line.startsWith('>>> EOF STDERR')) {
           _stderrDone();
         } else {
-          buffer.addAll("$line\n".charCodes);
+          buffer.addAll(decodeUtf8(line));
+          buffer.addAll("\n".charCodes);
         }
         line = stream.readLine();
       }
     }
     stream.onLine = onLineHandler;
   }
 
   ExitCodeEvent makeExitHandler(String status) {
     void handler(int exitCode) {
       if (active) {
@@ skipping 388 matching lines @@
         // the developer doesn't waste his or her time trying to fix a bunch of
         // tests that appear to be broken but were actually just flakes that
         // didn't get retried because there had already been one failure.
         bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
         new RunningProcess(test, allowRetry, this).start();
       }
       _numProcesses++;
     }
   }
 }