Call DumpRenderTree directly from test.dart instead of using the drt-trampoline.py indirection

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
  */
 library test_runner;
 
 import "dart:io";
 import "dart:isolate";
 import "dart:uri";
 import "status_file_parser.dart";
 import "test_progress.dart";
 import "test_suite.dart";
 
 const int NO_TIMEOUT = 0;
 const int SLOW_TIMEOUT_MULTIPLIER = 4;
 
 typedef void TestCaseEvent(TestCase testCase);
 typedef void ExitCodeEvent(int exitCode);
 typedef void EnqueueMoreWork(ProcessQueue queue);
 
+
+/**
+ * [areByteArraysEqual] compares a range of bytes from [buffer1] with a
+ * range of bytes from [buffer2].
+ *
+ * Returns [true] if the [count] bytes in [buffer1] (starting at
+ * [offset1]) match the [count] bytes in [buffer2] (starting at
+ * [offset2]).
+ * Otherwise [false] is returned.
+ */
+bool areByteArraysEqual(List<int> buffer1, int offset1,
+                        List<int> buffer2, int offset2,
+                        int count) {
+  if ((offset1 + count) > buffer1.length ||
+      (offset2 + count) > buffer2.length) {
+    return false;
+  }
+
+  for (var i = 0; i < count; i++) {
+    if (buffer1[offset1 + i] != buffer2[offset2 + i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * [findBytes] searches for [pattern] in [data] beginning at [startPos].
+ *
+ * Returns [true] if [pattern] was found in [data].
+ * Otherwise [false] is returned.
+ */
+int findBytes(List<int> data, List<int> pattern, [int startPos=0]) {
+  // TODO(kustermann): Use one of the fast string-matching algorithms!
+  for (int i=startPos; i < (data.length-pattern.length); i++) {
+    bool found = true;
+    for (int j=0; j<pattern.length; j++) {
+      if (data[i+j] != pattern[j]) {
+        found = false;
+      }
+    }
+    if (found) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+
 /** A command executed as a step in a test case. */
 class Command {
   /** Path to the executable of this command. */
   String executable;
 
   /** Command line arguments to the executable. */
   List<String> arguments;
+  
+  /** Environment for the command */
+  Map<String,String> environment;
 
   /** The actual command line that will be executed. */
   String commandLine;
 
-  Command(this.executable, this.arguments) {
+  Command(this.executable, this.arguments, [this.environment = null]) {
     if (Platform.operatingSystem == 'windows') {
       // Windows can't handle the first command if it is a .bat file or the like
       // with the slashes going the other direction.
       // TODO(efortuna): Remove this when fixed (Issue 1306).
       executable = executable.replaceAll('/', '\\');
     }
     commandLine = "$executable ${Strings.join(arguments, ' ')}";
   }
 
   String toString() => commandLine;
 
   Future<bool> get outputIsUpToDate => new Future.immediate(false);
+  Path get expectedOutputFile => null;
+  bool get isPixelTest => false;
 }
 
 class CompilationCommand extends Command {
   String _outputFile;
   bool _neverSkipCompilation;
   List<Uri> _bootstrapDependencies;
 
   CompilationCommand(this._outputFile,
                      this._neverSkipCompilation,
                      this._bootstrapDependencies,
@@ skipping 36 matching lines @@
             }
           }
           return true;
         }
       }
       return false;
     });
   }
 }
 
+class DumpRenderTreeCommand extends Command {
+  /**
+   * If [expectedOutputPath] is set, the output of DumpRenderTree is compared
+   * with the content of [expectedOutputPath].
+   * This is used for example for pixel tests, where [expectedOutputPath] points
+   * to a *png file.
+   */
+  Path expectedOutputPath;
+
+  DumpRenderTreeCommand(String executable,
+                        String htmlFile,
+                        List<String> options,
+                        List<String> dartFlags,
+                        Uri packageRootUri,
+                        Path this.expectedOutputPath)
+      : super(executable,
+              _getArguments(options, htmlFile),
+              _getEnvironment(dartFlags, packageRootUri));
+
+  static Map _getEnvironment(List<String> dartFlags, Uri packageRootUri) {
+    var needDartFlags = dartFlags != null && dartFlags.length > 0;
+    var needDartPackageRoot = packageRootUri != null;
+
+    var env = null;
+    if (needDartFlags || needDartPackageRoot) {
+      env = new Map.from(Platform.environment);
+      if (needDartFlags) {
+        env['DART_FLAGS'] = Strings.join(dartFlags, " ");
+      }
+      if (needDartPackageRoot) {
+        env['DART_PACKAGE_ROOT'] = packageRootUri.toString();
+      }
+    }
+
+    return env;
+  }
+
+  static List<String> _getArguments(List<String> options, String htmlFile) {
+    var arguments = new List.from(options);
+    arguments.add(htmlFile);
+    return arguments;
+  }
+  
+  Path get expectedOutputFile => expectedOutputPath;
+  bool get isPixelTest => (expectedOutputFile != null &&
+                           expectedOutputFile.filename.endsWith(".png"));
+}
+
+
 /**
  * TestCase contains all the information needed to run a test and evaluate
  * its output.  Running a test involves starting a separate process, with
  * the executable and arguments given by the TestCase, and recording its
  * stdout and stderr output streams, and its exit code.  TestCase only
  * contains static information about the test; actually running the test is
  * performed by [ProcessQueue] using a [RunningProcess] object.
  *
  * The output information is stored in a [CommandOutput] instance contained
  * in TestCase.commandOutputs. The last CommandOutput instance is responsible
@@ skipping 192 matching lines @@
  * code, the standard output and standard error, whether the process timed out,
  * and the time the process took to run.  It also contains a pointer to the
  * [TestCase] this is the output of.
  */
 abstract class CommandOutput {
   factory CommandOutput.fromCase(TestCase testCase,
                                  Command command,
                                  int exitCode,
                                  bool incomplete,
                                  bool timedOut,
-                                 List<String> stdout,
-                                 List<String> stderr,
+                                 List<int> stdout,
+                                 List<int> stderr,
                                  Duration time,
                                  bool compilationSkipped) {
     return new CommandOutputImpl.fromCase(testCase,
                                           command,
                                           exitCode,
                                           incomplete,
                                           timedOut,
                                           stdout,
                                           stderr,
                                           time,
                                           compilationSkipped);
   }
 
+  Command get command;
+
   bool get incomplete;
 
   String get result;
 
   bool get unexpectedOutput;
 
   bool get hasCrashed;
 
   bool get hasTimedOut;
 
   bool get didFail;
 
   bool requestRetry;
 
   Duration get time;
 
   int get exitCode;
 
-  List<String> get stdout;
+  List<int> get stdout;
 
-  List<String> get stderr;
+  List<int> get stderr;
 
   List<String> get diagnostics;
 
   bool get compilationSkipped;
 }
 
 class CommandOutputImpl implements CommandOutput {
+  Command command;
   TestCase testCase;
   int exitCode;
 
   /// Records if all commands were run, true if they weren't.
   final bool incomplete;
 
   bool timedOut;
   bool failed = false;
-  List<String> stdout;
-  List<String> stderr;
+  List<int> stdout;
+  List<int> stderr;
   Duration time;
   List<String> diagnostics;
   bool compilationSkipped;
 
   /**
    * A flag to indicate we have already printed a warning about ignoring the VM
    * crash, to limit the amount of output produced per test.
    */
   bool alreadyPrintedWarning = false;
 
   /**
    * Set to true if we encounter a condition in the output that indicates we
    * need to rerun this test.
    */
   bool requestRetry = false;
 
   // Don't call this constructor, call CommandOutput.fromCase() to
   // get a new TestOutput instance.
   CommandOutputImpl(TestCase this.testCase,
-                    Command command,
+                    Command this.command,
                     int this.exitCode,
                     bool this.incomplete,
                     bool this.timedOut,
-                    List<String> this.stdout,
-                    List<String> this.stderr,
+                    List<int> this.stdout,
+                    List<int> this.stderr,
                     Duration this.time,
                     bool this.compilationSkipped) {
     testCase.commandOutputs[command] = this;
     diagnostics = [];
   }
   factory CommandOutputImpl.fromCase(TestCase testCase,
                                      Command command,
                                      int exitCode,
                                      bool incomplete,
                                      bool timedOut,
-                                     List<String> stdout,
-                                     List<String> stderr,
+                                     List<int> stdout,
+                                     List<int> stderr,
                                      Duration time,
                                      bool compilationSkipped) {
     if (testCase is BrowserTestCase) {
       return new BrowserCommandOutputImpl(testCase,
                                           command,
                                           exitCode,
                                           incomplete,
                                           timedOut,
                                           stdout,
                                           stderr,
@@ skipping 72 matching lines @@
           command,
           exitCode,
           incomplete,
           timedOut,
           stdout,
           stderr,
           time,
           compilationSkipped);
 
   bool get didFail {
+    if (_failedBecauseOfMissingXDisplay) {
+      return true;
+    }
+
+    if (command.expectedOutputFile != null) {
+      // We are either doing a pixel test or a layout test with DumpRenderTree
+      return _failedBecauseOfUnexpectedDRTOutput;
+    }
+    return _browserTestFailure;
+  }
+
+  bool get _failedBecauseOfMissingXDisplay {
     // Browser case:
     // If the browser test failed, it may have been because DumpRenderTree
     // and the virtual framebuffer X server didn't hook up, or DRT crashed with
     // a core dump. Sometimes DRT crashes after it has set the stdout to PASS,
     // so we have to do this check first.
-    for (String line in super.stderr) {
+    var stderrLines = new String.fromCharCodes(super.stderr).split("\n");
+    for (String line in stderrLines) {
       if (line.contains('Gtk-WARNING **: cannot open display: :99') ||
-        line.contains('Failed to run command. return code=1')) {
+          line.contains('Failed to run command. return code=1')) {
         // If we get the X server error, or DRT crashes with a core dump, retry
         // the test.
         if ((testCase as BrowserTestCase).numRetries > 0) {
           requestRetry = true;
         }
         return true;
       }
     }
+    return false;
+  }
 
+  bool get _failedBecauseOfUnexpectedDRTOutput {
+    /*
+     * The output of DumpRenderTree is different for pixel tests than for
+     * layout tests.
+     *
+     * On a pixel test, the DRT output has the following format
+     *     ......
+     *     ......
+     *     Content-Length: ...\n
+     *     <*png data>
+     *     #EOF\n
+     * So we need to get the byte-range of the png data first, before
+     * comparing it with the content of the expected output file.
+     *
+     * On a layout tests, the DRT output is directly compared with the
+     * content of the expected output.
+     */
+    var stdout = testCase.commandOutputs[command].stdout;
+    var file = new File.fromPath(command.expectedOutputFile);
+    if (file.existsSync()) {
+      var bytesContentLength = "Content-Length:".charCodes;
+      var bytesNewLine = "\n".charCodes;
+      var bytesEOF = "#EOF\n".charCodes;
+
+      var expectedContent = file.readAsBytesSync();
+      if (command.isPixelTest) {
+        var startOfContentLength = findBytes(stdout, bytesContentLength);
+        if (startOfContentLength >= 0) {
+          var newLineAfterContentLength = findBytes(stdout,
+                                                    bytesNewLine,
+                                                    startOfContentLength);
+          if (newLineAfterContentLength > 0) {
+            var startPosition = newLineAfterContentLength +
+                bytesNewLine.length;
+            var endPosition = stdout.length - bytesEOF.length;
+
+            return !areByteArraysEqual(expectedContent,
+                                       0,
+                                       stdout,
+                                       startPosition,
+                                       endPosition - startPosition);
+          }
+        }
+        return true;
+      } else {
+        return !areByteArraysEqual(expectedContent, 0,
+                                   stdout, 0,
+                                   stdout.length);
+      }
+    }
+    return true;
+  }
+  
+  bool get _browserTestFailure {
     // Browser tests fail unless stdout contains
     // 'Content-Type: text/plain' followed by 'PASS'.
     bool has_content_type = false;
-    for (String line in super.stdout) {
+    var stdoutLines = new String.fromCharCodes(super.stdout).split("\n");
+    for (String line in stdoutLines) {
       switch (line) {
         case 'Content-Type: text/plain':
           has_content_type = true;
           break;
-
         case 'PASS':
           if (has_content_type) {
             return (exitCode != 0 && !hasCrashed);
           }
           break;
       }
     }
     return true;
   }
 }
@@ skipping 37 matching lines @@
     return failResult;
   }
 
   bool _didFail() {
     if (hasCrashed) return false;
 
     List<String> errors = [];
     List<String> staticWarnings = [];
 
     // Read the returned list of errors and stuff them away.
-    for (String line in super.stderr) {
+    var stderrLines = new String.fromCharCodes(super.stderr).split("\n");
+    for (String line in stderrLines) {
       if (line.length == 0) continue;
       List<String> fields = splitMachineError(line);
       if (fields[ERROR_LEVEL] == 'ERROR') {
         errors.add(fields[FORMATTED_ERROR]);
       } else if (fields[ERROR_LEVEL] == 'WARNING') {
         // We only care about testing Static type warnings
         // ignore all others
         if (fields[ERROR_TYPE] == 'STATIC_TYPE') {
           staticWarnings.add(fields[FORMATTED_ERROR]);
         }
@@ skipping 128 matching lines @@
  * the result; there are no pointers to it, so it should be available to
  * be garbage collected as soon as it is done.
  */
 class RunningProcess {
   ProcessQueue processQueue;
   Process process;
   TestCase testCase;
   bool timedOut = false;
   Date startTime;
   Timer timeoutTimer;
-  List<String> stdout;
-  List<String> stderr;
+  List<int> stdout;
+  List<int> stderr;
+  List<String> notifications;
   bool compilationSkipped;
   bool allowRetries;
 
   /** Which command of [testCase.commands] is currently being executed. */
   int currentStep;
 
   RunningProcess(TestCase this.testCase,
       [this.allowRetries = false, this.processQueue]);
 
   /**
    * Called when all commands are executed.
    */
   void testComplete(CommandOutput lastCommandOutput) {
+    var command = lastCommandOutput.command;
+
     if (timeoutTimer != null) {
       timeoutTimer.cancel();
     }
     if (lastCommandOutput.unexpectedOutput
         && testCase.configuration['verbose'] != null
         && testCase.configuration['verbose']) {
       print(testCase.displayName);
-      for (var line in lastCommandOutput.stderr) print(line);
-      for (var line in lastCommandOutput.stdout) print(line);
+
+      print(new String.fromCharCodes(lastCommandOutput.stderr));
+      if (!lastCommandOutput.command.isPixelTest) {
+        print(new String.fromCharCodes(lastCommandOutput.stdout));
+      } else {
+        print("DRT pixel test failed! stdout is not printed because it "
+              "contains binary data!");
+      }
+      print('');
+      if (notifications.length > 0) {
+        print("Notifications:");
+        for (var line in notifications) {
+          print(notifications);
+        }
+        print('');
+      }
     }
     if (allowRetries && testCase.usesWebDriver
         && lastCommandOutput.unexpectedOutput
         && (testCase as BrowserTestCase).numRetries > 0) {
       // Selenium tests can be flaky. Try rerunning.
       lastCommandOutput.requestRetry = true;
     }
     if (lastCommandOutput.requestRetry) {
       lastCommandOutput.requestRetry = false;
       this.timedOut = false;
@@ skipping 21 matching lines @@
     if (timedOut) {
       // Non-webdriver test timed out before it could complete. Webdriver tests
       // run their own timeouts by timing from the launch of the browser (which
       // could be delayed).
       testComplete(createCommandOutput(command, 0, true));
     } else if (currentStep == totalSteps) {
       // Done with all test commands.
       testComplete(createCommandOutput(command, exitCode, false));
     } else if (exitCode != 0) {
       // One of the steps failed.
-      stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n');
+      notifications.add('test.dart: Compilation failed$suffix, '
+                        'exit code $exitCode\n');
       testComplete(createCommandOutput(command, exitCode, true));
     } else {
       createCommandOutput(command, exitCode, true);
       // One compilation step successfully completed, move on to the
       // next step.
-      stderr.add('test.dart: Compilation finished $suffix\n');
-      stdout.add('test.dart: Compilation finished $suffix\n');
+      notifications.add('test.dart: Compilation finished $suffix\n\n');
       if (currentStep == totalSteps - 1 && testCase.usesWebDriver &&
           !testCase.configuration['noBatch']) {
         // Note: processQueue will always be non-null for runtime == ie9, ie10,
         // ff, safari, chrome, opera. (It is only null for runtime == vm)
         // This RunningProcess object is done, and hands over control to
         // BatchRunner.startTest(), which handles reporting, etc.
         if (timeoutTimer != null) {
           timeoutTimer.cancel();
         }
         processQueue._getBatchRunner(testCase).startTest(testCase);
@@ skipping 17 matching lines @@
         timedOut,
         stdout,
         stderr,
         new Date.now().difference(startTime),
         compilationSkipped);
     resetLocalOutputInformation();
     return commandOutput;
   }
 
   void resetLocalOutputInformation() {
-    stdout = new List<String>();
-    stderr = new List<String>();
+    stdout = new List<int>();
+    stderr = new List<int>();
+    notifications = new List<String>();
     compilationSkipped = false;
   }
 
-  VoidFunction makeReadHandler(StringInputStream source,
-                               List<String> destination) {
-    void handler () {
-      if (source.closed) return;  // TODO(whesse): Remove when bug is fixed.
-      var line = source.readLine();
-      while (null != line) {
-        destination.add(line);
-        line = source.readLine();
+  void drainStream(InputStream source, List<int> destination) {
+    void onDataHandler () {
+      if (source.closed) {
+        return;  // TODO(whesse): Remove when bug is fixed.
+      }
+      var data = source.read();
+      while (data != null) {
+        destination.addAll(data);
+        data = source.read();
       }
     }
-    return handler;
+    source.onData = onDataHandler;
+    source.onClosed = onDataHandler;
   }
 
   void start() {
     Expect.isFalse(testCase.expectedOutcomes.contains(SKIP));
     resetLocalOutputInformation();
     currentStep = 0;
     startTime = new Date.now();
     runCommand(testCase.commands[currentStep++], commandComplete);
   }
 
   void runCommand(Command command, void commandCompleteHandler(Command, int)) {
     void processExitHandler(int returnCode) {
       commandCompleteHandler(command, returnCode);
     }
 
     command.outputIsUpToDate.then((bool isUpToDate) {
       if (isUpToDate) {
-        stdout.add("Skipped compilation because the old output is "
-                   "still up to date!");
+        notifications.add("Skipped compilation because the old output is "
+                          "still up to date!");
         compilationSkipped = true;
         commandComplete(command, 0);
       } else {
         ProcessOptions options = new ProcessOptions();
-        options.environment =
-            new Map<String, String>.from(Platform.environment);
+        if (command.environment != null) {
+          options.environment =
+              new Map<String, String>.from(command.environment);
+        } else {
+          options.environment =
+              new Map<String, String>.from(Platform.environment);
+        }
+
         options.environment['DART_CONFIGURATION'] =
             TestUtils.configurationDir(testCase.configuration);
         Future processFuture = Process.start(command.executable,
                                              command.arguments,
                                              options);
         processFuture.then((Process p) {
           process = p;
           process.onExit = processExitHandler;
-          var stdoutStringStream = new StringInputStream(process.stdout);
-          var stderrStringStream = new StringInputStream(process.stderr);
-          stdoutStringStream.onLine =
-              makeReadHandler(stdoutStringStream, stdout);
-          stderrStringStream.onLine =
-              makeReadHandler(stderrStringStream, stderr);
+          drainStream(process.stdout, stdout);
+          drainStream(process.stderr, stderr);
           if (timeoutTimer == null) {
             // Create one timeout timer when starting test case, remove it at
             // the end.
             timeoutTimer = new Timer(1000 * testCase.timeout, timeoutHandler);
           }
           // If the timeout fired in between two commands, kill the just
           // started process immediately.
           if (timedOut) safeKill(process);
         });
         processFuture.handleException((e) {
@@ skipping 35 matching lines @@
 class BatchRunnerProcess {
   Command _command;
   String _executable;
   List<String> _batchArguments;
 
   Process _process;
   StringInputStream _stdoutStream;
   StringInputStream _stderrStream;
 
   TestCase _currentTest;
-  List<String> _testStdout;
-  List<String> _testStderr;
+  List<int> _testStdout;
+  List<int> _testStderr;
   String _status;
   bool _stdoutDrained = false;
   bool _stderrDrained = false;
   MutableValue<bool> _ignoreStreams;
   Date _startTime;
   Timer _timer;
 
   bool _isWebDriver;
 
   BatchRunnerProcess(TestCase testCase) {
@@ skipping 56 matching lines @@
   }
 
   void doStartTest(TestCase testCase) {
     _startTime = new Date.now();
     _testStdout = [];
     _testStderr = [];
     _status = null;
     _stdoutDrained = false;
     _stderrDrained = false;
     _ignoreStreams = new MutableValue<bool>(false);  // Captured by closures.
-    _stdoutStream.onLine = _readStdout(_stdoutStream, _testStdout);
-    _stderrStream.onLine = _readStderr(_stderrStream, _testStderr);
+    _readStdout(_stdoutStream, _testStdout);
+    _readStderr(_stderrStream, _testStderr);
     _timer = new Timer(testCase.timeout * 1000, _timeoutHandler);
+
+    if (testCase.commands.last.environment != null) {
+      print("Warning: command.environment != null, but we don't support custom "
+            "environments for batch runner tests!");
+    }
+
     var line = _createArgumentsLine(testCase.batchTestArguments);
     _process.stdin.onError = (err) {
       print('Error on batch runner input stream stdin');
       print('  Input line: $line');
       print('  Previous test\'s status: $_status');
       print('  Error: $err');
       throw err;
     };
     _process.stdin.write(line.charCodes);
   }
@@ skipping 29 matching lines @@
     // Move on when both stdout and stderr has been drained.
     if (_stdoutDrained) _reportResult();
   }
 
   void _stdoutDone() {
     _stdoutDrained = true;
     // Move on when both stdout and stderr has been drained.
     if (_stderrDrained) _reportResult();
   }
 
-  VoidFunction _readStdout(StringInputStream stream, List<String> buffer) {
+  void _readStdout(StringInputStream stream, List<int> buffer) {
     var ignoreStreams = _ignoreStreams;  // Capture this mutable object.
-    void reader() {
+    void onLineHandler() {
       if (ignoreStreams.value) {
          while (stream.readLine() != null) {
           // Do nothing.
         }
         return;
       }
       // Otherwise, process output and call _reportResult() when done.
       var line = stream.readLine();
       while (line != null) {
         if (line.startsWith('>>> TEST')) {
           _status = line;
         } else if (line.startsWith('>>> BATCH START')) {
           // ignore
         } else if (line.startsWith('>>> ')) {
           throw new Exception('Unexpected command from dartc batch runner.');
         } else {
-          buffer.add(line);
+          buffer.addAll("$line\n".charCodes);
         }
         line = stream.readLine();
       }
       if (_status != null) {
         _timer.cancel();
         _stdoutDone();
       }
     }
-    return reader;
+    stream.onLine = onLineHandler;
   }
 
-  VoidFunction _readStderr(StringInputStream stream, List<String> buffer) {
+  void _readStderr(StringInputStream stream, List<int> buffer) {
     var ignoreStreams = _ignoreStreams;  // Capture this mutable object.
-    void reader() {
+    void onLineHandler() {
       if (ignoreStreams.value) {
         while (stream.readLine() != null) {
           // Do nothing.
         }
         return;
       }
       // Otherwise, process output and call _reportResult() when done.
       var line = stream.readLine();
       while (line != null) {
         if (line.startsWith('>>> EOF STDERR')) {
           _stderrDone();
         } else {
-          buffer.add(line);
+          buffer.addAll("$line\n".charCodes);
         }
         line = stream.readLine();
       }
     }
-    return reader;
+    stream.onLine = onLineHandler;
   }
 
   ExitCodeEvent makeExitHandler(String status) {
     void handler(int exitCode) {
       if (active) {
         if (_timer != null) _timer.cancel();
         _status = status;
         // Read current content of streams, ignore any later output.
         _ignoreStreams.value = true;
         var line = _stdoutStream.readLine();
@@ skipping 375 matching lines @@
         // the developer doesn't waste his or her time trying to fix a bunch of
         // tests that appear to be broken but were actually just flakes that
         // didn't get retried because there had already been one failure.
         bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
         new RunningProcess(test, allowRetry, this).start();
       }
       _numProcesses++;
     }
   }
 }