Suppress timeouts on Windows browser bots when dart2js hangs.

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
@@ skipping 337 matching lines @@
       : super._("vm", executable, arguments, environmentOverrides);
 }
 
 class AdbPrecompilationCommand extends Command {
   final String precompiledRunnerFilename;
   final String precompiledTestDirectory;
   final List<String> arguments;
   final bool useBlobs;
 
   AdbPrecompilationCommand._(this.precompiledRunnerFilename,
-                             this.precompiledTestDirectory,
-                             this.arguments,
-                             this.useBlobs)
+      this.precompiledTestDirectory, this.arguments, this.useBlobs)
       : super._("adb_precompilation");
 
   void _buildHashCode(HashCodeBuilder builder) {
     super._buildHashCode(builder);
     builder.add(precompiledRunnerFilename);
     builder.add(precompiledTestDirectory);
     builder.add(arguments);
     builder.add(useBlobs);
   }
 
   bool _equal(AdbPrecompilationCommand other) =>
       super._equal(other) &&
       precompiledRunnerFilename == other.precompiledRunnerFilename &&
       useBlobs == other.useBlobs &&
       arguments == other.arguments &&
       precompiledTestDirectory == other.precompiledTestDirectory;
 
   String toString() => 'Steps to push precompiled runner and precompiled code '
-                       'to an attached device. Uses (and requires) adb.';
+      'to an attached device. Uses (and requires) adb.';
 }
 
 class JSCommandlineCommand extends ProcessCommand {
   JSCommandlineCommand._(
       String displayName, String executable, List<String> arguments,
       [Map<String, String> environmentOverrides = null])
       : super._(displayName, executable, arguments, environmentOverrides);
 }
 
 class PubCommand extends ProcessCommand {
@@ skipping 69 matching lines @@
       super._equal(other) &&
       _sourceDirectory == other._sourceDirectory &&
       _destinationDirectory == other._destinationDirectory;
 }
 
 class ModifyPubspecYamlCommand extends ScriptCommand {
   String _pubspecYamlFile;
   String _destinationFile;
   Map<String, Map> _dependencyOverrides;
 
-  ModifyPubspecYamlCommand._(this._pubspecYamlFile,
-      this._destinationFile, this._dependencyOverrides)
+  ModifyPubspecYamlCommand._(
+      this._pubspecYamlFile, this._destinationFile, this._dependencyOverrides)
       : super._("modify_pubspec") {
     assert(_pubspecYamlFile.endsWith("pubspec.yaml"));
     assert(_destinationFile.endsWith("pubspec.yaml"));
   }
 
   static Map<String, Map> _filterOverrides(
       String pubspec, Map<String, Map> overrides) {
     if (overrides.isEmpty) return overrides;
     var yaml = loadYaml(pubspec);
     var deps = yaml['dependencies'];
@@ skipping 179 matching lines @@
     return _getUniqueCommand(command);
   }
 
   VmCommand getVmCommand(String executable, List<String> arguments,
       Map<String, String> environmentOverrides) {
     var command = new VmCommand._(executable, arguments, environmentOverrides);
     return _getUniqueCommand(command);
   }
 
   AdbPrecompilationCommand getAdbPrecompiledCommand(String precompiledRunner,
-                                                    String testDirectory,
-                                                    List<String> arguments,
-                                                    bool useBlobs) {
+      String testDirectory, List<String> arguments, bool useBlobs) {
     var command = new AdbPrecompilationCommand._(
         precompiledRunner, testDirectory, arguments, useBlobs);
     return _getUniqueCommand(command);
   }
 
   Command getJSCommandlineCommand(String displayName, executable, arguments,
       [environment = null]) {
     var command = new JSCommandlineCommand._(
         displayName, executable, arguments, environment);
     return _getUniqueCommand(command);
@@ skipping 350 matching lines @@
   }
 
   // Reverse result of a negative test.
   bool hasFailed(TestCase testCase) {
     return testCase.isNegative ? !didFail(testCase) : didFail(testCase);
   }
 
   Expectation _negateOutcomeIfNegativeTest(
       Expectation outcome, bool isNegative) {
     if (!isNegative) return outcome;
-
+    if (outcome == Expectation.IGNORE) return outcome;
     if (outcome.canBeOutcomeOf(Expectation.FAIL)) {
       return Expectation.PASS;
     }
     return Expectation.FAIL;
   }
 }
 
 class BrowserCommandOutputImpl extends CommandOutputImpl {
   // Although tests are reported as passing, content shell sometimes exits with
   // a nonzero exitcode which makes our dartium builders extremely falky.
   // See: http://dartbug.com/15139.
   static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022;
   static bool isWindows = io.Platform.operatingSystem == 'windows';
   static bool _failedBecauseOfFlakyInfrastructure(List<int> stderrBytes) {
     // If the browser test failed, it may have been because content shell
     // and the virtual framebuffer X server didn't hook up, or it crashed with
     // a core dump. Sometimes content shell crashes after it has set the stdout
     // to PASS, so we have to do this check first.
     // Content shell also fails with a broken pipe message: Issue 26739
-    var zygoteCrash = new RegExp(
-        r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe");
+    var zygoteCrash =
+        new RegExp(r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe");
     var stderr = decodeUtf8(stderrBytes);
     // TODO(whesse): Issue: 7564
     // This may not be happening anymore.  Test by removing this suppression.
     if (stderr.contains(MESSAGE_CANNOT_OPEN_DISPLAY) ||
         stderr.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) {
       DebugLogger.warning(
           "Warning: Failure because of missing XDisplay. Test ignored");
       return true;
     }
     // Issue 26739
     if (zygoteCrash.hasMatch(stderr)) {
       DebugLogger.warning("Warning: Failure because of content_shell "
-                           "zygote crash. Test ignored");
+          "zygote crash. Test ignored");
       return true;
     }
     return false;
   }
 
   bool _infraFailure;
 
   BrowserCommandOutputImpl(
       command, exitCode, timedOut, stdout, stderr, time, compilationSkipped)
       : super(command, exitCode, timedOut, stdout, stderr, time,
-          compilationSkipped, 0),
+            compilationSkipped, 0),
         _infraFailure = _failedBecauseOfFlakyInfrastructure(stderr);
 
   Expectation result(TestCase testCase) {
     // Handle crashes and timeouts first
     if (hasCrashed) return Expectation.CRASH;
     if (hasTimedOut) return Expectation.TIMEOUT;
 
     if (_infraFailure) {
       return Expectation.IGNORE;
     }
@@ skipping 466 matching lines @@
       List<int> stdout,
       List<int> stderr,
       Duration time,
       bool compilationSkipped)
       : super(command, exitCode, timedOut, stdout, stderr, time,
             compilationSkipped, 0);
 
   Expectation result(TestCase testCase) {
     // Handle general crash/timeout detection.
     if (hasCrashed) return Expectation.CRASH;
-    if (hasTimedOut) return Expectation.TIMEOUT;
+    if (hasTimedOut) {
+      bool isWindows = io.Platform.operatingSystem == 'windows';
+      bool isBrowserTestCase =
+          testCase.commands.any((command) => command is BrowserTestCommand);
+      // TODO(26060) Dart2js batch mode hangs on Windows under heavy load.
+      return (isWindows && isBrowserTestCase)

[Søren Gjesse, 2016/06/28 14:40:08]

Don't you want to count the number of ignores, so that if something is causing a
lot of timeouts the bot will still fail?

Also the case of a single browser test that times out will now be masked on
Windows.

[Bill Hesse, 2016/06/28 14:45:59]

The number of ignores is counted at a higher level, when it is reported to a
progress monitor as the output status of this test case.  If there are more than
5 of them in a run, then the run fails.


It is only the dart2js compilation timing out, that falls in this case.  We can
create a Windows bot that only does dart2js compilation, and then the timeout
will be reported, since it will not be a browser test case.  I think we have
other dart2js bots that are slower than this, on non-windows platforms, so it is
unlikely it won't trigger them first.

[Søren Gjesse, 2016/06/28 15:12:19]

Thanks for the clarification.

+          ? Expectation.IGNORE
+          : Expectation.TIMEOUT;
+    }
 
     // Handle dart2js specific crash detection
     if (exitCode == DART2JS_EXITCODE_CRASH ||
         exitCode == VmCommandOutputImpl.DART_VM_EXITCODE_COMPILE_TIME_ERROR ||
         exitCode == VmCommandOutputImpl.DART_VM_EXITCODE_UNCAUGHT_EXCEPTION) {
       return Expectation.CRASH;
     }
 
     // Multitests are handled specially
     if (testCase.expectCompileError) {
@@ skipping 929 matching lines @@
       return _getBatchRunner("dart2js")
           .runCommand("dart2js", command, timeout, command.arguments);
     } else if (command is AnalysisCommand && batchMode) {
       return _getBatchRunner(command.flavor)
           .runCommand(command.flavor, command, timeout, command.arguments);
     } else if (command is ScriptCommand) {
       return command.run();
     } else if (command is AdbPrecompilationCommand) {
       assert(adbDevicePool != null);
       return adbDevicePool.acquireDevice().then((AdbDevice device) {
-        return _runAdbPrecompilationCommand(
-            device, command, timeout).whenComplete(() {
+        return _runAdbPrecompilationCommand(device, command, timeout)
+            .whenComplete(() {
           adbDevicePool.releaseDevice(device);
         });
       });
     } else {
       return new RunningProcess(command, timeout).run();
     }
   }
 
   Future<CommandOutput> _runAdbPrecompilationCommand(
       AdbDevice device, AdbPrecompilationCommand command, int timeout) async {
@@ skipping 10 matching lines @@
         .where((fse) => fse is io.File)
         .map((file) => file.path)
         .map((path) => path.substring(path.lastIndexOf('/') + 1))
         .toList();
 
     var timeoutDuration = new Duration(seconds: timeout);
 
     // All closures are of type "Future<AdbCommandResult> run()"
     List<Function> steps = [];
 
-    steps.add(() => device.runAdbShellCommand(
-          ['rm', '-Rf', deviceTestDir]));
-    steps.add(() => device.runAdbShellCommand(
-          ['mkdir', '-p', deviceTestDir]));
+    steps.add(() => device.runAdbShellCommand(['rm', '-Rf', deviceTestDir]));
+    steps.add(() => device.runAdbShellCommand(['mkdir', '-p', deviceTestDir]));
     // TODO: We should find a way for us to cache the runner binary and avoid
     // pushhing it for every single test (this is bad for SSD cycle time, test
     // timing).
     steps.add(() => device.runAdbCommand(
-          ['push', runner, '$devicedir/dart_precompiled_runtime']));
+        ['push', runner, '$devicedir/dart_precompiled_runtime']));
     steps.add(() => device.runAdbShellCommand(
-          ['chmod', '777', '$devicedir/dart_precompiled_runtime']));
+        ['chmod', '777', '$devicedir/dart_precompiled_runtime']));
 
     for (var file in files) {
-      steps.add(() => device.runAdbCommand(
-            ['push', '$testdir/$file', '$deviceTestDir/$file']));
+      steps.add(() => device
+          .runAdbCommand(['push', '$testdir/$file', '$deviceTestDir/$file']));
     }
 
     if (command.useBlobs) {
       steps.add(() => device.runAdbShellCommand(
-            ['$devicedir/dart_precompiled_runtime',
-             '--run-app-snapshot=$deviceTestDir',
-             '--use-blobs']..addAll(arguments),
-            timeout: timeoutDuration));
+          [
+            '$devicedir/dart_precompiled_runtime',
+            '--run-app-snapshot=$deviceTestDir',
+            '--use-blobs'
+          ]..addAll(arguments),
+          timeout: timeoutDuration));
     } else {
       steps.add(() => device.runAdbShellCommand(
-            ['$devicedir/dart_precompiled_runtime',
-             '--run-app-snapshot=$deviceTestDir'
-             ]..addAll(arguments),
-            timeout: timeoutDuration));
+          [
+            '$devicedir/dart_precompiled_runtime',
+            '--run-app-snapshot=$deviceTestDir'
+          ]..addAll(arguments),
+          timeout: timeoutDuration));
     }
 
     var stopwatch = new Stopwatch()..start();
     var writer = new StringBuffer();
 
     await device.waitForBootCompleted();
     await device.waitForDevice();
 
     AdbCommandResult result;
     for (var i = 0; i < steps.length; i++) {
       var fun = steps[i];
       var commandStopwatch = new Stopwatch()..start();
       result = await fun();
 
       writer.writeln("Executing ${result.command}");
       if (result.stdout.length > 0) {
         writer.writeln("Stdout:\n${result.stdout.trim()}");
       }
       if (result.stderr.length > 0) {
         writer.writeln("Stderr:\n${result.stderr.trim()}");
       }
       writer.writeln("ExitCode: ${result.exitCode}");
       writer.writeln("Time: ${commandStopwatch.elapsed}");
       writer.writeln("");
 
       // If one command fails, we stop processing the others and return
       // immediately.
       if (result.exitCode != 0) break;
     }
-    return createCommandOutput(
-        command, result.exitCode, result.timedOut, UTF8.encode('$writer'),
-        [], stopwatch.elapsed, false);
+    return createCommandOutput(command, result.exitCode, result.timedOut,
+        UTF8.encode('$writer'), [], stopwatch.elapsed, false);
   }
 
   BatchRunnerProcess _getBatchRunner(String identifier) {
     // Start batch processes if needed
     var runners = _batchProcesses[identifier];
     if (runners == null) {
       runners = new List<BatchRunnerProcess>(maxProcesses);
       for (int i = 0; i < maxProcesses; i++) {
         runners[i] = new BatchRunnerProcess();
       }
@@ skipping 335 matching lines @@
       var commandEnqueuer = new CommandEnqueuer(_graph);
 
       // CommandExecutor will execute commands
       var executor;
       if (recording) {
         executor = new RecordingCommandExecutor(new Path(recordingOutputFile));
       } else if (replaying) {
         executor = new ReplayingCommandExecutor(new Path(recordedInputFile));
       } else {
         executor = new CommandExecutorImpl(
-            _globalConfiguration, maxProcesses,
-            maxBrowserProcesses, adbDevicePool: adbDevicePool);
+            _globalConfiguration, maxProcesses, maxBrowserProcesses,
+            adbDevicePool: adbDevicePool);
       }
 
       // Run "runnable commands" using [executor] subject to
       // maxProcesses/maxBrowserProcesses constraint
       commandQueue = new CommandQueue(_graph, testCaseEnqueuer, executor,
           maxProcesses, maxBrowserProcesses, verbose);
 
       // Finish test cases when all commands were run (or some failed)
       var testCaseCompleter =
           new TestCaseCompleter(_graph, testCaseEnqueuer, commandQueue);
@@ skipping 55 matching lines @@
     }
   }
 
   void eventAllTestsDone() {
     for (var listener in _eventListener) {
       listener.allDone();
     }
     _allDone();
   }
 }