Start re-trying dartk compilations if they crash

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
@@ skipping 215 matching lines @@
       String displayName,
       String outputFile,
       bool neverSkipCompilation,
       List<Uri> bootstrapDependencies,
       String executable,
       List<String> arguments,
       Map<String, String> environmentOverrides)
       : super._(displayName, outputFile, neverSkipCompilation,
                 bootstrapDependencies, executable, arguments,
                 environmentOverrides);
+
+  int get maxNumRetries => 1;
 }
 
 /// This is just a Pair(String, Map) class with hashCode and operator ==
 class AddFlagsKey {
   final String flags;
   final Map env;
   AddFlagsKey(this.flags, this.env);
   // Just use object identity for environment map
   bool operator ==(other) =>
       other is AddFlagsKey && flags == other.flags && env == other.env;
@@ skipping 2588 matching lines @@
   if (!output.successful) {
     List<String> stdout, stderr;
 
     decodeOutput() {
       if (stdout == null && stderr == null) {
         stdout = decodeUtf8(output.stderr).split("\n");
         stderr = decodeUtf8(output.stderr).split("\n");
       }
     }
 
+    final command = output.command;
+
+    // The dartk batch compiler sometimes runs out of memory. In such a case we
+    // will retry running it.
+    if (command is KernelCompilationCommand) {
+      if (output.hasCrashed) {

[Vyacheslav Egorov (Google), 2016/12/14 13:49:49]

Can we grep output for out of memory instead of retrying all crashes?

Retrying all might hide bugs in the  long running VM/dartk

[kustermann, 2016/12/15 11:37:18]

Done.

+        return true;
+      }
+    }
+
+    // We currently rerun dartium tests, see issue 14074.
+    if (command is BrowserTestCommand &&
+        command.retry &&
+        command.browser == 'dartium') {
+      return true;
+    }
+
     if (io.Platform.operatingSystem == 'linux') {
       decodeOutput();
       // No matter which command we ran: If we get failures due to the
       // "xvfb-run" issue 7564, try re-running the test.
       bool containsFailureMsg(String line) {
         return line.contains(MESSAGE_CANNOT_OPEN_DISPLAY) ||
             line.contains(MESSAGE_FAILED_TO_RUN_COMMAND);
       }
       if (stdout.any(containsFailureMsg) || stderr.any(containsFailureMsg)) {
         return true;
       }
     }
 
-    // We currently rerun dartium tests, see issue 14074.
-    final command = output.command;
-    if (command is BrowserTestCommand &&
-        command.retry &&
-        command.browser == 'dartium') {
-      return true;
-    }
   }
   return false;
 }
 
 /*
  * [TestCaseCompleter] will listen for
  * NodeState.Processing -> NodeState.{Successful,Failed} state changes and
  * will complete a TestCase if it is finished.
  *
  * It provides a stream [finishedTestCases], which will stream all TestCases
@@ skipping 278 matching lines @@
     }
   }
 
   void eventAllTestsDone() {
     for (var listener in _eventListener) {
       listener.allDone();
     }
     _allDone();
   }
 }