Run test tearDowns and clean up temporary directories when a signal is caught.

test/runner/signal_test.dart

Index: test/runner/signal_test.dart
diff --git a/test/runner/signal_test.dart b/test/runner/signal_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..69c986117d2b291a2711a2cbd99605882a1ac0df
--- /dev/null
+++ b/test/runner/signal_test.dart
@@ -0,0 +1,277 @@
+// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+import 'package:test/test.dart';
+import 'package:test/src/util/io.dart';
+
+import '../io.dart';
+
+String _sandbox;
+
+String get _tempDir => p.join(_sandbox, "tmp");
+
+final _lines = UTF8.decoder.fuse(const LineSplitter());
+
+// This test is inherently prone to race conditions. If it fails, it will likely
+// do so flakily, but if it succeeds, it will succeed consistently. The tests
+// represent a best effort to kill the test runner at certain times during its
+// execution.
+void main() {
+  setUp(() {
+    _sandbox = createTempDir();
+  });
+
+  tearDown(() {
+    new Directory(_sandbox).deleteSync(recursive: true);
+  });
+
+  group("during loading,", () {
+    test("cleans up if killed while loading a VM test", () {

[kevmoo, 2015/04/03 00:49:45]

Got a failure on my mac, built w/ latest

06:17 +334 -1: test/runner/signal_test.dart: ... cleans up if killed while
loading a VM test
  Expected: empty
    Actual: [
    ?:Directory:
'/var/folders/j8/3kl8_xdn14x7mlbsq6_g8lfc0071_k/T/dart_test_YnbjgD/tmp/dart_test_O1h2TM'
  ]

  package:test                        expect
  test/runner/signal_test.dart 52:11  main.<fn>.<fn>.<fn>.<fn>
  ===== asynchronous gap ===========================
  dart:async                          _Future.then
  test/runner/signal_test.dart 51:12  main.<fn>.<fn>.<fn>
  ===== asynchronous gap ===========================
  dart:async                          _Future.then
  test/runner/signal_test.dart 46:44  main.<fn>.<fn>

[nweiz, 2015/04/03 20:56:22]

I added a couple zeros to the for loop. This should ensure that the signal
always comes in while loading the test (this wasn't happening previously), and
thus avoid the flake. I'm still not sure what causes the error in the first
place, but I suspect it may just be some sort of race condition in dart:io
itself. Without a consistent, debuggable reproduction, there's not much to do
but avoid it.

+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+void main() {
+  print("in test.dart");
+  // Spin for a long time so the test is probably killed while still loading.
+  for (var i = 0; i < 1000000; i++) {}
+}
+""");
+
+      return _startUnittest(["test.dart"]).then((process) {
+        return _lines.bind(process.stdout).first.then((line) {
+          expect(line, equals("in test.dart"));
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+
+    test("cleans up if killed while loading a browser test", () {
+      new File(p.join(_sandbox, "test.dart"))
+          .writeAsStringSync("void main() {}");
+
+      return _startUnittest(["-p", "chrome", "test.dart"]).then((process) {
+        return _lines.bind(process.stdout).first.then((line) {
+          expect(line, equals("Compiling test.dart..."));
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+
+    test("exits immediately if ^C is sent twice", () {
+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+void main() {
+  print("in test.dart");
+  while (true) {}
+}
+""");
+
+      return _startUnittest(["test.dart"]).then((process) {
+        return _lines.bind(process.stdout).first.then((line) {
+          expect(line, equals("in test.dart"));
+          process.kill();
+
+          // TODO(nweiz): Sending two signals in close succession can cause the
+          // second one to be ignored, so we wait a bit before the second
+          // one. Remove this hack when issue 23047 is fixed.
+          return new Future.delayed(new Duration(seconds: 1));
+        }).then((_) {
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+  });
+
+  group("during test running", () {
+    test("waits for a VM test to finish running", () {
+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+import 'dart:async';
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+void main() {
+  tearDown(() => new File("output").writeAsStringSync("ran teardown"));
+
+  test("test", () {
+    print("running test");
+    return new Future.delayed(new Duration(seconds: 1));
+  });
+}
+""");
+
+      return _startUnittest(["test.dart"]).then((process) {
+        return _lines.bind(process.stdout).skip(2).first.then((line) {
+          expect(line, equals("running test"));
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new File(p.join(_sandbox, "output")).readAsStringSync(),
+              equals("ran teardown"));
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+
+    test("kills a browser test immediately", () {
+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+  test("test", () {
+    print("running test");
+
+    // Allow an event loop to pass so the preceding print can be handled.
+    return new Future(() {
+      // Loop forever so that if the test isn't stopped while running, it never
+      // stops.
+      while (true) {}
+    });
+  });
+}
+""");
+
+      return _startUnittest(["-p", "chrome", "test.dart"]).then((process) {
+        return _lines.bind(process.stdout).skip(3).first.then((line) {
+          expect(line, equals("running test"));
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+
+    test("kills a VM test immediately if ^C is sent twice", () {
+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+import 'package:test/test.dart';
+
+void main() {
+  test("test", () {
+    print("running test");
+    while (true) {}
+  });
+}
+""");
+
+      return _startUnittest(["test.dart"]).then((process) {
+        return _lines.bind(process.stdout).skip(2).first.then((line) {
+          expect(line, equals("running test"));
+          process.kill();
+
+          // TODO(nweiz): Sending two signals in close succession can cause the
+          // second one to be ignored, so we wait a bit before the second
+          // one. Remove this hack when issue 23047 is fixed.
+          return new Future.delayed(new Duration(seconds: 1));
+        }).then((_) {
+          process.kill();
+          return process.exitCode;
+        });
+      });
+    });
+
+    test("causes expect() to always throw an error immediately", () {
+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+import 'dart:async';
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+void main() {
+  var expectThrewError = false;
+
+  tearDown(() {
+    new File("output").writeAsStringSync(expectThrewError.toString());
+  });
+
+  test("test", () {
+    print("running test");
+
+    return new Future.delayed(new Duration(seconds: 1)).then((_) {
+      try {
+        expect(true, isTrue);
+      } catch (_) {
+        expectThrewError = true;
+      }
+    });
+  });
+}
+""");
+
+      return _startUnittest(["test.dart"]).then((process) {
+        return _lines.bind(process.stdout).skip(2).first.then((line) {
+          expect(line, equals("running test"));
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new File(p.join(_sandbox, "output")).readAsStringSync(),
+              equals("true"));
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+
+    test("causes expectAsync() to always throw an error immediately", () {
+      new File(p.join(_sandbox, "test.dart")).writeAsStringSync("""
+import 'dart:async';
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+void main() {
+  var expectAsyncThrewError = false;
+
+  tearDown(() {
+    new File("output").writeAsStringSync(expectAsyncThrewError.toString());
+  });
+
+  test("test", () {
+    print("running test");
+
+    return new Future.delayed(new Duration(seconds: 1)).then((_) {
+      try {
+        expectAsync(() {});
+      } catch (_) {
+        expectAsyncThrewError = true;
+      }
+    });
+  });
+}
+""");
+
+      return _startUnittest(["test.dart"]).then((process) {
+        return _lines.bind(process.stdout).skip(2).first.then((line) {
+          expect(line, equals("running test"));
+          process.kill();
+          return process.exitCode;
+        }).then((_) {
+          expect(new File(p.join(_sandbox, "output")).readAsStringSync(),
+              equals("true"));
+          expect(new Directory(_tempDir).listSync(), isEmpty);
+        });
+      });
+    });
+  });
+}
+
+Future<Process> _startUnittest(List<String> args) {
+  new Directory(_tempDir).create();
+  return startUnittest(args, workingDirectory: _sandbox,
+      environment: {"_UNITTEST_TEMP_DIR": _tempDir});
+}