tools/test.dart: Add multitest support to Dart implementation of test runner.

tools/testing/dart/multitest.dart

+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("multitest");
+
+// Multitests are Dart test scripts containing lines of the form
+// " [some dart code] /// [key]: [error type]"
+//
+// For each key in the file, a new test file is made containing all
+// the normal lines of the file, and all of the multitest lines containing
+// that key, in the same order as in the source file.  The new test
+// is expected to fail if there is a non-empty error type listed, of
+// type 'compile-time error', 'runtime error', 'static type error', or
+// 'dynamic type error'.  The type error tests fail only in checked mode.
+// There is also a test created from only the untagged lines of the file,
+// with key "none", which is expected to pass.  This library extracts these
+// tests, writes them into a temporary directory, and passes them to the test
+// runner.  These tests may be referred to in the status files with the
+// pattern [test name]/[key].
+//
+// For example: file I_am_a_multitest.dart
+//   aaa
+//   bbb /// 02: runtime error
+//   ccc /// 02: continued
+//   ddd /// 07: static type error
+//   eee
+//
+// should create three tests:
+// I_am_a_multitest_none.dart
+//   aaa
+//   eee
+//
+// I_am_a_multitest_02.dart
+//   aaa
+//   bbb /// 02: runtime error
+//   ccc /// 02: continued
+//   eee
+//
+// and I_am_a_multitest_07.dart
+//   aaa
+//   ddd /// 07: static type error
+//   eee
+
+void ExtractTestsFromMultitest(String filename,
+                          Map<String, String> tests,

[Mads Ager (google), 2011/11/30 09:11:23]

Identation.

[Bill Hesse, 2011/11/30 10:01:39]

Done.

+                          Map<String, String> outcomes) {
+  // Read the entire file into a byte buffer and transform it to a
+  // String. This will treat the file as ascii but the only parts
+  // we are interested in will be ascii in any case.
+  File file = new File(filename);
+  file.openSync();
+  List chars = new List(file.lengthSync());
+  int offset = 0;
+  while (offset != chars.length) {
+    offset += file.readListSync(chars, offset, chars.length - offset);
+  }
+  file.closeSync();
+  String contents = new String.fromCharCodes(chars);
+  chars = null;
+  int first_newline = contents.indexOf('\n');
+  final String line_separator =
+      (first_newline == 0 || contents[first_newline - 1] != '\r')
+      ? '\n'
+      : '\r\n';
+  List<String> lines = contents.split(line_separator);
+  if (lines.last() == '') lines.removeLast();
+  contents = null;
+  Set<String> validMultitestOutcomes = new Set<String>.from(
+      ['compile-time error', 'runtime error',
+       'static type error', 'dynamic type error', '']);
+  
+  List<String> testTemplate = new List<String>();
+  testTemplate.add('// Test created from multitest named $filename.');
+  // Create the set of multitests, which will have a new test added each
+  // time we see a multitest line with a new key.
+  Map<String, List<String>> testsAsLines = new Map<String, List<String>>();
+  
+  for (String line in lines) {
+    if (line.contains('///')) {
+      var parts = line.split('///')[1].split(':');
+      var key = parts[0].trim();
+      var rest = parts[1].trim();
+      if (testsAsLines.containsKey(key)) {
+        Expect.equals('continued', rest);

[Mads Ager (google), 2011/11/30 09:11:23]

Should we always throw an exception here? Users will probably mostly run the
test scripts without using checked mode.

[Bill Hesse, 2011/11/30 10:01:39]

I was under the impression that Expect always checks the condition, in both
production and checked mode. assert only applies in checked mode.  I don't see
anything in Expect's implementation that disables it, and we use it in all the
tests.


On 2011/11/30 09:11:23, Mads Ager wrote:

+        testsAsLines[key].add(line);
+      } else {
+        (testsAsLines[key] = new List<String>.from(testTemplate)).add(line);
+        outcomes[key] = rest;
+        Expect.isTrue(validMultitestOutcomes.contains(rest));

[Mads Ager (google), 2011/11/30 09:11:23]

Ditto.

+      }
+    } else {
+      if ((line.startsWith('#import(') || line.startsWith('#source(')) &&
+          !line.contains("dart:")) {
+        // TODO(whesse): Rewrite relative paths to reflect temporary directory.

[Mads Ager (google), 2011/11/30 09:11:23]

Why does this work if we don't do this? Where are the tests generated? We should
make sure that they are generated in the output directory so we can clean
everything generated from the repository by deleting the output dir. 

We can do that in the next change.

[Bill Hesse, 2011/11/30 10:01:39]

Only one multitest has an #import, and so only that one fails, in the "none"
default test, if we don't do this.  We can leave it out, but then there is no
place to put the comment.

On 2011/11/30 09:11:23, Mads Ager wrote:

+        tests = [];
+        outcomes = [];
+        return;
+      }
+      testTemplate.add(line);
+      for (var test in testsAsLines.getValues()) test.add(line);
+    }
+  }
+  // Add the template, with no multitest lines, as a test with key 'none'.
+  testsAsLines['none'] = testTemplate;
+  outcomes['none'] = '';
+  
+  // Copy all the tests into the output map tests, as multiline strings.
+  for (String key in testsAsLines.getKeys()) {
+    tests[key] =
+        Strings.join(testsAsLines[key], line_separator) + line_separator;
+  }
+}
+
+
+void DoMultitest(String filename,
+                 Function doTest(List<String> args,
+                                 bool isNegative,
+                                 bool isNegativeIfChecked)) {
+  // Each new test is a single String value in the Map tests.
+  Map<String, String> tests = new Map<String, String>();
+  Map<String, String> outcomes = new Map<String, String>();
+  ExtractTestsFromMultitest(filename, tests, outcomes);
+  String pathSeparator = new Platform().pathSeparator();
+  int start = filename.lastIndexOf(pathSeparator) + 1;
+  int end = filename.indexOf('.dart', start);
+  String baseFilename = filename.substring(start, end);
+  Directory dir = new Directory("");

[Mads Ager (google), 2011/11/30 09:11:23]

Could you add a TODO here. We should make sure to point this at the build output
directory for test generation.

[Bill Hesse, 2011/11/30 10:01:39]

OK,  I didn't think of using the build directory.  Can we reliably find it?  Do
we really want to do this, and if we do, do we also want to delete the files and
directory automatically, and do we want to try to ensure that we do it if the
test is cancelled?  What if someone wants to debug using a generated multitest?

On 2011/11/30 09:11:23, Mads Ager wrote:

+  dir.errorHandler =
+      (error) { Expect.fail("Error creating temp directory: $error"); };

[Mads Ager (google), 2011/11/30 09:11:23]

Maybe just throw exception?

+
+  dir.createTempHandler = (){

[Mads Ager (google), 2011/11/30 09:11:23]

space between '()' and '{'

Also multiple occurrences below.

+    String path = dir.path + new Platform().pathSeparator();
+    Iterator currentKey = tests.getKeys().iterator();
+    WriteMultitestToFileAndQueueIt(tests, outcomes, currentKey,
+                                   '$path$baseFilename', doTest);
+  };
+
+  dir.createTemp();
+}
+
+
+// Write multiple tests to files, using tail recursion in a callback
+// to serialize the file operations, rather than opening all files at once.
+WriteMultitestToFileAndQueueIt(Map<String, String> tests,
+                               Map<String, String> outcomes,
+                               Iterator currentKey,
+                               String basePath,
+                               Function doTest) {
+  if (!currentKey.hasNext()) return;
+  final String key = currentKey.next();
+  final String filename = '${basePath}_$key.dart';
+  final File file = new File(filename);
+  file.errorHandler = (error) {
+    Expect.fail("Error creating temp file: $error");
+  };
+  file.createHandler = (){
+    file.open(writable: true);
+  };
+  file.openHandler =  (){
+    var bytes = tests[key].charCodes();
+    file.writeList(bytes, 0, bytes.length);
+  };
+  file.noPendingWriteHandler =(){
+    file.close();
+  };
+  file.closeHandler = (){
+    var outcome = outcomes[key];
+    bool isNegative = outcome.contains('compile-time error') ||
+    outcome.contains('runtime error');

[Mads Ager (google), 2011/11/30 09:11:23]

Indentation.

+    bool isNegativeIfChecked = outcome.contains('type error');
+    doTest(filename, isNegative, isNegativeIfChecked);
+    // TODO(whesse): Register files and directories to be deleted.
+    // They should be registered in a persistent list, so they can
+    // be deleted later even if the test script is interrupted.
+    WriteMultitestToFileAndQueueIt(tests, outcomes, currentKey,
+                                   basePath, doTest);
+  };
+  file.create();
+}