test.dart: add support for compiling multiple scripts for a single test.

tools/testing/dart/test_runner.dart

Index: tools/testing/dart/test_runner.dart
diff --git a/tools/testing/dart/test_runner.dart b/tools/testing/dart/test_runner.dart
index 7eddd3f73339cc679b4a7c3b8698506c21829d74..5aa3098ff514dbf770dbd285e8e116883b723a43 100644
--- a/tools/testing/dart/test_runner.dart
+++ b/tools/testing/dart/test_runner.dart
@@ -18,6 +18,21 @@
 
 final int NO_TIMEOUT = 0;
 
+/** A command executed as a step in a test case. */
+class Command {
+  /** Path to the executable of this command. */
+  String executable;
+
+  /** Command line arguments to the executable. */
+  List<String> arguments;
+
+  /** The actual command line that will be executed. */
+  String commandLine;
+
+  Command(this.executable, this.arguments) {
+    commandLine = "$executable ${Strings.join(arguments, ' ')}";
+  }
+}
 
 /**
  * TestCase contains all the information needed to run a test and evaluate
@@ -36,10 +51,15 @@ final int NO_TIMEOUT = 0;
  * the test is completed.
  */
 class TestCase {
-  String executablePath;
-  List<String> arguments;
+  /**
+   * A list of commands to execute. Most test cases have a single command. Frog
+   * tests have two commands, one to compilate the source and another to execute
+   * it. Some isolate tests might even have three, if they require compiling
+   * multiple sources that are run in isolation.
+   */
+  final List<Command> commands;
+
   Map configuration;
-  String commandLine;
   String displayName;
   TestOutput output;
   bool isNegative;
@@ -47,8 +67,7 @@ class TestCase {
   Function completedHandler;
 
   TestCase(this.displayName,
-           this.executablePath,
-           this.arguments,
+           this.commands,
            this.configuration,
            this.completedHandler,
            this.expectedOutcomes,
@@ -56,7 +75,6 @@ class TestCase {
     if (!isNegative) {
       this.isNegative = displayName.contains("NegativeTest");
     }
-    commandLine = "$executablePath ${Strings.join(arguments, ' ')}";
 
     // Special command handling. If a special command is specified
     // we have to completely rewrite the command that we are using.
@@ -69,24 +87,31 @@ class TestCase {
       var specialCommandSplit = specialCommand.split('@');
       var prefix = specialCommandSplit[0];
       var suffix = specialCommandSplit[1];
-      commandLine = '$prefix $commandLine $suffix';
-      var newArguments = [];
-      if (prefix.length > 0) {
-        var prefixSplit = prefix.split(' ');
-        var newExecutablePath = prefixSplit[0];
-        for (int i = 1; i < prefixSplit.length; i++) {
-          var current = prefixSplit[i];
-          if (!current.isEmpty()) newArguments.add(current);
+      List<Command> newCommands = [];
+      for (Command c in commands) {
+        var newExecutablePath;
+        var newArguments = [];
+
+        if (prefix.length > 0) {
+          var prefixSplit = prefix.split(' ');
+          newExecutablePath = prefixSplit[0];
+          for (int i = 1; i < prefixSplit.length; i++) {
+            var current = prefixSplit[i];
+            if (!current.isEmpty()) newArguments.add(current);
+          }
+          newArguments.add(c.executable);
         }
-        newArguments.add(executablePath);
-        executablePath = newExecutablePath;
+        newArguments.addAll(arguments);
+        var suffixSplit = suffix.split(' ');
+        suffixSplit.forEach((e) {
+          if (!e.isEmpty()) newArguments.add(e);
+        });
+        final newCommand = new Command(newExecutablePath, newArguments);
+        newCommands.add(newCommand);
+        Expect.stringEquals('$prefix ${c.commandLine} $suffix',
+            newCommand.commandLine);
       }
-      newArguments.addAll(arguments);
-      var suffixSplit = suffix.split(' ');
-      suffixSplit.forEach((e) {
-        if (!e.isEmpty()) newArguments.add(e);
-      });
-      arguments = newArguments;
+      commands = newCommand;
     }
   }
 
@@ -100,7 +125,7 @@ class TestCase {
   }
 
   List<String> get batchRunnerArguments() => ['-batch'];
-  List<String> get batchTestArguments() => arguments;
+  List<String> get batchTestArguments() => commands.last().arguments;
 
   void completed() { completedHandler(this); }
 }
@@ -113,46 +138,24 @@ class TestCase {
  */
 class BrowserTestCase extends TestCase {
   /**
-   * The executable that is run in a new process in the compilation phase.
-   */
-  String compilerPath;
-  /**
-   * The arguments for the compilation command.
-   */
-  List<String> compilerArguments;
-  /**
    * Indicates the number of potential retries remaining, to compensate for
    * flaky browser tests.
    */
   bool numRetries;
 
-  BrowserTestCase(displayName,
-                    this.compilerPath,
-                    this.compilerArguments,
-                    executablePath,
-                    arguments,
-                    configuration,
-                    completedHandler,
-                    expectedOutcomes,
-                    [isNegative = false]) : super(displayName,
-                                                  executablePath,
-                                                  arguments,
-                                                  configuration,
-                                                  completedHandler,
-                                                  expectedOutcomes,
-                                                  isNegative) {
-    if (compilerPath != null) {
-      commandLine = 'execution command: $commandLine';
-      String compilationCommand =
-          '$compilerPath ${Strings.join(compilerArguments, " ")}';
-      commandLine = 'compilation command: $compilationCommand\n$commandLine';
-    }
+  BrowserTestCase(displayName, commands, configuration, completedHandler,
+      expectedOutcomes, [isNegative = false])
+    : super(displayName, commands, configuration, completedHandler,
+        expectedOutcomes, isNegative) {
     numRetries = 2; // Allow two retries to compensate for flaky browser tests.
   }
 
-  List<String> get batchRunnerArguments() => [arguments[0], '--batch'];
+  List<String> get _lastArguments() => command.last().arguments;
+
+  List<String> get batchRunnerArguments() => [_lastArguments[0], '--batch'];
+
   List<String> get batchTestArguments() =>
-      arguments.getRange(1, arguments.length - 1);
+      _lastArguments.getRange(1, _lastArguments.length - 1);
 }
 
 
@@ -260,13 +263,20 @@ class RunningProcess {
   List<Function> handlers;
   bool allowRetries = false;
 
+  /** Which command of [testCase.commands] is currently being executed. */
+  int currentStep;
+
   RunningProcess(TestCase this.testCase,
       [this.allowRetries, this.processQueue]);
 
-  void exitHandler(int exitCode) {
+  /**
+   * Called when all commands are executed. [exitCode] is 0 if all command
+   * succeded, otherwise it will have the exit code of the first failing
+   * command.
+   */
+  void testComplete(int exitCode) {
     new TestOutput(testCase, exitCode, timedOut, stdout,
                    stderr, new Date.now().difference(startTime));
-    process.close();
     timeoutTimer.cancel();
     if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) {
       print(testCase.displayName);
@@ -283,27 +293,37 @@ class RunningProcess {
       testCase.output.requestRetry = false;
       this.timedOut = false;
       testCase.dynamic.numRetries--;
-      print("Potential flake. Re-running " + testCase.displayName);
+      print("Potential flake. Re-running ${testCase.displayName}");
       this.start();
     } else {
       testCase.completed();
     }
   }
 
-  void compilerExitHandler(int exitCode) {
-    if (exitCode != 0) {
+  /**
+   * Process exit handler called at the end of every command. It internally
+   * decides what is the most appropriate handler. In particular, all but the
+   * last command are compilation steps analyzed in [compilerExitHandler], the
+   * last command actually runs the test, which is analyzed in
+   * [testExitHandler].
+   */
+  void stepExitHandler(int exitCode) {
+    process.close();
+    if (currentStep == testCase.commands.length) { // done with test command
+      testComplete(exitCode);
+    } else if (exitCode != 0) {
       stderr.add('test.dart: Compilation step failed (exit code $exitCode)\n');
-      exitHandler(exitCode);
+      testComplete(exitCode);
     } else {
-      process.close();
-      stderr.add('test.dart: Compilation finished, starting execution\n');
-      stdout.add('test.dart: Compilation finished, starting execution\n');
-      if (testCase.configuration['component'] == 'webdriver') {
+      stderr.add('test.dart: Compilation step finished\n');

[Bill Hesse, 2012/02/28 16:41:56]

Could we add currentStep to the messages in stderr and stdout?

[Siggi Cherem (dart-lang), 2012/02/28 17:42:37]

Done

+      stdout.add('test.dart: Compilation step finished\n');
+      if (currentStep == testCase.commands.length - 1
+          && testCase.configuration['component'] == 'webdriver') {
         // Note: processQueue will always be non-null for component == webdriver
         // (It is only null for component == vm)
         processQueue._getBatchRunner(testCase).startTest(testCase);
       } else {
-        runCommand(testCase.executablePath, testCase.arguments, exitHandler);
+        runCommand(testCase.commands[currentStep++], stepExitHandler);
       }
     }
   }
@@ -323,25 +343,19 @@ class RunningProcess {
     Expect.isFalse(testCase.expectedOutcomes.contains(SKIP));
     stdout = new List<String>();
     stderr = new List<String>();
-    if (testCase is BrowserTestCase && testCase.dynamic.compilerPath != null) {
-      runCommand(testCase.dynamic.compilerPath,
-                 testCase.dynamic.compilerArguments,
-                 compilerExitHandler);
-    } else {
-      runCommand(testCase.executablePath, testCase.arguments, exitHandler);
-    }
+    currentStep = 0;
+    runCommand(testCase.commands[currentStep++], stepExitHandler);
   }
 
-  void runCommand(String executable,
-                  List<String> arguments,
+  void runCommand(Command command,
                   void exitHandler(int exitCode)) {
     if (new Platform().operatingSystem() == 'windows') {
       // Windows can't handle the first command if it is a .bat file or the like
       // with the slashes going the other direction.
       // TODO(efortuna): Remove this when fixed (Issue 1306).
-      executable = executable.replaceAll('/', '\\');
+      command.executable = command.executable.replaceAll('/', '\\');
     }
-    process = new Process.start(executable, arguments);
+    process = new Process.start(command.executable, command.arguments);
     process.exitHandler = exitHandler;
     startTime = new Date.now();
     InputStream stdoutStream = process.stdout;
@@ -378,7 +392,7 @@ class BatchRunnerProcess {
   bool _isWebDriver;
 
   BatchRunnerProcess(TestCase testCase) {
-    _executable = testCase.executablePath;
+    _executable = testCase.commands.last().executable;
     _batchArguments = testCase.batchRunnerArguments;
     _isWebDriver = testCase.configuration['component'] == 'webdriver';
   }
@@ -389,14 +403,14 @@ class BatchRunnerProcess {
     _currentTest = testCase;
     if (_process === null) {
       // Start process if not yet started.
-      _executable = testCase.executablePath;
+      _executable = testCase.commands.last().executable;
       _startProcess(() {
         doStartTest(testCase);
       });
-    } else if (testCase.executablePath != _executable) {
+    } else if (testCase.commands.last().executable != _executable) {
       // Restart this runner with the right executable for this test
       // if needed.
-      _executable = testCase.executablePath;
+      _executable = testCase.commands.last().executable;
       _batchArguments = testCase.batchRunnerArguments;
       _process.exitHandler = (exitCode) {
         _process.close();
@@ -728,13 +742,13 @@ class ProcessQueue {
     _checkDone();
     if (_numProcesses < _maxProcesses && !_tests.isEmpty()) {
       TestCase test = _tests.removeFirst();
-      if (_verbose) print(test.commandLine);
+      if (_verbose) print(test.commands.last().commandLine);
       if (_listTests) {
         final String tab = '\t';
         String outcomes =
             Strings.join(new List.from(test.expectedOutcomes), ',');
         print(test.displayName + tab + outcomes + tab + test.isNegative +
-              tab + Strings.join(test.arguments, tab));
+              tab + Strings.join(test.commands.last().arguments, tab));
         return;
       }
       _progress.start(test);