test.dart: add support for compiling multiple scripts for a single test.

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
  */
 #library("test_runner");
 
 #import("dart:io");
 #import("status_file_parser.dart");
 #import("test_progress.dart");
 #import("test_suite.dart");
 
 final int NO_TIMEOUT = 0;
 
+/** A command executed as a step in a test case. */
+class Command {
+  /** Path to the executable of this command. */
+  String executable;
+
+  /** Command line arguments to the executable. */
+  List<String> arguments;
+
+  /** The actual command line that will be executed. */
+  String commandLine;
+
+  Command(this.executable, this.arguments) {
+    commandLine = "$executable ${Strings.join(arguments, ' ')}";
+  }
+}
 
 /**
  * TestCase contains all the information needed to run a test and evaluate
  * its output.  Running a test involves starting a separate process, with
  * the executable and arguments given by the TestCase, and recording its
  * stdout and stderr output streams, and its exit code.  TestCase only
  * contains static information about the test; actually running the test is
  * performed by [ProcessQueue] using a [RunningProcess] object.
  *
  * The output information is stored in a [TestOutput] instance contained
  * in the TestCase. The TestOutput instance is responsible for evaluating
  * if the test has passed, failed, crashed, or timed out, and the TestCase
  * has information about what the expected result of the test should be.
  *
  * The TestCase has a callback function, [completedHandler], that is run when
  * the test is completed.
  */
 class TestCase {
-  String executablePath;
-  List<String> arguments;
+  /**
+   * A list of commands to execute. Most test cases have a single command. Frog
+   * tests have two commands, one to compilate the source and another to execute
+   * it. Some isolate tests might even have three, if they require compiling
+   * multiple sources that are run in isolation.
+   */
+  final List<Command> commands;
+
   Map configuration;
-  String commandLine;
   String displayName;
   TestOutput output;
   bool isNegative;
   Set<String> expectedOutcomes;
   Function completedHandler;
 
   TestCase(this.displayName,
-           this.executablePath,
-           this.arguments,
+           this.commands,
            this.configuration,
            this.completedHandler,
            this.expectedOutcomes,
            [this.isNegative = false]) {
     if (!isNegative) {
       this.isNegative = displayName.contains("NegativeTest");
     }
-    commandLine = "$executablePath ${Strings.join(arguments, ' ')}";
 
     // Special command handling. If a special command is specified
     // we have to completely rewrite the command that we are using.
     // We generate a new command-line that is the special command
     // where we replace '@' with the original command.
     var specialCommand = configuration['special-command'];
     if (!specialCommand.isEmpty()) {
       Expect.isTrue(specialCommand.contains('@'),
                     "special-command must contain a '@' char");
       var specialCommandSplit = specialCommand.split('@');
       var prefix = specialCommandSplit[0];
       var suffix = specialCommandSplit[1];
-      commandLine = '$prefix $commandLine $suffix';
-      var newArguments = [];
-      if (prefix.length > 0) {
-        var prefixSplit = prefix.split(' ');
-        var newExecutablePath = prefixSplit[0];
-        for (int i = 1; i < prefixSplit.length; i++) {
-          var current = prefixSplit[i];
-          if (!current.isEmpty()) newArguments.add(current);
+      List<Command> newCommands = [];
+      for (Command c in commands) {
+        var newExecutablePath;
+        var newArguments = [];
+
+        if (prefix.length > 0) {
+          var prefixSplit = prefix.split(' ');
+          newExecutablePath = prefixSplit[0];
+          for (int i = 1; i < prefixSplit.length; i++) {
+            var current = prefixSplit[i];
+            if (!current.isEmpty()) newArguments.add(current);
+          }
+          newArguments.add(c.executable);
         }
-        newArguments.add(executablePath);
-        executablePath = newExecutablePath;
+        newArguments.addAll(arguments);
+        var suffixSplit = suffix.split(' ');
+        suffixSplit.forEach((e) {
+          if (!e.isEmpty()) newArguments.add(e);
+        });
+        final newCommand = new Command(newExecutablePath, newArguments);
+        newCommands.add(newCommand);
+        Expect.stringEquals('$prefix ${c.commandLine} $suffix',
+            newCommand.commandLine);
       }
-      newArguments.addAll(arguments);
-      var suffixSplit = suffix.split(' ');
-      suffixSplit.forEach((e) {
-        if (!e.isEmpty()) newArguments.add(e);
-      });
-      arguments = newArguments;
+      commands = newCommand;
     }
   }
 
   int get timeout() => configuration['timeout'];
 
   String get configurationString() {
     final component = configuration['component'];
     final mode = configuration['mode'];
     final arch = configuration['arch'];
     return "$component ${mode}_$arch";
   }
 
   List<String> get batchRunnerArguments() => ['-batch'];
-  List<String> get batchTestArguments() => arguments;
+  List<String> get batchTestArguments() => commands.last().arguments;
 
   void completed() { completedHandler(this); }
 }
 
 
 /**
  * BrowserTestCase has an extra compilation command that is run in a separate
  * process, before the regular test is run as in the base class [TestCase].
  * If the compilation command fails, then the rest of the test is not run.
  */
 class BrowserTestCase extends TestCase {
   /**
-   * The executable that is run in a new process in the compilation phase.
-   */
-  String compilerPath;
-  /**
-   * The arguments for the compilation command.
-   */
-  List<String> compilerArguments;
-  /**
    * Indicates the number of potential retries remaining, to compensate for
    * flaky browser tests.
    */
   bool numRetries;
 
-  BrowserTestCase(displayName,
-                    this.compilerPath,
-                    this.compilerArguments,
-                    executablePath,
-                    arguments,
-                    configuration,
-                    completedHandler,
-                    expectedOutcomes,
-                    [isNegative = false]) : super(displayName,
-                                                  executablePath,
-                                                  arguments,
-                                                  configuration,
-                                                  completedHandler,
-                                                  expectedOutcomes,
-                                                  isNegative) {
-    if (compilerPath != null) {
-      commandLine = 'execution command: $commandLine';
-      String compilationCommand =
-          '$compilerPath ${Strings.join(compilerArguments, " ")}';
-      commandLine = 'compilation command: $compilationCommand\n$commandLine';
-    }
+  BrowserTestCase(displayName, commands, configuration, completedHandler,
+      expectedOutcomes, [isNegative = false])
+    : super(displayName, commands, configuration, completedHandler,
+        expectedOutcomes, isNegative) {
     numRetries = 2; // Allow two retries to compensate for flaky browser tests.
   }
 
-  List<String> get batchRunnerArguments() => [arguments[0], '--batch'];
+  List<String> get _lastArguments() => command.last().arguments;
+
+  List<String> get batchRunnerArguments() => [_lastArguments[0], '--batch'];
+
   List<String> get batchTestArguments() =>
-      arguments.getRange(1, arguments.length - 1);
+      _lastArguments.getRange(1, _lastArguments.length - 1);
 }
 
 
 /**
  * TestOutput records the output of a completed test: the process's exit code,
  * the standard output and standard error, whether the process timed out, and
  * the time the process took to run.  It also contains a pointer to the
  * [TestCase] this is the output of.
  */
 class TestOutput {
@@ skipping 87 matching lines @@
   Process process;
   TestCase testCase;
   bool timedOut = false;
   Date startTime;
   Timer timeoutTimer;
   List<String> stdout;
   List<String> stderr;
   List<Function> handlers;
   bool allowRetries = false;
 
+  /** Which command of [testCase.commands] is currently being executed. */
+  int currentStep;
+
   RunningProcess(TestCase this.testCase,
       [this.allowRetries, this.processQueue]);
 
-  void exitHandler(int exitCode) {
+  /**
+   * Called when all commands are executed. [exitCode] is 0 if all command
+   * succeded, otherwise it will have the exit code of the first failing
+   * command.
+   */
+  void testComplete(int exitCode) {
     new TestOutput(testCase, exitCode, timedOut, stdout,
                    stderr, new Date.now().difference(startTime));
-    process.close();
     timeoutTimer.cancel();
     if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) {
       print(testCase.displayName);
       for (var line in testCase.output.stderr) print(line);
       for (var line in testCase.output.stdout) print(line);
     }
     if (allowRetries != null && allowRetries
         && testCase.configuration['component'] == 'webdriver' &&
         testCase.output.unexpectedOutput && testCase.numRetries > 0) {
       // Selenium tests can be flaky. Try rerunning.
       testCase.output.requestRetry = true;
     }
     if (testCase.output.requestRetry) {
       testCase.output.requestRetry = false;
       this.timedOut = false;
       testCase.dynamic.numRetries--;
-      print("Potential flake. Re-running " + testCase.displayName);
+      print("Potential flake. Re-running ${testCase.displayName}");
       this.start();
     } else {
       testCase.completed();
     }
   }
 
-  void compilerExitHandler(int exitCode) {
-    if (exitCode != 0) {
+  /**
+   * Process exit handler called at the end of every command. It internally
+   * decides what is the most appropriate handler. In particular, all but the
+   * last command are compilation steps analyzed in [compilerExitHandler], the
+   * last command actually runs the test, which is analyzed in
+   * [testExitHandler].
+   */
+  void stepExitHandler(int exitCode) {
+    process.close();
+    if (currentStep == testCase.commands.length) { // done with test command
+      testComplete(exitCode);
+    } else if (exitCode != 0) {
       stderr.add('test.dart: Compilation step failed (exit code $exitCode)\n');
-      exitHandler(exitCode);
+      testComplete(exitCode);
     } else {
-      process.close();
-      stderr.add('test.dart: Compilation finished, starting execution\n');
-      stdout.add('test.dart: Compilation finished, starting execution\n');
-      if (testCase.configuration['component'] == 'webdriver') {
+      stderr.add('test.dart: Compilation step finished\n');

[Bill Hesse, 2012/02/28 16:41:56]

Could we add currentStep to the messages in stderr and stdout?

[Siggi Cherem (dart-lang), 2012/02/28 17:42:37]

Done

+      stdout.add('test.dart: Compilation step finished\n');
+      if (currentStep == testCase.commands.length - 1
+          && testCase.configuration['component'] == 'webdriver') {
         // Note: processQueue will always be non-null for component == webdriver
         // (It is only null for component == vm)
         processQueue._getBatchRunner(testCase).startTest(testCase);
       } else {
-        runCommand(testCase.executablePath, testCase.arguments, exitHandler);
+        runCommand(testCase.commands[currentStep++], stepExitHandler);
       }
     }
   }
 
   Function makeReadHandler(StringInputStream source, List<String> destination) {
     return () {
       if (source.closed) return;  // TODO(whesse): Remove when bug is fixed.
       var line = source.readLine();
       while (null != line) {
         destination.add(line);
         line = source.readLine();
       }
     };
   }
 
   void start() {
     Expect.isFalse(testCase.expectedOutcomes.contains(SKIP));
     stdout = new List<String>();
     stderr = new List<String>();
-    if (testCase is BrowserTestCase && testCase.dynamic.compilerPath != null) {
-      runCommand(testCase.dynamic.compilerPath,
-                 testCase.dynamic.compilerArguments,
-                 compilerExitHandler);
-    } else {
-      runCommand(testCase.executablePath, testCase.arguments, exitHandler);
-    }
+    currentStep = 0;
+    runCommand(testCase.commands[currentStep++], stepExitHandler);
   }
 
-  void runCommand(String executable,
-                  List<String> arguments,
+  void runCommand(Command command,
                   void exitHandler(int exitCode)) {
     if (new Platform().operatingSystem() == 'windows') {
       // Windows can't handle the first command if it is a .bat file or the like
       // with the slashes going the other direction.
       // TODO(efortuna): Remove this when fixed (Issue 1306).
-      executable = executable.replaceAll('/', '\\');
+      command.executable = command.executable.replaceAll('/', '\\');
     }
-    process = new Process.start(executable, arguments);
+    process = new Process.start(command.executable, command.arguments);
     process.exitHandler = exitHandler;
     startTime = new Date.now();
     InputStream stdoutStream = process.stdout;
     InputStream stderrStream = process.stderr;
     StringInputStream stdoutStringStream = new StringInputStream(stdoutStream);
     StringInputStream stderrStringStream = new StringInputStream(stderrStream);
     stdoutStringStream.lineHandler =
         makeReadHandler(stdoutStringStream, stdout);
     stderrStringStream.lineHandler =
         makeReadHandler(stderrStringStream, stderr);
@@ skipping 16 matching lines @@
 
   TestCase _currentTest;
   List<String> _testStdout;
   List<String> _testStderr;
   Date _startTime;
   Timer _timer;
 
   bool _isWebDriver;
 
   BatchRunnerProcess(TestCase testCase) {
-    _executable = testCase.executablePath;
+    _executable = testCase.commands.last().executable;
     _batchArguments = testCase.batchRunnerArguments;
     _isWebDriver = testCase.configuration['component'] == 'webdriver';
   }
 
   bool get active() => _currentTest != null;
 
   void startTest(TestCase testCase) {
     _currentTest = testCase;
     if (_process === null) {
       // Start process if not yet started.
-      _executable = testCase.executablePath;
+      _executable = testCase.commands.last().executable;
       _startProcess(() {
         doStartTest(testCase);
       });
-    } else if (testCase.executablePath != _executable) {
+    } else if (testCase.commands.last().executable != _executable) {
       // Restart this runner with the right executable for this test
       // if needed.
-      _executable = testCase.executablePath;
+      _executable = testCase.commands.last().executable;
       _batchArguments = testCase.batchRunnerArguments;
       _process.exitHandler = (exitCode) {
         _process.close();
         _startProcess(() {
           doStartTest(testCase);
         });
       };
       _process.kill();
     } else {
       doStartTest(testCase);
@@ skipping 311 matching lines @@
     for (var runner in runners) {
       if (!runner.active) return runner;
     }
     throw new Exception('Unable to find inactive batch runner.');
   }
 
   void _tryRunTest() {
     _checkDone();
     if (_numProcesses < _maxProcesses && !_tests.isEmpty()) {
       TestCase test = _tests.removeFirst();
-      if (_verbose) print(test.commandLine);
+      if (_verbose) print(test.commands.last().commandLine);
       if (_listTests) {
         final String tab = '\t';
         String outcomes =
             Strings.join(new List.from(test.expectedOutcomes), ',');
         print(test.displayName + tab + outcomes + tab + test.isNegative +
-              tab + Strings.join(test.arguments, tab));
+              tab + Strings.join(test.commands.last().arguments, tab));
         return;
       }
       _progress.start(test);
       Function oldCallback = test.completedHandler;
       Function wrapper = (TestCase test_arg) {
         _numProcesses--;
         _progress.done(test_arg);
         _tryRunTest();
         oldCallback(test_arg);
       };
       test.completedHandler = wrapper;
       if (test.configuration['component'] == 'dartc' &&
           test.displayName != 'dartc/junit_tests') {
         _getBatchRunner(test).startTest(test);
       } else {
         // Once we've actually failed a test, technically, we wouldn't need to
         // bother retrying any subsequent tests since the bot is already red.
         // However, we continue to retry tests until we have actually failed
         // four tests (arbitrarily chosen) for more debugable output, so that
         // the developer doesn't waste his or her time trying to fix a bunch of
         // tests that appear to be broken but were actually just flakes that
         // didn't get retried because there had already been one failure.
         bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
         new RunningProcess(test, allowRetry, this).start();
       }
       _numProcesses++;
     }
   }
 }