test.py: First step towards support of caching dart2js compilations across runtimes

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
  *   and creating [TestCase]s for those files that meet the relevant criteria.
  * - Preparing tests, including copying files and frameworks to temporary
  *   directories, and computing the command line and arguments to be run.
  */
 library test_suite;
 
 import "dart:async";
 import "dart:io";
 import "dart:isolate";
 import "drt_updater.dart";
 import "multitest.dart";
 import "status_file_parser.dart";
 import "test_runner.dart";
 import "utils.dart";
 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR;
 
 part "browser_test.dart";
 
 
-// TODO(rnystrom): Add to dart:core?
 /**
  * A simple function that tests [arg] and returns `true` or `false`.
  */
 typedef bool Predicate<T>(T arg);
 
 typedef void CreateTest(Path filePath,
                         bool hasCompileError,
                         bool hasRuntimeError,
                         {bool isNegativeIfChecked,
                          bool hasFatalTypeErrors,
@@ skipping 62 matching lines @@
  *
  * Most TestSuites represent a directory or directory tree containing tests,
  * and a status file containing the expected results when these tests are run.
  */
 abstract class TestSuite {
   final Map configuration;
   final String suiteName;
 
   TestSuite(this.configuration, this.suiteName);
 
+  String get configurationDir {
+    return TestUtils.configurationDir(configuration);
+  }
+
   /**
    * Whether or not binaries should be found in the root build directory or
    * in the built SDK.
    */
   bool get useSdk {
     // The pub suite always uses the SDK.
     // TODO(rnystrom): Eventually, all test suites should run out of the SDK
     // and this check should go away.
     if (suiteName == 'pub') return true;
 
@@ skipping 177 matching lines @@
  * The tests are compiled into a monolithic executable by the build step.
  * The executable lists its tests when run with the --list command line flag.
  * Individual tests are run by specifying them on the command line.
  */
 class CCTestSuite extends TestSuite {
   final String testPrefix;
   String targetRunnerPath;
   String hostRunnerPath;
   final String dartDir;
   List<String> statusFilePaths;
-  TestCaseEvent doTest;
+  Function doTest;
   VoidFunction doDone;
   ReceivePort receiveTestName;
   TestExpectations testExpectations;
 
   CCTestSuite(Map configuration,
               String suiteName,
               String runnerName,
               List<String> this.statusFilePaths,
               {this.testPrefix: ''})
       : super(configuration, suiteName),
@@ skipping 29 matching lines @@
 
       if (configuration["report"]) {
         SummaryReport.add(expectations);
       }
 
       if (expectations.contains(SKIP)) return;
 
       var args = TestUtils.standardOptions(configuration);
       args.add(testName);
 
+      var command = CommandBuilder.instance.getCommand(
+          'run_vm_unittest', targetRunnerPath, args, configurationDir);
       doTest(
           new TestCase(constructedName,
-                       [new Command('run_vm_unittest', targetRunnerPath, args)],
+                       [command],
                        configuration,
-                       completeHandler,
                        expectations));
     }
   }
 
-  void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]) {
+  void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) {
     doTest = onTest;
     doDone = onDone;
 
     var filesRead = 0;
     void statusFileRead() {
       filesRead++;
       if (filesRead == statusFilePaths.length) {
         receiveTestName = new ReceivePort();
         var port = spawnFunction(ccTestLister);
         port.send(hostRunnerPath, receiveTestName.toSendPort());
         receiveTestName.receive(testNameHandler);
       }
     }
 
     testExpectations = new TestExpectations();
     for (var statusFilePath in statusFilePaths) {
       ReadTestExpectationsInto(testExpectations,
                                '$dartDir/$statusFilePath',
                                configuration,
                                statusFileRead);
     }
   }
-
-  void completeHandler(TestCase testCase) {
-  }
 }
 
 
 class TestInformation {
   Path filePath;
   Map optionsFromFile;
   bool hasCompileError;
   bool hasRuntimeError;
   bool isNegativeIfChecked;
   bool hasFatalTypeErrors;
   Set<String> multitestOutcome;
 
   TestInformation(this.filePath, this.optionsFromFile,
                   this.hasCompileError, this.hasRuntimeError,
                   this.isNegativeIfChecked, this.hasFatalTypeErrors,
                   this.multitestOutcome) {
     assert(filePath.isAbsolute);
   }
 }
 
 /**
  * A standard [TestSuite] implementation that searches for tests in a
  * directory, and creates [TestCase]s that compile and/or run them.
  */
 class StandardTestSuite extends TestSuite {
   final Path suiteDir;
   final List<String> statusFilePaths;
-  TestCaseEvent doTest;
+  Function doTest;
   TestExpectations testExpectations;
   List<TestInformation> cachedTests;
   final Path dartDir;
   Predicate<String> isTestFilePredicate;
   final bool listRecursively;
   final extraVmOptions;
 
   static final RegExp multiTestRegExp = new RegExp(r"/// [0-9][0-9]:(.*)");
 
   StandardTestSuite(Map configuration,
@@ skipping 65 matching lines @@
    */
   bool isTestFile(String filename) {
     // Use the specified predicate, if provided.
     if (isTestFilePredicate != null) return isTestFilePredicate(filename);
 
     return filename.endsWith("Test.dart");
   }
 
   List<String> additionalOptions(Path filePath) => [];
 
-  void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]) {
+  void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) {
     updateDartium().then((_) {
       doTest = onTest;
 
       return readExpectations();
     }).then((expectations) {
       testExpectations = expectations;
 
       // Checked if we have already found and generated the tests for
       // this suite.
       if (!testCache.containsKey(suiteName)) {
@@ skipping 217 matching lines @@
 
     for (var vmOptions in vmOptionsList) {
       var allVmOptions = vmOptions;
       if (!extraVmOptions.isEmpty) {
         allVmOptions = new List.from(vmOptions)..addAll(extraVmOptions);
       }
 
       doTest(new TestCase('$suiteName/$testName',
                           makeCommands(info, allVmOptions, commonArguments),
                           configuration,
-                          completeHandler,
                           expectations,
                           isNegative: isNegative,
                           info: info));
     }
   }
 
   List<Command> makeCommands(TestInformation info, var vmOptions, var args) {
     var compiler = configuration['compiler'];
     switch (compiler) {
     case 'dart2js':
       args = new List.from(args);
       String tempDir = createOutputDirectory(info.filePath, '');
       args.add('--out=$tempDir/out.js');
 
-       List<Command> commands =
-           <Command>[new CompilationCommand(compiler,
-                                            "$tempDir/out.js",
-                                            !useSdk,
-                                            dart2JsBootstrapDependencies,
-                                            compilerPath,
-                                            args)];
+      var command = CommandBuilder.instance.getCompilationCommand(
+          compiler, "$tempDir/out.js", !useSdk,
+          dart2JsBootstrapDependencies, compilerPath, args, configurationDir);
+
+      List<Command> commands = <Command>[command];
       if (info.hasCompileError) {
         // Do not attempt to run the compiled result. A compilation
         // error should be reported by the compilation command.
       } else if (configuration['runtime'] == 'd8') {
-        commands.add(new Command("d8", d8FileName, ['$tempDir/out.js']));
+        commands.add(CommandBuilder.instance.getCommand(
+            "d8", d8FileName, ['$tempDir/out.js'], configurationDir));
       } else if (configuration['runtime'] == 'jsshell') {
-        commands.add(
-            new Command("jsshell", jsShellFileName, ['$tempDir/out.js']));
+        commands.add(CommandBuilder.instance.getCommand(
+            "jsshell", jsShellFileName, ['$tempDir/out.js'], configurationDir));
       }
       return commands;
 
     case 'dart2dart':
       args = new List.from(args);
       args.add('--output-type=dart');
       String tempDir = createOutputDirectory(info.filePath, '');
       args.add('--out=$tempDir/out.dart');
 
       List<Command> commands =
-          <Command>[new CompilationCommand(compiler,
-                                           "$tempDir/out.dart",
-                                           !useSdk,
-                                           dart2JsBootstrapDependencies,
-                                           compilerPath,
-                                           args)];
+          <Command>[CommandBuilder.instance.getCompilationCommand(
+              compiler, "$tempDir/out.dart", !useSdk,
+              dart2JsBootstrapDependencies, compilerPath, args,
+              configurationDir)];
       if (info.hasCompileError) {
         // Do not attempt to run the compiled result. A compilation
         // error should be reported by the compilation command.
       } else if (configuration['runtime'] == 'vm') {
         // TODO(antonm): support checked.
         var vmArguments = new List.from(vmOptions);
         vmArguments.addAll([
             '--ignore-unrecognized-flags', '$tempDir/out.dart']);
-        commands.add(new Command("vm", vmFileName, vmArguments));
+        commands.add(CommandBuilder.instance.getCommand(
+            "vm", vmFileName, vmArguments, configurationDir));
       } else {
         throw 'Unsupported runtime ${configuration["runtime"]} for dart2dart';
       }
       return commands;
 
     case 'none':
+      var arguments = new List.from(vmOptions);
+      arguments.addAll(args);
+      return <Command>[CommandBuilder.instance.getCommand(
+          'vm', dartShellFileName, arguments, configurationDir)];
+
     case 'dartanalyzer':
     case 'dart2analyzer':
-      var displayName = (configuration['compiler'] == 'none'
-          ? 'vm' : configuration['compiler']);
-      var arguments = new List.from(vmOptions);
-      arguments.addAll(args);
-      return <Command>[
-          new Command(displayName, dartShellFileName, arguments)];
+      return <Command>[CommandBuilder.instance.getAnalysisCommand(
+          compiler, dartShellFileName, args, configurationDir,
+          flavor: compiler)];
 
     default:
       throw 'Unknown compiler ${configuration["compiler"]}';
     }
   }
 
   CreateTest makeTestCaseCreator(RegExp pattern, Map optionsFromFile) {
     return (Path filePath,
             bool hasCompileError,
             bool hasRuntimeError,
@@ skipping 71 matching lines @@
     File file = new File(dartWrapperFilename);
     RandomAccessFile dartWrapper = file.openSync(mode: FileMode.WRITE);
 
     var usePackageImport = dartLibraryFilename.segments().contains("pkg");
     var libraryPathComponent = _createUrlPathFromFile(dartLibraryFilename);
     dartWrapper.writeStringSync(dartTestWrapper(usePackageImport,
                                                 libraryPathComponent));
     dartWrapper.closeSync();
   }
 
-
   /**
    * The [StandardTestSuite] has support for tests that
    * compile a test from Dart to JavaScript, and then run the resulting
    * JavaScript.  This function creates a working directory to hold the
    * JavaScript version of the test, and copies the appropriate framework
    * files to that directory.  It creates a [BrowserTestCase], which has
    * two sequential steps to be run by the [ProcessQueue] when the test is
    * executed: a compilation step and an execution step, both with the
    * appropriate executable and arguments. The [expectations] object can be
    * either a Set<String> if the test is a regular test, or a Map<String
@@ skipping 87 matching lines @@
           // compiled, move the input scripts over with the script so they can
           // be accessed.
           String result = new File.fromPath(fromPath).readAsStringSync();
           new File('$tempDir/$baseName.dart').writeAsStringSync(result);
         }
       }
 
 
       // Variables for browser multi-tests.
       List<String> subtestNames = info.optionsFromFile['subtestNames'];
-      BrowserTestCase multitestParentTest;
       int subtestIndex = 0;
       // Construct the command that executes the browser test
       do {
         List<Command> commandSet = new List<Command>.from(commands);
-        if (subtestIndex != 0) {
-          // NOTE: The first time we enter this loop, all the compilation
-          // commands will be executed. On subsequent loop iterations, we
-          // don't need to do any compilations. Thus we set "commandSet = []".
-          commandSet = [];
-        }
 
         var htmlPath_subtest = _createUrlPathFromFile(new Path(htmlPath));
         var fullHtmlPath = _getUriForBrowserTest(info, htmlPath_subtest,
                                                  subtestNames, subtestIndex);
 
         List<String> args = <String>[];
 
         if (configuration['use_browser_controller']) {
           // This command is not actually run, it is used for reproducing
           // the failure.
           args = ['tools/testing/dart/launch_browser.dart',
                   runtime,
                   fullHtmlPath];
-          commandSet.add(new Command(runtime,
-                                     TestUtils.dartTestExecutable.toString(),
-                                     args));
+          commandSet.add(CommandBuilder.instance.getBrowserTestCommand(
+              runtime, fullHtmlPath,
+              TestUtils.dartTestExecutable.toString(), args, configurationDir));
         } else if (TestUtils.usesWebDriver(runtime)) {
           args = [
               dartDir.append('tools/testing/run_selenium.py').toNativePath(),
               '--browser=$runtime',
               // NOTE: This value will be overridden by the test runner
               '--timeout=${configuration['timeout']}',
               '--out=$fullHtmlPath'];
           if (runtime == 'dartium') {
             args.add('--executable=$dartiumFilename');
           }
           if (subtestIndex != 0) {
             args.add('--force-refresh');
           }
-          commandSet.add(new Command(runtime, 'python', args));
+          commandSet.add(CommandBuilder.instance.getSeleniumTestCommand(
+                  runtime, fullHtmlPath, 'python', args, configurationDir));
         } else {
-          if (runtime != "drt") {
-            print("Unknown runtime $runtime");
-            exit(1);
-          }
+          assert(runtime == "drt");
 
           var dartFlags = [];
           var contentShellOptions = [];
 
           contentShellOptions.add('--no-timeout');
           contentShellOptions.add('--dump-render-tree');
 
           if (compiler == 'none' || compiler == 'dart2dart') {
             dartFlags.add('--ignore-unrecognized-flags');
             if (configuration["checked"]) {
               dartFlags.add('--enable_asserts');
               dartFlags.add("--enable_type_checks");
             }
             dartFlags.addAll(vmOptions);
           }
 
           if (expectedOutput != null) {
             if (expectedOutput.toNativePath().endsWith('.png')) {
               // pixel tests are specified by running DRT "foo.html'-p"
               contentShellOptions.add('--notree');
               fullHtmlPath = "${fullHtmlPath}'-p";
             }
           }
-          commandSet.add(new ContentShellCommand(contentShellFilename,
-                                                 fullHtmlPath,
-                                                 contentShellOptions,
-                                                 dartFlags,
-                                                 expectedOutput));
+          commandSet.add(CommandBuilder.instance.getContentShellCommand(
+              contentShellFilename, fullHtmlPath, contentShellOptions,
+              dartFlags, expectedOutput, configurationDir));
         }
 
         // Create BrowserTestCase and queue it.
         String testDisplayName = '$suiteName/$testName';
         var testCase;
         if (info.optionsFromFile['isMultiHtmlTest']) {
           testDisplayName = '$testDisplayName/${subtestNames[subtestIndex]}';
           testCase = new BrowserTestCase(testDisplayName,
-              commandSet, configuration, completeHandler,
+              commandSet, configuration,
               expectations['$testName/${subtestNames[subtestIndex]}'],
-              info, info.hasCompileError || info.hasRuntimeError, fullHtmlPath,
-              subtestIndex != 0);
+              info, info.hasCompileError || info.hasRuntimeError, fullHtmlPath);
         } else {
           testCase = new BrowserTestCase(testDisplayName,
-              commandSet, configuration, completeHandler, expectations,
-              info, info.hasCompileError || info.hasRuntimeError, fullHtmlPath,
-              false);
-        }
-        if (subtestIndex == 0) {
-          multitestParentTest = testCase;
-        } else {
-          multitestParentTest.addObserver(testCase);
+              commandSet, configuration, expectations,
+              info, info.hasCompileError || info.hasRuntimeError, fullHtmlPath);
         }
 
         doTest(testCase);
         subtestIndex++;
       } while(subtestIndex < subtestNames.length);
     }
   }
 
   /** Helper to create a compilation command for a single input file. */
   Command _compileCommand(String inputFile, String outputFile,
       String compiler, String dir, vmOptions, optionsFromFile) {
     assert (['dart2js', 'dart2dart'].contains(compiler));
     String executable = compilerPath;
     List<String> args = TestUtils.standardOptions(configuration);
     String packageRoot =
       packageRootArgument(optionsFromFile['packageRoot']);
     if (packageRoot != null) {
       args.add(packageRoot);
     }
     args.add('--out=$outputFile');
     args.add(inputFile);
     if (executable.endsWith('.dart')) {
       // Run the compiler script via the Dart VM.
       args.insert(0, executable);
       executable = dartShellFileName;
     }
-    return new CompilationCommand(compiler,
-                                  outputFile,
-                                  !useSdk,
-                                  dart2JsBootstrapDependencies,
-                                  compilerPath,
-                                  args);
+    return CommandBuilder.instance.getCompilationCommand(
+        compiler, outputFile, !useSdk,
+        dart2JsBootstrapDependencies, compilerPath, args, configurationDir);
   }
 
   /**
    * Create a directory for the generated test.  If a Dart language test
    * needs to be run in a browser, the Dart test needs to be embedded in
    * an HTML page, with a testing framework based on scripting and DOM events.
    * These scripts and pages are written to a generated_test directory
    * inside the build directory of the checkout.
    *
    * Those tests which are already HTML web applications (web tests), with
@@ skipping 71 matching lines @@
     if (configuration['dartium'] != '') {
       return configuration['dartium'];
     }
     if (Platform.operatingSystem == 'macos') {
       return dartDir.append('client/tests/dartium/Chromium.app/Contents/'
                             'MacOS/Chromium').toNativePath();
     }
     return dartDir.append('client/tests/dartium/chrome').toNativePath();
   }
 
-  void completeHandler(TestCase testCase) {
-  }
-
   List<String> commonArgumentsFromFile(Path filePath, Map optionsFromFile) {
     List args = TestUtils.standardOptions(configuration);
 
     String packageRoot = packageRootArgument(optionsFromFile['packageRoot']);
     if (packageRoot != null) {
       args.add(packageRoot);
     }
     args.addAll(additionalOptions(filePath));
     if (configuration['analyzer']) {
       args.add('--machine');
@@ skipping 416 matching lines @@
         'org.junit.runner.JUnitCore'];
     args.addAll(testClasses);
 
     // Lengthen the timeout for JUnit tests.  It is normal for them
     // to run for a few minutes.
     Map updatedConfiguration = new Map();
     configuration.forEach((key, value) {
       updatedConfiguration[key] = value;
     });
     updatedConfiguration['timeout'] *= 3;
+    var command = CommandBuilder.instance.getCommand(
+        'junit_test', 'java', args, configurationDir);
     doTest(new TestCase(suiteName,
-                        [new Command('junit_test', 'java', args)],
+                        [command],
                         updatedConfiguration,
-                        completeHandler,
                         new Set<String>.from([PASS])));
     doDone();
   }
 
-  void completeHandler(TestCase testCase) {
-  }
-
   void computeClassPath() {
     classPath =
         ['$buildDir/analyzer/util/analyzer/dart_analyzer.jar',
          '$buildDir/analyzer/dart_analyzer_tests.jar',
          // Third party libraries.
          '$dartDir/third_party/args4j/2.0.12/args4j-2.0.12.jar',
          '$dartDir/third_party/guava/r13/guava-13.0.1.jar',
          '$dartDir/third_party/rhino/1_7R3/js.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-core-1.3.0RC2.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-generator-1.3.0RC2.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-integration-1.3.0RC2.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-library-1.3.0RC2.jar',
          '$dartDir/third_party/junit/v4_8_2/junit.jar']
-        .join(Platform.operatingSystem == 'windows'? ';': ':');  // Path separator.
+        .join(Platform.operatingSystem == 'windows'? ';': ':');
   }
 }
 
 class LastModifiedCache {
   Map<String, DateTime> _cache = <String, DateTime>{};
 
   /**
    * Returns the last modified date of the given [uri].
    *
    * The return value will be cached for future queries. If [uri] is a local
@@ skipping 275 matching lines @@
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
  * $compileErrorSkip tests are skipped on browsers due to compile-time error
 """;
     print(report);
    }
 }