cleanup to tools/testing/dart

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
  *   and creating [TestCase]s for those files that meet the relevant criteria.
  * - Preparing tests, including copying files and frameworks to temporary
  *   directories, and computing the command line and arguments to be run.
  */
 library test_suite;
 
-import "dart:async";
-import "dart:io";
-import "drt_updater.dart";
-import "html_test.dart" as htmlTest;
-import "path.dart";
-import "multitest.dart";
-import "status_file_parser.dart";
-import "test_runner.dart";
-import "utils.dart";
-import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR;
+import 'dart:async';
+import 'dart:io';
 
-import "compiler_configuration.dart" show
-    CommandArtifact,
-    CompilerConfiguration;
-
-import "runtime_configuration.dart" show
-    RuntimeConfiguration;
-
-import 'browser_test.dart';
+import 'compiler_configuration.dart' show CompilerConfiguration;
+import 'lib/browser_test.dart';
+import 'lib/command.dart';
+import 'lib/drt_updater.dart';
+import 'lib/html_test.dart' as htmlTest;
+import 'lib/http_server.dart' show PREFIX_BUILDDIR, PREFIX_DARTDIR;
+import 'lib/multitest.dart';
+import 'lib/path.dart';
+import 'lib/status_file_parser.dart';
+import 'lib/summary_report.dart';
+import 'lib/test_case.dart';
+import 'lib/test_information.dart';
+import 'lib/test_utils.dart';
+import 'lib/utils.dart';
+import 'runtime_configuration.dart' show CommandArtifact, RuntimeConfiguration;
 
 
 RegExp multiHtmlTestGroupRegExp = new RegExp(r"\s*[^/]\s*group\('[^,']*");
 RegExp multiHtmlTestRegExp = new RegExp(r"useHtmlIndividualConfiguration()");
 // Require at least one non-space character before '///'
 RegExp multiTestRegExp = new RegExp(r"\S *"
                                     r"/// \w+:(.*)");
-RegExp dartExtension = new RegExp(r'\.dart$');
 
 /**
  * A simple function that tests [arg] and returns `true` or `false`.
  */
 typedef bool Predicate<T>(T arg);
 
-typedef void CreateTest(Path filePath,
-                        Path originTestPath,
-                        bool hasCompileError,
-                        bool hasRuntimeError,
-                        {bool isNegativeIfChecked,
-                         bool hasCompileErrorIfChecked,
-                         bool hasStaticWarning,
-                         String multitestKey});
-
 typedef void VoidFunction();
 
 /**
  * Calls [function] asynchronously. Returns a future that completes with the
  * result of the function. If the function is `null`, returns a future that
  * completes immediately with `null`.
  */
 Future asynchronously(function()) {
   if (function == null) return new Future.value(null);
 
@@ skipping 404 matching lines @@
   Map buildPubspecDependencyOverrides(Map packageDirectories) {
     Map overrides = {};
     packageDirectories.forEach((String packageName, String fullPath) {
       overrides[packageName] = { 'path' : fullPath };
     });
     return overrides;
   }
 
 }
 
-
 Future<Iterable<String>> ccTestLister(String runnerPath) {
   return Process.run(runnerPath, ["--list"]).then((ProcessResult result) {
     if (result.exitCode != 0) {
       throw "Failed to list tests: '$runnerPath --list'. "
             "Process exited with ${result.exitCode}";
     }
     return result.stdout
       .split('\n')
       .map((line) => line.trim())
       .where((name) => name.length > 0);
   });
 }
 
-
 /**
  * A specialized [TestSuite] that runs tests written in C to unit test
  * the Dart virtual machine and its API.
  *
  * The tests are compiled into a monolithic executable by the build step.
  * The executable lists its tests when run with the --list command line flag.
  * Individual tests are run by specifying them on the command line.
  */
 class CCTestSuite extends TestSuite {
   final String testPrefix;
@@ skipping 52 matching lines @@
         doTest = null;
         if (onDone != null) onDone();
       }).catchError((error) {
         print("Fatal error occured: $error");
         exit(1);
       });
     });
   }
 }
 
-
-class TestInformation {
-  Path filePath;
-  Path originTestPath;
-  Map optionsFromFile;
-  bool hasCompileError;
-  bool hasRuntimeError;
-  bool isNegativeIfChecked;
-  bool hasCompileErrorIfChecked;
-  bool hasStaticWarning;
-  String multitestKey;
-
-  TestInformation(this.filePath, this.originTestPath, this.optionsFromFile,
-                  this.hasCompileError, this.hasRuntimeError,
-                  this.isNegativeIfChecked, this.hasCompileErrorIfChecked,
-                  this.hasStaticWarning,
-                  {this.multitestKey: ''}) {
-    assert(filePath.isAbsolute);
-  }
-}
-
-
-class HtmlTestInformation extends TestInformation {
-  List<String> expectedMessages;
-  List<String> scripts;
-
-  HtmlTestInformation(Path filePath, this.expectedMessages, this.scripts)
-      : super(filePath, filePath,
-              {'isMultitest': false, 'isMultiHtmlTest': false},
-              false, false, false, false, false) {}
-}
-
-
 /**
  * A standard [TestSuite] implementation that searches for tests in a
  * directory, and creates [TestCase]s that compile and/or run them.
  */
 class StandardTestSuite extends TestSuite {
   final Path suiteDir;
   final List<String> statusFilePaths;
   TestExpectations testExpectations;
   List<TestInformation> cachedTests;
   final Path dartDir;
@@ skipping 1396 matching lines @@
     ReadTestExpectations(statusFiles, configuration).then((expectations) {
       Future.wait([discoverPackagesInRepository(),
                    discoverSamplesInRepository()]).then((List results) {
         Map packageDirectories = results[0];
         Map sampleDirectories = results[1];
         enqueueTestCases(packageDirectories, sampleDirectories, expectations);
       });
     });
   }
 }
-
-
-class LastModifiedCache {
-  Map<String, DateTime> _cache = <String, DateTime>{};
-
-  /**
-   * Returns the last modified date of the given [uri].
-   *
-   * The return value will be cached for future queries. If [uri] is a local
-   * file, it's last modified [Date] will be returned. If the file does not
-   * exist, null will be returned instead.
-   * In case [uri] is not a local file, this method will always return
-   * the current date.
-   */
-  DateTime getLastModified(Uri uri) {
-    if (uri.scheme == "file") {
-      if (_cache.containsKey(uri.path)) {
-        return _cache[uri.path];
-      }
-      var file = new File(new Path(uri.path).toNativePath());
-      _cache[uri.path] = file.existsSync() ? file.lastModifiedSync() : null;
-      return _cache[uri.path];
-    }
-    return new DateTime.now();
-  }
-}
-
-
-class ExistsCache {
-  Map<String, bool> _cache = <String, bool>{};
-
-  /**
-   * Returns true if the file in [path] exists, false otherwise.
-   *
-   * The information will be cached.
-   */
-  bool doesFileExist(String path) {
-    if (!_cache.containsKey(path)) {
-      _cache[path] = new File(path).existsSync();
-    }
-    return _cache[path];
-  }
-}
-
-
-class TestUtils {
-  /**
-   * Any script using TestUtils must set dartDirUri to a file:// URI
-   * pointing to the root of the Dart checkout.
-   */
-  static setDartDirUri(uri) {
-    dartDirUri = uri;
-    dartDir = new Path(uri.toFilePath());
-  }
-  static Uri dartDirUri;
-  static Path dartDir;
-  static LastModifiedCache lastModifiedCache = new LastModifiedCache();
-  static ExistsCache existsCache = new ExistsCache();
-  static Path currentWorkingDirectory =
-      new Path(Directory.current.path);
-
-  /**
-   * Creates a directory using a [relativePath] to an existing
-   * [base] directory if that [relativePath] does not already exist.
-   */
-  static Directory mkdirRecursive(Path base, Path relativePath) {
-    if (relativePath.isAbsolute) {
-      base = new Path('/');
-    }
-    Directory dir = new Directory(base.toNativePath());
-    assert(dir.existsSync());
-    var segments = relativePath.segments();
-    for (String segment in segments) {
-      base = base.append(segment);
-      if (base.toString() == "/$segment" &&
-          segment.length == 2 &&
-          segment.endsWith(':')) {
-        // Skip the directory creation for a path like "/E:".
-        continue;
-      }
-      dir = new Directory(base.toNativePath());
-      if (!dir.existsSync()) {
-        dir.createSync();
-      }
-      assert(dir.existsSync());
-    }
-    return dir;
-  }
-
-  /**
-   * Copy a [source] file to a new place.
-   * Assumes that the directory for [dest] already exists.
-   */
-  static Future copyFile(Path source, Path dest) {
-    return new File(source.toNativePath()).openRead()
-        .pipe(new File(dest.toNativePath()).openWrite());
-  }
-
-  static Future copyDirectory(String source, String dest) {
-    source = new Path(source).toNativePath();
-    dest = new Path(dest).toNativePath();
-
-    var executable = 'cp';
-    var args = ['-Rp', source, dest];
-    if (Platform.operatingSystem == 'windows') {
-      executable = 'xcopy';
-      args = [source, dest, '/e', '/i'];
-    }
-    return Process.run(executable, args).then((ProcessResult result) {
-      if (result.exitCode != 0) {
-        throw new Exception("Failed to execute '$executable "
-            "${args.join(' ')}'.");
-      }
-    });
-  }
-
-  static Future deleteDirectory(String path) {
-    // We are seeing issues with long path names on windows when
-    // deleting them. Use the system tools to delete our long paths.
-    // See issue 16264.
-    if (Platform.operatingSystem == 'windows') {
-      var native_path = new Path(path).toNativePath();
-      // Running this in a shell sucks, but rmdir is not part of the standard
-      // path.
-      return Process.run('rmdir', ['/s', '/q', native_path], runInShell: true)
-        .then((ProcessResult result) {
-          if (result.exitCode != 0) {
-            throw new Exception('Can\'t delete path $native_path. '
-                                'This path might be too long');
-          }
-        });
-    } else {
-      var dir = new Directory(path);
-      return dir.delete(recursive: true);
-    }
-  }
-
-  static Path debugLogfile() {
-    return new Path(".debug.log");
-  }
-
-  static String flakyFileName() {
-    // If a flaky test did fail, infos about it (i.e. test name, stdin, stdout)
-    // will be written to this file. This is useful for the debugging of
-    // flaky tests.
-    // When running on a built bot, the file can be made visible in the
-    // waterfall UI.
-    return ".flaky.log";
-  }
-
-  static String testOutcomeFileName() {
-    // If test.py was invoked with '--write-test-outcome-log it will write
-    // test outcomes to this file.
-    return ".test-outcome.log";
-  }
-
-  static void ensureExists(String filename, Map configuration) {
-    if (!configuration['list'] && !existsCache.doesFileExist(filename)) {
-      throw "'$filename' does not exist";
-    }
-  }
-
-  static Path absolutePath(Path path) {
-    if (!path.isAbsolute) {
-      return currentWorkingDirectory.join(path);
-    }
-    return path;
-  }
-
-  static String outputDir(Map configuration) {
-    var result = '';
-    var system = configuration['system'];
-    if (system == 'linux') {
-      result = 'out/';
-    } else if (system == 'macos') {
-      result = 'xcodebuild/';
-    } else if (system == 'windows') {
-      result = 'build/';
-    }
-    return result;
-  }
-
-  static List<String> standardOptions(Map configuration) {
-    List args = ["--ignore-unrecognized-flags"];
-    if (configuration["checked"]) {
-      args.add('--enable_asserts');
-      args.add("--enable_type_checks");
-    }
-    String compiler = configuration["compiler"];
-    if (compiler == "dart2js" || compiler == "dart2dart") {
-      args = [];
-      if (configuration["checked"]) {
-        args.add('--enable-checked-mode');
-      }
-      // args.add("--verbose");
-      if (!isBrowserRuntime(configuration['runtime'])) {
-        args.add("--allow-mock-compilation");
-        args.add("--categories=all");
-      }
-    }
-    if ((compiler == "dart2js" || compiler == "dart2dart") &&
-        configuration["minified"]) {
-      args.add("--minify");
-    }
-    if (compiler == "dartanalyzer" || compiler == "dart2analyzer") {
-      args.add("--show-package-warnings");
-      args.add("--enable-async");
-    }
-    return args;
-  }
-
-  static bool isBrowserRuntime(String runtime) {
-    const BROWSERS = const [
-      'drt',
-      'dartium',
-      'ie9',
-      'ie10',
-      'ie11',
-      'safari',
-      'opera',
-      'chrome',
-      'ff',
-      'chromeOnAndroid',
-      'safarimobilesim',
-      'ContentShellOnAndroid',
-      'DartiumOnAndroid'
-    ];
-    return BROWSERS.contains(runtime);
-  }
-
-  static bool isJsCommandLineRuntime(String runtime) =>
-      const ['d8', 'jsshell'].contains(runtime);
-
-  static bool isCommandLineAnalyzer(String compiler) =>
-      compiler == 'dartanalyzer' || compiler == 'dart2analyzer';
-
-  static String buildDir(Map configuration) {
-    // FIXME(kustermann,ricow): Our code assumes that the returned 'buildDir'
-    // is relative to the current working directory.
-    // Thus, if we pass in an absolute path (e.g. '--build-directory=/tmp/out')
-    // we get into trouble.
-    if (configuration['build_directory'] != '') {
-      return configuration['build_directory'];
-    }
-
-    return "${outputDir(configuration)}${configurationDir(configuration)}";
-  }
-
-  static getValidOutputDir(Map configuration, String mode, String arch) {
-    // We allow our code to have been cross compiled, i.e., that there
-    // is an X in front of the arch. We don't allow both a cross compiled
-    // and a normal version to be present (except if you specifically pass
-    // in the build_directory).
-    var normal = '$mode$arch';
-    var cross = '${mode}X$arch';
-    var outDir = outputDir(configuration);
-    var normalDir = new Directory(new Path('$outDir$normal').toNativePath());
-    var crossDir = new Directory(new Path('$outDir$cross').toNativePath());
-    if (normalDir.existsSync() && crossDir.existsSync()) {
-      throw "You can't have both $normalDir and $crossDir, we don't know which"
-            " binary to use";
-    }
-    if (crossDir.existsSync()) {
-      return cross;
-    }
-    return normal;
-  }
-
-  static String configurationDir(Map configuration) {
-    // For regular dart checkouts, the configDir by default is mode+arch.
-    // For Dartium, the configDir by default is mode (as defined by the Chrome
-    // build setup). We can detect this because in the dartium checkout, the
-    // "output" directory is a sibling of the dart directory instead of a child.
-    var mode = (configuration['mode'] == 'debug') ? 'Debug' : 'Release';
-    var arch = configuration['arch'].toUpperCase();
-    if (currentWorkingDirectory != dartDir) {
-      return getValidOutputDir(configuration, mode, arch);
-    } else {
-      return mode;
-    }
-  }
-
-  /**
-   * Returns the path to the dart binary checked into the repo, used for
-   * bootstrapping test.dart.
-   */
-  static Path get dartTestExecutable {
-    var path = '$dartDir/tools/testing/bin/'
-        '${Platform.operatingSystem}/dart';
-    if (Platform.operatingSystem == 'windows') {
-      path = '$path.exe';
-    }
-    return new Path(path);
-  }
-
-  /**
-   * Gets extra options under [key] passed to the testing script.
-   */
-  static List<String> getExtraOptions(Map configuration, String key) {
-    if (configuration[key] == null) return <String>[];
-    return configuration[key]
-        .split(" ")
-        .map((s) => s.trim())
-        .where((s) => s.isNotEmpty)
-        .toList();
-  }
-
-  /**
-   * Gets extra vm options passed to the testing script.
-   */
-  static List<String> getExtraVmOptions(Map configuration) =>
-      getExtraOptions(configuration, 'vm_options');
-
-  static String getShortName(String path) {
-    final PATH_REPLACEMENTS = const {
-      "pkg_polymer_e2e_test_bad_import_test": "polymer_bi",
-      "pkg_polymer_e2e_test_canonicalization_test": "polymer_c16n",
-      "pkg_polymer_e2e_test_experimental_boot_test": "polymer_boot",
-      "pkg_polymer_e2e_test_good_import_test": "polymer_gi",
-      "tests_co19_src_Language_12_Expressions_14_Function_Invocation_":
-          "co19_fn_invoke_",
-      "tests_co19_src_LayoutTests_fast_css_getComputedStyle_getComputedStyle-":
-          "co19_css_getComputedStyle_",
-      "tests_co19_src_LayoutTests_fast_dom_Document_CaretRangeFromPoint_"
-          "caretRangeFromPoint-": "co19_caretrangefrompoint_",
-      "tests_co19_src_LayoutTests_fast_dom_Document_CaretRangeFromPoint_"
-          "hittest-relative-to-viewport_": "co19_caretrange_hittest_",
-      "tests_co19_src_LayoutTests_fast_dom_HTMLLinkElement_link-onerror-"
-          "stylesheet-with-": "co19_dom_link-",
-      "tests_co19_src_LayoutTests_fast_dom_": "co19_dom",
-      "tests_co19_src_LayoutTests_fast_canvas_webgl": "co19_canvas_webgl",
-      "tests_co19_src_LibTest_core_AbstractClassInstantiationError_"
-          "AbstractClassInstantiationError_": "co19_abstract_class_",
-      "tests_co19_src_LibTest_core_IntegerDivisionByZeroException_"
-          "IntegerDivisionByZeroException_": "co19_division_by_zero",
-      "tests_co19_src_WebPlatformTest_html_dom_documents_dom-tree-accessors_":
-          "co19_dom_accessors_",
-      "tests_co19_src_WebPlatformTest_html_semantics_embedded-content_"
-          "media-elements_": "co19_media_elements",
-      "tests_co19_src_WebPlatformTest_html_semantics_": "co19_semantics_",
-
-      "tests_co19_src_WebPlatformTest_html-templates_additions-to-"
-          "the-steps-to-clone-a-node_": "co19_htmltemplates_clone_",
-      "tests_co19_src_WebPlatformTest_html-templates_definitions_"
-          "template-contents-owner": "co19_htmltemplates_contents",
-      "tests_co19_src_WebPlatformTest_html-templates_parsing-html-"
-          "templates_additions-to-": "co19_htmltemplates_add_",
-      "tests_co19_src_WebPlatformTest_html-templates_parsing-html-"
-          "templates_appending-to-a-template_": "co19_htmltemplates_append_",
-      "tests_co19_src_WebPlatformTest_html-templates_parsing-html-"
-          "templates_clearing-the-stack-back-to-a-given-context_":
-          "co19_htmltemplates_clearstack_",
-      "tests_co19_src_WebPlatformTest_html-templates_parsing-html-"
-          "templates_creating-an-element-for-the-token_":
-          "co19_htmltemplates_create_",
-      "tests_co19_src_WebPlatformTest_html-templates_template-element"
-          "_template-": "co19_htmltemplates_element-",
-      "tests_co19_src_WebPlatformTest_html-templates_": "co19_htmltemplate_",
-
-      "tests_co19_src_WebPlatformTest_shadow-dom_shadow-trees_":
-          "co19_shadow-trees_",
-      "tests_co19_src_WebPlatformTest_shadow-dom_elements-and-dom-objects_":
-          "co19_shadowdom_",
-      "tests_co19_src_WebPlatformTest_shadow-dom_html-elements-in-"
-          "shadow-trees_": "co19_shadow_html_",
-      "tests_co19_src_WebPlatformTest_html_webappapis_system-state-and-"
-          "capabilities_the-navigator-object": "co19_webappapis_navigator_",
-      "tests_co19_src_WebPlatformTest_DOMEvents_approved_": "co19_dom_approved_"
-    };
-
-    // Some tests are already in [build_dir]/generated_tests.
-    String GEN_TESTS = 'generated_tests/';
-    if (path.contains(GEN_TESTS)) {
-      int index = path.indexOf(GEN_TESTS) + GEN_TESTS.length;
-      path = 'multitest/${path.substring(index)}';
-    }
-    path = path.replaceAll('/', '_');
-    final int WINDOWS_SHORTEN_PATH_LIMIT = 58;
-    if (Platform.operatingSystem == 'windows' &&
-        path.length > WINDOWS_SHORTEN_PATH_LIMIT) {
-      for (var key in PATH_REPLACEMENTS.keys) {
-        if (path.startsWith(key)) {
-          path = path.replaceFirst(key, PATH_REPLACEMENTS[key]);
-          break;
-        }
-      }
-    }
-    return path;
-  }
-}
-
-
-class SummaryReport {
-  static int total = 0;
-  static int skipped = 0;
-  static int skippedByDesign = 0;
-  static int noCrash = 0;
-  static int flakyCrash = 0;
-  static int pass = 0;
-  static int failOk = 0;
-  static int fail = 0;
-  static int crash = 0;
-  static int timeout = 0;
-  static int compileErrorSkip = 0;
-
-  static List<TestCase> nonStandardTestCases = [];
-
-  static void add(TestCase testCase) {
-    var expectations = testCase.expectedOutcomes;
-
-    bool containsFail = expectations.any(
-        (expectation) => expectation.canBeOutcomeOf(Expectation.FAIL));
-    bool containsPass = expectations.contains(Expectation.PASS);
-    bool containsSkip = expectations.contains(Expectation.SKIP);
-    bool containsSkipByDesign = 
-        expectations.contains(Expectation.SKIP_BY_DESIGN);
-    bool containsCrash = expectations.contains(Expectation.CRASH);
-    bool containsOK = expectations.contains(Expectation.OK);
-    bool containsSlow = expectations.contains(Expectation.SLOW);
-    bool containsTimeout = expectations.contains(Expectation.TIMEOUT);
-
-    ++total;
-    if (containsSkip) {
-      ++skipped;
-    } else if (containsSkipByDesign) {
-      ++skipped;
-      ++skippedByDesign;
-    } else {
-      // We don't do if-else below because the buckets should be exclusive.
-      // We keep a count around to guarantee that
-      int markers = 0;
-
-      // Counts the number of flaky tests.
-      if (containsFail && containsPass && !containsCrash && !containsOK) {
-        ++noCrash;
-        ++markers;
-      }
-      if (containsCrash && !containsOK &&  expectations.length > 1) {
-        ++flakyCrash;
-        ++markers;
-      }
-      if ((containsPass && expectations.length == 1) ||
-          (containsPass && containsSlow && expectations.length == 2)) {
-        ++pass;
-        ++markers;
-      }
-      if (containsFail && containsOK) {
-        ++failOk;
-        ++markers;
-      }
-      if ((containsFail && expectations.length == 1) ||
-          (containsFail && containsSlow && expectations.length == 2)) {
-        ++fail;
-        ++markers;
-      }
-      if ((containsCrash && expectations.length == 1) ||
-          (containsCrash && containsSlow && expectations.length == 2)) {
-        ++crash;
-        ++markers;
-      }
-      if (containsTimeout && expectations.length == 1) {
-        ++timeout;
-        ++markers;
-      }
-      if (markers != 1) {
-        nonStandardTestCases.add(testCase);
-      }
-    }
-  }
-
-  static void addCompileErrorSkipTest() {
-    total++;
-    compileErrorSkip++;
-  }
-
-  static void printReport() {
-    if (total == 0) return;
-    var bogus = nonStandardTestCases.length;
-    String report = """Total: $total tests
- * $skipped tests will be skipped ($skippedByDesign skipped by design)
- * $noCrash tests are expected to be flaky but not crash
- * $flakyCrash tests are expected to flaky crash
- * $pass tests are expected to pass
- * $failOk tests are expected to fail that we won't fix
- * $fail tests are expected to fail that we should fix
- * $crash tests are expected to crash that we should fix
- * $timeout tests are allowed to timeout
- * $compileErrorSkip tests are skipped on browsers due to compile-time error
- * $bogus could not be categorized or are in multiple categories
-""";
-    print(report);
-  }
-}