Make test runner more flexible for running fast tests.

tools/run-tests.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 35 matching lines @@
 from testrunner.objects import context
 
 
 ARCH_GUESS = utils.DefaultArch()
 DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"]
 TIMEOUT_DEFAULT = 60
 TIMEOUT_SCALEFACTOR = {"debug"   : 4,
                        "release" : 1 }
 
 # Use this to run several variants of the tests.
-VARIANT_FLAGS = [[],
-                 ["--stress-opt", "--always-opt"],
-                 ["--nocrankshaft"]]
+VARIANT_FLAGS = {
+    "default": [],
+    "stress": ["--stress-opt", "--always-opt"],
+    "nocrankshaft": ["--nocrankshaft"]}
+
+VARIANTS = ["default", "stress", "nocrankshaft"]
+
 MODE_FLAGS = {
     "debug"   : ["--nobreak-on-abort", "--nodead-code-elimination",
                  "--nofold-constants", "--enable-slow-asserts",
                  "--debug-code", "--verify-heap"],
     "release" : ["--nobreak-on-abort", "--nodead-code-elimination",
                  "--nofold-constants"]}
 
 SUPPORTED_ARCHS = ["android_arm",
                    "android_ia32",
                    "arm",
@@ skipping 21 matching lines @@
                     help="Architecture and mode in the format 'arch.mode'",
                     default=None)
   result.add_option("--buildbot",
                     help="Adapt to path structure used on buildbots",
                     default=False, action="store_true")
   result.add_option("--cat", help="Print the source of the tests",
                     default=False, action="store_true")
   result.add_option("--flaky-tests",
                     help="Regard tests marked as flaky (run|skip|dontcare)",
                     default="dontcare")
+  result.add_option("--slow-tests",
+                    help="Regard slow tests (run|skip|dontcare)",
+                    default="dontcare")
+  result.add_option("--pass-fail-tests",
+                    help="Regard pass|fail tests (run|skip|dontcare)",
+                    default="dontcare")
   result.add_option("--command-prefix",
                     help="Prepended to each shell command used to run a test",
                     default="")
   result.add_option("--download-data", help="Download missing test suite data",
                     default=False, action="store_true")
   result.add_option("--extra-flags",
                     help="Additional flags to pass to each test command",
                     default="")
   result.add_option("--isolates", help="Whether to test isolates",
                     default=False, action="store_true")
@@ skipping 11 matching lines @@
                     dest="no_network", action="store_true")
   result.add_option("--no-presubmit", "--nopresubmit",
                     help='Skip presubmit checks',
                     default=False, dest="no_presubmit", action="store_true")
   result.add_option("--no-stress", "--nostress",
                     help="Don't run crankshaft --always-opt --stress-op test",
                     default=False, dest="no_stress", action="store_true")
   result.add_option("--no-variants", "--novariants",
                     help="Don't run any testing variants",
                     default=False, dest="no_variants", action="store_true")
+  result.add_option("--variants",
+                    help="Comma-separated list of testing variants")
   result.add_option("--outdir", help="Base directory with compile output",
                     default="out")
   result.add_option("-p", "--progress",
                     help=("The style of progress indicator"
                           " (verbose, dots, color, mono)"),
                     choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
   result.add_option("--report", help="Print a summary of the tests to be run",
                     default=False, action="store_true")
   result.add_option("--shard-count",
                     help="Split testsuites into this number of shards",
@@ skipping 19 matching lines @@
                     default=False, action="store_true")
   result.add_option("--junitout", help="File name of the JUnit output")
   result.add_option("--junittestsuite",
                     help="The testsuite name in the JUnit output file",
                     default="v8tests")
   return result
 
 
 def ProcessOptions(options):
   global VARIANT_FLAGS
+  global VARIANTS
 
   # Architecture and mode related stuff.
   if options.arch_and_mode:
     tokens = options.arch_and_mode.split(".")
     options.arch = tokens[0]
     options.mode = tokens[1]
   options.mode = options.mode.split(",")
   for mode in options.mode:
     if not mode.lower() in ["debug", "release"]:
       print "Unknown mode %s" % mode
@@ skipping 18 matching lines @@
     options.no_network = True
   options.command_prefix = shlex.split(options.command_prefix)
   options.extra_flags = shlex.split(options.extra_flags)
   if options.j == 0:
     options.j = multiprocessing.cpu_count()
 
   def excl(*args):
     """Returns true if zero or one of multiple arguments are true."""
     return reduce(lambda x, y: x + y, args) <= 1
 
-  if not excl(options.no_stress, options.stress_only, options.no_variants):
-    print "Use only one of --no-stress, --stress-only or --no-variants."
+  if not excl(options.no_stress, options.stress_only, options.no_variants,
+              bool(options.variants)):
+    print("Use only one of --no-stress, --stress-only, --no-variants or "
+          "--variants.")
     return False
   if options.no_stress:
-    VARIANT_FLAGS = [[], ["--nocrankshaft"]]
+    VARIANTS = ["default", "nocrankshaft"]
   if options.no_variants:
-    VARIANT_FLAGS = [[]]
+    VARIANTS = ["default"]
+  if options.stress_only:
+    VARIANTS = ["stress"]
+  if options.variants:
+    VARIANTS = options.variants.split(",")
+    if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
+      print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
+      return False
   if not options.shell_dir:
     if options.shell:
       print "Warning: --shell is deprecated, use --shell-dir instead."
       options.shell_dir = os.path.dirname(options.shell)
-  if options.stress_only:
-    VARIANT_FLAGS = [["--stress-opt", "--always-opt"]]
   if options.valgrind:
     run_valgrind = os.path.join("tools", "run-valgrind.py")
     # This is OK for distributed running, so we don't need to set no_network.
     options.command_prefix = (["python", "-u", run_valgrind] +
                               options.command_prefix)
-  if not options.flaky_tests in ["run", "skip", "dontcare"]:
-    print "Unknown flaky test mode %s" % options.flaky_tests
+  def CheckTestMode(name, option):
+    if not option in ["run", "skip", "dontcare"]:
+      print "Unknown %s mode %s" % (name, option)
+      return False
+    return True
+  if not CheckTestMode("flaky test", options.flaky_tests):
+    return False
+  if not CheckTestMode("slow test", options.slow_tests):
+    return False
+  if not CheckTestMode("pass|fail test", options.pass_fail_tests):
     return False
   if not options.no_i18n:
     DEFAULT_TESTS.append("intl")
   return True
 
 
 def ShardTests(tests, shard_count, shard_run):
   if shard_count < 2:
     return tests
   if shard_run < 1 or shard_run > shard_count:
@@ skipping 96 matching lines @@
   }
   all_tests = []
   num_tests = 0
   test_id = 0
   for s in suites:
     s.ReadStatusFile(variables)
     s.ReadTestCases(ctx)
     if len(args) > 0:
       s.FilterTestCasesByArgs(args)
     all_tests += s.tests
-    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests)
+    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
+                              options.slow_tests, options.pass_fail_tests)
     if options.cat:
       verbose.PrintTestSource(s.tests)
       continue
+    variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
     s.tests = [ t.CopyAddingFlags(v)
                 for t in s.tests
-                for v in s.VariantFlags(t, VARIANT_FLAGS) ]
+                for v in s.VariantFlags(t, variant_flags) ]
     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
     num_tests += len(s.tests)
     for t in s.tests:
       t.id = test_id
       test_id += 1
 
   if options.cat:
     return 0  # We're done here.
 
   if options.report:
@@ skipping 43 matching lines @@
   except KeyboardInterrupt:
     return 1
 
   if options.time:
     verbose.PrintTestDurations(suites, overall_duration)
   return exit_code
 
 
 if __name__ == "__main__":
   sys.exit(Main())