Merge bleeding_edge 18021:18297

tools/run-tests.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 10 matching lines @@
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import itertools
 import multiprocessing
 import optparse
 import os
 from os.path import join
 import shlex
 import subprocess
 import sys
 import time
 
 from testrunner.local import execution
@@ skipping 47 matching lines @@
                     help="Architecture and mode in the format 'arch.mode'",
                     default=None)
   result.add_option("--buildbot",
                     help="Adapt to path structure used on buildbots",
                     default=False, action="store_true")
   result.add_option("--cat", help="Print the source of the tests",
                     default=False, action="store_true")
   result.add_option("--flaky-tests",
                     help="Regard tests marked as flaky (run|skip|dontcare)",
                     default="dontcare")
+  result.add_option("--slow-tests",
+                    help="Regard slow tests (run|skip|dontcare)",
+                    default="dontcare")
+  result.add_option("--pass-fail-tests",
+                    help="Regard pass|fail tests (run|skip|dontcare)",
+                    default="dontcare")
   result.add_option("--command-prefix",
                     help="Prepended to each shell command used to run a test",
                     default="")
   result.add_option("--download-data", help="Download missing test suite data",
                     default=False, action="store_true")
   result.add_option("--extra-flags",
                     help="Additional flags to pass to each test command",
                     default="")
   result.add_option("--isolates", help="Whether to test isolates",
                     default=False, action="store_true")
@@ skipping 11 matching lines @@
                     dest="no_network", action="store_true")
   result.add_option("--no-presubmit", "--nopresubmit",
                     help='Skip presubmit checks',
                     default=False, dest="no_presubmit", action="store_true")
   result.add_option("--no-stress", "--nostress",
                     help="Don't run crankshaft --always-opt --stress-op test",
                     default=False, dest="no_stress", action="store_true")
   result.add_option("--no-variants", "--novariants",
                     help="Don't run any testing variants",
                     default=False, dest="no_variants", action="store_true")
+  result.add_option("--variants",
+                    help="Comma-separated list of testing variants")
   result.add_option("--outdir", help="Base directory with compile output",
                     default="out")
   result.add_option("-p", "--progress",
                     help=("The style of progress indicator"
                           " (verbose, dots, color, mono)"),
                     choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
+  result.add_option("--quickcheck", default=False, action="store_true",
+                    help=("Quick check mode (skip slow/flaky tests)"))
   result.add_option("--report", help="Print a summary of the tests to be run",
                     default=False, action="store_true")
   result.add_option("--shard-count",
                     help="Split testsuites into this number of shards",
                     default=1, type="int")
   result.add_option("--shard-run",
                     help="Run this shard from the split up tests.",
                     default=1, type="int")
   result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
   result.add_option("--shell-dir", help="Directory containing executables",
@@ skipping 13 matching lines @@
                     default=False, action="store_true")
   result.add_option("--junitout", help="File name of the JUnit output")
   result.add_option("--junittestsuite",
                     help="The testsuite name in the JUnit output file",
                     default="v8tests")
   return result
 
 
 def ProcessOptions(options):
   global VARIANT_FLAGS
+  global VARIANTS
 
   # Architecture and mode related stuff.
   if options.arch_and_mode:
-    tokens = options.arch_and_mode.split(".")
-    options.arch = tokens[0]
-    options.mode = tokens[1]
+    options.arch_and_mode = [arch_and_mode.split(".")
+        for arch_and_mode in options.arch_and_mode.split(",")]
+    options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
+    options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
   options.mode = options.mode.split(",")
   for mode in options.mode:
-    if not mode.lower() in ["debug", "release"]:
+    if not mode.lower() in ["debug", "release", "optdebug"]:
       print "Unknown mode %s" % mode
       return False
   if options.arch in ["auto", "native"]:
     options.arch = ARCH_GUESS
   options.arch = options.arch.split(",")
   for arch in options.arch:
     if not arch in SUPPORTED_ARCHS:
       print "Unknown architecture %s" % arch
       return False
 
+  # Store the final configuration in arch_and_mode list. Don't overwrite
+  # predefined arch_and_mode since it is more expressive than arch and mode.
+  if not options.arch_and_mode:
+    options.arch_and_mode = itertools.product(options.arch, options.mode)
+
   # Special processing of other options, sorted alphabetically.
 
   if options.buildbot:
     # Buildbots run presubmit tests as a separate step.
     options.no_presubmit = True
     options.no_network = True
   if options.command_prefix:
     print("Specifying --command-prefix disables network distribution, "
           "running tests locally.")
     options.no_network = True
   options.command_prefix = shlex.split(options.command_prefix)
   options.extra_flags = shlex.split(options.extra_flags)
   if options.j == 0:
     options.j = multiprocessing.cpu_count()
 
   def excl(*args):
     """Returns true if zero or one of multiple arguments are true."""
     return reduce(lambda x, y: x + y, args) <= 1
 
-  if not excl(options.no_stress, options.stress_only, options.no_variants):
-    print "Use only one of --no-stress, --stress-only or --no-variants."
+  if not excl(options.no_stress, options.stress_only, options.no_variants,
+              bool(options.variants), options.quickcheck):
+    print("Use only one of --no-stress, --stress-only, --no-variants, "
+          "--variants, or --quickcheck.")
     return False
   if options.no_stress:
-    VARIANT_FLAGS = [[], ["--nocrankshaft"]]
+    VARIANTS = ["default", "nocrankshaft"]
   if options.no_variants:
-    VARIANT_FLAGS = [[]]
+    VARIANTS = ["default"]
+  if options.stress_only:
+    VARIANTS = ["stress"]
+  if options.variants:
+    VARIANTS = options.variants.split(",")
+    if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
+      print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
+      return False
+  if options.quickcheck:
+    VARIANTS = ["default", "stress"]
+    options.flaky_tests = "skip"
+    options.slow_tests = "skip"
+    options.pass_fail_tests = "skip"
+
   if not options.shell_dir:
     if options.shell:
       print "Warning: --shell is deprecated, use --shell-dir instead."
       options.shell_dir = os.path.dirname(options.shell)
-  if options.stress_only:
-    VARIANT_FLAGS = [["--stress-opt", "--always-opt"]]
   if options.valgrind:
     run_valgrind = os.path.join("tools", "run-valgrind.py")
     # This is OK for distributed running, so we don't need to set no_network.
     options.command_prefix = (["python", "-u", run_valgrind] +
                               options.command_prefix)
-  if not options.flaky_tests in ["run", "skip", "dontcare"]:
-    print "Unknown flaky test mode %s" % options.flaky_tests
+  def CheckTestMode(name, option):
+    if not option in ["run", "skip", "dontcare"]:
+      print "Unknown %s mode %s" % (name, option)
+      return False
+    return True
+  if not CheckTestMode("flaky test", options.flaky_tests):
+    return False
+  if not CheckTestMode("slow test", options.slow_tests):
+    return False
+  if not CheckTestMode("pass|fail test", options.pass_fail_tests):
     return False
   return True
 
 
 def ShardTests(tests, shard_count, shard_run):
   if shard_count < 2:
     return tests
   if shard_run < 1 or shard_run > shard_count:
     print "shard-run not a valid number, should be in [1:shard-count]"
     print "defaulting back to running all tests"
@@ skipping 18 matching lines @@
   workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
   if not options.no_presubmit:
     print ">>> running presubmit tests"
     code = subprocess.call(
         [sys.executable, join(workspace, "tools", "presubmit.py")])
     exit_code = code
 
   suite_paths = utils.GetSuitePaths(join(workspace, "test"))
 
   if len(args) == 0:
-    suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
+    suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ]
   else:
     args_suites = set()
     for arg in args:
       suite = arg.split(os.path.sep)[0]
       if not suite in args_suites:
         args_suites.add(suite)
-    suite_paths = [ s for s in suite_paths if s in args_suites ]
+    suite_paths = [ s for s in args_suites if s in suite_paths ]
 
   suites = []
   for root in suite_paths:
     suite = testsuite.TestSuite.LoadTestSuite(
         os.path.join(workspace, "test", root))
     if suite:
       suites.append(suite)
 
   if options.download_data:
     for s in suites:
       s.DownloadData()
 
-  for mode in options.mode:
-    for arch in options.arch:
+  for (arch, mode) in options.arch_and_mode:
+    try:
       code = Execute(arch, mode, args, options, suites, workspace)
-      exit_code = exit_code or code
+    except KeyboardInterrupt:
+      return 2
+    exit_code = exit_code or code
   return exit_code
 
 
 def Execute(arch, mode, args, options, suites, workspace):
   print(">>> Running tests for %s.%s" % (arch, mode))
 
   shell_dir = options.shell_dir
   if not shell_dir:
     if options.buildbot:
       shell_dir = os.path.join(workspace, options.outdir, mode)
       mode = mode.lower()
     else:
       shell_dir = os.path.join(workspace, options.outdir,
                                "%s.%s" % (arch, mode))
   shell_dir = os.path.relpath(shell_dir)
 
+  if mode == "optdebug":
+    mode = "debug"  # "optdebug" is just an alias.
+
   # Populate context object.
   mode_flags = MODE_FLAGS[mode]
   timeout = options.timeout
   if timeout == -1:
     # Simulators are slow, therefore allow a longer default timeout.
     if arch in SLOW_ARCHS:
       timeout = 2 * TIMEOUT_DEFAULT;
     else:
       timeout = TIMEOUT_DEFAULT;
 
@@ skipping 16 matching lines @@
   }
   all_tests = []
   num_tests = 0
   test_id = 0
   for s in suites:
     s.ReadStatusFile(variables)
     s.ReadTestCases(ctx)
     if len(args) > 0:
       s.FilterTestCasesByArgs(args)
     all_tests += s.tests
-    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests)
+    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
+                              options.slow_tests, options.pass_fail_tests)
     if options.cat:
       verbose.PrintTestSource(s.tests)
       continue
+    variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
     s.tests = [ t.CopyAddingFlags(v)
                 for t in s.tests
-                for v in s.VariantFlags(t, VARIANT_FLAGS) ]
+                for v in s.VariantFlags(t, variant_flags) ]
     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
     num_tests += len(s.tests)
     for t in s.tests:
       t.id = test_id
       test_id += 1
 
   if options.cat:
     return 0  # We're done here.
 
   if options.report:
@@ skipping 34 matching lines @@
       runner = network_execution.NetworkedRunner(suites, progress_indicator,
                                                  ctx, peers, workspace)
     else:
       runner = execution.Runner(suites, progress_indicator, ctx)
 
     exit_code = runner.Run(options.j)
     if runner.terminate:
       return exit_code
     overall_duration = time.time() - start_time
   except KeyboardInterrupt:
-    return 1
+    raise
 
   if options.time:
     verbose.PrintTestDurations(suites, overall_duration)
   return exit_code
 
 
 if __name__ == "__main__":
   sys.exit(Main())