[test] Add random seed stress mode to test runner.

tools/run-tests.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 285 matching lines @@
   result.add_option("--valgrind", help="Run tests through valgrind",
                     default=False, action="store_true")
   result.add_option("--warn-unused", help="Report unused rules",
                     default=False, action="store_true")
   result.add_option("--junitout", help="File name of the JUnit output")
   result.add_option("--junittestsuite",
                     help="The testsuite name in the JUnit output file",
                     default="v8tests")
   result.add_option("--random-seed", default=0, dest="random_seed",
                     help="Default seed for initializing random generator")
+  result.add_option("--random-seed-stress-count", default=1, type="int",
+                    dest="random_seed_stress_count",
+                    help="Number of runs with different random seeds")
   result.add_option("--msan",
                     help="Regard test expectations for MSAN",
                     default=False, action="store_true")
   return result
 
 
+def RandomSeed():
+  seed = 0
+  while not seed:
+    seed = random.SystemRandom().randint(-2147483648, 2147483647)
+  return seed
+
+
 def ProcessOptions(options):
   global VARIANT_FLAGS
   global VARIANTS
 
   # Architecture and mode related stuff.
   if options.arch_and_mode:
     options.arch_and_mode = [arch_and_mode.split(".")
         for arch_and_mode in options.arch_and_mode.split(",")]
     options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
     options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
@@ skipping 44 matching lines @@
     VARIANTS = ["default"]
     suppressions_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                      'sanitizers', 'tsan_suppressions.txt')
     tsan_options = '%s suppressions=%s' % (
         os.environ.get('TSAN_OPTIONS', ''), suppressions_file)
     os.environ['TSAN_OPTIONS'] = tsan_options
 
   if options.j == 0:
     options.j = multiprocessing.cpu_count()
 
-  while options.random_seed == 0:
-    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
+  if options.random_seed_stress_count <= 1:

[Jakob Kummerow, 2015/06/11 08:50:35]

Shouldn't this condition include a check that options.random_seed == 0?
Otherwise it's a bit surprising that any explicitly specified random seed will
silently be overwritten unless the random seed stress count is > 1.

[Michael Achenbach, 2015/06/11 09:45:05]

Good catch!

+    options.random_seed = RandomSeed()
 
   def excl(*args):
     """Returns true if zero or one of multiple arguments are true."""
     return reduce(lambda x, y: x + y, args) <= 1
 
   if not excl(options.no_stress, options.stress_only, options.no_variants,
               bool(options.variants)):
     print("Use only one of --no-stress, --stress-only, --no-variants, "
           "or --variants.")
     return False
@@ skipping 157 matching lines @@
   verbose_output = (
       options.verbose or
       utils.IsWindows() and options.progress == "verbose"
   )
   ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir,
                         mode_flags, verbose_output,
                         timeout, options.isolates,
                         options.command_prefix,
                         options.extra_flags,
                         options.no_i18n,
                         options.random_seed,

[Jakob Kummerow, 2015/06/11 08:50:35]

In case of --random-seed=1 --random-seed-stress-count=2, this will lead to
duplicate --random-seed flags on the final test command line, right? One coming
from the test case (cf. CopyAddingFlags below), and one coming from the context.

[Michael Achenbach, 2015/06/11 09:45:05]

Done.

                         options.no_sorting,
                         options.rerun_failures_count,
                         options.rerun_failures_max,
                         options.predictable,
                         options.no_harness)
 
   # TODO(all): Combine "simulator" and "simulator_run".
   simulator_run = not options.dont_skip_simulator_slow_tests and \
       arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64el', \
                'ppc', 'ppc64'] and \
@@ skipping 23 matching lines @@
     s.ReadTestCases(ctx)
     if len(args) > 0:
       s.FilterTestCasesByArgs(args)
     all_tests += s.tests
     s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
                               options.slow_tests, options.pass_fail_tests)
     if options.cat:
       verbose.PrintTestSource(s.tests)
       continue
     variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
-    s.tests = [ t.CopyAddingFlags(v)
-                for t in s.tests
-                for v in s.VariantFlags(t, variant_flags) ]
+    variant_tests = [ t.CopyAddingFlags(v)
+                      for t in s.tests
+                      for v in s.VariantFlags(t, variant_flags) ]
+
+    if options.random_seed_stress_count > 1:
+      # Duplicate test for random seed stress mode.
+      def iter_seed_flags():
+        for i in range(0, options.random_seed_stress_count):
+          # Use given random seed for all runs or a new random seed if none
+          # is specified.
+          seed = options.random_seed or RandomSeed()
+          yield ["--random-seed=%s" % str(seed)]

[Jakob Kummerow, 2015/06/11 08:50:35]

nit: explicit str() is not necessary when using %s (arguably %d would be more
appropriate, and  still wouldn't need manual conversion)

[Michael Achenbach, 2015/06/11 09:45:05]

Done.

+      s.tests = [
+        t.CopyAddingFlags(v)
+        for t in variant_tests
+        for v in iter_seed_flags()
+      ]
+    else:
+      s.tests = variant_tests
+
     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
     num_tests += len(s.tests)
 
   if options.cat:
     return 0  # We're done here.
 
   if options.report:
     verbose.PrintReport(all_tests)
 
   # Run the tests, either locally or distributed on the network.
@@ skipping 35 matching lines @@
   exit_code = runner.Run(options.j)
   overall_duration = time.time() - start_time
 
   if options.time:
     verbose.PrintTestDurations(suites, overall_duration)
   return exit_code
 
 
 if __name__ == "__main__":
   sys.exit(Main())