Add flaky test classification feature to test suites.

tools/run-tests.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 76 matching lines @@
                           "'auto' or 'native' for auto-detect"),
                     default="ia32,x64,arm")
   result.add_option("--arch-and-mode",
                     help="Architecture and mode in the format 'arch.mode'",
                     default=None)
   result.add_option("--buildbot",
                     help="Adapt to path structure used on buildbots",
                     default=False, action="store_true")
   result.add_option("--cat", help="Print the source of the tests",
                     default=False, action="store_true")
+  result.add_option("--flaky-tests",
+                    help="Regard tests marked as flaky (run|skip|dontcare)",
+                    default="dontcare")
   result.add_option("--command-prefix",
                     help="Prepended to each shell command used to run a test",
                     default="")
   result.add_option("--download-data", help="Download missing test suite data",
                     default=False, action="store_true")
   result.add_option("--extra-flags",
                     help="Additional flags to pass to each test command",
                     default="")
   result.add_option("--isolates", help="Whether to test isolates",
                     default=False, action="store_true")
@@ skipping 90 matching lines @@
     if options.shell:
       print "Warning: --shell is deprecated, use --shell-dir instead."
       options.shell_dir = os.path.dirname(options.shell)
   if options.stress_only:
     VARIANT_FLAGS = [["--stress-opt", "--always-opt"]]
   if options.valgrind:
     run_valgrind = os.path.join("tools", "run-valgrind.py")
     # This is OK for distributed running, so we don't need to set no_network.
     options.command_prefix = (["python", "-u", run_valgrind] +
                               options.command_prefix)
+  if not options.flaky_tests in ["run", "skip", "dontcare"]:
+    print "Unknown flaky test mode %s" % options.flaky_tests
+    return False
   return True
 
 
 def ShardTests(tests, shard_count, shard_run):
   if shard_count < 2:
     return tests
   if shard_run < 1 or shard_run > shard_count:
     print "shard-run not a valid number, should be in [1:shard-count]"
     print "defaulting back to running all tests"
     return tests
@@ skipping 91 matching lines @@
   }
   all_tests = []
   num_tests = 0
   test_id = 0
   for s in suites:
     s.ReadStatusFile(variables)
     s.ReadTestCases(ctx)
     if len(args) > 0:
       s.FilterTestCasesByArgs(args)
     all_tests += s.tests
-    s.FilterTestCasesByStatus(options.warn_unused)
+    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests)
     if options.cat:
       verbose.PrintTestSource(s.tests)
       continue
     variant_flags = s.VariantFlags() or VARIANT_FLAGS
     s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in variant_flags ]
     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
     num_tests += len(s.tests)
     for t in s.tests:
       t.id = test_id
       test_id += 1
@@ skipping 48 matching lines @@
   except KeyboardInterrupt:
     return 1
 
   if options.time:
     verbose.PrintTestDurations(suites, overall_duration)
   return exit_code
 
 
 if __name__ == "__main__":
   sys.exit(Main())