Add flaky test classification feature to test suites.

tools/testrunner/local/testsuite.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 48 matching lines @@
     return "d8"
 
   def suffix(self):
     return ".js"
 
   def status_file(self):
     return "%s/%s.status" % (self.root, self.name)
 
   # Used in the status file and for stdout printing.
   def CommonTestName(self, testcase):
-    return testcase.path
+    if utils.IsWindows():
+      return testcase.path.replace("\\", "/")
+    else:
+      return testcase.path
 
   def ListTests(self, context):
     raise NotImplementedError
 
   def VariantFlags(self):
     return None
 
   def DownloadData(self):
     pass
 
   def ReadStatusFile(self, variables):
     (self.rules, self.wildcards) = \
         statusfile.ReadStatusFile(self.status_file(), variables)
 
   def ReadTestCases(self, context):
     self.tests = self.ListTests(context)
 
-  def FilterTestCasesByStatus(self, warn_unused_rules):
+  @staticmethod
+  def _FilterFlaky(flaky, mode):
+    return (mode == "run" and not flaky) or (mode == "skip" and flaky)
+
+  def FilterTestCasesByStatus(self, warn_unused_rules, flaky_tests="dontcare"):
     filtered = []
     used_rules = set()
     for t in self.tests:
+      flaky = False
       testname = self.CommonTestName(t)
-      if utils.IsWindows():
-        testname = testname.replace("\\", "/")
       if testname in self.rules:
         used_rules.add(testname)
-        outcomes = self.rules[testname]
-        t.outcomes = outcomes  # Even for skipped tests, as the TestCase
-        # object stays around and PrintReport() uses it.
-        if statusfile.DoSkip(outcomes):
+        # Even for skipped tests, as the TestCase object stays around and
+        # PrintReport() uses it.
+        t.outcomes = self.rules[testname]
+        if statusfile.DoSkip(t.outcomes):
           continue  # Don't add skipped tests to |filtered|.
-      if len(self.wildcards) != 0:
-        skip = False
-        for rule in self.wildcards:
-          assert rule[-1] == '*'
-          if testname.startswith(rule[:-1]):
-            used_rules.add(rule)
-            outcomes = self.wildcards[rule]
-            t.outcomes = outcomes
-            if statusfile.DoSkip(outcomes):
-              skip = True
-              break  # "for rule in self.wildcards"
-        if skip: continue  # "for t in self.tests"
+        flaky = statusfile.IsFlaky(t.outcomes)
+      skip = False
+      for rule in self.wildcards:
+        assert rule[-1] == '*'
+        if testname.startswith(rule[:-1]):
+          used_rules.add(rule)
+          t.outcomes = self.wildcards[rule]
+          if statusfile.DoSkip(t.outcomes):
+            skip = True
+            break  # "for rule in self.wildcards"
+          flaky = flaky or statusfile.IsFlaky(t.outcomes)
+      if skip or self._FilterFlaky(flaky, flaky_tests):
+        continue  # "for t in self.tests"
       filtered.append(t)
     self.tests = filtered
 
     if not warn_unused_rules:
       return
 
     for rule in self.rules:
       if rule not in used_rules:
         print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
     for rule in self.wildcards:
@@ skipping 55 matching lines @@
   def StripOutputForTransmit(self, testcase):
     if not self.HasUnexpectedOutput(testcase):
       testcase.output.stdout = ""
       testcase.output.stderr = ""
 
   def CalculateTotalDuration(self):
     self.total_duration = 0.0
     for t in self.tests:
       self.total_duration += t.duration
     return self.total_duration