[test] Enable test status filtering by variant

tools/testrunner/local/testsuite.py

Index: tools/testrunner/local/testsuite.py
diff --git a/tools/testrunner/local/testsuite.py b/tools/testrunner/local/testsuite.py
index b4e1856bf6e5b89f4ca6b4b285c76e22b263de77..fc29ce9e6a2b2abc21f524e0e6fbd20f8e082522 100644
--- a/tools/testrunner/local/testsuite.py
+++ b/tools/testrunner/local/testsuite.py
@@ -33,33 +33,9 @@ from . import commands
 from . import statusfile
 from . import utils
 from ..objects import testcase
+from variants import ALL_VARIANTS, ALL_VARIANT_FLAGS, FAST_VARIANT_FLAGS
+
 
-# Use this to run several variants of the tests.
-ALL_VARIANT_FLAGS = {
-  "default": [[]],
-  "stress": [["--stress-opt", "--always-opt"]],
-  "turbofan": [["--turbo"]],
-  "turbofan_opt": [["--turbo", "--always-opt"]],
-  "nocrankshaft": [["--nocrankshaft"]],
-  "ignition": [["--ignition"]],
-  "ignition_turbofan": [["--ignition-staging", "--turbo"]],
-  "preparser": [["--min-preparse-length=0"]],
-}
-
-# FAST_VARIANTS implies no --always-opt.
-FAST_VARIANT_FLAGS = {
-  "default": [[]],
-  "stress": [["--stress-opt"]],
-  "turbofan": [["--turbo"]],
-  "nocrankshaft": [["--nocrankshaft"]],
-  "ignition": [["--ignition"]],
-  "ignition_turbofan": [["--ignition-staging", "--turbo"]],
-  "preparser": [["--min-preparse-length=0"]],
-}
-
-ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt",
-                    "nocrankshaft", "ignition", "ignition_turbofan",
-                    "preparser"])
 FAST_VARIANTS = set(["default", "turbofan"])
 STANDARD_VARIANT = set(["default"])
 IGNITION_VARIANT = set(["ignition"])
@@ -153,8 +129,9 @@ class TestSuite(object):
     pass
 
   def ReadStatusFile(self, variables):
-    (self.rules, self.wildcards) = \
-        statusfile.ReadStatusFile(self.status_file(), variables)
+    with open(self.status_file()) as f:
+      self.rules, self.wildcards = (
+          statusfile.ReadStatusFile(f.read(), variables))
 
   def ReadTestCases(self, context):
     self.tests = self.ListTests(context)
@@ -169,18 +146,40 @@ class TestSuite(object):
 
   def FilterTestCasesByStatus(self, warn_unused_rules,
                               slow_tests="dontcare",
-                              pass_fail_tests="dontcare"):
+                              pass_fail_tests="dontcare",
+                              variants=False):
+
+    # Use only variants-dependent rules and wildcards when filtering
+    # respective test cases and generic rules when filtering generic test
+    # cases.
+    if not variants:
+      rules = self.rules[""]
+      wildcards = self.wildcards[""]
+    else:
+      # We set rules and wildcards to a variant-specific version for each test
+      # below.
+      rules = {}
+      wildcards = {}
+
     filtered = []
+
+    # Remember used rules as tuples of (rule, variant), where variant is "" for
+    # variant-independent rules.
     used_rules = set()
+
     for t in self.tests:
       slow = False
       pass_fail = False
       testname = self.CommonTestName(t)
-      if testname in self.rules:
-        used_rules.add(testname)
+      variant = t.variant or ""
+      if variants:
+        rules = self.rules[variant]
+        wildcards = self.wildcards[variant]
+      if testname in rules:
+        used_rules.add((testname, variant))
         # Even for skipped tests, as the TestCase object stays around and
         # PrintReport() uses it.
-        t.outcomes = self.rules[testname]
+        t.outcomes = rules[testname]

[Jakob Kummerow, 2016/08/04 13:08:31]

s/=/|=/ here (cf line 195 below) should address the FIXME in the test.

         if statusfile.DoSkip(t.outcomes):
           continue  # Don't add skipped tests to |filtered|.
         for outcome in t.outcomes:
@@ -189,14 +188,14 @@ class TestSuite(object):
         slow = statusfile.IsSlow(t.outcomes)
         pass_fail = statusfile.IsPassOrFail(t.outcomes)
       skip = False
-      for rule in self.wildcards:
+      for rule in wildcards:
         assert rule[-1] == '*'
         if testname.startswith(rule[:-1]):
-          used_rules.add(rule)
-          t.outcomes |= self.wildcards[rule]
+          used_rules.add((rule, variant))
+          t.outcomes |= wildcards[rule]
           if statusfile.DoSkip(t.outcomes):
             skip = True
-            break  # "for rule in self.wildcards"
+            break  # "for rule in wildcards"
           slow = slow or statusfile.IsSlow(t.outcomes)
           pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
       if (skip
@@ -209,12 +208,26 @@ class TestSuite(object):
     if not warn_unused_rules:
       return
 
-    for rule in self.rules:
-      if rule not in used_rules:
-        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
-    for rule in self.wildcards:
-      if rule not in used_rules:
-        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
+    if not variants:
+      for rule in self.rules[""]:
+        if (rule, "") not in used_rules:
+          print("Unused rule: %s -> %s (variant independent)" % (
+              rule, self.rules[""][rule]))
+      for rule in self.wildcards[""]:
+        if (rule, "") not in used_rules:
+          print("Unused rule: %s -> %s (variant independent)" % (
+              rule, self.wildcards[""][rule]))
+    else:
+      for variant in ALL_VARIANTS:
+        for rule in self.rules[variant]:
+          if (rule, variant) not in used_rules:
+            print("Unused rule: %s -> %s (variant: %s)" % (
+                rule, self.rules[variant][rule], variant))
+        for rule in self.wildcards[variant]:
+          if (rule, variant) not in used_rules:
+            print("Unused rule: %s -> %s (variant: %s)" % (
+                rule, self.wildcards[variant][rule], variant))
+
 
   def FilterTestCasesByArgs(self, args):
     """Filter test cases based on command-line arguments.