[test] Key variant flags by variant name everywhere.

test/benchmarks/testcfg.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 17 matching lines @@
 
 import os
 import shutil
 import subprocess
 import tarfile
 
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
+class BenchmarksVariantFlagsBuilder(testsuite.VariantFlagsBuilder):
+  def AllVariants(self, testcase):
+    # Both --nocrankshaft and --stressopt are very slow. Add TF but without
+    # always opt to match the way the benchmarks are run for performance
+    # testing.
+    return self.FastVariants(testcase)
+
+
 class BenchmarksTestSuite(testsuite.TestSuite):
 
   def __init__(self, name, root):
     super(BenchmarksTestSuite, self).__init__(name, root)
     self.testroot = root
 
   def ListTests(self, context):
     tests = []
     for test in [
         "kraken/ai-astar",
@@ skipping 127 matching lines @@
         ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/"
          "kraken-1.1/"),
         "8", "kraken")
 
     self._DownloadIfNecessary(
         "http://octane-benchmark.googlecode.com/svn/trunk/",
         "26", "octane")
 
     os.chdir(old_cwd)
 
-  def VariantFlags(self, testcase, default_flags):
-    # Both --nocrankshaft and --stressopt are very slow. Add TF but without
-    # always opt to match the way the benchmarks are run for performance
-    # testing.
-    return [[], ["--turbo"]]

[Michael Achenbach, 2015/07/27 12:37:45]

This completely ignored the status file. The new approach will e.g. allow to
limit a single benchmark to no-variants in the status file.

+  def VariantFlagsBuilder(self):
+    return BenchmarksVariantFlagsBuilder
 
 
 def GetSuite(name, root):
   return BenchmarksTestSuite(name, root)