Refactor variations_combinations.py to use testing configs

tools/chrome_proxy/webdriver/variations_combinations.py

Index: tools/chrome_proxy/webdriver/variations_combinations.py
diff --git a/tools/chrome_proxy/webdriver/variations_combinations.py b/tools/chrome_proxy/webdriver/variations_combinations.py
index 3471a418acd2a44887806c0aab0c0c678fadf48e..56f890f6f727076f3fc5d78ce341dec93539c88a 100644
--- a/tools/chrome_proxy/webdriver/variations_combinations.py
+++ b/tools/chrome_proxy/webdriver/variations_combinations.py
@@ -2,60 +2,67 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import io

[tbansal1, 2017/05/30 23:31:23]

May be add:
import fieldtrial_util

Similar to
https://cs.chromium.org/chromium/src/tools/perf/core/perf_benchmark.py?type=c...

[Robert Ogden, 2017/05/30 23:44:35]

This is done below, after the sys.path.append. Should I move both lines up?

[tbansal1, 2017/05/30 23:50:12]

Aah, ok. This l-g-t-m.

 import os
-import re
+import platform
 import sys
 import time
 import unittest
 
 import common
 
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+  os.pardir, 'tools', 'variations'))
+import fieldtrial_util
 
-combinations = [
-  # One object for each set of tests to run with the given variations.
-  {
-    'label': 'dummy example',
-    'tests': [
-      # Of the form <file_name>.<class_name>.<method_name>
-      # Also accepts wildcard (*) as matching anything.
-      "lite_page.LitePage.testLitePage",
-      "lite_page.LitePage.testLitePageFallback",
-      "quic*"
-    ],
-    'variations': [
-      "DataReductionProxyUseQuic/Enabled",
-      "DataCompressionProxyLoFi/Enabled_Preview",
-      "DataCompressionProxyLitePageFallback/Enabled"
-    ],
-    'variations-params': [
-      "DataCompressionProxyLoFi.Enabled_Preview:effective_connection_type/4G"
-    ]
-  }
+test_blacklist = [
+  # These tests set their own field trials and should be ignored.
+  'lofi.LoFi.testLoFiSlowConnection',
+  'lofi.LoFi.testLoFiIfHeavyFastConnection',
+  'quic.Quic.testCheckPageWithQuicProxy',
+  'quic.Quic.testCheckPageWithQuicProxyTransaction',
+  'lite_page.LitePage.testLitePageFallback',
 ]
 
+def GetExperimentArgs():
+  """Returns a list of arguments with all tested field trials.
 
-def GetAllTestsFromRegexList(test_list, test_suite_iter):
-  """A helper function to make a test suite from tests matching the given list.
+  This function is a simple wrapper around the variation team's fieldtrail_util
+  script that generates command line arguments to test Chromium field trials.
 
-  Args:
-    test_list: a string list of all tests to run, allowing for simple regex
-    test_suite_iter: An iterator of all test suites to search
   Returns:
-    a test suite with all the tests specified by the test_list
+    an array of command line arguments to pass to chrome
+  """
+  config_path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+    os.pardir, 'testing', 'variations', 'fieldtrial_testing_config.json')
+  my_platform = ''
+  if common.ParseFlags().android:
+    my_platform = 'android'
+  elif platform.system().lower() == 'linux':
+    my_platform = 'linux'
+  elif platform.system().lower() == 'windows':
+    my_platform = 'win'
+  elif platform.system().lower() == 'darwin':
+    my_platform = 'mac'

[tbansal1, 2017/05/30 23:31:23]

add chromeos?

[Robert Ogden, 2017/05/30 23:44:35]

our integration test framework doesn't support ChromeOS yet

[tbansal1, 2017/05/30 23:50:12]

Acknowledged.

+  return fieldtrial_util.GenerateArgs(config_path, my_platform)

[tbansal1, 2017/05/30 23:31:23]

return an error if the platform is not supported>

[Robert Ogden, 2017/05/30 23:44:35]

Done.

+
+def GenerateTestSuites():
+  """A generator function that yields non-blacklisted tests to run.
+
+  This function yeilds test suites each with a single test case whose id is not
+  blacklisted in the array at the top of this file.
+
+  Yields:
+    non-blacklisted test suites to run
   """
-  id_to_test_map = {}
-  for test_suite in test_suite_iter:
+  loader = unittest.TestLoader()
+  for test_suite in loader.discover(os.path.dirname(__file__), pattern='*.py'):
     for test_case in test_suite:
       for test_method in test_case:
-        id_to_test_map[test_method.id()] = test_method
-  my_test_suite = unittest.TestSuite()
-  for test_spec in test_list:
-    regex = re.compile('^' + test_spec.replace('.', '\\.').replace('*', '.*')
-      + '$')
-    for test_id in sorted(id_to_test_map):
-      if regex.match(test_id):
-        my_test_suite.addTest(id_to_test_map[test_id])
-  return my_test_suite
+        if test_method.id() not in test_blacklist:
+          ts = unittest.TestSuite()
+          ts.addTest(test_method)
+          yield (ts, test_method.id())
 
 def ParseFlagsWithExtraBrowserArgs(extra_args):
   """Generates a function to override common.ParseFlags.
@@ -76,26 +83,36 @@ def ParseFlagsWithExtraBrowserArgs(extra_args):
   return AddExtraBrowserArgs
 
 def main():
-  """Runs each set of tests against its set of variations.
+  """Runs all non-blacklisted tests against Chromium field trials.
 
-  For each test combination, the above variation specifications will be used to
-  setup the browser arguments for each test given above that will be run.
+  This script run all chrome proxy integration tests that haven't been
+  blacklisted against the field trial testing configuration used by Chromium
+  perf bots.
   """
   flags = common.ParseFlags()
-  for variation_test in combinations:
-    # Set browser arguments to use the given variations.
-    extra_args = '--force-fieldtrials=' + '/'.join(variation_test['variations'])
-    extra_args += ' --force-fieldtrial-params=' + ','.join(
-      variation_test['variations-params'])
-    common.ParseFlags = ParseFlagsWithExtraBrowserArgs(extra_args)
-    # Run the given tests.
-    loader = unittest.TestLoader()
-    test_suite_iter = loader.discover(os.path.dirname(__file__), pattern='*.py')
-    my_test_suite = GetAllTestsFromRegexList(variation_test['tests'],
-      test_suite_iter)
-    testRunner = unittest.runner.TextTestRunner(verbosity=2,
-      failfast=flags.failfast, buffer=(not flags.disable_buffer))
-    testRunner.run(my_test_suite)
+  experiment_args = ' '.join(GetExperimentArgs())
+  common.ParseFlags = ParseFlagsWithExtraBrowserArgs(experiment_args)
+  # Each test is wrapped in its own test suite so results can be evaluated
+  # individually.
+  for test_suite, test_id in GenerateTestSuites():
+    buf = io.BytesIO()
+    sys.stdout.write('%s... ' % test_id)
+    sys.stdout.flush()
+    testRunner = unittest.runner.TextTestRunner(stream=buf, verbosity=2,
+      buffer=(not flags.disable_buffer))
+    result = testRunner.run(test_suite)
+    if result.wasSuccessful():
+      print 'ok'
+    else:
+      print 'failed'
+      print buf.getvalue()
+      print 'To repeat this test, run: '
+      print "%s %s %s --test_filter=%s --browser_args='%s'" % (sys.executable,
+        os.path.join(os.path.dirname(__file__), 'run_all_tests.py'),
+        ' '.join(sys.argv[1:]), '.'.join(test_id.split('.')[1:]),
+        experiment_args)
+      if flags.failfast:
+        return
 
 if __name__ == '__main__':
   main()