Add CommandLineArgumentParameter and use it for WebView tests

build/android/pylib/instrumentation/test_runner.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Class for running instrumentation tests on a single device."""
 
 import logging
 import os
 import re
 import sys
 import time
+from collections import namedtuple
 
 from devil.android import device_errors
 from pylib import constants
 from pylib import flag_changer
 from pylib import valgrind_tools
 from pylib.base import base_test_result
 from pylib.base import base_test_runner
 from pylib.instrumentation import instrumentation_test_instance
 from pylib.instrumentation import json_perf_parser
 from pylib.instrumentation import test_result
@@ skipping 94 matching lines @@
           stripped_flags = (l.strip() for l in device_flags_file)
           flags_to_add.extend([flag for flag in stripped_flags if flag])
       self.flags.AddFlags(flags_to_add)
 
   def TearDown(self):
     """Cleans up the test harness and saves outstanding data from test run."""
     if self.flags:
       self.flags.Restore()
     super(TestRunner, self).TearDown()
 
-  def TestSetup(self, test):
+  def TestSetup(self, test, flag_modifiers):
     """Sets up the test harness for running a particular test.
 
     Args:
       test: The name of the test that will be run.
     """
     self.SetupPerfMonitoringIfNeeded(test)
     self._SetupIndividualTestTimeoutScale(test)
     self.tool.SetupEnvironment()
 
-    if self.flags and self._IsFreTest(test):
-      self.flags.RemoveFlags(['--disable-fre'])
+    if self.flags:
+      self.flags.PushFlags(add=flag_modifiers.add, remove=flag_modifiers.remove)
 
     # Make sure the forwarder is still running.
     self._RestartHttpServerForwarderIfNecessary()
 
     if self.coverage_dir:
       coverage_basename = '%s.ec' % test
       self.coverage_device_file = '%s/%s/%s' % (
           self.device.GetExternalStoragePath(),
           TestRunner._DEVICE_COVERAGE_DIR, coverage_basename)
       self.coverage_host_file = os.path.join(
@@ skipping 16 matching lines @@
     """Determines whether a test is a performance test.
 
     Args:
       test: The name of the test to be checked.
 
     Returns:
       Whether the test is annotated as a performance test.
     """
     return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test)
 
+  def _GetTestCmdlineParameters(self, test):
+    """Determines whether the test is parameterized to be run with different
+       command-line flags.
+
+    Args:
+      test: The name of the test to be checked.
+
+    Returns:
+      The list of parameters.
+    """
+    annotations = self.test_pkg.GetTestAnnotations(test)
+    params = instrumentation_test_instance.ParseCommandLineFlagParameters(
+      annotations)
+    if not params:
+      params = [namedtuple('Dummy', ['add', 'remove'])([], [])]
+    return params
+
   def SetupPerfMonitoringIfNeeded(self, test):
     """Sets up performance monitoring if the specified test requires it.
 
     Args:
       test: The name of the test to be run.
     """
     if not self._IsPerfTest(test):
       return
     self.device.RunShellCommand(
         ['rm', TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX])
     self._logcat_monitor = self.device.GetLogcatMonitor()
     self._logcat_monitor.Start()
 
-  def TestTeardown(self, test, result):
+  def TestTeardown(self, test, results):
     """Cleans up the test harness after running a particular test.
 
     Depending on the options of this TestRunner this might handle performance
     tracking.  This method will only be called if the test passed.
 
     Args:
       test: The name of the test that was just run.
-      result: result for this test.
+      results: results for this test.
     """
 
     self.tool.CleanUpEnvironment()
 
-    if self.flags and self._IsFreTest(test):
+    if self.flags:
       self.flags.Restore()
 
-    if not result:
+    if not results:
       return
-    if result.DidRunPass():
+    if results.DidRunPass():
       self.TearDownPerfMonitoring(test)
 
       if self.coverage_dir:
         self.device.PullFile(
             self.coverage_device_file, self.coverage_host_file)
         self.device.RunShellCommand(
             'rm -f %s' % self.coverage_device_file)
     elif self.package_info:
       self.device.ClearApplicationState(self.package_info.package)
 
@@ skipping 122 matching lines @@
     return test_result.InstrumentationTestResult(
         test, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
 
   #override
   def RunTest(self, test):
     results = base_test_result.TestRunResults()
     timeout = (self._GetIndividualTestTimeoutSecs(test) *
                self._GetIndividualTestTimeoutScale(test) *
                self.tool.GetTimeoutScale())
 
-    start_ms = 0
-    duration_ms = 0
-    try:
-      self.TestSetup(test)
+    cmdline_parameters = self._GetTestCmdlineParameters(test)
+    for flag_modifiers in cmdline_parameters:
+      start_ms = 0
+      duration_ms = 0
+      try:
+        if self._IsFreTest(test):
+          flag_modifiers.remove.append('--disable-fre')
+        self.TestSetup(test, flag_modifiers)
 
-      try:
-        self.device.GoHome()
-      except device_errors.CommandTimeoutError:
-        logging.exception('Failed to focus the launcher.')
+        try:
+          self.device.GoHome()
+        except device_errors.CommandTimeoutError:
+          logging.exception('Failed to focus the launcher.')
 
-      time_ms = lambda: int(time.time() * 1000)
-      start_ms = time_ms()
-      raw_output = self._RunTest(test, timeout)
-      duration_ms = time_ms() - start_ms
+        time_ms = lambda: int(time.time() * 1000)
+        start_ms = time_ms()
+        raw_output = self._RunTest(test, timeout)
+        duration_ms = time_ms() - start_ms
 
-      # Parse the test output
-      result_code, result_bundle, statuses = (
-          instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output))
-      result = self._GenerateTestResult(
-          test, result_code, result_bundle, statuses, start_ms, duration_ms)
-      if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
-          self.test_pkg.GetPackageName(), self.device):
-        result.SetType(base_test_result.ResultType.CRASH)
+        # Parse the test output
+        result_code, result_bundle, statuses = (
+            instrumentation_test_instance.ParseAmInstrumentRawOutput(
+              raw_output))
+        result = self._GenerateTestResult(
+            test, result_code, result_bundle, statuses, start_ms, duration_ms)
+        if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
+            self.test_pkg.GetPackageName(), self.device):
+          result.SetType(base_test_result.ResultType.CRASH)
+      except device_errors.CommandTimeoutError as e:
+        result = test_result.InstrumentationTestResult(
+          test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms,
+          log=str(e) or 'No information')
+        if self.package_info:
+          self.device.ForceStop(self.package_info.package)
+          self.device.ForceStop(self.package_info.test_package)
+      except device_errors.DeviceUnreachableError as e:
+        result = test_result.InstrumentationTestResult(
+            test, base_test_result.ResultType.CRASH, start_ms, duration_ms,
+            log=str(e) or 'No information')
+      if len(cmdline_parameters) > 1:
+        # Specify commandline flag modifications used in the test run
+        result_name = result.GetName()
+        if flag_modifiers.add:
+          result_name = '%s with {%s}' % (
+            result_name, ' '.join(flag_modifiers.add))
+        if flag_modifiers.remove:
+          result_name = '%s without {%s}' % (
+            result_name, ' '.join(flag_modifiers.remove))
+        result.SetName(result_name)
       results.AddResult(result)
-    except device_errors.CommandTimeoutError as e:
-      results.AddResult(test_result.InstrumentationTestResult(
-          test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms,
-          log=str(e) or 'No information'))
-      if self.package_info:
-        self.device.ForceStop(self.package_info.package)
-        self.device.ForceStop(self.package_info.test_package)
-    except device_errors.DeviceUnreachableError as e:
-      results.AddResult(test_result.InstrumentationTestResult(
-          test, base_test_result.ResultType.CRASH, start_ms, duration_ms,
-          log=str(e) or 'No information'))
-    self.TestTeardown(test, results)
+
+      self.TestTeardown(test, results)
+
     return (results, None if results.DidRunPass() else test)