Add a snapshot of flutter/engine/src/build to our sdk

build/android/pylib/instrumentation/test_runner.py

+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Class for running instrumentation tests on a single device."""
+
+import logging
+import os
+import re
+import sys
+import time
+
+from pylib import constants
+from pylib import flag_changer
+from pylib import valgrind_tools
+from pylib.base import base_test_result
+from pylib.base import base_test_runner
+from pylib.device import device_errors
+from pylib.instrumentation import instrumentation_test_instance
+from pylib.instrumentation import json_perf_parser
+from pylib.instrumentation import test_result
+from pylib.local.device import local_device_instrumentation_test_run
+
+sys.path.append(os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
+                             'common'))
+import perf_tests_results_helper # pylint: disable=F0401
+
+
+_PERF_TEST_ANNOTATION = 'PerfTest'
+
+
+class TestRunner(base_test_runner.BaseTestRunner):
+  """Responsible for running a series of tests connected to a single device."""
+
+  _DEVICE_COVERAGE_DIR = 'chrome/test/coverage'
+  _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
+  _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
+                                       '/chrome-profile*')
+
+  def __init__(self, test_options, device, shard_index, test_pkg,
+               additional_flags=None):
+    """Create a new TestRunner.
+
+    Args:
+      test_options: An InstrumentationOptions object.
+      device: Attached android device.
+      shard_index: Shard index.
+      test_pkg: A TestPackage object.
+      additional_flags: A list of additional flags to add to the command line.
+    """
+    super(TestRunner, self).__init__(device, test_options.tool)
+    self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
+    self._logcat_monitor = None
+
+    self.coverage_device_file = None
+    self.coverage_dir = test_options.coverage_dir
+    self.coverage_host_file = None
+    self.options = test_options
+    self.test_pkg = test_pkg
+    # Use the correct command line file for the package under test.
+    cmdline_file = [a.cmdline_file for a in constants.PACKAGE_INFO.itervalues()
+                    if a.test_package == self.test_pkg.GetPackageName()]
+    assert len(cmdline_file) < 2, 'Multiple packages have the same test package'
+    if len(cmdline_file) and cmdline_file[0]:
+      self.flags = flag_changer.FlagChanger(self.device, cmdline_file[0])
+      if additional_flags:
+        self.flags.AddFlags(additional_flags)
+    else:
+      self.flags = None
+
+  #override
+  def InstallTestPackage(self):
+    self.test_pkg.Install(self.device)
+
+  def _GetInstrumentationArgs(self):
+    ret = {}
+    if self.options.wait_for_debugger:
+      ret['debug'] = 'true'
+    if self.coverage_dir:
+      ret['coverage'] = 'true'
+      ret['coverageFile'] = self.coverage_device_file
+
+    return ret
+
+  def _TakeScreenshot(self, test):
+    """Takes a screenshot from the device."""
+    screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % test)
+    logging.info('Taking screenshot named %s', screenshot_name)
+    self.device.TakeScreenshot(screenshot_name)
+
+  def SetUp(self):
+    """Sets up the test harness and device before all tests are run."""
+    super(TestRunner, self).SetUp()
+    if not self.device.HasRoot():
+      logging.warning('Unable to enable java asserts for %s, non rooted device',
+                      str(self.device))
+    else:
+      if self.device.SetJavaAsserts(self.options.set_asserts):
+        # TODO(jbudorick) How to best do shell restart after the
+        #                 android_commands refactor?
+        self.device.RunShellCommand('stop')
+        self.device.RunShellCommand('start')
+        self.device.WaitUntilFullyBooted()
+
+    # We give different default value to launch HTTP server based on shard index
+    # because it may have race condition when multiple processes are trying to
+    # launch lighttpd with same port at same time.
+    self.LaunchTestHttpServer(
+        os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port)
+    if self.flags:
+      self.flags.AddFlags(['--disable-fre', '--enable-test-intents'])
+      if self.options.device_flags:
+        with open(self.options.device_flags) as device_flags_file:
+          stripped_flags = (l.strip() for l in device_flags_file)
+          self.flags.AddFlags([flag for flag in stripped_flags if flag])
+
+  def TearDown(self):
+    """Cleans up the test harness and saves outstanding data from test run."""
+    if self.flags:
+      self.flags.Restore()
+    super(TestRunner, self).TearDown()
+
+  def TestSetup(self, test):
+    """Sets up the test harness for running a particular test.
+
+    Args:
+      test: The name of the test that will be run.
+    """
+    self.SetupPerfMonitoringIfNeeded(test)
+    self._SetupIndividualTestTimeoutScale(test)
+    self.tool.SetupEnvironment()
+
+    if self.flags and self._IsFreTest(test):
+      self.flags.RemoveFlags(['--disable-fre'])
+
+    # Make sure the forwarder is still running.
+    self._RestartHttpServerForwarderIfNecessary()
+
+    if self.coverage_dir:
+      coverage_basename = '%s.ec' % test
+      self.coverage_device_file = '%s/%s/%s' % (
+          self.device.GetExternalStoragePath(),
+          TestRunner._DEVICE_COVERAGE_DIR, coverage_basename)
+      self.coverage_host_file = os.path.join(
+          self.coverage_dir, coverage_basename)
+
+  def _IsFreTest(self, test):
+    """Determines whether a test is a first run experience test.
+
+    Args:
+      test: The name of the test to be checked.
+
+    Returns:
+      Whether the feature being tested is FirstRunExperience.
+    """
+    annotations = self.test_pkg.GetTestAnnotations(test)
+    return 'FirstRunExperience' == annotations.get('Feature', None)
+
+  def _IsPerfTest(self, test):
+    """Determines whether a test is a performance test.
+
+    Args:
+      test: The name of the test to be checked.
+
+    Returns:
+      Whether the test is annotated as a performance test.
+    """
+    return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test)
+
+  def SetupPerfMonitoringIfNeeded(self, test):
+    """Sets up performance monitoring if the specified test requires it.
+
+    Args:
+      test: The name of the test to be run.
+    """
+    if not self._IsPerfTest(test):
+      return
+    self.device.RunShellCommand(
+        ['rm', TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX])
+    self._logcat_monitor = self.device.GetLogcatMonitor()
+    self._logcat_monitor.Start()
+
+  def TestTeardown(self, test, result):
+    """Cleans up the test harness after running a particular test.
+
+    Depending on the options of this TestRunner this might handle performance
+    tracking.  This method will only be called if the test passed.
+
+    Args:
+      test: The name of the test that was just run.
+      result: result for this test.
+    """
+
+    self.tool.CleanUpEnvironment()
+
+    # The logic below relies on the test passing.
+    if not result or not result.DidRunPass():
+      return
+
+    self.TearDownPerfMonitoring(test)
+
+    if self.flags and self._IsFreTest(test):
+      self.flags.AddFlags(['--disable-fre'])
+
+    if self.coverage_dir:
+      self.device.PullFile(
+          self.coverage_device_file, self.coverage_host_file)
+      self.device.RunShellCommand(
+          'rm -f %s' % self.coverage_device_file)
+
+  def TearDownPerfMonitoring(self, test):
+    """Cleans up performance monitoring if the specified test required it.
+
+    Args:
+      test: The name of the test that was just run.
+    Raises:
+      Exception: if there's anything wrong with the perf data.
+    """
+    if not self._IsPerfTest(test):
+      return
+    raw_test_name = test.split('#')[1]
+
+    # Wait and grab annotation data so we can figure out which traces to parse
+    regex = self._logcat_monitor.WaitFor(
+        re.compile(r'\*\*PERFANNOTATION\(' + raw_test_name + r'\)\:(.*)'))
+
+    # If the test is set to run on a specific device type only (IE: only
+    # tablet or phone) and it is being run on the wrong device, the test
+    # just quits and does not do anything.  The java test harness will still
+    # print the appropriate annotation for us, but will add --NORUN-- for
+    # us so we know to ignore the results.
+    # The --NORUN-- tag is managed by ChromeTabbedActivityTestBase.java
+    if regex.group(1) != '--NORUN--':
+
+      # Obtain the relevant perf data.  The data is dumped to a
+      # JSON formatted file.
+      json_string = self.device.ReadFile(
+          '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt',
+          as_root=True)
+
+      if not json_string:
+        raise Exception('Perf file is empty')
+
+      if self.options.save_perf_json:
+        json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
+        with open(json_local_file, 'w') as f:
+          f.write(json_string)
+        logging.info('Saving Perf UI JSON from test ' +
+                     test + ' to ' + json_local_file)
+
+      raw_perf_data = regex.group(1).split(';')
+
+      for raw_perf_set in raw_perf_data:
+        if raw_perf_set:
+          perf_set = raw_perf_set.split(',')
+          if len(perf_set) != 3:
+            raise Exception('Unexpected number of tokens in perf annotation '
+                            'string: ' + raw_perf_set)
+
+          # Process the performance data
+          result = json_perf_parser.GetAverageRunInfoFromJSONString(json_string,
+                                                                    perf_set[0])
+          perf_tests_results_helper.PrintPerfResult(perf_set[1], perf_set[2],
+                                                    [result['average']],
+                                                    result['units'])
+
+  def _SetupIndividualTestTimeoutScale(self, test):
+    timeout_scale = self._GetIndividualTestTimeoutScale(test)
+    valgrind_tools.SetChromeTimeoutScale(self.device, timeout_scale)
+
+  def _GetIndividualTestTimeoutScale(self, test):
+    """Returns the timeout scale for the given |test|."""
+    annotations = self.test_pkg.GetTestAnnotations(test)
+    timeout_scale = 1
+    if 'TimeoutScale' in annotations:
+      try:
+        timeout_scale = int(annotations['TimeoutScale'])
+      except ValueError:
+        logging.warning('Non-integer value of TimeoutScale ignored. (%s)'
+                        % annotations['TimeoutScale'])
+    if self.options.wait_for_debugger:
+      timeout_scale *= 100
+    return timeout_scale
+
+  def _GetIndividualTestTimeoutSecs(self, test):
+    """Returns the timeout in seconds for the given |test|."""
+    annotations = self.test_pkg.GetTestAnnotations(test)
+    if 'Manual' in annotations:
+      return 10 * 60 * 60
+    if 'IntegrationTest' in annotations:
+      return 30 * 60
+    if 'External' in annotations:
+      return 10 * 60
+    if 'EnormousTest' in annotations:
+      return 10 * 60
+    if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
+      return 5 * 60
+    if 'MediumTest' in annotations:
+      return 3 * 60
+    if 'SmallTest' in annotations:
+      return 1 * 60
+
+    logging.warn(("Test size not found in annotations for test '%s', using " +
+                  "1 minute for timeout.") % test)
+    return 1 * 60
+
+  def _RunTest(self, test, timeout):
+    """Runs a single instrumentation test.
+
+    Args:
+      test: Test class/method.
+      timeout: Timeout time in seconds.
+
+    Returns:
+      The raw output of am instrument as a list of lines.
+    """
+    extras = self._GetInstrumentationArgs()
+    extras['class'] = test
+    return self.device.StartInstrumentation(
+        '%s/%s' % (self.test_pkg.GetPackageName(), self.options.test_runner),
+        raw=True, extras=extras, timeout=timeout, retries=3)
+
+  def _GenerateTestResult(self, test, instr_result_code, instr_result_bundle,
+                          statuses, start_ms, duration_ms):
+    results = instrumentation_test_instance.GenerateTestResults(
+        instr_result_code, instr_result_bundle, statuses, start_ms, duration_ms)
+    for r in results:
+      if r.GetName() == test:
+        return r
+    logging.error('Could not find result for test: %s', test)
+    return test_result.InstrumentationTestResult(
+        test, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
+
+  #override
+  def RunTest(self, test):
+    results = base_test_result.TestRunResults()
+    timeout = (self._GetIndividualTestTimeoutSecs(test) *
+               self._GetIndividualTestTimeoutScale(test) *
+               self.tool.GetTimeoutScale())
+
+    start_ms = 0
+    duration_ms = 0
+    try:
+      self.TestSetup(test)
+
+      try:
+        self.device.GoHome()
+      except device_errors.CommandTimeoutError:
+        logging.exception('Failed to focus the launcher.')
+
+      time_ms = lambda: int(time.time() * 1000)
+      start_ms = time_ms()
+      raw_output = self._RunTest(test, timeout)
+      duration_ms = time_ms() - start_ms
+
+      # Parse the test output
+      result_code, result_bundle, statuses = (
+          instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output))
+      result = self._GenerateTestResult(
+          test, result_code, result_bundle, statuses, start_ms, duration_ms)
+      if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
+          self.test_pkg.GetPackageName(), self.device):
+        result.SetType(base_test_result.ResultType.CRASH)
+      results.AddResult(result)
+    except device_errors.CommandTimeoutError as e:
+      results.AddResult(test_result.InstrumentationTestResult(
+          test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms,
+          log=str(e) or 'No information'))
+    except device_errors.DeviceUnreachableError as e:
+      results.AddResult(test_result.InstrumentationTestResult(
+          test, base_test_result.ResultType.CRASH, start_ms, duration_ms,
+          log=str(e) or 'No information'))
+    self.TestTeardown(test, results)
+    return (results, None if results.DidRunPass() else test)