Index: build/android/pylib/instrumentation/test_runner.py |
diff --git a/build/android/pylib/instrumentation/test_runner.py b/build/android/pylib/instrumentation/test_runner.py |
deleted file mode 100644 |
index 0f2e53fea733e032314bf5cee7ccd6d711e3df3b..0000000000000000000000000000000000000000 |
--- a/build/android/pylib/instrumentation/test_runner.py |
+++ /dev/null |
@@ -1,374 +0,0 @@ |
-# Copyright (c) 2012 The Chromium Authors. All rights reserved. |
-# Use of this source code is governed by a BSD-style license that can be |
-# found in the LICENSE file. |
- |
-"""Class for running instrumentation tests on a single device.""" |
- |
-import logging |
-import os |
-import re |
-import sys |
-import time |
- |
-from pylib import constants |
-from pylib import flag_changer |
-from pylib import valgrind_tools |
-from pylib.base import base_test_result |
-from pylib.base import base_test_runner |
-from pylib.device import device_errors |
-from pylib.instrumentation import instrumentation_test_instance |
-from pylib.instrumentation import json_perf_parser |
-from pylib.instrumentation import test_result |
-from pylib.local.device import local_device_instrumentation_test_run |
- |
-sys.path.append(os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', |
- 'common')) |
-import perf_tests_results_helper # pylint: disable=F0401 |
- |
- |
-_PERF_TEST_ANNOTATION = 'PerfTest' |
- |
- |
-class TestRunner(base_test_runner.BaseTestRunner): |
- """Responsible for running a series of tests connected to a single device.""" |
- |
- _DEVICE_COVERAGE_DIR = 'chrome/test/coverage' |
- _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
- _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
- '/chrome-profile*') |
- |
- def __init__(self, test_options, device, shard_index, test_pkg, |
- additional_flags=None): |
- """Create a new TestRunner. |
- |
- Args: |
- test_options: An InstrumentationOptions object. |
- device: Attached android device. |
- shard_index: Shard index. |
- test_pkg: A TestPackage object. |
- additional_flags: A list of additional flags to add to the command line. |
- """ |
- super(TestRunner, self).__init__(device, test_options.tool) |
- self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
- self._logcat_monitor = None |
- |
- self.coverage_device_file = None |
- self.coverage_dir = test_options.coverage_dir |
- self.coverage_host_file = None |
- self.options = test_options |
- self.test_pkg = test_pkg |
- # Use the correct command line file for the package under test. |
- cmdline_file = [a.cmdline_file for a in constants.PACKAGE_INFO.itervalues() |
- if a.test_package == self.test_pkg.GetPackageName()] |
- assert len(cmdline_file) < 2, 'Multiple packages have the same test package' |
- if len(cmdline_file) and cmdline_file[0]: |
- self.flags = flag_changer.FlagChanger(self.device, cmdline_file[0]) |
- if additional_flags: |
- self.flags.AddFlags(additional_flags) |
- else: |
- self.flags = None |
- |
- #override |
- def InstallTestPackage(self): |
- self.test_pkg.Install(self.device) |
- |
- def _GetInstrumentationArgs(self): |
- ret = {} |
- if self.options.wait_for_debugger: |
- ret['debug'] = 'true' |
- if self.coverage_dir: |
- ret['coverage'] = 'true' |
- ret['coverageFile'] = self.coverage_device_file |
- |
- return ret |
- |
- def _TakeScreenshot(self, test): |
- """Takes a screenshot from the device.""" |
- screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % test) |
- logging.info('Taking screenshot named %s', screenshot_name) |
- self.device.TakeScreenshot(screenshot_name) |
- |
- def SetUp(self): |
- """Sets up the test harness and device before all tests are run.""" |
- super(TestRunner, self).SetUp() |
- if not self.device.HasRoot(): |
- logging.warning('Unable to enable java asserts for %s, non rooted device', |
- str(self.device)) |
- else: |
- if self.device.SetJavaAsserts(self.options.set_asserts): |
- # TODO(jbudorick) How to best do shell restart after the |
- # android_commands refactor? |
- self.device.RunShellCommand('stop') |
- self.device.RunShellCommand('start') |
- self.device.WaitUntilFullyBooted() |
- |
- # We give different default value to launch HTTP server based on shard index |
- # because it may have race condition when multiple processes are trying to |
- # launch lighttpd with same port at same time. |
- self.LaunchTestHttpServer( |
- os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port) |
- if self.flags: |
- self.flags.AddFlags(['--disable-fre', '--enable-test-intents']) |
- if self.options.device_flags: |
- with open(self.options.device_flags) as device_flags_file: |
- stripped_flags = (l.strip() for l in device_flags_file) |
- self.flags.AddFlags([flag for flag in stripped_flags if flag]) |
- |
- def TearDown(self): |
- """Cleans up the test harness and saves outstanding data from test run.""" |
- if self.flags: |
- self.flags.Restore() |
- super(TestRunner, self).TearDown() |
- |
- def TestSetup(self, test): |
- """Sets up the test harness for running a particular test. |
- |
- Args: |
- test: The name of the test that will be run. |
- """ |
- self.SetupPerfMonitoringIfNeeded(test) |
- self._SetupIndividualTestTimeoutScale(test) |
- self.tool.SetupEnvironment() |
- |
- if self.flags and self._IsFreTest(test): |
- self.flags.RemoveFlags(['--disable-fre']) |
- |
- # Make sure the forwarder is still running. |
- self._RestartHttpServerForwarderIfNecessary() |
- |
- if self.coverage_dir: |
- coverage_basename = '%s.ec' % test |
- self.coverage_device_file = '%s/%s/%s' % ( |
- self.device.GetExternalStoragePath(), |
- TestRunner._DEVICE_COVERAGE_DIR, coverage_basename) |
- self.coverage_host_file = os.path.join( |
- self.coverage_dir, coverage_basename) |
- |
- def _IsFreTest(self, test): |
- """Determines whether a test is a first run experience test. |
- |
- Args: |
- test: The name of the test to be checked. |
- |
- Returns: |
- Whether the feature being tested is FirstRunExperience. |
- """ |
- annotations = self.test_pkg.GetTestAnnotations(test) |
- return 'FirstRunExperience' == annotations.get('Feature', None) |
- |
- def _IsPerfTest(self, test): |
- """Determines whether a test is a performance test. |
- |
- Args: |
- test: The name of the test to be checked. |
- |
- Returns: |
- Whether the test is annotated as a performance test. |
- """ |
- return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) |
- |
- def SetupPerfMonitoringIfNeeded(self, test): |
- """Sets up performance monitoring if the specified test requires it. |
- |
- Args: |
- test: The name of the test to be run. |
- """ |
- if not self._IsPerfTest(test): |
- return |
- self.device.RunShellCommand( |
- ['rm', TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX]) |
- self._logcat_monitor = self.device.GetLogcatMonitor() |
- self._logcat_monitor.Start() |
- |
- def TestTeardown(self, test, result): |
- """Cleans up the test harness after running a particular test. |
- |
- Depending on the options of this TestRunner this might handle performance |
- tracking. This method will only be called if the test passed. |
- |
- Args: |
- test: The name of the test that was just run. |
- result: result for this test. |
- """ |
- |
- self.tool.CleanUpEnvironment() |
- |
- # The logic below relies on the test passing. |
- if not result or not result.DidRunPass(): |
- return |
- |
- self.TearDownPerfMonitoring(test) |
- |
- if self.flags and self._IsFreTest(test): |
- self.flags.AddFlags(['--disable-fre']) |
- |
- if self.coverage_dir: |
- self.device.PullFile( |
- self.coverage_device_file, self.coverage_host_file) |
- self.device.RunShellCommand( |
- 'rm -f %s' % self.coverage_device_file) |
- |
- def TearDownPerfMonitoring(self, test): |
- """Cleans up performance monitoring if the specified test required it. |
- |
- Args: |
- test: The name of the test that was just run. |
- Raises: |
- Exception: if there's anything wrong with the perf data. |
- """ |
- if not self._IsPerfTest(test): |
- return |
- raw_test_name = test.split('#')[1] |
- |
- # Wait and grab annotation data so we can figure out which traces to parse |
- regex = self._logcat_monitor.WaitFor( |
- re.compile(r'\*\*PERFANNOTATION\(' + raw_test_name + r'\)\:(.*)')) |
- |
- # If the test is set to run on a specific device type only (IE: only |
- # tablet or phone) and it is being run on the wrong device, the test |
- # just quits and does not do anything. The java test harness will still |
- # print the appropriate annotation for us, but will add --NORUN-- for |
- # us so we know to ignore the results. |
- # The --NORUN-- tag is managed by ChromeTabbedActivityTestBase.java |
- if regex.group(1) != '--NORUN--': |
- |
- # Obtain the relevant perf data. The data is dumped to a |
- # JSON formatted file. |
- json_string = self.device.ReadFile( |
- '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt', |
- as_root=True) |
- |
- if not json_string: |
- raise Exception('Perf file is empty') |
- |
- if self.options.save_perf_json: |
- json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name |
- with open(json_local_file, 'w') as f: |
- f.write(json_string) |
- logging.info('Saving Perf UI JSON from test ' + |
- test + ' to ' + json_local_file) |
- |
- raw_perf_data = regex.group(1).split(';') |
- |
- for raw_perf_set in raw_perf_data: |
- if raw_perf_set: |
- perf_set = raw_perf_set.split(',') |
- if len(perf_set) != 3: |
- raise Exception('Unexpected number of tokens in perf annotation ' |
- 'string: ' + raw_perf_set) |
- |
- # Process the performance data |
- result = json_perf_parser.GetAverageRunInfoFromJSONString(json_string, |
- perf_set[0]) |
- perf_tests_results_helper.PrintPerfResult(perf_set[1], perf_set[2], |
- [result['average']], |
- result['units']) |
- |
- def _SetupIndividualTestTimeoutScale(self, test): |
- timeout_scale = self._GetIndividualTestTimeoutScale(test) |
- valgrind_tools.SetChromeTimeoutScale(self.device, timeout_scale) |
- |
- def _GetIndividualTestTimeoutScale(self, test): |
- """Returns the timeout scale for the given |test|.""" |
- annotations = self.test_pkg.GetTestAnnotations(test) |
- timeout_scale = 1 |
- if 'TimeoutScale' in annotations: |
- try: |
- timeout_scale = int(annotations['TimeoutScale']) |
- except ValueError: |
- logging.warning('Non-integer value of TimeoutScale ignored. (%s)' |
- % annotations['TimeoutScale']) |
- if self.options.wait_for_debugger: |
- timeout_scale *= 100 |
- return timeout_scale |
- |
- def _GetIndividualTestTimeoutSecs(self, test): |
- """Returns the timeout in seconds for the given |test|.""" |
- annotations = self.test_pkg.GetTestAnnotations(test) |
- if 'Manual' in annotations: |
- return 10 * 60 * 60 |
- if 'IntegrationTest' in annotations: |
- return 30 * 60 |
- if 'External' in annotations: |
- return 10 * 60 |
- if 'EnormousTest' in annotations: |
- return 10 * 60 |
- if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: |
- return 5 * 60 |
- if 'MediumTest' in annotations: |
- return 3 * 60 |
- if 'SmallTest' in annotations: |
- return 1 * 60 |
- |
- logging.warn(("Test size not found in annotations for test '%s', using " + |
- "1 minute for timeout.") % test) |
- return 1 * 60 |
- |
- def _RunTest(self, test, timeout): |
- """Runs a single instrumentation test. |
- |
- Args: |
- test: Test class/method. |
- timeout: Timeout time in seconds. |
- |
- Returns: |
- The raw output of am instrument as a list of lines. |
- """ |
- extras = self._GetInstrumentationArgs() |
- extras['class'] = test |
- return self.device.StartInstrumentation( |
- '%s/%s' % (self.test_pkg.GetPackageName(), self.options.test_runner), |
- raw=True, extras=extras, timeout=timeout, retries=3) |
- |
- def _GenerateTestResult(self, test, instr_result_code, instr_result_bundle, |
- statuses, start_ms, duration_ms): |
- results = instrumentation_test_instance.GenerateTestResults( |
- instr_result_code, instr_result_bundle, statuses, start_ms, duration_ms) |
- for r in results: |
- if r.GetName() == test: |
- return r |
- logging.error('Could not find result for test: %s', test) |
- return test_result.InstrumentationTestResult( |
- test, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms) |
- |
- #override |
- def RunTest(self, test): |
- results = base_test_result.TestRunResults() |
- timeout = (self._GetIndividualTestTimeoutSecs(test) * |
- self._GetIndividualTestTimeoutScale(test) * |
- self.tool.GetTimeoutScale()) |
- |
- start_ms = 0 |
- duration_ms = 0 |
- try: |
- self.TestSetup(test) |
- |
- try: |
- self.device.GoHome() |
- except device_errors.CommandTimeoutError: |
- logging.exception('Failed to focus the launcher.') |
- |
- time_ms = lambda: int(time.time() * 1000) |
- start_ms = time_ms() |
- raw_output = self._RunTest(test, timeout) |
- duration_ms = time_ms() - start_ms |
- |
- # Parse the test output |
- result_code, result_bundle, statuses = ( |
- instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) |
- result = self._GenerateTestResult( |
- test, result_code, result_bundle, statuses, start_ms, duration_ms) |
- if local_device_instrumentation_test_run.DidPackageCrashOnDevice( |
- self.test_pkg.GetPackageName(), self.device): |
- result.SetType(base_test_result.ResultType.CRASH) |
- results.AddResult(result) |
- except device_errors.CommandTimeoutError as e: |
- results.AddResult(test_result.InstrumentationTestResult( |
- test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms, |
- log=str(e) or 'No information')) |
- except device_errors.DeviceUnreachableError as e: |
- results.AddResult(test_result.InstrumentationTestResult( |
- test, base_test_result.ResultType.CRASH, start_ms, duration_ms, |
- log=str(e) or 'No information')) |
- self.TestTeardown(test, results) |
- return (results, None if results.DidRunPass() else test) |