Reorganize Android's test scripts into packages.

build/android/pylib/run_java_tests.py

Index: build/android/pylib/run_java_tests.py
diff --git a/build/android/pylib/run_java_tests.py b/build/android/pylib/run_java_tests.py
deleted file mode 100644
index 07b45e0ea9dfa857bafeb563699dcabfd4c14806..0000000000000000000000000000000000000000
--- a/build/android/pylib/run_java_tests.py
+++ /dev/null
@@ -1,593 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Runs the Java tests. See more information on run_instrumentation_tests.py."""
-
-import fnmatch
-import logging
-import os
-import re
-import shutil
-import sys
-import time
-
-import android_commands
-import apk_info
-from base_test_runner import BaseTestRunner
-from base_test_sharder import BaseTestSharder, SetTestsContainer
-import cmd_helper
-import constants
-import errors
-from forwarder import Forwarder
-from json_perf_parser import GetAverageRunInfoFromJSONString
-from perf_tests_helper import PrintPerfResult
-import sharded_tests_queue
-from test_result import SingleTestResult, TestResults
-import valgrind_tools
-
-_PERF_TEST_ANNOTATION = 'PerfTest'
-
-
-class FatalTestException(Exception):
-  """A fatal test exception."""
-  pass
-
-
-def _TestNameToExpectation(test_name):
-  # A test name is a Package.Path.Class#testName; convert to what we use in
-  # the expectation file.
-  return '.'.join(test_name.replace('#', '.').split('.')[-2:])
-
-
-def FilterTests(test_names, pattern_list, inclusive):
-  """Filters |test_names| using a list of patterns.
-
-  Args:
-    test_names: A list of test names.
-    pattern_list: A list of patterns.
-    inclusive: If True, returns the tests that match any pattern. if False,
-               returns the tests that do not match any pattern.
-  Returns:
-    A list of test names.
-  """
-  ret = []
-  for t in test_names:
-    has_match = False
-    for pattern in pattern_list:
-      has_match = has_match or fnmatch.fnmatch(_TestNameToExpectation(t),
-                                               pattern)
-    if has_match == inclusive:
-      ret += [t]
-  return ret
-
-
-class TestRunner(BaseTestRunner):
-  """Responsible for running a series of tests connected to a single device."""
-
-  _DEVICE_DATA_DIR = 'chrome/test/data'
-  _EMMA_JAR = os.path.join(os.environ.get('ANDROID_BUILD_TOP', ''),
-                           'external/emma/lib/emma.jar')
-  _COVERAGE_MERGED_FILENAME = 'unittest_coverage.es'
-  _COVERAGE_WEB_ROOT_DIR = os.environ.get('EMMA_WEB_ROOTDIR')
-  _COVERAGE_FILENAME = 'coverage.ec'
-  _COVERAGE_RESULT_PATH = ('/data/data/com.google.android.apps.chrome/files/' +
-                           _COVERAGE_FILENAME)
-  _COVERAGE_META_INFO_PATH = os.path.join(os.environ.get('ANDROID_BUILD_TOP',
-                                                         ''),
-                                          'out/target/common/obj/APPS',
-                                          'Chrome_intermediates/coverage.em')
-  _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
-  _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
-                                       '/chrome-profile*')
-  _DEVICE_HAS_TEST_FILES = {}
-
-  def __init__(self, options, device, tests_iter, coverage, shard_index, apks,
-               ports_to_forward):
-    """Create a new TestRunner.
-
-    Args:
-      options: An options object with the following required attributes:
-      -  build_type: 'Release' or 'Debug'.
-      -  install_apk: Re-installs the apk if opted.
-      -  save_perf_json: Whether or not to save the JSON file from UI perf
-            tests.
-      -  screenshot_failures: Take a screenshot for a test failure
-      -  tool: Name of the Valgrind tool.
-      -  wait_for_debugger: blocks until the debugger is connected.
-      -  disable_assertions: Whether to disable java assertions on the device.
-      device: Attached android device.
-      tests_iter: A list of tests to be run.
-      coverage: Collects coverage information if opted.
-      shard_index: shard # for this TestRunner, used to create unique port
-          numbers.
-      apks: A list of ApkInfo objects need to be installed. The first element
-            should be the tests apk, the rests could be the apks used in test.
-            The default is ChromeTest.apk.
-      ports_to_forward: A list of port numbers for which to set up forwarders.
-                        Can be optionally requested by a test case.
-    Raises:
-      FatalTestException: if coverage metadata is not available.
-    """
-    BaseTestRunner.__init__(
-        self, device, options.tool, shard_index, options.build_type)
-
-    if not apks:
-      apks = [apk_info.ApkInfo(options.test_apk_path,
-                               options.test_apk_jar_path)]
-
-    self.build_type = options.build_type
-    self.install_apk = options.install_apk
-    self.test_data = options.test_data
-    self.save_perf_json = options.save_perf_json
-    self.screenshot_failures = options.screenshot_failures
-    self.wait_for_debugger = options.wait_for_debugger
-    self.disable_assertions = options.disable_assertions
-
-    self.tests_iter = tests_iter
-    self.coverage = coverage
-    self.apks = apks
-    self.test_apk = apks[0]
-    self.instrumentation_class_path = self.test_apk.GetPackageName()
-    self.ports_to_forward = ports_to_forward
-
-    self.test_results = TestResults()
-    self.forwarder = None
-
-    if self.coverage:
-      if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
-        os.remove(TestRunner._COVERAGE_MERGED_FILENAME)
-      if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH):
-        raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
-                                 ' : Coverage meta info [' +
-                                 TestRunner._COVERAGE_META_INFO_PATH +
-                                 '] does not exist.')
-      if (not TestRunner._COVERAGE_WEB_ROOT_DIR or
-          not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)):
-        raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
-                                 ' : Path specified in $EMMA_WEB_ROOTDIR [' +
-                                 TestRunner._COVERAGE_WEB_ROOT_DIR +
-                                 '] does not exist.')
-
-  def _GetTestsIter(self):
-    if not self.tests_iter:
-      # multiprocessing.Queue can't be pickled across processes if we have it as
-      # a member set during constructor.  Grab one here instead.
-      self.tests_iter = (BaseTestSharder.tests_container)
-    assert self.tests_iter
-    return self.tests_iter
-
-  def CopyTestFilesOnce(self):
-    """Pushes the test data files to the device. Installs the apk if opted."""
-    if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
-      logging.warning('Already copied test files to device %s, skipping.',
-                      self.device)
-      return
-    for dest_host_pair in self.test_data:
-      dst_src = dest_host_pair.split(':',1)
-      dst_layer = dst_src[0]
-      host_src = dst_src[1]
-      host_test_files_path = constants.CHROME_DIR + '/' + host_src
-      if os.path.exists(host_test_files_path):
-        self.adb.PushIfNeeded(host_test_files_path,
-                              self.adb.GetExternalStorage() + '/' +
-                              TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
-    if self.install_apk:
-      for apk in self.apks:
-        self.adb.ManagedInstall(apk.GetApkPath(),
-                                package_name=apk.GetPackageName())
-    self.tool.CopyFiles()
-    TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
-
-  def SaveCoverageData(self, test):
-    """Saves the Emma coverage data before it's overwritten by the next test.
-
-    Args:
-      test: the test whose coverage data is collected.
-    """
-    if not self.coverage:
-      return
-    if not self.adb.Adb().Pull(TestRunner._COVERAGE_RESULT_PATH,
-                               constants.CHROME_DIR):
-      logging.error('ERROR: Unable to find file ' +
-                    TestRunner._COVERAGE_RESULT_PATH +
-                    ' on the device for test ' + test)
-    pulled_coverage_file = os.path.join(constants.CHROME_DIR,
-                                        TestRunner._COVERAGE_FILENAME)
-    if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
-      cmd = ['java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'merge',
-             '-in', pulled_coverage_file,
-             '-in', TestRunner._COVERAGE_MERGED_FILENAME,
-             '-out', TestRunner._COVERAGE_MERGED_FILENAME]
-      cmd_helper.RunCmd(cmd)
-    else:
-      shutil.copy(pulled_coverage_file,
-                  TestRunner._COVERAGE_MERGED_FILENAME)
-    os.remove(pulled_coverage_file)
-
-  def GenerateCoverageReportIfNeeded(self):
-    """Uses the Emma to generate a coverage report and a html page."""
-    if not self.coverage:
-      return
-    cmd = ['java', '-classpath', TestRunner._EMMA_JAR,
-           'emma', 'report', '-r', 'html',
-           '-in', TestRunner._COVERAGE_MERGED_FILENAME,
-           '-in', TestRunner._COVERAGE_META_INFO_PATH]
-    cmd_helper.RunCmd(cmd)
-    new_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
-                           time.strftime('Coverage_for_%Y_%m_%d_%a_%H:%M'))
-    shutil.copytree('coverage', new_dir)
-
-    latest_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
-                              'Latest_Coverage_Run')
-    if os.path.exists(latest_dir):
-      shutil.rmtree(latest_dir)
-    os.mkdir(latest_dir)
-    webserver_new_index = os.path.join(new_dir, 'index.html')
-    webserver_new_files = os.path.join(new_dir, '_files')
-    webserver_latest_index = os.path.join(latest_dir, 'index.html')
-    webserver_latest_files = os.path.join(latest_dir, '_files')
-    # Setup new softlinks to last result.
-    os.symlink(webserver_new_index, webserver_latest_index)
-    os.symlink(webserver_new_files, webserver_latest_files)
-    cmd_helper.RunCmd(['chmod', '755', '-R', latest_dir, new_dir])
-
-  def _GetInstrumentationArgs(self):
-    ret = {}
-    if self.coverage:
-      ret['coverage'] = 'true'
-    if self.wait_for_debugger:
-      ret['debug'] = 'true'
-    return ret
-
-  def _TakeScreenshot(self, test):
-    """Takes a screenshot from the device."""
-    screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png')
-    logging.info('Taking screenshot named %s', screenshot_name)
-    self.adb.TakeScreenshot(screenshot_name)
-
-  def SetUp(self):
-    """Sets up the test harness and device before all tests are run."""
-    super(TestRunner, self).SetUp()
-    if not self.adb.IsRootEnabled():
-      logging.warning('Unable to enable java asserts for %s, non rooted device',
-                      self.device)
-    else:
-      if self.adb.SetJavaAssertsEnabled(enable=not self.disable_assertions):
-        self.adb.Reboot(full_reboot=False)
-
-    # We give different default value to launch HTTP server based on shard index
-    # because it may have race condition when multiple processes are trying to
-    # launch lighttpd with same port at same time.
-    http_server_ports = self.LaunchTestHttpServer(
-        os.path.join(constants.CHROME_DIR),
-        (constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index))
-    if self.ports_to_forward:
-      port_pairs = [(port, port) for port in self.ports_to_forward]
-      # We need to remember which ports the HTTP server is using, since the
-      # forwarder will stomp on them otherwise.
-      port_pairs.append(http_server_ports)
-      self.forwarder = Forwarder(self.adb, self.build_type)
-      self.forwarder.Run(port_pairs, self.tool, '127.0.0.1')
-    self.CopyTestFilesOnce()
-    self.flags.AddFlags(['--enable-test-intents'])
-
-  def TearDown(self):
-    """Cleans up the test harness and saves outstanding data from test run."""
-    if self.forwarder:
-      self.forwarder.Close()
-    self.GenerateCoverageReportIfNeeded()
-    super(TestRunner, self).TearDown()
-
-  def TestSetup(self, test):
-    """Sets up the test harness for running a particular test.
-
-    Args:
-      test: The name of the test that will be run.
-    """
-    self.SetupPerfMonitoringIfNeeded(test)
-    self._SetupIndividualTestTimeoutScale(test)
-    self.tool.SetupEnvironment()
-
-    # Make sure the forwarder is still running.
-    self.RestartHttpServerForwarderIfNecessary()
-
-  def _IsPerfTest(self, test):
-    """Determines whether a test is a performance test.
-
-    Args:
-      test: The name of the test to be checked.
-
-    Returns:
-      Whether the test is annotated as a performance test.
-    """
-    return _PERF_TEST_ANNOTATION in self.test_apk.GetTestAnnotations(test)
-
-  def SetupPerfMonitoringIfNeeded(self, test):
-    """Sets up performance monitoring if the specified test requires it.
-
-    Args:
-      test: The name of the test to be run.
-    """
-    if not self._IsPerfTest(test):
-      return
-    self.adb.Adb().SendCommand('shell rm ' +
-                               TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX)
-    self.adb.StartMonitoringLogcat()
-
-  def TestTeardown(self, test, test_result):
-    """Cleans up the test harness after running a particular test.
-
-    Depending on the options of this TestRunner this might handle coverage
-    tracking or performance tracking.  This method will only be called if the
-    test passed.
-
-    Args:
-      test: The name of the test that was just run.
-      test_result: result for this test.
-    """
-
-    self.tool.CleanUpEnvironment()
-
-    # The logic below relies on the test passing.
-    if not test_result or test_result.GetStatusCode():
-      return
-
-    self.TearDownPerfMonitoring(test)
-    self.SaveCoverageData(test)
-
-  def TearDownPerfMonitoring(self, test):
-    """Cleans up performance monitoring if the specified test required it.
-
-    Args:
-      test: The name of the test that was just run.
-    Raises:
-      FatalTestException: if there's anything wrong with the perf data.
-    """
-    if not self._IsPerfTest(test):
-      return
-    raw_test_name = test.split('#')[1]
-
-    # Wait and grab annotation data so we can figure out which traces to parse
-    regex = self.adb.WaitForLogMatch(re.compile('\*\*PERFANNOTATION\(' +
-                                                raw_test_name +
-                                                '\)\:(.*)'), None)
-
-    # If the test is set to run on a specific device type only (IE: only
-    # tablet or phone) and it is being run on the wrong device, the test
-    # just quits and does not do anything.  The java test harness will still
-    # print the appropriate annotation for us, but will add --NORUN-- for
-    # us so we know to ignore the results.
-    # The --NORUN-- tag is managed by MainActivityTestBase.java
-    if regex.group(1) != '--NORUN--':
-
-      # Obtain the relevant perf data.  The data is dumped to a
-      # JSON formatted file.
-      json_string = self.adb.GetFileContents(
-          '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt')
-
-      if json_string:
-        json_string = '\n'.join(json_string)
-      else:
-        raise FatalTestException('Perf file does not exist or is empty')
-
-      if self.save_perf_json:
-        json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
-        with open(json_local_file, 'w') as f:
-          f.write(json_string)
-        logging.info('Saving Perf UI JSON from test ' +
-                     test + ' to ' + json_local_file)
-
-      raw_perf_data = regex.group(1).split(';')
-
-      for raw_perf_set in raw_perf_data:
-        if raw_perf_set:
-          perf_set = raw_perf_set.split(',')
-          if len(perf_set) != 3:
-            raise FatalTestException('Unexpected number of tokens in '
-                                     'perf annotation string: ' + raw_perf_set)
-
-          # Process the performance data
-          result = GetAverageRunInfoFromJSONString(json_string, perf_set[0])
-
-          PrintPerfResult(perf_set[1], perf_set[2],
-                          [result['average']], result['units'])
-
-  def _SetupIndividualTestTimeoutScale(self, test):
-    timeout_scale = self._GetIndividualTestTimeoutScale(test)
-    valgrind_tools.SetChromeTimeoutScale(self.adb, timeout_scale)
-
-  def _GetIndividualTestTimeoutScale(self, test):
-    """Returns the timeout scale for the given |test|."""
-    annotations = self.apks[0].GetTestAnnotations(test)
-    timeout_scale = 1
-    if 'TimeoutScale' in annotations:
-      for annotation in annotations:
-        scale_match = re.match('TimeoutScale:([0-9]+)', annotation)
-        if scale_match:
-          timeout_scale = int(scale_match.group(1))
-    if self.wait_for_debugger:
-      timeout_scale *= 100
-    return timeout_scale
-
-  def _GetIndividualTestTimeoutSecs(self, test):
-    """Returns the timeout in seconds for the given |test|."""
-    annotations = self.apks[0].GetTestAnnotations(test)
-    if 'Manual' in annotations:
-      return 600 * 60
-    if 'External' in annotations:
-      return 10 * 60
-    if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
-      return 5 * 60
-    if 'MediumTest' in annotations:
-      return 3 * 60
-    return 1 * 60
-
-  def RunTests(self):
-    """Runs the tests, generating the coverage if needed.
-
-    Returns:
-      A TestResults object.
-    """
-    instrumentation_path = (self.instrumentation_class_path +
-                            '/android.test.InstrumentationTestRunner')
-    instrumentation_args = self._GetInstrumentationArgs()
-    for test in self._GetTestsIter():
-      test_result = None
-      start_date_ms = None
-      try:
-        self.TestSetup(test)
-        start_date_ms = int(time.time()) * 1000
-        args_with_filter = dict(instrumentation_args)
-        args_with_filter['class'] = test
-        # |test_results| is a list that should contain
-        # a single TestResult object.
-        logging.warn(args_with_filter)
-        (test_results, _) = self.adb.Adb().StartInstrumentation(
-            instrumentation_path=instrumentation_path,
-            instrumentation_args=args_with_filter,
-            timeout_time=(self._GetIndividualTestTimeoutSecs(test) *
-                          self._GetIndividualTestTimeoutScale(test) *
-                          self.tool.GetTimeoutScale()))
-        duration_ms = int(time.time()) * 1000 - start_date_ms
-        assert len(test_results) == 1
-        test_result = test_results[0]
-        status_code = test_result.GetStatusCode()
-        if status_code:
-          log = test_result.GetFailureReason()
-          if not log:
-            log = 'No information.'
-          if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0:
-            self._TakeScreenshot(test)
-          self.test_results.failed += [SingleTestResult(test, start_date_ms,
-                                                        duration_ms, log)]
-        else:
-          result = [SingleTestResult(test, start_date_ms, duration_ms)]
-          self.test_results.ok += result
-      # Catch exceptions thrown by StartInstrumentation().
-      # See ../../third_party/android/testrunner/adb_interface.py
-      except (errors.WaitForResponseTimedOutError,
-              errors.DeviceUnresponsiveError,
-              errors.InstrumentationError), e:
-        if start_date_ms:
-          duration_ms = int(time.time()) * 1000 - start_date_ms
-        else:
-          start_date_ms = int(time.time()) * 1000
-          duration_ms = 0
-        message = str(e)
-        if not message:
-          message = 'No information.'
-        self.test_results.crashed += [SingleTestResult(test, start_date_ms,
-                                                       duration_ms,
-                                                       message)]
-        test_result = None
-      self.TestTeardown(test, test_result)
-    return self.test_results
-
-
-class TestSharder(BaseTestSharder):
-  """Responsible for sharding the tests on the connected devices."""
-
-  def __init__(self, attached_devices, options, tests, apks):
-    BaseTestSharder.__init__(self, attached_devices, options.build_type)
-    self.options = options
-    self.tests = tests
-    self.apks = apks
-
-  def SetupSharding(self, tests):
-    """Called before starting the shards."""
-    SetTestsContainer(sharded_tests_queue.ShardedTestsQueue(
-        len(self.attached_devices), tests))
-
-  def CreateShardedTestRunner(self, device, index):
-    """Creates a sharded test runner.
-
-    Args:
-      device: Device serial where this shard will run.
-      index: Index of this device in the pool.
-
-    Returns:
-      A TestRunner object.
-    """
-    return TestRunner(self.options, device, None, False, index, self.apks, [])
-
-
-def DispatchJavaTests(options, apks):
-  """Dispatches Java tests onto connected device(s).
-
-  If possible, this method will attempt to shard the tests to
-  all connected devices. Otherwise, dispatch and run tests on one device.
-
-  Args:
-    options: Command line options.
-    apks: list of APKs to use.
-
-  Returns:
-    A TestResults object holding the results of the Java tests.
-
-  Raises:
-    FatalTestException: when there's no attached the devices.
-  """
-  test_apk = apks[0]
-  # The default annotation for tests which do not have any sizes annotation.
-  default_size_annotation = 'SmallTest'
-
-  def _GetTestsMissingAnnotation(test_apk):
-    test_size_annotations = frozenset(['Smoke', 'SmallTest', 'MediumTest',
-                                       'LargeTest', 'EnormousTest', 'FlakyTest',
-                                       'DisabledTest', 'Manual', 'PerfTest'])
-    tests_missing_annotations = []
-    for test_method in test_apk.GetTestMethods():
-      annotations = frozenset(test_apk.GetTestAnnotations(test_method))
-      if (annotations.isdisjoint(test_size_annotations) and
-          not apk_info.ApkInfo.IsPythonDrivenTest(test_method)):
-        tests_missing_annotations.append(test_method)
-    return sorted(tests_missing_annotations)
-
-  if options.annotation:
-    available_tests = test_apk.GetAnnotatedTests(options.annotation)
-    if options.annotation.count(default_size_annotation) > 0:
-      tests_missing_annotations = _GetTestsMissingAnnotation(test_apk)
-      if tests_missing_annotations:
-        logging.warning('The following tests do not contain any annotation. '
-                        'Assuming "%s":\n%s',
-                        default_size_annotation,
-                        '\n'.join(tests_missing_annotations))
-        available_tests += tests_missing_annotations
-  else:
-    available_tests = [m for m in test_apk.GetTestMethods()
-                       if not apk_info.ApkInfo.IsPythonDrivenTest(m)]
-  coverage = os.environ.get('EMMA_INSTRUMENT') == 'true'
-
-  tests = []
-  if options.test_filter:
-    # |available_tests| are in adb instrument format: package.path.class#test.
-    filter_without_hash = options.test_filter.replace('#', '.')
-    tests = [t for t in available_tests
-             if filter_without_hash in t.replace('#', '.')]
-  else:
-    tests = available_tests
-
-  if not tests:
-    logging.warning('No Java tests to run with current args.')
-    return TestResults()
-
-  tests *= options.number_of_runs
-
-  attached_devices = android_commands.GetAttachedDevices()
-  test_results = TestResults()
-
-  if not attached_devices:
-    raise FatalTestException('You have no devices attached or visible!')
-  if options.device:
-    attached_devices = [options.device]
-
-  logging.info('Will run: %s', str(tests))
-
-  if len(attached_devices) > 1 and (coverage or options.wait_for_debugger):
-    logging.warning('Coverage / debugger can not be sharded, '
-                    'using first available device')
-    attached_devices = attached_devices[:1]
-  sharder = TestSharder(attached_devices, options, tests, apks)
-  test_results = sharder.RunShardedTests()
-  return test_results