List Java Instru Test Information From JUnit Runner

build/android/pylib/instrumentation/instrumentation_test_instance.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
+import json
 import logging
 import os
 import pickle
 import re
 
 from devil.android import apk_helper
+from devil.android import device_temp_file
 from devil.android import md5sum
 from pylib import constants
 from pylib.base import base_test_result
 from pylib.base import test_exception
 from pylib.base import test_instance
 from pylib.constants import host_paths
 from pylib.instrumentation import test_result
 from pylib.instrumentation import instrumentation_parser
 from pylib.utils import dexdump
 from pylib.utils import instrumentation_tracing
 from pylib.utils import proguard
 from pylib.utils import shared_preference_utils
+from py_utils import tempfile_ext
 
 with host_paths.SysPath(host_paths.BUILD_COMMON_PATH):
   import unittest_util # pylint: disable=import-error
 
 # Ref: http://developer.android.com/reference/android/app/Activity.html
 _ACTIVITY_RESULT_CANCELED = 0
 _ACTIVITY_RESULT_OK = -1
 
 _COMMAND_LINE_PARAMETER = 'cmdlinearg-parameter'
 _DEFAULT_ANNOTATIONS = [
     'SmallTest', 'MediumTest', 'LargeTest', 'EnormousTest', 'IntegrationTest']
 _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [
     'DisabledTest', 'FlakyTest']
 _VALID_ANNOTATIONS = set(['Manual'] + _DEFAULT_ANNOTATIONS +
                          _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS)
 _EXTRA_DRIVER_TEST_LIST = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
 _EXTRA_DRIVER_TEST_LIST_FILE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile')
 _EXTRA_DRIVER_TARGET_PACKAGE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage')
 _EXTRA_DRIVER_TARGET_CLASS = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass')
 _EXTRA_TIMEOUT_SCALE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TimeoutScale')
+_EXTRA_TEST_LIST = (
+    'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList')
 
 _SKIP_PARAMETERIZATION = 'SkipCommandLineParameterization'
 _COMMANDLINE_PARAMETERIZATION = 'CommandLineParameter'
 _NATIVE_CRASH_RE = re.compile('(process|native) crash', re.IGNORECASE)
 _CMDLINE_NAME_SEGMENT_RE = re.compile(
     r' with(?:out)? \{[^\}]*\}')
 _PICKLE_FORMAT_VERSION = 12
 
 
 class MissingSizeAnnotationError(test_exception.TestException):
   def __init__(self, class_name):
     super(MissingSizeAnnotationError, self).__init__(class_name +
         ': Test method is missing required size annotation. Add one of: ' +
         ', '.join('@' + a for a in _VALID_ANNOTATIONS))
 
 
 class TestListPickleException(test_exception.TestException):
   pass
 
 
 # TODO(jbudorick): Make these private class methods of
-# InstrumentationTestInstance once the instrumentation test_runner is
+# InstrumentationTestInstance once the instrumentation junit3_runner_class is
 # deprecated.
 def ParseAmInstrumentRawOutput(raw_output):
   """Parses the output of an |am instrument -r| call.
 
   Args:
     raw_output: the output of an |am instrument -r| call as a list of lines
   Returns:
     A 3-tuple containing:
       - the instrumentation code as an integer
       - the instrumentation result as a list of lines
@@ skipping 75 matching lines @@
 
   return results
 
 
 def FilterTests(tests, test_filter=None, annotations=None,
                 excluded_annotations=None):
   """Filter a list of tests
 
   Args:
     tests: a list of tests. e.g. [
-           {'annotations": {}, 'class': 'com.example.TestA', 'methods':[]},
-           {'annotations": {}, 'class': 'com.example.TestB', 'methods':[]}]
+           {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'},
+           {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}]
     test_filter: googletest-style filter string.
     annotations: a dict of wanted annotations for test methods.
     exclude_annotations: a dict of annotations to exclude.
 
   Return:
     A list of filtered tests
   """
   def gtest_filter(t):
     if not test_filter:
       return True
     # Allow fully-qualified name as well as an omitted package.
     unqualified_class_test = {
       'class': t['class'].split('.')[-1],
       'method': t['method']
     }
     names = [
       GetTestName(t, sep='.'),
       GetTestName(unqualified_class_test, sep='.'),
       GetUniqueTestName(t, sep='.')
     ]
 
+    if t['is_junit4']:
+      names += [
+          GetTestNameWithoutParameterPostfix(t, sep='.'),
+          GetTestNameWithoutParameterPostfix(unqualified_class_test, sep='.')
+      ]
+
     pattern_groups = test_filter.split('-')
     if len(pattern_groups) > 1:
       negative_filter = pattern_groups[1]
       if unittest_util.FilterTestNames(names, negative_filter):
         return []
 
     positive_filter = pattern_groups[0]
     return unittest_util.FilterTestNames(names, positive_filter)
 
   def annotation_filter(all_annotations):
@@ skipping 34 matching lines @@
 
     if (not annotation_filter(t['annotations'])
         or not excluded_annotation_filter(t['annotations'])):
       continue
 
     filtered_tests.append(t)
 
   return filtered_tests
 
 
+def GetAllTestsFromRunner(device, test_apk, test_package, junit4_runner_class):
+  pickle_path = '%s-runner.pickle' % test_apk.path
+  try:
+    tests = _GetTestsFromPickle(pickle_path, test_apk.path)
+  except TestListPickleException as e:
+    logging.info('Could not get tests from pickle: %s', e)
+    logging.info('Getting tests by print tests in instrumentation runner')

[jbudorick, 2017/07/19 20:43:34]

nit: 'Getting tests by having %s list them.' % junit4_runner_class

[the real yoland, 2017/07/20 02:04:33]

Done.

+    tests = _GetAllTestsFromRunner(device, test_package, junit4_runner_class)
+    _SaveTestsToPickle(pickle_path, test_apk.path, tests)
+  return tests
+
+
+# TODO(yolandyan): remove this once the tests are converted to junit4
 def GetAllTestsFromJar(test_jar):
   pickle_path = '%s-proguard.pickle' % test_jar
   try:
     tests = _GetTestsFromPickle(pickle_path, test_jar)
   except TestListPickleException as e:
     logging.info('Could not get tests from pickle: %s', e)
     logging.info('Getting tests from JAR via proguard.')
     tests = _GetTestsFromProguard(test_jar)
     _SaveTestsToPickle(pickle_path, test_jar, tests)
   return tests
@@ skipping 78 matching lines @@
     for class_name, class_info in package_info['classes'].iteritems():
       if class_name.endswith('Test'):
         tests.append({
             'class': '%s.%s' % (package_name, class_name),
             'annotations': {},
             'methods': get_test_methods(class_info['methods']),
             'superclass': class_info['superclass'],
         })
   return tests
 
+def _GetAllTestsFromRunner(device, test_package, junit4_runner_class):

[jbudorick, 2017/07/19 20:43:34]

Sorry, missed this the first time around. Device-specific behavior does not
belong in the test instance. This should be handled in the test_run.

[the real yoland, 2017/07/20 02:04:33]

Done

+  with device_temp_file.DeviceTempFile(
+      device.adb, suffix='.json', dir=device.GetExternalStoragePath()) as f:

[jbudorick, 2017/07/19 20:43:34]

nit: this needs a more descriptive name than "f" given the extent of the scope
below.

[the real yoland, 2017/07/20 02:04:33]

Done

+    extras = {}
+    extras[_EXTRA_TEST_LIST] = f.name
+    extras['log'] = 'true'
+    extras['package'] = '.'.join(test_package.split('.')[:2])
+    target = '%s/%s' % (test_package, junit4_runner_class)
+    output_string = ''.join(device.StartInstrumentation(
+        target, extras=extras, timeout=30, retries=2))

[jbudorick, 2017/07/19 20:43:34]

I mentioned retries here, but I should say that neither timeout nor retries
needs to be overridden here.

[the real yoland, 2017/07/20 02:04:33]

Done

+    if output_string:
+      raise Exception('Test listing through %s failed on device:\n%s' % (
+          junit4_runner_class, output_string))
+    with tempfile_ext.NamedTemporaryDirectory() as host_dir:
+      host_file = os.path.join(host_dir, 'list_tests.json')
+      device.PullFile(f.name, host_file)
+      with open(host_file, 'r') as host_file:
+        json_string = host_file.read()
+        return json.loads(json_string)
+
 
 def _SaveTestsToPickle(pickle_path, jar_path, tests):
   jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
   pickle_data = {
     'VERSION': _PICKLE_FORMAT_VERSION,
     'JAR_MD5SUM': jar_md5,
     'TEST_METHODS': tests,
   }
   with open(pickle_path, 'w') as pickle_file:
     pickle.dump(pickle_data, pickle_file)
@@ skipping 23 matching lines @@
 
   Args:
     test: the instrumentation test dict.
     sep: the character(s) that should join the class name and the method name.
   Returns:
     The test name as a string.
   """
   return '%s%s%s' % (test['class'], sep, test['method'])
 
 
+def GetTestNameWithoutParameterPostfix(
+      test, sep='#', parameterization_sep='__'):
+  """Gets the name of the given JUnit4 test without parameter postfix.
+
+  For most WebView JUnit4 javatests, each test is parameterizatized with
+  "__sandboxed_mode" to run in both non-sandboxed mode and sandboxed mode.
+
+  This function returns the name of the test without parameterization
+  so test filters can match both parameterized and non-parameterized tests.
+
+  Args:
+    test: the instrumentation test dict.
+    sep: the character(s) that should join the class name and the method name.
+    parameterization_sep: the character(s) that seperate method name and method
+                          parameterization postfix.
+  Returns:
+    The test name without parameter postfix as a string.
+  """
+  name = GetTestName(test, sep=sep)
+  return name.split(parameterization_sep)[0]
+
+
 def GetUniqueTestName(test, sep='#'):
   """Gets the unique name of the given test.
 
   This will include text to disambiguate between tests for which GetTestName
   would return the same name.
 
   Args:
     test: the instrumentation test dict.
     sep: the character(s) that should join the class name and the method name.
   Returns:
@@ skipping 12 matching lines @@
 
     self._additional_apks = []
     self._apk_under_test = None
     self._apk_under_test_incremental_install_script = None
     self._package_info = None
     self._suite = None
     self._test_apk = None
     self._test_apk_incremental_install_script = None
     self._test_jar = None
     self._test_package = None
-    self._test_runner = None
-    self._test_runner_junit4 = None
+    self._junit3_runner_class = None
+    self._junit4_runner_class = None
     self._test_support_apk = None
     self._initializeApkAttributes(args, error_func)
 
     self._data_deps = None
     self._data_deps_delegate = None
     self._runtime_deps_path = None
     self._initializeDataDependencyAttributes(args, data_deps_delegate)
 
     self._annotations = None
     self._excluded_annotations = None
@@ skipping 86 matching lines @@
       error_func('Unable to find test APK: %s' % self._test_apk.path)
     if not self._test_jar:
       logging.warning('Test jar not specified. Test runner will not have '
                       'Java annotation info available. May not handle test '
                       'timeouts correctly.')
     elif not os.path.exists(self._test_jar):
       error_func('Unable to find test JAR: %s' % self._test_jar)
 
     self._test_package = self._test_apk.GetPackageName()
     all_instrumentations = self._test_apk.GetAllInstrumentations()
-    test_runners = [
+    all_junit3_runner_classes = [
         x for x in all_instrumentations if ('true' not in x.get(
             'chromium-junit4', ''))]
-    test_runners_junit4 = [
+    all_junit4_test_runner_classes = [
         x for x in all_instrumentations if ('true' in x.get(
             'chromium-junit4', ''))]
 
-    if len(test_runners) > 1:
+    if len(all_junit3_runner_classes) > 1:
       logging.warning('This test apk has more than one JUnit3 instrumentation')
-    if len(test_runners_junit4) > 1:
+    if len(all_junit4_test_runner_classes) > 1:
       logging.warning('This test apk has more than one JUnit4 instrumentation')
 
-    self._test_runner = (
-      test_runners[0]['android:name'] if test_runners else
-      self.test_apk.GetInstrumentationName())
-    self._test_runner_junit4 = (
-      test_runners_junit4[0]['android:name'] if test_runners_junit4 else None)
+    self._junit3_runner_class = (
+      all_junit3_runner_classes[0]['android:name']
+      if all_junit3_runner_classes else self.test_apk.GetInstrumentationName())
+
+    self._junit4_runner_class = (
+      all_junit4_test_runner_classes[0]['android:name']
+      if all_junit4_test_runner_classes else None)
 
     self._package_info = None
     if self._apk_under_test:
       package_under_test = self._apk_under_test.GetPackageName()
       for package_info in constants.PACKAGE_INFO.itervalues():
         if package_under_test == package_info.package:
           self._package_info = package_info
           break
     if not self._package_info:
       logging.warning('Unable to find package info for %s', self._test_package)
@@ skipping 181 matching lines @@
 
   @property
   def test_support_apk(self):
     return self._test_support_apk
 
   @property
   def test_package(self):
     return self._test_package
 
   @property
-  def test_runner(self):
-    return self._test_runner
+  def junit3_runner_class(self):
+    return self._junit3_runner_class
 
   @property
-  def test_runner_junit4(self):
-    return self._test_runner_junit4
+  def junit4_runner_class(self):
+    return self._junit4_runner_class
 
   @property
   def timeout_scale(self):
     return self._timeout_scale
 
   @property
   def total_external_shards(self):
     return self._total_external_shards
 
   @property
   def ui_screenshot_dir(self):
     return self._ui_screenshot_dir
 
   #override
   def TestType(self):
     return 'instrumentation'
 
   #override
   def SetUp(self):
     self._data_deps.extend(
         self._data_deps_delegate(self._runtime_deps_path))
 
   def GetDataDependencies(self):
     return self._data_deps
 
-  def GetTests(self):
-    if self.test_jar:
+  def GetTests(self, device=None):
+    if self._junit4_runner_class:
+      tests = GetAllTestsFromRunner(
+          device, self.test_apk, self.test_package, self.junit4_runner_class)
+    elif self.test_jar:
       tests = GetAllTestsFromJar(self.test_jar)
     else:
       tests = GetAllTestsFromApk(self.test_apk.path)
     inflated_tests = self._ParameterizeTestsWithFlags(self._InflateTests(tests))
-    if self._test_runner_junit4 is None and any(
+    if self._junit4_runner_class is None and any(
         t['is_junit4'] for t in inflated_tests):
       raise MissingJUnit4RunnerException()
     filtered_tests = FilterTests(
         inflated_tests, self._test_filter, self._annotations,
         self._excluded_annotations)
     if self._test_filter and not filtered_tests:
       for t in inflated_tests:
         logging.debug('  %s', GetUniqueTestName(t))
       raise UnmatchedFilterException(self._test_filter)
     return filtered_tests
@@ skipping 26 matching lines @@
         for p in parameters[1:]:
           parameterized_t = copy.copy(t)
           parameterized_t['flags'] = ['--%s' % p]
           new_tests.append(parameterized_t)
     return tests + new_tests
 
   def GetDriverEnvironmentVars(
       self, test_list=None, test_list_file_path=None):
     env = {
       _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package,
-      _EXTRA_DRIVER_TARGET_CLASS: self.test_runner,
+      _EXTRA_DRIVER_TARGET_CLASS: self.junit3_runner_class,
       _EXTRA_TIMEOUT_SCALE: self._timeout_scale,
     }
 
     if test_list:
       env[_EXTRA_DRIVER_TEST_LIST] = ','.join(test_list)
 
     if test_list_file_path:
       env[_EXTRA_DRIVER_TEST_LIST_FILE] = (
           os.path.basename(test_list_file_path))
 
     return env
 
   @staticmethod
   def ParseAmInstrumentRawOutput(raw_output):
     return ParseAmInstrumentRawOutput(raw_output)
 
   @staticmethod
   def GenerateTestResults(
       result_code, result_bundle, statuses, start_ms, duration_ms):
     return GenerateTestResults(result_code, result_bundle, statuses,
                                start_ms, duration_ms)
 
   #override
   def TearDown(self):
     pass