List Java Instru Test Information From JUnit Runner

build/android/pylib/local/device/local_device_instrumentation_test_run.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import contextlib
+import json
 import logging
 import os
 import posixpath
 import re
 import sys
 import tempfile
 import time
 
 from devil.android import crash_handler
 from devil.android import device_errors
@@ skipping 41 matching lines @@
 
 LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v', 'DEBUG:I',
                   'StrictMode:D', '%s:I' % _TAG]
 
 EXTRA_SCREENSHOT_FILE = (
     'org.chromium.base.test.ScreenshotOnFailureStatement.ScreenshotFile')
 
 EXTRA_UI_CAPTURE_DIR = (
     'org.chromium.base.test.util.Screenshooter.ScreenshotDir')
 
+_EXTRA_TEST_LIST = (
+    'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList')
+
 UI_CAPTURE_DIRS = ['chromium_tests_root', 'UiCapture']
 
 FEATURE_ANNOTATION = 'Feature'
 RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest'
 
 # This needs to be kept in sync with formatting in |RenderUtils.imageName|
 RE_RENDER_IMAGE_NAME = re.compile(
       r'(?P<test_class>\w+)\.'
       r'(?P<description>\w+)\.'
       r'(?P<device_model>\w+)\.'
       r'(?P<orientation>port|land)\.png')
 
 @contextlib.contextmanager
 def _LogTestEndpoints(device, test_name):
   device.RunShellCommand(
       ['log', '-p', 'i', '-t', _TAG, 'START %s' % test_name],
       check_return=True)
   try:
     yield
   finally:
     device.RunShellCommand(
         ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
         check_return=True)
 
-# TODO(jbudorick): Make this private once the instrumentation test_runner is
-# deprecated.
+# TODO(jbudorick): Make this private once the instrumentation test_runner
+# is deprecated.
 def DidPackageCrashOnDevice(package_name, device):
   # Dismiss any error dialogs. Limit the number in case we have an error
   # loop or we are failing to dismiss.
   try:
     for _ in xrange(10):
       package = device.DismissCrashDialogIfNeeded()
       if not package:
         return False
       # Assume test package convention of ".test" suffix
       if package in package_name:
@@ skipping 196 matching lines @@
     if not str(device) in self._flag_changers:
       self._flag_changers[str(device)] = flag_changer.FlagChanger(
         device, self._test_instance.package_info.cmdline_file)
 
   #override
   def _CreateShards(self, tests):
     return tests
 
   #override
   def _GetTests(self):
-    tests = self._test_instance.GetTests()
+    tests = None
+    if self._test_instance.junit4_runner_class:
+      raw_tests = self._GetTestsFromRunner()
+      tests = self._test_instance.ProcessRawTests(raw_tests)
+    else:
+      tests = self._test_instance.GetTests()
     tests = self._ApplyExternalSharding(
         tests, self._test_instance.external_shard_index,
         self._test_instance.total_external_shards)
     return tests
 
   #override
   def _GetUniqueTestName(self, test):
     return instrumentation_test_instance.GetUniqueTestName(test)
 
   #override
@@ skipping 44 matching lines @@
       extras.update(
           self._test_instance.GetDriverEnvironmentVars(
               test_list=test_names))
       timeout = sum(timeouts)
     else:
       test_name = instrumentation_test_instance.GetTestName(test)
       test_display_name = self._GetUniqueTestName(test)
       if test['is_junit4']:
         target = '%s/%s' % (
             self._test_instance.test_package,
-            self._test_instance.test_runner_junit4)
+            self._test_instance.junit4_runner_class)
       else:
         target = '%s/%s' % (
-            self._test_instance.test_package, self._test_instance.test_runner)
+            self._test_instance.test_package,
+            self._test_instance.junit3_runner_class)
       extras['class'] = test_name
       if 'flags' in test and test['flags']:
         flags_to_add.extend(test['flags'])
       timeout = self._GetTimeoutFromAnnotations(
         test['annotations'], test_display_name)
 
       test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
           test['annotations'])
       if test_timeout_scale and test_timeout_scale != 1:
         valgrind_tools.SetChromeTimeoutScale(
@@ skipping 156 matching lines @@
                 time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
                 device.serial)
             tombstones_url = logdog_helper.text(
                 stream_name, '\n'.join(resolved_tombstones))
           result.SetLink('tombstones', tombstones_url)
 
     if self._env.concurrent_adb:
       post_test_step_thread_group.JoinAll()
     return results, None
 
+  def _GetTestsFromRunner(self):
+    test_apk_path = self._test_instance.test_apk.path
+    pickle_path = '%s-runner.pickle' % test_apk_path
+    try:
+      raw_tests = instrumentation_test_instance.GetTestsFromPickle(
+          pickle_path, test_apk_path)
+    except instrumentation_test_instance.TestListPickleException as e:
+      junit4_runner_class = self._test_instance.junit4_runner_class
+      test_package = self._test_instance.test_package
+      logging.info('Could not get tests from pickle: %s', e)
+      logging.info('Getting tests by having %s list them.',
+                   self._test_instance.junit4_runner_class)
+      def list_tests(dev):

[jbudorick, 2017/07/24 22:24:38]

nit: this is a lot to do in an exception; could you instead tweak it to be:

  try:
    return GetTestsFromPickle...
  except ...:
    logging.info('Could not get tests from pickle')

  logging.info('Getting tests by...')
  def list_tests(dev):
    ...

  ...
  return raw_tests

?

[the real yoland, 2017/07/24 23:25:42]

Done

+        with device_temp_file.DeviceTempFile(
+            dev.adb, suffix='.json',
+            dir=dev.GetExternalStoragePath()) as dev_test_list_json:
+          extras = {}
+          extras['log'] = 'true'
+          extras['package'] = '.'.join(
+              self._test_instance.test_package.split('.')[:2])
+          extras[_EXTRA_TEST_LIST] = dev_test_list_json.name
+          target = '%s/%s' % (test_package, junit4_runner_class)
+          output_string = ''.join(dev.StartInstrumentation(
+              target, extras=extras))
+          if output_string:
+            raise device_errors.CommandFailedError(
+                ('Test listing through %s failed on dev:\n%s. Are you using'

[jbudorick, 2017/07/24 22:24:38]

nit: for a multi-line string like this with multiple replacements, use .format
rather than % and consider building it prior to raising the exception

[the real yoland, 2017/07/24 23:25:42]

Done

+                 + ' the right JUnit runner (org.chromium.base.test.'
+                 + 'BaseChromiumAndroidJUnitRunner)?') % (
+                junit4_runner_class, output_string), dev.serial)
+          with tempfile_ext.NamedTemporaryDirectory() as host_dir:
+            host_file = os.path.join(host_dir, 'list_tests.json')
+            dev.PullFile(dev_test_list_json.name, host_file)
+            with open(host_file, 'r') as host_file:
+                return json.load(host_file)
+
+      raw_test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None)
+
+      # If all devices failed to list tests, raise an exception.
+      # Check that tl is not None and is not empty.
+      if all(not tl for tl in raw_test_lists):
+        raise device_errors.CommandFailedError(
+            'Failed to list tests on any device')
+
+      raw_tests = [tl for tl in raw_test_lists if tl][0]

[the real yoland, 2017/07/24 21:08:34]

Here I am simply getting the first viable raw test list, since it's hard to
compare and get union of multiple list of dictionaries, and they will all be the
same anyway.

+
+      instrumentation_test_instance.SaveTestsToPickle(
+          pickle_path, test_apk_path, raw_tests)
+    return raw_tests
+
   def _SaveScreenshot(self, device, screenshot_host_dir, screenshot_device_file,
                       test_name, results):
     if screenshot_host_dir:
       screenshot_host_file = os.path.join(
           screenshot_host_dir,
           '%s-%s.png' % (
               test_name,
               time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime())))
       if device.FileExists(screenshot_device_file.name):
         try:
@@ skipping 149 matching lines @@
     timeout *= cls._GetTimeoutScaleFromAnnotations(annotations)
 
     return timeout
 
 def _IsRenderTest(test):
   """Determines if a test or list of tests has a RenderTest amongst them."""
   if not isinstance(test, list):
     test = [test]
   return any([RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get(
               FEATURE_ANNOTATION, {}).get('value', ()) for t in test])