List Java Instru Test Information From JUnit Runner

build/android/pylib/local/device/local_device_instrumentation_test_run.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import contextlib
+import json
 import logging
 import os
 import posixpath
 import re
 import sys
 import tempfile
 import time
 
 from devil.android import crash_handler
 from devil.android import device_errors
@@ skipping 41 matching lines @@
 
 LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v', 'DEBUG:I',
                   'StrictMode:D', '%s:I' % _TAG]
 
 EXTRA_SCREENSHOT_FILE = (
     'org.chromium.base.test.ScreenshotOnFailureStatement.ScreenshotFile')
 
 EXTRA_UI_CAPTURE_DIR = (
     'org.chromium.base.test.util.Screenshooter.ScreenshotDir')
 
+_EXTRA_TEST_LIST = (
+    'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList')
+
 UI_CAPTURE_DIRS = ['chromium_tests_root', 'UiCapture']
 
 FEATURE_ANNOTATION = 'Feature'
 RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest'
 
 # This needs to be kept in sync with formatting in |RenderUtils.imageName|
 RE_RENDER_IMAGE_NAME = re.compile(
       r'(?P<test_class>\w+)\.'
       r'(?P<description>\w+)\.'
       r'(?P<device_model>\w+)\.'
       r'(?P<orientation>port|land)\.png')
 
 @contextlib.contextmanager
 def _LogTestEndpoints(device, test_name):
   device.RunShellCommand(
       ['log', '-p', 'i', '-t', _TAG, 'START %s' % test_name],
       check_return=True)
   try:
     yield
   finally:
     device.RunShellCommand(
         ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
         check_return=True)
 
-# TODO(jbudorick): Make this private once the instrumentation test_runner is
-# deprecated.
+# TODO(jbudorick): Make this private once the instrumentation test_runner
+# is deprecated.
 def DidPackageCrashOnDevice(package_name, device):
   # Dismiss any error dialogs. Limit the number in case we have an error
   # loop or we are failing to dismiss.
   try:
     for _ in xrange(10):
       package = device.DismissCrashDialogIfNeeded()
       if not package:
         return False
       # Assume test package convention of ".test" suffix
       if package in package_name:
@@ skipping 196 matching lines @@
     if not str(device) in self._flag_changers:
       self._flag_changers[str(device)] = flag_changer.FlagChanger(
         device, self._test_instance.package_info.cmdline_file)
 
   #override
   def _CreateShards(self, tests):
     return tests
 
   #override
   def _GetTests(self):
+    if self._test_instance.junit4_runner_class:
+      self._SaveTestsFromRunner()

[jbudorick, 2017/07/21 19:35:08]

This is more complicated than it needs to be. Rather than saving the tests into
the instance and then getting them right back on the next line, just get them
here. Don't worry about saving them up into the instance in between.

[the real yoland, 2017/07/21 22:24:09]

Done

     tests = self._test_instance.GetTests()
     tests = self._ApplyExternalSharding(
         tests, self._test_instance.external_shard_index,
         self._test_instance.total_external_shards)
     return tests
 
   #override
   def _GetUniqueTestName(self, test):
     return instrumentation_test_instance.GetUniqueTestName(test)
 
@@ skipping 45 matching lines @@
       extras.update(
           self._test_instance.GetDriverEnvironmentVars(
               test_list=test_names))
       timeout = sum(timeouts)
     else:
       test_name = instrumentation_test_instance.GetTestName(test)
       test_display_name = self._GetUniqueTestName(test)
       if test['is_junit4']:
         target = '%s/%s' % (
             self._test_instance.test_package,
-            self._test_instance.test_runner_junit4)
+            self._test_instance.junit4_runner_class)
       else:
         target = '%s/%s' % (
-            self._test_instance.test_package, self._test_instance.test_runner)
+            self._test_instance.test_package,
+            self._test_instance.junit3_runner_class)
       extras['class'] = test_name
       if 'flags' in test and test['flags']:
         flags_to_add.extend(test['flags'])
       timeout = self._GetTimeoutFromAnnotations(
         test['annotations'], test_display_name)
 
       test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
           test['annotations'])
       if test_timeout_scale and test_timeout_scale != 1:
         valgrind_tools.SetChromeTimeoutScale(
@@ skipping 156 matching lines @@
                 time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
                 device.serial)
             tombstones_url = logdog_helper.text(
                 stream_name, '\n'.join(resolved_tombstones))
           result.SetLink('tombstones', tombstones_url)
 
     if self._env.concurrent_adb:
       post_test_step_thread_group.JoinAll()
     return results, None
 
+  def _SaveTestsFromRunner(self):

[jbudorick, 2017/07/21 19:35:08]

This should be _GetTestsFromRunner

[the real yoland, 2017/07/21 22:24:09]

Done

+    try:
+      raw_tests = self._test_instance.GetAllTestsFromRunnerPickle()
+      self._test_instance.SetTestsFromRunner(raw_tests)
+    except instrumentation_test_instance.TestListPickleException as e:
+      junit4_runner_class = self._test_instance.junit4_runner_class
+      test_package = self._test_instance.test_package
+      logging.info('Could not get tests from pickle: %s', e)
+      logging.info('Getting tests by having %s list them.',
+                   self._test_instance.junit4_runner_class)
+      device = self._env.devices[0]
+      with device_temp_file.DeviceTempFile(
+          device.adb, suffix='.json',
+          dir=device.GetExternalStoragePath()) as device_test_list_json:
+        extras = {}
+        extras[_EXTRA_TEST_LIST] = device_test_list_json.name
+        extras['log'] = 'true'
+        extras['package'] = '.'.join(
+            self._test_instance.test_package.split('.')[:2])
+        target = '%s/%s' % (test_package, junit4_runner_class)
+        output_string = ''.join(device.StartInstrumentation(
+            target, extras=extras))
+        if output_string:

[jbudorick, 2017/07/21 19:35:08]

What could this output that would indicate failure in a way that wouldn't be
caught by our normal error detection mechanisms?

[the real yoland, 2017/07/21 22:24:09]

I had incidents where it printed out the tests because I didn't set up JUnit
runner right.

+          raise Exception('Test listing through %s failed on device:\n%s' % (

[jbudorick, 2017/07/21 19:35:08]

I don't think this should be a raw Exception. Maybe a CommandFailedError, or a
subclass thereof?

[the real yoland, 2017/07/21 22:24:09]

Done

+              junit4_runner_class, output_string))
+        with tempfile_ext.NamedTemporaryDirectory() as host_dir:
+          host_file = os.path.join(host_dir, 'list_tests.json')
+          device.PullFile(device_test_list_json.name, host_file)
+          with open(host_file, 'r') as host_file:
+            json_string = host_file.read()
+            raw_tests = json.loads(json_string)
+            self._test_instance.SaveTestsToPickle(raw_tests)

[jbudorick, 2017/07/21 19:35:08]

nit: do this outside of the temporary context managers.

[the real yoland, 2017/07/21 22:24:09]

Done

+            self._test_instance.SetTestsFromRunner(raw_tests)
+
   def _SaveScreenshot(self, device, screenshot_host_dir, screenshot_device_file,
                       test_name, results):
     if screenshot_host_dir:
       screenshot_host_file = os.path.join(
           screenshot_host_dir,
           '%s-%s.png' % (
               test_name,
               time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime())))
       if device.FileExists(screenshot_device_file.name):
         try:
@@ skipping 149 matching lines @@
     timeout *= cls._GetTimeoutScaleFromAnnotations(annotations)
 
     return timeout
 
 def _IsRenderTest(test):
   """Determines if a test or list of tests has a RenderTest amongst them."""
   if not isinstance(test, list):
     test = [test]
   return any([RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get(
               FEATURE_ANNOTATION, {}).get('value', ()) for t in test])