Find annotated tests by exposing API in instrumentation_test_instance

tools/android/find_disabled_tests.py

+#!/usr/bin/env python

[mikecase (-- gone --), 2016/06/10 20:59:03]

as mentioned, rename script to something more general since it isnt specific to
disabled tests.

[the real yoland, 2016/06/10 22:20:39]

Done

+# Copyright (c) 2016 The Chromium Authors. All rights reserved.

[mikecase (-- gone --), 2016/06/10 20:59:02]

No (c) in the Copyright notice.

[the real yoland, 2016/06/10 22:20:40]

Done

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Finds all the tracked(disabled/flaky) tests from proguard dump"""

[mikecase (-- gone --), 2016/06/10 20:59:03]

offline you said this was for all annotations. Still says disabled/flaky here.

[the real yoland, 2016/06/10 22:20:40]

Done

+
+import argparse
+import datetime
+import json
+import linecache
+import logging
+import os
+import pprint
+import re
+import sys
+import time
+
+_SRC_DIR = os.path.abspath(os.path.join(
+      os.path.dirname(__file__), '..', '..'))
+
+sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
+from pylib import constants
+from pylib.instrumentation import instrumentation_test_instance
+
+sys.path.append(os.path.join(_SRC_DIR, 'third_party', 'catapult', 'devil'))
+from devil.utils import cmd_helper
+
+_DEFAULT_APK_OUTPUT_DIR = os.path.join('out', 'Debug')

[mikecase (-- gone --), 2016/06/10 20:59:03]

Use pylib/constants like GetBuildDirectory or something instead of this.

+
+_GIT_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+_EXPORT_TIME_FORMAT = '%Y%m%dT%H%M%S'
+
+_GIT_LOG_TIME_PATTERN = re.compile(r'\d+')
+_GIT_LOG_MESSAGE_PATTERN = r'Cr-Commit-Position: refs/heads/master@{#(\d+)}'
+
+_CRBUG_ID_PATTERN = re.compile(r'crbug(?:.com)?/(\d+)')
+
+_TEST_MASTER_KEY = 'tests'
+_REPORT_MASTER_KEY = 'metadata'
+_REVISION_KEY = 'revision'
+_COMMIT_POS_KEY = 'commit_pos'
+_ANNOTATIONS_KEY = 'annotations'
+_TEST_NAME_KEY = 'test_name'
+_TEST_APK_KEY = 'test_apk_name'
+_CRBUG_KEY = 'bug_id'
+_CLASS_PATH_KEY = 'class_path'
+_CLASS_NAME_KEY = 'class_name'
+_TOTAL_TEST_COUNT_KEY = 'total_test_count'
+_UTC_SCRIPT_RUNTIME= 'script_runtime'
+_UTC_REVISIONTIME_KEY = 'revision_time'
+_PLATFORM_KEY = 'platform'
+_PLATFORM_VALUE = 'android'
+
+
+def _GetBugId(message):
+  """Validate bug message format and get bug id"""
+  result = re.search(_CRBUG_ID_PATTERN, message)
+  if result:
+    return int(result.group(1))
+  else:
+    return None
+
+
+def _GetAnnotations(test_annotations):
+  """Store annotations in the existing anntation_dict and return bug id"""
+  bug_id = None
+  annotations_dict = {}
+  for annotation, content in test_annotations.iteritems():
+    if content is not None and content.get('message') is not None:
+        bug_id = _GetBugId(content.get('message'))

[mikecase (-- gone --), 2016/06/10 20:59:02]

Ive seen some tests with multiple bugs attached, like
@FlakyTest(message="crbug/1,crbug/2") I think. Will this work for that?

[the real yoland, 2016/06/10 22:20:40]

That would, except I will do some parsing on the server side when importing or
when displaying the information

+    annotations_dict.update({annotation: content})
+  return bug_id, annotations_dict
+
+
+def _GetTests(test_apks, apk_output_dir):
+  """Return a list of all annotated tests and total test count"""
+  result = []
+  total_test_count = 0
+  for test_apk in test_apks:
+    logging.info('Current test apk: %s', test_apk)
+    test_jar = os.path.join(
+        apk_output_dir, constants.SDK_BUILD_TEST_JAVALIB_DIR,
+        '%s.jar' % test_apk)
+    all_tests = instrumentation_test_instance.GetAllTests(test_jar=test_jar)
+    for test_class in all_tests:
+      class_path = test_class['class']
+      class_name = test_class['class'].split('.')[-1]
+
+      class_bug_id, class_annotation = _GetAnnotations(
+          test_class['annotations'])
+      for test_method in test_class['methods']:
+        total_test_count += 1
+        # getting annotation of each test case
+        test_bug_id, test_annotations = _GetAnnotations(
+            test_method['annotations'])
+        test_bug_id = test_bug_id if test_bug_id else class_bug_id
+        test_annotations.update(class_annotation)
+        # getting test method name of each test
+        test_name = test_method['method']
+        test_dict = {
+            _CRBUG_KEY: test_bug_id,
+            _ANNOTATIONS_KEY: test_annotations,
+            _TEST_NAME_KEY: test_name,
+            _TEST_APK_KEY: test_apk,
+            _CLASS_NAME_KEY: class_name,
+            _CLASS_PATH_KEY: class_path
+        }
+        result.append(test_dict)
+  logging.info('Total count of tests in all test apks: %d', total_test_count)
+  return result, total_test_count
+
+
+def _GetReportMeta(utc_script_runtime_string, total_test_count):
+  """Returns a dictionary of the report's metadata"""
+  revision = cmd_helper.GetCmdOutput(['git', 'rev-parse', 'HEAD']).strip()
+  raw_string = cmd_helper.GetCmdOutput(
+      ['git', 'log', '--pretty=format:%at', '--max-count=1', 'HEAD'])
+  time_string_search = re.search(_GIT_LOG_TIME_PATTERN, raw_string)
+  if time_string_search is None:
+    raise Exception('Timestamp format incorrect, expected all digits, got %s'
+                    % raw_string)
+
+  raw_string = cmd_helper.GetCmdOutput(

[mikecase (-- gone --), 2016/06/10 20:59:02]

Should probably explicitly set the cwd arg to somewhere inside Chromium when
running a git command

+      ['git', 'log', '--pretty=format:%b', '--max-count=1', 'HEAD'])
+  commit_pos_search = re.search(_GIT_LOG_MESSAGE_PATTERN, raw_string)
+  if commit_pos_search is None:
+    raise Exception('Cr-Commit-Position is not found, potentially running with '
+                    'uncommited HEAD')
+  commit_pos = int(commit_pos_search.group(1))
+
+  utc_revision_time = datetime.datetime.utcfromtimestamp(
+      int(time_string_search.group(0)))
+  utc_revision_time = utc_revision_time.strftime(_EXPORT_TIME_FORMAT)
+  logging.info(
+      'revision is %s, revision time is %s', revision, utc_revision_time)
+
+  return {
+      _REVISION_KEY: revision,
+      _COMMIT_POS_KEY: commit_pos,
+      _UTC_SCRIPT_RUNTIME: utc_script_runtime_string,
+      _UTC_REVISIONTIME_KEY: utc_revision_time,
+      _PLATFORM_KEY: _PLATFORM_VALUE,
+      _TOTAL_TEST_COUNT_KEY: total_test_count
+  }
+
+
+def _GetReport(test_apks, script_runtime_string, apk_output_dir):
+  """Generate the dictionary of report data
+
+  Args:
+    test_apks: a list of apks for search for tests
+    script_runtime_string: the time when the script is run at
+                           format: '%Y%m%dT%H%M%S'
+  """
+
+  test_data, total_test_count = _GetTests(test_apks, apk_output_dir)
+  report_meta = _GetReportMeta(script_runtime_string, total_test_count)
+  report_data = {
+      _REPORT_MASTER_KEY: report_meta,
+      _TEST_MASTER_KEY: test_data}
+  return report_data
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-t', '--test-apks', nargs='+', dest='test_apks',
+                      required=True,
+                      help='List all test apks file name that the script uses '
+                           'to fetch tracked tests from')
+  parser.add_argument('--json-output-dir', required=True,
+                      help='JSON file output directory')
+  parser.add_argument('--apk-output-dir', default=_DEFAULT_APK_OUTPUT_DIR,
+                      help='The output directory of test apks')
+  parser.add_argument('-v', '--verbose', action='store_true', default=False,
+                      help='INFO verbosity')
+
+  arguments = parser.parse_args(sys.argv[1:])
+  logging.basicConfig(
+      level=logging.INFO if arguments.verbose else logging.WARNING)
+
+  script_runtime = datetime.datetime.utcnow()
+  script_runtime_string = script_runtime.strftime(_EXPORT_TIME_FORMAT)
+  logging.info('Build time is %s', script_runtime_string)
+  apk_output_dir = os.path.abspath(os.path.join(
+    constants.DIR_SOURCE_ROOT, arguments.apk_output_dir))
+  report_data = _GetReport(
+      arguments.test_apks, script_runtime_string, apk_output_dir)
+
+  temp_output_dir = arguments.json_output_dir
+  json_output_path = os.path.join(
+      temp_output_dir, '%s-android-chrome.json' % script_runtime_string)
+  with open(json_output_path, 'w') as f:
+    json.dump(report_data, f, sort_keys=True, separators=(',',': '))
+    logging.info('Saved json output file to %s', json_output_path)
+
+if __name__ == '__main__':
+  main()