Find annotated tests by exposing API in instrumentation_test_instance

tools/android/find_disabled_tests.py

+#!/usr/bin/env python
+# Copyright (c) 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Finds all the disabled tests from proguard dump"""
+
+import argparse
+import datetime
+import json
+import linecache
+import logging
+import os
+import re
+import sys
+import time
+import pprint
+
+
+_SRC_DIR = os.path.abspath(os.path.join(
+      os.path.dirname(__file__), '..', '..'))
+
+sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
+
+from pylib import constants
+from pylib.instrumentation import instrumentation_test_instance
+
+sys.path.append(os.path.join(_SRC_DIR, 'third_party', 'catapult', 'devil'))
+from devil.utils import cmd_helper
+
+_DISABLED_TESTS_ANNOTATION_LIST = {'DisabledTest': None, 'FlakyTest': None}
+_GIT_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+_EXPORT_TIME_FORMAT = '%Y%m%dT%H%M%S'
+
+class JSONKeyName(object):
+  TEST_MASTER_KEY = 'tests'
+  RECORD_MASTER_KEY = 'records'
+  REVISION_KEY = 'revision'
+  STATUS_KEY = 'status'
+  TEST_NAME_KEY = 'name'
+  CRBUG_KEY = 'bug_id'
+  CLASS_PATH_KEY = 'class_path'
+  CLASS_NAME_KEY = 'class_name'
+  DF_TEST_AMOUNT_KEY = 'df_test_amount'
+  TOTAL_TEST_AMOUNT_KEY = 'total_test_amount'
+  UTC_BUILDTIME_KEY = 'utc_buildtime'
+  UTC_REVISIONTIME_KEY = 'utc_revisiontime'
+  PLATFORM_KEY = 'platform'
+  PLATFORM_VALUE = 'android'
+
+def _SerializeTests(arguments):
+  """Create instrumentation test instance and extract disabled tests info"""
+  total_test_amount = 0
+  disabled_test_list = []
+  for test_apk in arguments.test_apks:
+    arguments.test_apk = test_apk
+    test_jar = os.path.join(
+        constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
+        '%s.jar' % arguments.test_apk)
+    temp_disabled_test_list = instrumentation_test_instance.GetFilteredTests(
+        test_jar=test_jar,
+        test_filter=None,
+        annotations=arguments.annotations,
+        exclude_annotations=None
+        )
+    disabled_test_list.extend(temp_disabled_test_list)
+    total_test_amount += instrumentation_test_instance.TotalTestAmount(test_jar)
+  result = []
+  for test_class in disabled_test_list:
+    class_path = test_class['class']
+    class_name = test_class['class'].split('.')[-1]
+    for test_method in test_class['methods']:
+      # getting annotation of each test case
+      bug_id = None
+      status = []
+      test_annotations = test_method['annotations']
+      assert len(test_annotations) != 0
+      for annotation, content in test_annotations.iteritems():
+        if content is not None:
+          bug_id = content['message']
+        status.append(annotation)
+      # getting test method name of each test
+      test_name = test_method['method']
+      test_dict = {
+          JSONKeyName.CRBUG_KEY: bug_id,
+          JSONKeyName.STATUS_KEY: status,
+          JSONKeyName.TEST_NAME_KEY: test_name,
+          JSONKeyName.CLASS_NAME_KEY: class_name,
+          JSONKeyName.CLASS_PATH_KEY: class_path
+          }
+      result.append(test_dict)
+  return result, total_test_amount
+
+def _AddRecord(disabled_test_list, utc_buildtime_string, total_test_amount):
+  disabled_test_number = len(disabled_test_list)
+
+  revision = cmd_helper.GetCmdOutput(['git', 'rev-parse', 'HEAD']).strip()
+  raw_time_string = cmd_helper.GetCmdOutput(
+      ['git', 'log', '--pretty=format:%aI', '--max-count=1', 'HEAD'])
+  time_string_search = re.search(r'\d+-\d+-\d+T\d+:\d+:\d+', raw_time_string)
+  if time_string_search is None:
+    raise Exception('Time format incorrect')
+
+  revision_time = time.strptime(time_string_search.group(0), _GIT_TIME_FORMAT)
+  utc_revision_time = datetime.datetime.utcfromtimestamp(
+      time.mktime(revision_time))
+  utc_revision_time = utc_revision_time.strftime(_EXPORT_TIME_FORMAT)
+
+  record_data = [{
+      JSONKeyName.REVISION_KEY: revision,
+      JSONKeyName.UTC_BUILDTIME_KEY: utc_buildtime_string,
+      JSONKeyName.UTC_REVISIONTIME_KEY: utc_revision_time,
+      JSONKeyName.PLATFORM_KEY: JSONKeyName.PLATFORM_VALUE,
+      JSONKeyName.DF_TEST_AMOUNT_KEY: disabled_test_number,
+      JSONKeyName.TOTAL_TEST_AMOUNT_KEY: total_test_amount
+      }]
+  return record_data
+
+def main():
+  default_build_type = os.environ.get('BUILDTYPE', 'Debug')
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-t', '--test-apks', action='append', default=[],
+                      dest='test_apks', help='List all test apks file name ' +
+                           'that the script uses to fetch disabled tests from')
+  # TODO(yolandyan): if builder uses incremental script, jar path changes
+  parser.add_argument('--test_apk_incremental_install_script')
+  parser.add_argument(
+      '--debug', action='store_const', const='Debug', dest='build_type',
+      default=default_build_type,
+      help=('If set, run test suites under out/Debug. '
+            'Default is env var BUILDTYPE or Debug.'))
+  parser.add_argument('-o', '--output-path',
+                      help='JSON file output to be uploaded on to gcs')
+  parser.add_argument('-a', '--annotations', dest='annotations',
+                      action='append', default=['DisabledTest', 'FlakyTest'],
+                      help='Disabled tests annotations, seperated by ,' +
+                          'e.g. --annotation-str=DisabledTest,FlakyTest')
+
+  arguments = parser.parse_args(sys.argv[1:])
+  arguments.annotations = dict((x, None) for x in arguments.annotations)
+  constants.SetBuildType(arguments.build_type)
+
+  buildtime = datetime.datetime.utcnow()
+  buildtime_string = buildtime.strftime('%Y%m%dT%H%M%S')
+  test_data, total_test_amount = _SerializeTests(arguments)
+  record_data = _AddRecord(test_data, buildtime_string, total_test_amount)
+
+  if arguments.output_path is None:
+    output_path = constants.GetOutDirectory()
+  else:
+    output_path = arguments.output_path
+  json_output_path = os.path.join(output_path,
+                                  '%s-android-chrome.json' % buildtime_string)
+  export_data = {
+      JSONKeyName.RECORD_MASTER_KEY: record_data,
+      JSONKeyName.TEST_MASTER_KEY: test_data
+      }
+  with open(json_output_path, 'w') as f:
+    json.dump(export_data, f, indent=2, sort_keys=True)
+
+if __name__ == "__main__":
+  main()