Find annotated tests by exposing API in instrumentation_test_instance

tools/android/find_disabled_tests.py

Index: tools/android/find_disabled_tests.py
diff --git a/tools/android/find_disabled_tests.py b/tools/android/find_disabled_tests.py
new file mode 100755
index 0000000000000000000000000000000000000000..b1f983da2350f2e5e81804a5e8f94ed35f9b4f0e
--- /dev/null
+++ b/tools/android/find_disabled_tests.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# Copyright (c) 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Finds all the tracked(disabled/flaky) tests from proguard dump"""
+
+import argparse
+import datetime
+import json
+import linecache
+import logging
+import os
+import pprint
+import re
+import sys
+import time
+
+_SRC_DIR = os.path.abspath(os.path.join(
+      os.path.dirname(__file__), '..', '..'))
+
+sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
+
+from pylib import constants
+from pylib.instrumentation import instrumentation_test_instance
+
+sys.path.append(os.path.join(_SRC_DIR, 'third_party', 'catapult', 'devil'))
+from devil.utils import cmd_helper
+
+_GIT_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+_EXPORT_TIME_FORMAT = '%Y%m%dT%H%M%S'
+_EXCLUDE_ANNOTATION_LIST = ["CommandLineFlags$Add"]

[perezju, 2016/04/25 12:45:08]

why excluded?

[Yoland Yan(Google), 2016/04/26 01:10:08]

Because I thought annotation also has string element. which can be confused as
bug id for a test
But not true, removed

+
+class _JSONKeyName(object):

[perezju, 2016/04/25 12:45:08]

I would still vote for removing this constant definitions and just use the
strings directly below.

[Yoland Yan(Google), 2016/04/26 01:10:08]

Got it

+  TEST_MASTER_KEY = 'tests'
+  REPORT_MASTER_KEY = 'metadata'
+  REVISION_KEY = 'revision'
+  COMMIT_POS_KEY = 'commit_pos'
+  ANNOTATIONS_KEY = 'annotations'
+  TEST_NAME_KEY = 'test_name'
+  TEST_APK_KEY = 'test_apk_name'
+  CRBUG_KEY = 'bug_id'
+  CLASS_PATH_KEY = 'class_path'
+  CLASS_NAME_KEY = 'class_name'
+  TRACKED_TEST_COUNT_KEY = 'tracked_test_count'
+  DISABLED_TEST_COUNT_KEY = 'disabled_test_count'
+  FLAKY_TEST_COUNT_KEY = 'flaky_test_count'
+  TOTAL_TEST_COUNT_KEY = 'total_test_count'
+  UTC_BUILDTIME_KEY = 'build_time'
+  UTC_REVISIONTIME_KEY = 'revision_time'
+  PLATFORM_KEY = 'platform'
+  PLATFORM_VALUE = 'android'
+
+
+def _GetBugId(message):
+  """Validate bug message format and get bug id"""
+  result = re.search(r'crbug(?:.com)?/(\d+)', message)
+  if result:
+    return int(result.group(1))
+  else:
+    return None
+
+def _GetAnnotations(test_annotations, annotations_dict):

[perezju, 2016/04/25 12:45:08]

don't pass annotations_dict; looks like it's always empty. Instead return the
pair: bug_id, annotations_dict

[Yoland Yan(Google), 2016/04/26 01:10:08]

Done.

+  """Store annotations in the existing anntation_dict and return bug id"""
+  bug_id = None
+  for annotation, content in test_annotations.iteritems():
+    if annotation in _EXCLUDE_ANNOTATION_LIST:
+      continue
+    if content is not None and content.get('message') is not None:
+        bug_id = _GetBugId(content.get('message'))
+    annotations_dict.update({annotation: content})
+  return bug_id
+
+
+def _GetTests(test_apks):
+  """All the all the tests"""
+  result = []
+  total_test_count = 0
+  for test_apk in test_apks:
+    logging.info('Current test apk: %s', test_apk)
+    test_jar = os.path.join(
+        constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
+        '%s.jar' % test_apk)
+    all_test = instrumentation_test_instance.GetAllTests(test_jar=test_jar)

[perezju, 2016/04/25 12:45:08]

nit: all_tests

[Yoland Yan(Google), 2016/04/26 01:10:08]

Done.

+    for test_class in all_test:
+      class_path = test_class['class']
+      class_name = test_class['class'].split('.')[-1]
+
+      class_annotation = {}
+      bug_id = _GetAnnotations(test_class['annotations'], class_annotation)
+      for test_method in test_class['methods']:
+        total_test_count += 1
+        # getting annotation of each test case
+        test_annotations = {}
+        bug_id = _GetAnnotations(test_method['annotations'], test_annotations)
+        test_annotations.update(class_annotation)
+        # getting test method name of each test
+        test_name = test_method['method']
+        test_dict = {
+            _JSONKeyName.CRBUG_KEY: bug_id,
+            _JSONKeyName.ANNOTATIONS_KEY: test_annotations,
+            _JSONKeyName.TEST_NAME_KEY: test_name,
+            _JSONKeyName.TEST_APK_KEY: test_apk,
+            _JSONKeyName.CLASS_NAME_KEY: class_name,
+            _JSONKeyName.CLASS_PATH_KEY: class_path
+        }
+        result.append(test_dict)
+  logging.info('Total count of tests in all test apks: %d', total_test_count)
+  return result, total_test_count
+
+
+def _GetReportMeta(utc_buildtime_string, total_test_count):
+  """Returns a dictionary of the report's metadata"""
+  revision = cmd_helper.GetCmdOutput(['git', 'rev-parse', 'HEAD']).strip()
+  raw_string = cmd_helper.GetCmdOutput(
+      ['git', 'log', '--pretty=format:%aI', '--max-count=1', 'HEAD'])
+  time_string_search = re.search(r'\d+-\d+-\d+T\d+:\d+:\d+', raw_string)

[perezju, 2016/04/25 12:45:08]

define a constant for this regex

[Yoland Yan(Google), 2016/04/26 01:10:08]

Done.

+  if time_string_search is None:
+    raise Exception('Time format incorrect')
+
+  raw_string = cmd_helper.GetCmdOutput(
+      ['git', 'log', '--pretty=format:%b', '--max-count=1', 'HEAD'])
+  commit_pos_search = re.search(r'Cr-Commit-Position: (.*)', raw_string)

[perezju, 2016/04/25 12:45:08]

ditto

[Yoland Yan(Google), 2016/04/26 01:10:08]

Done.

+  if commit_pos_search is None:
+    raise Exception('Cr commit position is not found, potentially running with '
+                    'uncommited HEAD')
+  commit_pos = commit_pos_search.group(1)
+
+  revision_time = time.strptime(time_string_search.group(0), _GIT_TIME_FORMAT)
+  utc_revision_time = datetime.datetime.utcfromtimestamp(
+      time.mktime(revision_time))
+  utc_revision_time = utc_revision_time.strftime(_EXPORT_TIME_FORMAT)
+  logging.info(
+      'revision is %s, revision time is %s', revision, utc_revision_time)
+
+  record_data = {
+      _JSONKeyName.REVISION_KEY: revision,
+      _JSONKeyName.COMMIT_POS_KEY: commit_pos,
+      _JSONKeyName.UTC_BUILDTIME_KEY: utc_buildtime_string,
+      _JSONKeyName.UTC_REVISIONTIME_KEY: utc_revision_time,
+      _JSONKeyName.PLATFORM_KEY: _JSONKeyName.PLATFORM_VALUE,
+      _JSONKeyName.TOTAL_TEST_COUNT_KEY: total_test_count}
+  return record_data

[perezju, 2016/04/25 12:45:07]

nit: just

  return { ... }

[Yoland Yan(Google), 2016/04/26 01:10:08]

Done.

+
+
+def _GetReport(test_apks, buildtime_string):
+  """Generate the dictionary of report data
+
+  Args:
+    test_apks: a list of apks for search for tests
+    buildtime_string: the time when the script is run at
+                      format: '%Y%m%dT%H%M%S'
+  """
+
+  test_data, total_test_count = _GetTests(test_apks)
+  report_meta = _GetReportMeta(buildtime_string, total_test_count)
+  report_data = {
+      _JSONKeyName.REPORT_MASTER_KEY: report_meta,
+      _JSONKeyName.TEST_MASTER_KEY: test_data}
+  return report_data
+
+
+def main():
+  default_build_type = os.environ.get('BUILDTYPE', 'Debug')
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-t', '--test-apks', nargs='+', dest='test_apks',
+                      help='List all test apks file name that the script uses '
+                           'to fetch tracked tests from')
+  parser.add_argument(
+      '--debug', action='store_const', const='Debug', dest='build_type',
+      default=default_build_type,
+      help=('If set, run test suites under out/Debug. '
+            'Default is env var BUILDTYPE or Debug.'))
+  parser.add_argument(
+      '--release', action='store_const', const='Release', dest='build_type',
+      help=('If set, run test suites under out/Release. '
+            'Default is env var BUILDTYPE or Debug.'))
+  parser.add_argument('-o', '--output-path',
+                      help='JSON file output to be uploaded on to gcs')
+  parser.add_argument('-v', '--verbose', action='store_true', default=False,
+                      help='DEBUG verbosity')
+
+  arguments = parser.parse_args(sys.argv[1:])
+  logging.basicConfig(
+      level=logging.DEBUG if arguments.verbose else logging.WARNING)
+  constants.SetBuildType(arguments.build_type)
+  logging.info('Using jar from build type: %s', arguments.build_type)
+
+  buildtime = datetime.datetime.utcnow()
+  buildtime_string = buildtime.strftime(_EXPORT_TIME_FORMAT)
+  logging.info('Build time is %s', buildtime_string)
+  report_data = _GetReport(arguments.test_apks, buildtime_string)
+
+  if arguments.output_path is None:
+    output_path = constants.GetOutDirectory()
+  else:
+    output_path = arguments.output_path
+  json_output_path = os.path.join(output_path,
+                                  '%s-android-chrome.json' % buildtime_string)
+  with open(json_output_path, 'w') as f:
+    json.dump(report_data, f, indent=2, sort_keys=True, separators=(',',':'))

[perezju, 2016/04/25 12:45:08]

':' -> ': ' (missing a space)

[Yoland Yan(Google), 2016/04/26 01:10:08]

woah, done!

+    logging.info('Saved json output file to %s', json_output_path)
+
+if __name__ == '__main__':
+  main()