[Android] Buildbot changes for EMMA code coverage

build/android/buildbot/bb_device_steps.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import glob
 import multiprocessing
 import os
 import shutil
 import sys
+import time
 
 import bb_utils
 import bb_annotations
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import provision_devices
 from pylib import android_commands
 from pylib import constants
 from pylib.gtest import gtest_config
 
 sys.path.append(os.path.join(
     constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner'))
 import errors
 
 
 CHROME_SRC = constants.DIR_SOURCE_ROOT
 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat')
+REVISION_FILENAME = 'FULL_BUILD_REVISION'
 
 # Describes an instrumation test suite:
 #   test: Name of test we're running.
 #   apk: apk to be installed.
 #   apk_package: package for the apk to be installed.
 #   test_apk: apk to run tests on.
 #   test_data: data folder in format destination:source.
 #   host_driven_root: The host-driven test root directory.
 #   annotation: Annotation of the tests to include.
 #   exclude_annotation: The annotation of the tests to exclude.
 I_TEST = collections.namedtuple('InstrumentationTest', [
-    'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root',
-    'annotation', 'exclude_annotation', 'extra_flags'])
+    'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'coverage',
+    'host_driven_root', 'annotation', 'exclude_annotation', 'extra_flags'])
 
-def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None,
-      annotation=None, exclude_annotation=None, extra_flags=None):
-  return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root,
-                annotation, exclude_annotation, extra_flags)
+def I(name, apk, apk_package, test_apk, test_data, coverage=True,
+      host_driven_root=None, annotation=None, exclude_annotation=None,
+      extra_flags=None):
+  return I_TEST(name, apk, apk_package, test_apk, test_data, coverage,
+                host_driven_root, annotation, exclude_annotation, extra_flags)
 
 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
     I('ContentShell',
       'ContentShell.apk',
       'org.chromium.content_shell_apk',
       'ContentShellTest',
       'content:content/test/data/android/device_files'),
     I('ChromiumTestShell',
       'ChromiumTestShell.apk',
       'org.chromium.chrome.testshell',
@@ skipping 101 matching lines @@
   InstallApk(options, test)
   args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
           '--verbose']
   if options.target == 'Release':
     args.append('--release')
   if options.asan:
     args.append('--tool=asan')
   if options.flakiness_server:
     args.append('--flakiness-dashboard-server=%s' %
                 options.flakiness_server)
+  if options.coverage_bucket and test.coverage:
+    args.append('--coverage-dir=%s' % options.coverage_dir)
   if test.host_driven_root:
     args.append('--host-driven-root=%s' % test.host_driven_root)
   if test.annotation:
     args.extend(['-A', test.annotation])
   if test.exclude_annotation:
     args.extend(['-E', test.exclude_annotation])
   if test.extra_flags:
     args.extend(test.extra_flags)
   if python_only:
     args.append('-p')
@@ skipping 85 matching lines @@
     ('provision_devices', ProvisionDevices),
     ('device_status_check', DeviceStatusCheck)
   ]
 
 
 def RunUnitTests(options):
   RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)
 
 
 def RunInstrumentationTests(options):
+  if options.coverage_bucket:
+    shutil.rmtree(options.coverage_dir, ignore_errors=True)

[Isaac (away), 2013/08/20 23:28:10]

not needed for triggered bots.  Also, I'd prefer we avoid silent shell
manipulations.

[gkanwar1, 2013/08/20 23:40:29]

Good to know, removed. Not sure what you mean by silent shell manipulations?

[Isaac (away), 2013/08/22 18:10:37]

I mean, if you're going to delete a folder, print something like
"Removing <>".  Actually it might be better to use RunCmd() because it will both
print the command and skip running it during testing.

[gkanwar1, 2013/08/22 18:12:11]

Ah, got it. Good to know for the future. For this CL, this is just entirely
removed.

+    os.mkdir(options.coverage_dir)
+
   for test in INSTRUMENTATION_TESTS.itervalues():
     RunInstrumentationSuite(options, test)
 
+  if options.coverage_bucket:

[Isaac (away), 2013/08/20 23:28:10]

Does this need to be called from RunInstrumentationTests?

[gkanwar1, 2013/08/20 23:40:29]

Nope. It was there because it was acting sort of as a 'cleanup' for
instrumentation tests, but now that it's broken into its own step, it probably
would best be placed in MainTestWrapper. Moved.

+    GenerateJavaCoverageReport(options)
+
 
 def RunWebkitTests(options):
   RunTestSuites(options, ['webkit_unit_tests'])
   RunWebkitLint(options.target)
 
 
 def RunWebRTCTests(options):
   RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES)
 
 
 def GetTestStepCmds():
   return [
       ('chromedriver', RunChromeDriverTests),
       ('unit', RunUnitTests),
       ('ui', RunInstrumentationTests),
       ('webkit', RunWebkitTests),
       ('webkit_layout', RunWebkitLayoutTests),
       ('webrtc', RunWebRTCTests),
   ]
 
 
+def UploadCoverageData(options, path, coverage_type):
+  revision_file = os.path.join(CHROME_SRC, 'out', options.target,
+                               REVISION_FILENAME)

[Isaac (away), 2013/08/20 23:28:10]

Would be better to use revision from build_properties.  

options.build_properties.get('got_revision', 'testing')

[gkanwar1, 2013/08/20 23:40:29]

Ah, didn't realize that was in build_properties. Done.

+  # If the revision file doesn't exist, we're just testing the bot
+  if os.path.exists(revision_file):
+    with open(revision_file) as f:
+      revision = f.read()
+  else:
+    revision = 'testing'
+  bot_id = options.build_properties.get('buildername', 'unknown')
+
+  timehash = hash(time.time())
+
+  boto_config = os.path.join(bb_utils.BB_BUILD_DIR, 'site_config', '.boto')
+  os.environ['AWS_CREDENTIAL_FILE'] = boto_config

[Isaac (away), 2013/08/20 23:28:10]

These are set automatically on bots; this isn't needed.  But also you cannot
modify os.environ here.

[gkanwar1, 2013/08/20 23:40:29]

Good to know, removed.

+  os.environ['BOTO_CONFIG'] = boto_config
+  RunCmd([os.path.join(bb_utils.BB_BUILD_DIR, 'third_party',

[Isaac (away), 2013/08/20 23:28:10]

let's make the gsutil path a constant in bb_utils.

[gkanwar1, 2013/08/20 23:40:30]

Done.

+                       'gsutil', 'gsutil'), 'cp', '-R',
+          path, 'gs://%s/%s/%s/%s/%s' %
+          (options.coverage_bucket, coverage_type,
+           bot_id, revision, timehash)])
+  bb_annotations.PrintLink(
+      'Coverage report',
+      'https://storage.googleapis.com/%s/%s/%s/%s/%s/index.html'
+      % (options.coverage_bucket, coverage_type, bot_id, revision, timehash))
+
+
+def GenerateJavaCoverageReport(options):
+  bb_annotations.PrintNamedStep('java_coverage_report')
+
+  coverage_html = os.path.join(options.coverage_dir, 'coverage_html')
+  RunCmd(['build/android/generate_emma_html.py',
+          '--coverage-dir', options.coverage_dir,
+          '--metadata-dir', os.path.join(CHROME_SRC, 'out', options.target),
+          '--output', os.path.join(coverage_html, 'coverage.html')])
+  UploadCoverageData(options, coverage_html, 'java')
+
+
 def LogcatDump(options):
   # Print logcat, kill logcat monitor
   bb_annotations.PrintNamedStep('logcat_dump')
   logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log')
   with open(logcat_file, 'w') as f:
     RunCmd([
         os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'),
         LOGCAT_DIR], stdout=f)
   RunCmd(['cat', logcat_file])
 
@@ skipping 40 matching lines @@
                     help='Run experiemental tests')
   parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
                     action='append',
                     help=('Run a test suite. Test suites: "%s"' %
                           '", "'.join(VALID_TESTS)))
   parser.add_option('--asan', action='store_true', help='Run tests with asan.')
   parser.add_option('--install', metavar='<apk name>',
                     help='Install an apk by name')
   parser.add_option('--reboot', action='store_true',
                     help='Reboot devices before running tests')
+  parser.add_option('--coverage-bucket',
+                    help=('Bucket name to store coverage results. Coverage is '
+                          'only run if this is set.'))
   parser.add_option(
       '--flakiness-server',
       help='The flakiness dashboard server to which the results should be '
            'uploaded.')
   parser.add_option(
       '--auto-reconnect', action='store_true',
       help='Push script to device which restarts adbd on disconnections.')
   parser.add_option(
       '--logcat-dump-output',
       help='The logcat dump output will be "tee"-ed into this file')
 
   return parser
 
 
 def main(argv):
   parser = GetDeviceStepsOptParser()
   options, args = parser.parse_args(argv[1:])
 
   if args:
     return sys.exit('Unused args %s' % args)
 
   unknown_tests = set(options.test_filter) - VALID_TESTS
   if unknown_tests:
     return sys.exit('Unknown tests %s' % list(unknown_tests))
 
   setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
+  if options.coverage_bucket:
+    setattr(options, 'coverage_dir',
+            os.path.join(CHROME_SRC, 'out', options.target, 'coverage'))
 
   MainTestWrapper(options)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))