🎯 Fail if an instrumentation test is missing size annotation

build/android/pylib/instrumentation/instrumentation_test_instance.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import logging
 import os
 import pickle
 import re
@@ skipping 15 matching lines @@
 # Ref: http://developer.android.com/reference/android/app/Activity.html
 _ACTIVITY_RESULT_CANCELED = 0
 _ACTIVITY_RESULT_OK = -1
 
 _COMMAND_LINE_PARAMETER = 'cmdlinearg-parameter'
 _DEFAULT_ANNOTATIONS = [
     'Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
     'EnormousTest', 'IntegrationTest']
 _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [
     'DisabledTest', 'FlakyTest']
+_VALID_ANNOTATIONS = set(['Manual'] + _DEFAULT_ANNOTATIONS +
+                         _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS)
 _EXTRA_DRIVER_TEST_LIST = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
 _EXTRA_DRIVER_TEST_LIST_FILE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile')
 _EXTRA_DRIVER_TARGET_PACKAGE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage')
 _EXTRA_DRIVER_TARGET_CLASS = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass')
 _EXTRA_TIMEOUT_SCALE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TimeoutScale')
 
 _PARAMETERIZED_TEST_ANNOTATION = 'ParameterizedTest'
 _PARAMETERIZED_TEST_SET_ANNOTATION = 'ParameterizedTest$Set'
 _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE)
 _PICKLE_FORMAT_VERSION = 10
 
 
+class MissingSizeAnnotationError(Exception):
+  def __init__(self, class_name):
+    super(MissingSizeAnnotationError, self).__init__(class_name +
+        ': Test method is missing required size annotation. Add one of: ' +
+        ', '.join('@' + a for a in _VALID_ANNOTATIONS))
+
+
 # TODO(jbudorick): Make these private class methods of
 # InstrumentationTestInstance once the instrumentation test_runner is
 # deprecated.
 def ParseAmInstrumentRawOutput(raw_output):
   """Parses the output of an |am instrument -r| call.
 
   Args:
     raw_output: the output of an |am instrument -r| call as a list of lines
   Returns:
     A 3-tuple containing:
@@ skipping 484 matching lines @@
 
     def gtest_filter(c, m):
       if not self._test_filter:
         return True
       # Allow fully-qualified name as well as an omitted package.
       names = ['%s.%s' % (c['class'], m['method']),
                '%s.%s' % (c['class'].split('.')[-1], m['method'])]
       return unittest_util.FilterTestNames(names, self._test_filter)
 
     def annotation_filter(all_annotations):
-      if not self._annotations:
+      if not self._annotations or not all_annotations:

[jbudorick, 2016/04/29 00:38:34]

Can `not all_annotations` ever be true here?

[agrieve, 2016/04/29 19:26:02]

Whoops, meant to revert that part! Done.

         return True
       return any_annotation_matches(self._annotations, all_annotations)
 
     def excluded_annotation_filter(all_annotations):
       if not self._excluded_annotations:
         return True
       return not any_annotation_matches(self._excluded_annotations,
                                         all_annotations)
 
     def any_annotation_matches(annotations, all_annotations):
       return any(
           ak in all_annotations and (av is None or av == all_annotations[ak])
           for ak, av in annotations.iteritems())
 
     filtered_classes = []
     for c in tests:
       filtered_methods = []
       for m in c['methods']:
         # Gtest filtering
         if not gtest_filter(c, m):
           continue
 
         all_annotations = dict(c['annotations'])
         all_annotations.update(m['annotations'])
+
+        # Enforce that all tests declare their size.
+        if not any(a in _VALID_ANNOTATIONS for a in all_annotations):
+          raise MissingSizeAnnotationError('%s.%s' % (c['class'], m['method']))
+
         if (not annotation_filter(all_annotations)
             or not excluded_annotation_filter(all_annotations)):
           continue
 
         filtered_methods.append(m)
 
       if filtered_methods:
         filtered_class = dict(c)
         filtered_class['methods'] = filtered_methods
         filtered_classes.append(filtered_class)
@@ skipping 50 matching lines @@
   def GenerateTestResults(
       result_code, result_bundle, statuses, start_ms, duration_ms):
     return GenerateTestResults(result_code, result_bundle, statuses,
                                start_ms, duration_ms)
 
   #override
   def TearDown(self):
     if self._isolate_delegate:
       self._isolate_delegate.Clear()