🎯 Fail if an instrumentation test is missing size annotation

build/android/pylib/instrumentation/instrumentation_test_instance.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import logging
 import os
 import pickle
 import re
@@ skipping 15 matching lines @@
 # Ref: http://developer.android.com/reference/android/app/Activity.html
 _ACTIVITY_RESULT_CANCELED = 0
 _ACTIVITY_RESULT_OK = -1
 
 _COMMAND_LINE_PARAMETER = 'cmdlinearg-parameter'
 _DEFAULT_ANNOTATIONS = [
     'Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
     'EnormousTest', 'IntegrationTest']
 _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [
     'DisabledTest', 'FlakyTest']
+_VALID_ANNOTATIONS = set(['Manual', 'PerfTest'] + _DEFAULT_ANNOTATIONS +
+                         _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS)
 _EXTRA_DRIVER_TEST_LIST = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
 _EXTRA_DRIVER_TEST_LIST_FILE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile')
 _EXTRA_DRIVER_TARGET_PACKAGE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage')
 _EXTRA_DRIVER_TARGET_CLASS = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass')
 _EXTRA_TIMEOUT_SCALE = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TimeoutScale')
 
 _PARAMETERIZED_TEST_ANNOTATION = 'ParameterizedTest'
 _PARAMETERIZED_TEST_SET_ANNOTATION = 'ParameterizedTest$Set'
 _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE)
 _PICKLE_FORMAT_VERSION = 10
 
 
+class MissingSizeAnnotationError(Exception):
+  def __init__(self, class_name):
+    super(MissingSizeAnnotationError, self).__init__(class_name +
+        ': Test method is missing required size annotation. Add one of: ' +
+        ', '.join('@' + a for a in _VALID_ANNOTATIONS))
+
+
 # TODO(jbudorick): Make these private class methods of
 # InstrumentationTestInstance once the instrumentation test_runner is
 # deprecated.
 def ParseAmInstrumentRawOutput(raw_output):
   """Parses the output of an |am instrument -r| call.
 
   Args:
     raw_output: the output of an |am instrument -r| call as a list of lines
   Returns:
     A 3-tuple containing:
@@ skipping 509 matching lines @@
     filtered_classes = []
     for c in tests:
       filtered_methods = []
       for m in c['methods']:
         # Gtest filtering
         if not gtest_filter(c, m):
           continue
 
         all_annotations = dict(c['annotations'])
         all_annotations.update(m['annotations'])
+
+        # Enforce that all tests declare their size.
+        if not any(a in _VALID_ANNOTATIONS for a in all_annotations):
+          raise MissingSizeAnnotationError('%s.%s' % (c['class'], m['method']))
+
         if (not annotation_filter(all_annotations)
             or not excluded_annotation_filter(all_annotations)):
           continue
 
         filtered_methods.append(m)
 
       if filtered_methods:
         filtered_class = dict(c)
         filtered_class['methods'] = filtered_methods
         filtered_classes.append(filtered_class)
@@ skipping 50 matching lines @@
   def GenerateTestResults(
       result_code, result_bundle, statuses, start_ms, duration_ms):
     return GenerateTestResults(result_code, result_bundle, statuses,
                                start_ms, duration_ms)
 
   #override
   def TearDown(self):
     if self._isolate_delegate:
       self._isolate_delegate.Clear()