WebRTC: Make WebRTC's Test more similar to Chromium's GTestTest

scripts/slave/recipe_modules/webrtc/api.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from recipe_engine.types import freeze
 from recipe_engine import recipe_api
 from . import builders
 from . import steps
 
 
 class WebRTCApi(recipe_api.RecipeApi):
   def __init__(self, **kwargs):
     super(WebRTCApi, self).__init__(**kwargs)
     self._env = {}
     self._isolated_targets = None
 
     # Keep track of working directory (which contains the checkout).
     # None means "default value".
     self._working_dir = None
 
   BUILDERS = builders.BUILDERS
   RECIPE_CONFIGS = builders.RECIPE_CONFIGS
 
   NORMAL_TESTS = freeze({
     'audio_decoder_unittests': {},
     'common_audio_unittests': {},
     'common_video_unittests': {},
     'modules_tests': {
-      'shards': 2,
+      'swarming_shards': 2,
     },
     'modules_unittests': {
-      'shards': 6,
+      'swarming_shards': 6,
     },
     'peerconnection_unittests': {
-      'shards': 4,
+      'swarming_shards': 4,
     },
     'rtc_media_unittests': {},
     'rtc_pc_unittests': {},
     'rtc_stats_unittests': {},
     'rtc_unittests': {
-      'shards': 6,
+      'swarming_shards': 6,
     },
     'system_wrappers_unittests': {},
     'test_support_unittests': {},
     'tools_unittests': {},
     'video_engine_tests': {
-      'shards': 4,
+      'swarming_shards': 4,
     },
     'voice_engine_unittests': {},
     'webrtc_nonparallel_tests': {
       'parallel': False,
     },
     'xmllite_xmpp_unittests': {},
   })
 
-  ANDROID_DEVICE_TESTS = freeze({
-    'audio_decoder_unittests': {},
-    'common_audio_unittests': {},
-    'common_video_unittests': {},
-    'modules_tests': {},
-    'modules_unittests': {},
-    'peerconnection_unittests': {},
-    'rtc_stats_unittests': {},
-    'rtc_unittests': {},
-    'system_wrappers_unittests': {},
-    'test_support_unittests': {},
-    'tools_unittests': {},
-    'video_engine_tests': {},
-    'voice_engine_unittests': {},
-    'webrtc_nonparallel_tests': {},
-  })
+  ANDROID_DEVICE_TESTS = (
+    'audio_decoder_unittests',
+    'common_audio_unittests',
+    'common_video_unittests',
+    'modules_tests',
+    'modules_unittests',
+    'peerconnection_unittests',
+    'rtc_stats_unittests',
+    'rtc_unittests',
+    'system_wrappers_unittests',
+    'test_support_unittests',
+    'tools_unittests',
+    'video_engine_tests',
+    'voice_engine_unittests',
+    'webrtc_nonparallel_tests',
+  )
 
-  ANDROID_INSTRUMENTATION_TESTS = freeze({
-    'AppRTCMobileTest': {},
-    'libjingle_peerconnection_android_unittest': {},
-  })
+  ANDROID_INSTRUMENTATION_TESTS = (
+    'AppRTCMobileTest',
+    'libjingle_peerconnection_android_unittest',
+  )
 
-  ANDROID_JUNIT_TESTS = freeze({
-    'android_junit_tests': {},
-  })
+  ANDROID_JUNIT_TESTS = (
+    'android_junit_tests',
+  )
 
   DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com'
 
   @property
   def should_build(self):
     return self.bot_type in ('builder', 'builder_tester')
 
   @property
   def should_test(self):
     return self.bot_type in ('tester', 'builder_tester')
@@ skipping 50 matching lines @@
       self.m.gclient.apply_config(c)
 
     if self.m.tryserver.is_tryserver:
       self.m.chromium.apply_config('trybot_flavor')
 
   def configure_swarming(self):
     self.c.use_isolate = self.bot_config.get('use_isolate')
     self.c.enable_swarming = self.bot_config.get('enable_swarming')
     if self.c.use_isolate:
       self.m.isolate.set_isolate_environment(self.m.chromium.c)
-      self._isolated_targets = []
       if self.c.TEST_SUITE == 'webrtc':
-        self._isolated_targets += self.NORMAL_TESTS.keys()
-      if self.c.TEST_SUITE == 'android_swarming':
-        self._isolated_targets += (self.ANDROID_JUNIT_TESTS.keys() +
-                                   self.ANDROID_DEVICE_TESTS.keys() +
-                                   self.ANDROID_INSTRUMENTATION_TESTS.keys())
+        self._isolated_targets = (self.NORMAL_TESTS.keys())
+      elif self.c.TEST_SUITE == 'android_swarming':
+        self._isolated_targets = (self.ANDROID_DEVICE_TESTS +
+                                  self.ANDROID_INSTRUMENTATION_TESTS)
       self._isolated_targets = sorted(self._isolated_targets)
       if not self._isolated_targets: # pragma: no cover
         raise self.m.step.StepFailure('Isolation and swarming are only '
                                       'supported for webrtc and '
                                       'android_swarming test suites.')
 
     self.c.enable_swarming = self.bot_config.get('enable_swarming')
     if self.c.enable_swarming:
       self.m.chromium_swarming.configure_swarming(
           'webrtc',
@@ skipping 60 matching lines @@
       with self.m.step.defer_results():
         if tests:
           run_android_device_steps = (not self.c.enable_swarming and
               self.m.chromium.c.TARGET_PLATFORM == 'android' and
               self.c.TEST_SUITE != 'android_swarming')
 
           if run_android_device_steps:
             self.m.chromium_android.common_tests_setup_steps()
 
           for test in tests:
+            test.pre_run(self, suffix='')

[kjellander_chromium, 2016/10/21 11:04:07]

https://cs.chromium.org/chromium/build/scripts/slave/recipe_modules/chromium_...
has a lot of try+except clauses around theirs. Do you know if we should have
that too if it's not a big deal. 
I guess as long as a failed test doesn't abort the build (i.e. other tests
continue to run and complete), then there shouldn't be a problem without them.

[ehmaldonado_chromium, 2016/10/21 13:12:49]

Good point.
This doesn't change the behavior we had before. When I start using Chromium's
classes, I'll make sure of it.

+
+          for test in tests:
             test.run(self, suffix='')
 
+          for test in tests:
+            test.post_run(self, suffix='')
+
           if run_android_device_steps:
             self.m.chromium_android.shutdown_device_monitor()
             self.m.chromium_android.logcat_dump(
                 gs_bucket=self.master_config.get('build_gs_bucket'))
             self.m.chromium_android.stack_tool_steps(force_latest_version=True)
             self.m.chromium_android.test_report()
 
       with self.m.step.defer_results():
         for test in tests:
           if test.enable_swarming:
@@ skipping 117 matching lines @@
   def clean_test_output(self):
     """Remove all test output in out/, since we have tests leaking files."""
     out_dir = self.m.path['checkout'].join('out')
     self.m.python('clean test output files',
                   script=self.resource('cleanup_files.py'),
                   args=[out_dir],
                   infra_step=True)
 
   def virtual_webcam_check(self):
     self.m.python('webcam_check', self.resource('ensure_webcam_is_running.py'))