Handle Telemetry's TimeoutException in retries of flaky tests.

content/test/gpu/gpu_tests/gpu_test_base.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 
 from telemetry import benchmark as benchmark_module
+from telemetry.core import exceptions
 from telemetry.page import page as page_module
 from telemetry.page import page_test
 from telemetry.page import shared_page_state as shared_page_state_module
 from telemetry.testing import fakes
 
 from gpu_tests import exception_formatter
 from gpu_tests import gpu_test_expectations
 
 class TestBase(benchmark_module.Benchmark):
   """Base classes for all GPU tests in this directory. Implements
@@ skipping 35 matching lines @@
   def ValidateAndMeasurePage(self, page, tab, result):
     pass
 
 def _CanRunOnBrowser(browser_info, page):
   expectations = page.GetExpectations()
   return expectations.GetExpectationForPage(
     browser_info.browser, page) != 'skip'
 
 def RunStoryWithRetries(cls, shared_page_state, results):
   page = shared_page_state.current_page
+
+  # Save this away for use in _DidRunStory, below.
+  shared_page_state._last_cached_page = page
+
   expectations = page.GetExpectations()
   expectation = 'pass'
   if expectations:
     expectation = expectations.GetExpectationForPage(
       shared_page_state.browser, page)
   if expectation == 'skip':
     raise Exception(
       'Skip expectations should have been handled in CanRunOnBrowser')
   try:
     super(cls, shared_page_state).RunStory(results)
@@ skipping 23 matching lines @@
         break
       except Exception:
         # Squelch any exceptions from any but the last retry.
         if ii == num_retries - 1:
           raise
   else:
     if expectation == 'fail':
       logging.warning(
           '%s was expected to fail, but passed.\n', page.display_name)
 
+def _DidRunStory(cls, shared_page_state, results,
+                 force_timeout_exception=False):
+  try:
+    super(cls, shared_page_state).DidRunStory(results)
+    if force_timeout_exception:
+      raise exceptions.TimeoutException('Deliberate timeout')
+  except Exception:
+    # See whether we need to squelch this exception because there's
+    # a failure or flaky expectation.
+    page = shared_page_state._last_cached_page
+    expectations = page.GetExpectations()
+    expectation = 'pass'
+    if expectations:
+      expectation = expectations.GetExpectationForPage(
+        shared_page_state.browser, page)
+    # Unlike RunStoryWithRetries, it's possible for this entry point
+    # to see skip expectations, in particular if an exception was
+    # thrown during the test run.
+    if expectation != 'pass':

[nednguyen, 2016/05/04 17:58:25]

I think you may want exercise the code path of tearing down the browser &
restarting it here since the story_runner will no longer do that for you
https://github.com/catapult-project/catapult/blob/master/telemetry/telemetry/...

+      # Expect that RunStory probably also failed, so don't worry
+      # about counting the number of failures seen in DidRunStory.
+      # Just squelch the exception and continue.
+      msg = 'Expected exception in DidRunStory for %s' % page.display_name
+      exception_formatter.PrintFormattedException(msg=msg)
+      return
+    raise
+
 class GpuSharedPageState(shared_page_state_module.SharedPageState):
   def CanRunOnBrowser(self, browser_info, page):
     return _CanRunOnBrowser(browser_info, page)
 
   def RunStory(self, results):
     RunStoryWithRetries(GpuSharedPageState, self, results)
 
+  def DidRunStory(self, results):
+    _DidRunStory(GpuSharedPageState, self, results)
+
 
 # TODO(kbr): re-evaluate the need for this SharedPageState
 # subclass. It's only used by the WebGL conformance suite.
 class DesktopGpuSharedPageState(
     shared_page_state_module.SharedDesktopPageState):
   def CanRunOnBrowser(self, browser_info, page):
     return _CanRunOnBrowser(browser_info, page)
 
   def RunStory(self, results):
     RunStoryWithRetries(DesktopGpuSharedPageState, self, results)
 
+  def DidRunStory(self, results):
+    _DidRunStory(DesktopGpuSharedPageState, self, results)
+
 
 class FakeGpuSharedPageState(fakes.FakeSharedPageState):
   def CanRunOnBrowser(self, browser_info, page):
     return _CanRunOnBrowser(browser_info, page)
 
   def RunStory(self, results):
     RunStoryWithRetries(FakeGpuSharedPageState, self, results)
 
+  def DidRunStory(self, results,   #pylint: disable=arguments-differ
+      force_timeout_exception=False):
+    _DidRunStory(FakeGpuSharedPageState, self, results,
+                 force_timeout_exception=force_timeout_exception)
+
 
 class PageBase(page_module.Page):
   # The convention is that pages subclassing this class must be
   # configured with the test expectations.
   def __init__(self, url, page_set=None, base_dir=None, name='',
                shared_page_state_class=GpuSharedPageState,
                make_javascript_deterministic=True,
                expectations=None):
     super(PageBase, self).__init__(
       url=url, page_set=page_set, base_dir=base_dir, name=name,
@@ skipping 14 matching lines @@
 
 
     # TODO(kbr): this is fragile -- if someone changes the
     # shared_page_state_class to something that doesn't handle skip
     # expectations, then they'll hit the exception in
     # RunStoryWithRetries, above. Need to rethink.
     self._expectations = expectations
 
   def GetExpectations(self):
     return self._expectations