Handle Telemetry's TimeoutException in retries of flaky tests.

content/test/gpu/gpu_tests/gpu_test_base.py

Index: content/test/gpu/gpu_tests/gpu_test_base.py
diff --git a/content/test/gpu/gpu_tests/gpu_test_base.py b/content/test/gpu/gpu_tests/gpu_test_base.py
index 758cccb11849ac7dd35bcdfacee532fe63896757..fa1a0479d96a7e23e1a1b05cccf32767a4376c3c 100644
--- a/content/test/gpu/gpu_tests/gpu_test_base.py
+++ b/content/test/gpu/gpu_tests/gpu_test_base.py
@@ -5,6 +5,7 @@
 import logging
 
 from telemetry import benchmark as benchmark_module
+from telemetry.core import exceptions
 from telemetry.page import page as page_module
 from telemetry.page import page_test
 from telemetry.page import shared_page_state as shared_page_state_module
@@ -60,6 +61,10 @@ def _CanRunOnBrowser(browser_info, page):
 
 def RunStoryWithRetries(cls, shared_page_state, results):
   page = shared_page_state.current_page
+
+  # Save this away for use in _DidRunStory, below.
+  shared_page_state._last_cached_page = page
+
   expectations = page.GetExpectations()
   expectation = 'pass'
   if expectations:
@@ -103,6 +108,33 @@ def RunStoryWithRetries(cls, shared_page_state, results):
       logging.warning(
           '%s was expected to fail, but passed.\n', page.display_name)
 
+def _DidRunStory(cls, shared_page_state, results,
+                 force_timeout_exception=False):
+  try:
+    super(cls, shared_page_state).DidRunStory(results)
+    if force_timeout_exception:
+      raise exceptions.TimeoutException('Deliberate timeout')
+  except Exception:
+    # See whether we need to squelch this exception because there's
+    # a failure or flaky expectation.
+    page = shared_page_state._last_cached_page
+    expectations = page.GetExpectations()
+    expectation = 'pass'
+    if expectations:
+      expectation = expectations.GetExpectationForPage(
+        shared_page_state.browser, page)
+    # Unlike RunStoryWithRetries, it's possible for this entry point
+    # to see skip expectations, in particular if an exception was
+    # thrown during the test run.
+    if expectation != 'pass':

[nednguyen, 2016/05/04 17:58:25]

I think you may want exercise the code path of tearing down the browser &
restarting it here since the story_runner will no longer do that for you
https://github.com/catapult-project/catapult/blob/master/telemetry/telemetry/...

+      # Expect that RunStory probably also failed, so don't worry
+      # about counting the number of failures seen in DidRunStory.
+      # Just squelch the exception and continue.
+      msg = 'Expected exception in DidRunStory for %s' % page.display_name
+      exception_formatter.PrintFormattedException(msg=msg)
+      return
+    raise
+
 class GpuSharedPageState(shared_page_state_module.SharedPageState):
   def CanRunOnBrowser(self, browser_info, page):
     return _CanRunOnBrowser(browser_info, page)
@@ -110,6 +142,9 @@ class GpuSharedPageState(shared_page_state_module.SharedPageState):
   def RunStory(self, results):
     RunStoryWithRetries(GpuSharedPageState, self, results)
 
+  def DidRunStory(self, results):
+    _DidRunStory(GpuSharedPageState, self, results)
+
 
 # TODO(kbr): re-evaluate the need for this SharedPageState
 # subclass. It's only used by the WebGL conformance suite.
@@ -121,6 +156,9 @@ class DesktopGpuSharedPageState(
   def RunStory(self, results):
     RunStoryWithRetries(DesktopGpuSharedPageState, self, results)
 
+  def DidRunStory(self, results):
+    _DidRunStory(DesktopGpuSharedPageState, self, results)
+
 
 class FakeGpuSharedPageState(fakes.FakeSharedPageState):
   def CanRunOnBrowser(self, browser_info, page):
@@ -129,6 +167,11 @@ class FakeGpuSharedPageState(fakes.FakeSharedPageState):
   def RunStory(self, results):
     RunStoryWithRetries(FakeGpuSharedPageState, self, results)
 
+  def DidRunStory(self, results,   #pylint: disable=arguments-differ
+      force_timeout_exception=False):
+    _DidRunStory(FakeGpuSharedPageState, self, results,
+                 force_timeout_exception=force_timeout_exception)
+
 
 class PageBase(page_module.Page):
   # The convention is that pages subclassing this class must be