[telemetry] Try to print browser logs and capture a screenshot upon story failure

telemetry/telemetry/internal/story_runner_unittest.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import os
 import StringIO
 import sys
 import unittest
 
 from catapult_base import cloud_storage  # pylint: disable=import-error
 
 from telemetry import benchmark
 from telemetry.core import exceptions
 from telemetry.core import util
 from telemetry import decorators
+from telemetry.internal.actions import page_action
+from telemetry.internal.results import page_test_results
 from telemetry.internal.results import results_options
 from telemetry.internal import story_runner
 from telemetry.internal.util import exception_formatter as ex_formatter_module
 from telemetry.page import page as page_module
 from telemetry.page import legacy_page_test
 from telemetry import story as story_module
 from telemetry.testing import options_for_unittests
 from telemetry.testing import system_stub
 import mock
+from telemetry.value import failure
 from telemetry.value import improvement_direction
 from telemetry.value import list_of_scalar_values
 from telemetry.value import scalar
+from telemetry.value import skip
 from telemetry.value import summary as summary_module
 from telemetry.web_perf import story_test
 from telemetry.web_perf import timeline_based_measurement
 from telemetry.wpr import archive_info
 
 # This linter complains if we define classes nested inside functions.
 # pylint: disable=bad-super-call
 
 
 class FakePlatform(object):
@@ skipping 27 matching lines @@
 
   def RunStory(self, results):
     raise NotImplementedError
 
   def DidRunStory(self, results):
     pass
 
   def TearDownState(self):
     pass
 
+  def DumpStateUponFailure(self, story, results):
+    pass
+
 
 class TestSharedPageState(TestSharedState):
   def RunStory(self, results):
     self._test.RunPage(self._current_story, None, results)
 
 
 class FooStoryState(TestSharedPageState):
   pass
 
 
@@ skipping 59 matching lines @@
 
 
 def GetNumberOfSuccessfulPageRuns(results):
   return len([run for run in results.all_page_runs if run.ok or run.skipped])
 
 
 class TestOnlyException(Exception):
   pass
 
 
+class FailureValueMatcher(object):
+
+  def __init__(self, expected_exception_message):
+    self._expected_exception_message = expected_exception_message
+
+  def __eq__(self, other):
+    return (isinstance(other, failure.FailureValue) and
+            other.exc_info[1].message == self._expected_exception_message)
+
+
+class SkipValueMatcher(object):
+
+  def __eq__(self, other):
+    return isinstance(other, skip.SkipValue)
+
+
 class StoryRunnerTest(unittest.TestCase):
 
   def setUp(self):
     self.fake_stdout = StringIO.StringIO()
     self.actual_stdout = sys.stdout
     sys.stdout = self.fake_stdout
     self.options = _GetOptionForUnittest()
     self.results = results_options.CreateResults(
         EmptyMetadataForTest(), self.options)
     self._story_runner_logging_stub = None
@@ skipping 336 matching lines @@
 
     unit_test_events = []  # track what was called when
     class DidRunTestError(Exception):
       pass
 
     class TestTearDownSharedState(TestSharedPageState):
       def TearDownState(self):
         unit_test_events.append('tear-down-state')
         raise DidRunTestError
 
+      def DumpStateUponFailure(self, story, results):
+        unit_test_events.append('dump-state')
+
 
     class Test(legacy_page_test.LegacyPageTest):
       def __init__(self, *args):
         super(Test, self).__init__(*args)
         self.run_count = 0
 
       def RunPage(self, *_):
         old_run_count = self.run_count
         self.run_count += 1
         if old_run_count == 0:
           unit_test_events.append('app-crash')
           raise exceptions.AppCrashException
 
       def ValidateAndMeasurePage(self, page, tab, results):
         pass
 
     story_set.AddStory(DummyLocalStory(TestTearDownSharedState))
     story_set.AddStory(DummyLocalStory(TestTearDownSharedState))
     test = Test()
 
     with self.assertRaises(DidRunTestError):
       story_runner.Run(
           test, story_set, self.options, self.results)
-    self.assertEqual(['app-crash', 'tear-down-state'], unit_test_events)
+    self.assertEqual(['app-crash', 'dump-state', 'tear-down-state'],
+                     unit_test_events)
     # The AppCrashException gets added as a failure.
     self.assertEquals(1, len(self.results.failures))
 
   def testPagesetRepeat(self):
     story_set = story_module.StorySet()
 
     # TODO(eakuefner): Factor this out after flattening page ref in Value
     blank_story = DummyLocalStory(TestSharedPageState, name='blank')
     green_story = DummyLocalStory(TestSharedPageState, name='green')
     story_set.AddStory(blank_story)
@@ skipping 127 matching lines @@
 
       def DidRunStory(self, results):
         pass
 
       def CanRunStory(self, story):
         return True
 
       def TearDownState(self):
         pass
 
+      def DumpStateUponFailure(self, story, results):
+        pass
+
     class FailingStory(story_module.Story):
       def __init__(self):
         super(FailingStory, self).__init__(
             shared_state_class=SimpleSharedState,
             is_local=True)
         self.was_run = False
 
       def Run(self, shared_state):
         self.was_run = True
         raise legacy_page_test.Failure
@@ skipping 32 matching lines @@
         num_failing_stories=5, runner_max_failures=3,
         options_max_failures=None, expected_num_failures=4)
 
   def testMaxFailuresOption(self):
     # Runs up to max_failures+1 failing tests before stopping, since
     # every tests after max_failures failures have been encountered
     # may all be passing.
     self._testMaxFailuresOptionIsRespectedAndOverridable(
         num_failing_stories=5, runner_max_failures=3,
         options_max_failures=1, expected_num_failures=2)
+
+  def _CreateErrorProcessingMock(self, method_exceptions=None,
+                                 legacy_test=False):
+    if legacy_test:
+      test_class = legacy_page_test.LegacyPageTest
+    else:
+      test_class = story_test.StoryTest
+
+    root_mock = mock.NonCallableMock(
+        story=mock.NonCallableMagicMock(story_module.Story),
+        results=mock.NonCallableMagicMock(page_test_results.PageTestResults),
+        test=mock.NonCallableMagicMock(test_class),
+        state=mock.NonCallableMagicMock(
+            story_module.SharedState,
+            CanRunStory=mock.Mock(return_value=True)))
+
+    if method_exceptions:
+      root_mock.configure_mock(**{
+          path + '.side_effect': exception
+          for path, exception in method_exceptions.iteritems()})
+
+    return root_mock
+
+  def testRunStoryAndProcessErrorIfNeeded_success(self):
+    root_mock = self._CreateErrorProcessingMock()
+
+    story_runner._RunStoryAndProcessErrorIfNeeded(
+        root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.RunStory(root_mock.results),
+      mock.call.test.Measure(root_mock.state.platform, root_mock.results),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_successLegacy(self):
+    root_mock = self._CreateErrorProcessingMock(legacy_test=True)
+
+    story_runner._RunStoryAndProcessErrorIfNeeded(
+        root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.RunStory(root_mock.results),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunPage(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryTimeout(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'state.WillRunStory': exceptions.TimeoutException('foo')
+    })
+
+    story_runner._RunStoryAndProcessErrorIfNeeded(
+        root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
+      mock.call.results.AddValue(FailureValueMatcher('foo')),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryError(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'state.CanRunStory': exceptions.Error('foo')
+    })
+
+    with self.assertRaisesRegexp(exceptions.Error, 'foo'):
+      story_runner._RunStoryAndProcessErrorIfNeeded(
+          root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
+      mock.call.results.AddValue(FailureValueMatcher('foo')),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryUnsupportedAction(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'state.RunStory': page_action.PageActionNotSupported('foo')
+    })
+
+    story_runner._RunStoryAndProcessErrorIfNeeded(
+        root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.RunStory(root_mock.results),
+      mock.call.results.AddValue(SkipValueMatcher()),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryUnhandlable(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'test.WillRunStory': Exception('foo')
+    })
+
+    with self.assertRaisesRegexp(Exception, 'foo'):
+      story_runner._RunStoryAndProcessErrorIfNeeded(
+          root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
+      mock.call.results.AddValue(FailureValueMatcher('foo')),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_finallyException(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'state.DidRunStory': Exception('bar')
+    })
+
+    with self.assertRaisesRegexp(Exception, 'bar'):
+      story_runner._RunStoryAndProcessErrorIfNeeded(
+          root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.RunStory(root_mock.results),
+      mock.call.test.Measure(root_mock.state.platform, root_mock.results),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryTimeout_finallyException(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'state.RunStory': exceptions.TimeoutException('foo'),
+      'state.DidRunStory': Exception('bar')
+    })
+
+    story_runner._RunStoryAndProcessErrorIfNeeded(
+        root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.RunStory(root_mock.results),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
+      mock.call.results.AddValue(FailureValueMatcher('foo')),
+      mock.call.state.DidRunStory(root_mock.results)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryError_finallyException(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'state.WillRunStory': exceptions.Error('foo'),
+      'test.DidRunStory': Exception('bar')
+    })
+
+    with self.assertRaisesRegexp(exceptions.Error, 'foo'):
+      story_runner._RunStoryAndProcessErrorIfNeeded(
+          root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
+      mock.call.results.AddValue(FailureValueMatcher('foo')),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryUnsupportedAction_finallyException(
+      self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'test.WillRunStory': page_action.PageActionNotSupported('foo'),
+      'state.DidRunStory': Exception('bar')
+    })
+
+    story_runner._RunStoryAndProcessErrorIfNeeded(
+        root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.results.AddValue(SkipValueMatcher()),
+      mock.call.state.DidRunStory(root_mock.results)
+    ])
+
+  def testRunStoryAndProcessErrorIfNeeded_tryUnhandlable_finallyException(self):
+    root_mock = self._CreateErrorProcessingMock(method_exceptions={
+      'test.Measure': Exception('foo'),
+      'test.DidRunStory': Exception('bar')
+    })
+
+    with self.assertRaisesRegexp(Exception, 'foo'):
+      story_runner._RunStoryAndProcessErrorIfNeeded(
+          root_mock.story, root_mock.results, root_mock.state, root_mock.test)
+
+    self.assertEquals(root_mock.method_calls, [
+      mock.call.test.WillRunStory(root_mock.state.platform),
+      mock.call.state.WillRunStory(root_mock.story),
+      mock.call.state.CanRunStory(root_mock.story),
+      mock.call.state.RunStory(root_mock.results),
+      mock.call.test.Measure(root_mock.state.platform, root_mock.results),
+      mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
+      mock.call.results.AddValue(FailureValueMatcher('foo')),
+      mock.call.state.DidRunStory(root_mock.results),
+      mock.call.test.DidRunStory(root_mock.state.platform)
+    ])