OLD | NEW |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import logging | 5 import logging |
6 | 6 |
7 from telemetry import benchmark as benchmark_module | 7 from telemetry import benchmark as benchmark_module |
8 from telemetry.core import exceptions | |
9 from telemetry.page import page as page_module | 8 from telemetry.page import page as page_module |
10 from telemetry.page import page_test | 9 from telemetry.page import page_test |
11 from telemetry.page import shared_page_state | 10 from telemetry.page import shared_page_state as shared_page_state_module |
12 from telemetry.testing import fakes | 11 from telemetry.testing import fakes |
13 from telemetry.value import skip | |
14 | 12 |
15 import exception_formatter | 13 from gpu_tests import exception_formatter |
16 import gpu_test_expectations | 14 from gpu_tests import gpu_test_expectations |
17 | |
18 """Base classes for all GPU tests in this directory. Implements | |
19 support for per-page test expectations.""" | |
20 | 15 |
21 class TestBase(benchmark_module.Benchmark): | 16 class TestBase(benchmark_module.Benchmark): |
| 17 """Base classes for all GPU tests in this directory. Implements |
| 18 support for per-page test expectations.""" |
| 19 |
22 def __init__(self, max_failures=None): | 20 def __init__(self, max_failures=None): |
23 super(TestBase, self).__init__(max_failures=max_failures) | 21 super(TestBase, self).__init__(max_failures=max_failures) |
24 self._cached_expectations = None | 22 self._cached_expectations = None |
25 | 23 |
26 def GetExpectations(self): | 24 def GetExpectations(self): |
27 """Returns the expectations that apply to this test.""" | 25 """Returns the expectations that apply to this test.""" |
28 if not self._cached_expectations: | 26 if not self._cached_expectations: |
29 self._cached_expectations = self._CreateExpectations() | 27 self._cached_expectations = self._CreateExpectations() |
30 if not isinstance(self._cached_expectations, | 28 if not isinstance(self._cached_expectations, |
31 gpu_test_expectations.GpuTestExpectations): | 29 gpu_test_expectations.GpuTestExpectations): |
(...skipping 13 matching lines...) Expand all Loading... |
45 # future. | 43 # future. |
46 class ValidatorBase(page_test.PageTest): | 44 class ValidatorBase(page_test.PageTest): |
47 def __init__(self, | 45 def __init__(self, |
48 needs_browser_restart_after_each_page=False, | 46 needs_browser_restart_after_each_page=False, |
49 clear_cache_before_each_run=False): | 47 clear_cache_before_each_run=False): |
50 super(ValidatorBase, self).__init__( | 48 super(ValidatorBase, self).__init__( |
51 needs_browser_restart_after_each_page=\ | 49 needs_browser_restart_after_each_page=\ |
52 needs_browser_restart_after_each_page, | 50 needs_browser_restart_after_each_page, |
53 clear_cache_before_each_run=clear_cache_before_each_run) | 51 clear_cache_before_each_run=clear_cache_before_each_run) |
54 | 52 |
| 53 def ValidateAndMeasurePage(self, page, tab, result): |
| 54 pass |
55 | 55 |
56 def _CanRunOnBrowser(browser_info, page): | 56 def _CanRunOnBrowser(browser_info, page): |
57 expectations = page.GetExpectations() | 57 expectations = page.GetExpectations() |
58 return expectations.GetExpectationForPage( | 58 return expectations.GetExpectationForPage( |
59 browser_info.browser, page) != 'skip' | 59 browser_info.browser, page) != 'skip' |
60 | 60 |
61 def RunStoryWithRetries(cls, shared_page_state, results): | 61 def RunStoryWithRetries(cls, shared_page_state, results): |
62 page = shared_page_state.current_page | 62 page = shared_page_state.current_page |
63 expectations = page.GetExpectations() | 63 expectations = page.GetExpectations() |
64 expectation = 'pass' | 64 expectation = 'pass' |
65 if expectations: | 65 if expectations: |
66 expectation = expectations.GetExpectationForPage( | 66 expectation = expectations.GetExpectationForPage( |
67 shared_page_state.browser, page) | 67 shared_page_state.browser, page) |
68 if expectation == 'skip': | 68 if expectation == 'skip': |
69 raise Exception( | 69 raise Exception( |
70 'Skip expectations should have been handled in CanRunOnBrowser') | 70 'Skip expectations should have been handled in CanRunOnBrowser') |
71 try: | 71 try: |
72 super(cls, shared_page_state).RunStory(results) | 72 super(cls, shared_page_state).RunStory(results) |
73 except Exception: | 73 except Exception: |
74 if expectation == 'pass': | 74 if expectation == 'pass': |
75 raise | 75 raise |
76 elif expectation == 'fail': | 76 elif expectation == 'fail': |
77 msg = 'Expected exception while running %s' % page.display_name | 77 msg = 'Expected exception while running %s' % page.display_name |
78 exception_formatter.PrintFormattedException(msg=msg) | 78 exception_formatter.PrintFormattedException(msg=msg) |
79 return | 79 return |
80 if expectation != 'flaky': | 80 if expectation != 'flaky': |
81 logging.warning( | 81 logging.warning( |
82 'Unknown expectation %s while handling exception for %s' % | 82 'Unknown expectation %s while handling exception for %s', |
83 (expectation, page.display_name)) | 83 expectation, page.display_name) |
84 raise | 84 raise |
85 # Flaky tests are handled here. | 85 # Flaky tests are handled here. |
86 num_retries = expectations.GetFlakyRetriesForPage( | 86 num_retries = expectations.GetFlakyRetriesForPage( |
87 shared_page_state.browser, page) | 87 shared_page_state.browser, page) |
88 if not num_retries: | 88 if not num_retries: |
89 # Re-raise the exception. | 89 # Re-raise the exception. |
90 raise | 90 raise |
91 # Re-run the test up to |num_retries| times. | 91 # Re-run the test up to |num_retries| times. |
92 for ii in xrange(0, num_retries): | 92 for ii in xrange(0, num_retries): |
93 print 'FLAKY TEST FAILURE, retrying: ' + page.display_name | 93 print 'FLAKY TEST FAILURE, retrying: ' + page.display_name |
94 try: | 94 try: |
95 super(cls, shared_page_state).RunStory(results) | 95 super(cls, shared_page_state).RunStory(results) |
96 break | 96 break |
97 except Exception: | 97 except Exception: |
98 # Squelch any exceptions from any but the last retry. | 98 # Squelch any exceptions from any but the last retry. |
99 if ii == num_retries - 1: | 99 if ii == num_retries - 1: |
100 raise | 100 raise |
101 else: | 101 else: |
102 if expectation == 'fail': | 102 if expectation == 'fail': |
103 logging.warning( | 103 logging.warning( |
104 '%s was expected to fail, but passed.\n', page.display_name) | 104 '%s was expected to fail, but passed.\n', page.display_name) |
105 | 105 |
106 class GpuSharedPageState(shared_page_state.SharedPageState): | 106 class GpuSharedPageState(shared_page_state_module.SharedPageState): |
107 def CanRunOnBrowser(self, browser_info, page): | 107 def CanRunOnBrowser(self, browser_info, page): |
108 return _CanRunOnBrowser(browser_info, page) | 108 return _CanRunOnBrowser(browser_info, page) |
109 | 109 |
110 def RunStory(self, results): | 110 def RunStory(self, results): |
111 RunStoryWithRetries(GpuSharedPageState, self, results) | 111 RunStoryWithRetries(GpuSharedPageState, self, results) |
112 | 112 |
113 | 113 |
114 # TODO(kbr): re-evaluate the need for this SharedPageState | 114 # TODO(kbr): re-evaluate the need for this SharedPageState |
115 # subclass. It's only used by the WebGL conformance suite. | 115 # subclass. It's only used by the WebGL conformance suite. |
116 class DesktopGpuSharedPageState( | 116 class DesktopGpuSharedPageState( |
117 shared_page_state.SharedDesktopPageState): | 117 shared_page_state_module.SharedDesktopPageState): |
118 def CanRunOnBrowser(self, browser_info, page): | 118 def CanRunOnBrowser(self, browser_info, page): |
119 return _CanRunOnBrowser(browser_info, page) | 119 return _CanRunOnBrowser(browser_info, page) |
120 | 120 |
121 def RunStory(self, results): | 121 def RunStory(self, results): |
122 RunStoryWithRetries(DesktopGpuSharedPageState, self, results) | 122 RunStoryWithRetries(DesktopGpuSharedPageState, self, results) |
123 | 123 |
124 | 124 |
125 class FakeGpuSharedPageState(fakes.FakeSharedPageState): | 125 class FakeGpuSharedPageState(fakes.FakeSharedPageState): |
126 def CanRunOnBrowser(self, browser_info, page): | 126 def CanRunOnBrowser(self, browser_info, page): |
127 return _CanRunOnBrowser(browser_info, page) | 127 return _CanRunOnBrowser(browser_info, page) |
(...skipping 28 matching lines...) Expand all Loading... |
156 | 156 |
157 | 157 |
158 # TODO(kbr): this is fragile -- if someone changes the | 158 # TODO(kbr): this is fragile -- if someone changes the |
159 # shared_page_state_class to something that doesn't handle skip | 159 # shared_page_state_class to something that doesn't handle skip |
160 # expectations, then they'll hit the exception in | 160 # expectations, then they'll hit the exception in |
161 # RunStoryWithRetries, above. Need to rethink. | 161 # RunStoryWithRetries, above. Need to rethink. |
162 self._expectations = expectations | 162 self._expectations = expectations |
163 | 163 |
164 def GetExpectations(self): | 164 def GetExpectations(self): |
165 return self._expectations | 165 return self._expectations |
OLD | NEW |