Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import logging | 5 import logging |
| 6 | 6 |
| 7 from telemetry import benchmark as benchmark_module | 7 from telemetry import benchmark as benchmark_module |
| 8 from telemetry.core import exceptions | |
| 9 from telemetry.page import page as page_module | 8 from telemetry.page import page as page_module |
| 10 from telemetry.page import page_test | 9 from telemetry.page import page_test |
| 11 from telemetry.page import shared_page_state | 10 from telemetry.page import shared_page_state |
| 12 from telemetry.testing import fakes | 11 from telemetry.testing import fakes |
| 13 from telemetry.value import skip | |
| 14 | 12 |
| 15 import exception_formatter | 13 from gpu_tests import exception_formatter |
| 16 import gpu_test_expectations | 14 from gpu_tests import gpu_test_expectations |
| 17 | |
| 18 """Base classes for all GPU tests in this directory. Implements | |
| 19 support for per-page test expectations.""" | |
| 20 | 15 |
| 21 class TestBase(benchmark_module.Benchmark): | 16 class TestBase(benchmark_module.Benchmark): |
| 17 """Base classes for all GPU tests in this directory. Implements | |
| 18 support for per-page test expectations.""" | |
| 19 | |
| 22 def __init__(self, max_failures=None): | 20 def __init__(self, max_failures=None): |
| 23 super(TestBase, self).__init__(max_failures=max_failures) | 21 super(TestBase, self).__init__(max_failures=max_failures) |
| 24 self._cached_expectations = None | 22 self._cached_expectations = None |
| 25 | 23 |
| 26 def GetExpectations(self): | 24 def GetExpectations(self): |
| 27 """Returns the expectations that apply to this test.""" | 25 """Returns the expectations that apply to this test.""" |
| 28 if not self._cached_expectations: | 26 if not self._cached_expectations: |
| 29 self._cached_expectations = self._CreateExpectations() | 27 self._cached_expectations = self._CreateExpectations() |
| 30 if not isinstance(self._cached_expectations, | 28 if not isinstance(self._cached_expectations, |
| 31 gpu_test_expectations.GpuTestExpectations): | 29 gpu_test_expectations.GpuTestExpectations): |
| (...skipping 13 matching lines...) Expand all Loading... | |
| 45 # future. | 43 # future. |
| 46 class ValidatorBase(page_test.PageTest): | 44 class ValidatorBase(page_test.PageTest): |
| 47 def __init__(self, | 45 def __init__(self, |
| 48 needs_browser_restart_after_each_page=False, | 46 needs_browser_restart_after_each_page=False, |
| 49 clear_cache_before_each_run=False): | 47 clear_cache_before_each_run=False): |
| 50 super(ValidatorBase, self).__init__( | 48 super(ValidatorBase, self).__init__( |
| 51 needs_browser_restart_after_each_page=\ | 49 needs_browser_restart_after_each_page=\ |
| 52 needs_browser_restart_after_each_page, | 50 needs_browser_restart_after_each_page, |
| 53 clear_cache_before_each_run=clear_cache_before_each_run) | 51 clear_cache_before_each_run=clear_cache_before_each_run) |
| 54 | 52 |
| 53 def ValidateAndMeasurePage(self, page, tab, result): | |
|
Ken Russell (switch to Gerrit)
2015/10/21 22:12:08
Again, unfortunate that useless code is required t
Corentin Wallez
2015/10/21 23:07:25
Acknowledged.
| |
| 54 pass | |
| 55 | 55 |
| 56 def _CanRunOnBrowser(browser_info, page): | 56 def _CanRunOnBrowser(browser_info, page): |
| 57 expectations = page.GetExpectations() | 57 expectations = page.GetExpectations() |
| 58 return expectations.GetExpectationForPage( | 58 return expectations.GetExpectationForPage( |
| 59 browser_info.browser, page) != 'skip' | 59 browser_info.browser, page) != 'skip' |
| 60 | 60 |
| 61 def RunStoryWithRetries(cls, shared_page_state, results): | 61 def RunStoryWithRetries(cls, shared_state, results): |
|
Ken Russell (switch to Gerrit)
2015/10/21 22:12:08
Consider importing shared_page_state as shared_pag
Corentin Wallez
2015/10/21 23:07:25
Done.
| |
| 62 page = shared_page_state.current_page | 62 page = shared_state.current_page |
| 63 expectations = page.GetExpectations() | 63 expectations = page.GetExpectations() |
| 64 expectation = 'pass' | 64 expectation = 'pass' |
| 65 if expectations: | 65 if expectations: |
| 66 expectation = expectations.GetExpectationForPage( | 66 expectation = expectations.GetExpectationForPage( |
| 67 shared_page_state.browser, page) | 67 shared_state.browser, page) |
| 68 if expectation == 'skip': | 68 if expectation == 'skip': |
| 69 raise Exception( | 69 raise Exception( |
| 70 'Skip expectations should have been handled in CanRunOnBrowser') | 70 'Skip expectations should have been handled in CanRunOnBrowser') |
| 71 try: | 71 try: |
| 72 super(cls, shared_page_state).RunStory(results) | 72 super(cls, shared_state).RunStory(results) |
| 73 except Exception: | 73 except Exception: |
| 74 if expectation == 'pass': | 74 if expectation == 'pass': |
| 75 raise | 75 raise |
| 76 elif expectation == 'fail': | 76 elif expectation == 'fail': |
| 77 msg = 'Expected exception while running %s' % page.display_name | 77 msg = 'Expected exception while running %s' % page.display_name |
| 78 exception_formatter.PrintFormattedException(msg=msg) | 78 exception_formatter.PrintFormattedException(msg=msg) |
| 79 return | 79 return |
| 80 if expectation != 'flaky': | 80 if expectation != 'flaky': |
| 81 logging.warning( | 81 logging.warning( |
| 82 'Unknown expectation %s while handling exception for %s' % | 82 'Unknown expectation %s while handling exception for %s', |
| 83 (expectation, page.display_name)) | 83 expectation, page.display_name) |
| 84 raise | 84 raise |
| 85 # Flaky tests are handled here. | 85 # Flaky tests are handled here. |
| 86 num_retries = expectations.GetFlakyRetriesForPage( | 86 num_retries = expectations.GetFlakyRetriesForPage( |
| 87 shared_page_state.browser, page) | 87 shared_state.browser, page) |
| 88 if not num_retries: | 88 if not num_retries: |
| 89 # Re-raise the exception. | 89 # Re-raise the exception. |
| 90 raise | 90 raise |
| 91 # Re-run the test up to |num_retries| times. | 91 # Re-run the test up to |num_retries| times. |
| 92 for ii in xrange(0, num_retries): | 92 for ii in xrange(0, num_retries): |
| 93 print 'FLAKY TEST FAILURE, retrying: ' + page.display_name | 93 print 'FLAKY TEST FAILURE, retrying: ' + page.display_name |
| 94 try: | 94 try: |
| 95 super(cls, shared_page_state).RunStory(results) | 95 super(cls, shared_state).RunStory(results) |
| 96 break | 96 break |
| 97 except Exception: | 97 except Exception: |
| 98 # Squelch any exceptions from any but the last retry. | 98 # Squelch any exceptions from any but the last retry. |
| 99 if ii == num_retries - 1: | 99 if ii == num_retries - 1: |
| 100 raise | 100 raise |
| 101 else: | 101 else: |
| 102 if expectation == 'fail': | 102 if expectation == 'fail': |
| 103 logging.warning( | 103 logging.warning( |
| 104 '%s was expected to fail, but passed.\n', page.display_name) | 104 '%s was expected to fail, but passed.\n', page.display_name) |
| 105 | 105 |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 156 | 156 |
| 157 | 157 |
| 158 # TODO(kbr): this is fragile -- if someone changes the | 158 # TODO(kbr): this is fragile -- if someone changes the |
| 159 # shared_page_state_class to something that doesn't handle skip | 159 # shared_page_state_class to something that doesn't handle skip |
| 160 # expectations, then they'll hit the exception in | 160 # expectations, then they'll hit the exception in |
| 161 # RunStoryWithRetries, above. Need to rethink. | 161 # RunStoryWithRetries, above. Need to rethink. |
| 162 self._expectations = expectations | 162 self._expectations = expectations |
| 163 | 163 |
| 164 def GetExpectations(self): | 164 def GetExpectations(self): |
| 165 return self._expectations | 165 return self._expectations |
| OLD | NEW |