| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 import glob | |
| 5 import json | |
| 6 import os | |
| 7 import re | |
| 8 | |
| 9 from gpu_tests import cloud_storage_test_base | |
| 10 from gpu_tests import pixel_expectations | |
| 11 import page_sets | |
| 12 | |
| 13 from py_utils import cloud_storage | |
| 14 from telemetry.page import page_test | |
| 15 from telemetry.util import image_util | |
| 16 | |
| 17 | |
| 18 test_data_dir = os.path.abspath(os.path.join( | |
| 19 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) | |
| 20 | |
| 21 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') | |
| 22 | |
| 23 test_harness_script = r""" | |
| 24 var domAutomationController = {}; | |
| 25 | |
| 26 domAutomationController._succeeded = false; | |
| 27 domAutomationController._finished = false; | |
| 28 | |
| 29 domAutomationController.setAutomationId = function(id) {} | |
| 30 | |
| 31 domAutomationController.send = function(msg) { | |
| 32 domAutomationController._finished = true; | |
| 33 | |
| 34 if(msg.toLowerCase() == "success") { | |
| 35 domAutomationController._succeeded = true; | |
| 36 } else { | |
| 37 domAutomationController._succeeded = false; | |
| 38 } | |
| 39 } | |
| 40 | |
| 41 window.domAutomationController = domAutomationController; | |
| 42 """ | |
| 43 | |
| 44 class PixelTestFailure(Exception): | |
| 45 pass | |
| 46 | |
| 47 def _DidTestSucceed(tab): | |
| 48 return tab.EvaluateJavaScript('domAutomationController._succeeded') | |
| 49 | |
| 50 class PixelValidator(cloud_storage_test_base.ValidatorBase): | |
| 51 def CustomizeBrowserOptions(self, options): | |
| 52 # --test-type=gpu is used only to suppress the "Google API Keys are missing" | |
| 53 # infobar, which causes flakiness in tests. | |
| 54 options.AppendExtraBrowserArgs(['--enable-gpu-benchmarking', | |
| 55 '--test-type=gpu']) | |
| 56 | |
| 57 def ValidateAndMeasurePage(self, page, tab, results): | |
| 58 if not _DidTestSucceed(tab): | |
| 59 raise page_test.Failure('Page indicated a failure') | |
| 60 | |
| 61 if not tab.screenshot_supported: | |
| 62 raise page_test.Failure('Browser does not support screenshot capture') | |
| 63 | |
| 64 screenshot = tab.Screenshot(5) | |
| 65 | |
| 66 if screenshot is None: | |
| 67 raise page_test.Failure('Could not capture screenshot') | |
| 68 | |
| 69 dpr = tab.EvaluateJavaScript('window.devicePixelRatio') | |
| 70 | |
| 71 if hasattr(page, 'test_rect'): | |
| 72 screenshot = image_util.Crop( | |
| 73 screenshot, page.test_rect[0] * dpr, page.test_rect[1] * dpr, | |
| 74 page.test_rect[2] * dpr, page.test_rect[3] * dpr) | |
| 75 | |
| 76 if hasattr(page, 'expected_colors'): | |
| 77 # Use expected pixels instead of ref images for validation. | |
| 78 expected_colors_file = os.path.abspath(os.path.join( | |
| 79 os.path.dirname(__file__), page.expected_colors)) | |
| 80 expected_colors = self._ReadPixelExpectations(expected_colors_file) | |
| 81 self._ValidateScreenshotSamples( | |
| 82 tab, page.display_name, screenshot, expected_colors, dpr) | |
| 83 return | |
| 84 | |
| 85 image_name = self._UrlToImageName(page.display_name) | |
| 86 | |
| 87 if self.options.upload_refimg_to_cloud_storage: | |
| 88 if self._ConditionallyUploadToCloudStorage(image_name, page, tab, | |
| 89 screenshot): | |
| 90 # This is the new reference image; there's nothing to compare against. | |
| 91 ref_png = screenshot | |
| 92 else: | |
| 93 # There was a preexisting reference image, so we might as well | |
| 94 # compare against it. | |
| 95 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) | |
| 96 elif self.options.download_refimg_from_cloud_storage: | |
| 97 # This bot doesn't have the ability to properly generate a | |
| 98 # reference image, so download it from cloud storage. | |
| 99 try: | |
| 100 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) | |
| 101 except cloud_storage.NotFoundError: | |
| 102 # There is no reference image yet in cloud storage. This | |
| 103 # happens when the revision of the test is incremented or when | |
| 104 # a new test is added, because the trybots are not allowed to | |
| 105 # produce reference images, only the bots on the main | |
| 106 # waterfalls. Report this as a failure so the developer has to | |
| 107 # take action by explicitly suppressing the failure and | |
| 108 # removing the suppression once the reference images have been | |
| 109 # generated. Otherwise silent failures could happen for long | |
| 110 # periods of time. | |
| 111 raise page_test.Failure('Could not find image %s in cloud storage' % | |
| 112 image_name) | |
| 113 else: | |
| 114 # Legacy path using on-disk results. | |
| 115 ref_png = self._GetReferenceImage(self.options.reference_dir, | |
| 116 image_name, page.revision, screenshot) | |
| 117 | |
| 118 # Test new snapshot against existing reference image | |
| 119 if not image_util.AreEqual(ref_png, screenshot, tolerance=page.tolerance): | |
| 120 if self.options.test_machine_name: | |
| 121 self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png) | |
| 122 else: | |
| 123 self._WriteErrorImages(self.options.generated_dir, image_name, | |
| 124 screenshot, ref_png) | |
| 125 raise page_test.Failure('Reference image did not match captured screen') | |
| 126 | |
| 127 def _DeleteOldReferenceImages(self, ref_image_path, cur_revision): | |
| 128 if not cur_revision: | |
| 129 return | |
| 130 | |
| 131 old_revisions = glob.glob(ref_image_path + "_*.png") | |
| 132 for rev_path in old_revisions: | |
| 133 m = re.match(r'^.*_(\d+)\.png$', rev_path) | |
| 134 if m and int(m.group(1)) < cur_revision: | |
| 135 print 'Found deprecated reference image. Deleting rev ' + m.group(1) | |
| 136 os.remove(rev_path) | |
| 137 | |
| 138 def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot): | |
| 139 if not cur_revision: | |
| 140 cur_revision = 0 | |
| 141 | |
| 142 image_path = os.path.join(img_dir, img_name) | |
| 143 | |
| 144 self._DeleteOldReferenceImages(image_path, cur_revision) | |
| 145 | |
| 146 image_path = image_path + '_' + str(cur_revision) + '.png' | |
| 147 | |
| 148 try: | |
| 149 ref_png = image_util.FromPngFile(image_path) | |
| 150 # This can raise a couple of exceptions including IOError and ValueError. | |
| 151 except Exception: | |
| 152 ref_png = None | |
| 153 | |
| 154 if ref_png is not None: | |
| 155 return ref_png | |
| 156 | |
| 157 print ('Reference image not found. Writing tab contents as reference to: ' + | |
| 158 image_path) | |
| 159 | |
| 160 self._WriteImage(image_path, screenshot) | |
| 161 return screenshot | |
| 162 | |
| 163 def _ReadPixelExpectations(self, json_file_path): | |
| 164 with open(json_file_path, 'r') as f: | |
| 165 json_contents = json.load(f) | |
| 166 return json_contents | |
| 167 | |
| 168 class Pixel(cloud_storage_test_base.CloudStorageTestBase): | |
| 169 test = PixelValidator | |
| 170 | |
| 171 @classmethod | |
| 172 def Name(cls): | |
| 173 return 'pixel' | |
| 174 | |
| 175 @classmethod | |
| 176 def AddBenchmarkCommandLineArgs(cls, group): | |
| 177 super(Pixel, cls).AddBenchmarkCommandLineArgs(group) | |
| 178 group.add_option('--reference-dir', | |
| 179 help='Overrides the default on-disk location for reference images ' | |
| 180 '(only used for local testing without a cloud storage account)', | |
| 181 default=default_reference_image_dir) | |
| 182 | |
| 183 def CreateStorySet(self, options): | |
| 184 story_set = page_sets.PixelTestsStorySet(self.GetExpectations(), | |
| 185 try_es3=True) | |
| 186 for page in story_set: | |
| 187 page.script_to_evaluate_on_commit = test_harness_script | |
| 188 return story_set | |
| 189 | |
| 190 def _CreateExpectations(self): | |
| 191 return pixel_expectations.PixelExpectations() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2363343002:
Ported pixel_test to new gpu_integration_test harness. (Closed)
