OLD | NEW |
| (Empty) |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 import glob | |
5 import json | |
6 import os | |
7 import re | |
8 | |
9 from gpu_tests import cloud_storage_test_base | |
10 from gpu_tests import pixel_expectations | |
11 import page_sets | |
12 | |
13 from py_utils import cloud_storage | |
14 from telemetry.page import page_test | |
15 from telemetry.util import image_util | |
16 | |
17 | |
18 test_data_dir = os.path.abspath(os.path.join( | |
19 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) | |
20 | |
21 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') | |
22 | |
23 test_harness_script = r""" | |
24 var domAutomationController = {}; | |
25 | |
26 domAutomationController._succeeded = false; | |
27 domAutomationController._finished = false; | |
28 | |
29 domAutomationController.setAutomationId = function(id) {} | |
30 | |
31 domAutomationController.send = function(msg) { | |
32 domAutomationController._finished = true; | |
33 | |
34 if(msg.toLowerCase() == "success") { | |
35 domAutomationController._succeeded = true; | |
36 } else { | |
37 domAutomationController._succeeded = false; | |
38 } | |
39 } | |
40 | |
41 window.domAutomationController = domAutomationController; | |
42 """ | |
43 | |
44 class PixelTestFailure(Exception): | |
45 pass | |
46 | |
47 def _DidTestSucceed(tab): | |
48 return tab.EvaluateJavaScript('domAutomationController._succeeded') | |
49 | |
50 class PixelValidator(cloud_storage_test_base.ValidatorBase): | |
51 def CustomizeBrowserOptions(self, options): | |
52 # --test-type=gpu is used only to suppress the "Google API Keys are missing" | |
53 # infobar, which causes flakiness in tests. | |
54 options.AppendExtraBrowserArgs(['--enable-gpu-benchmarking', | |
55 '--test-type=gpu']) | |
56 | |
57 def ValidateAndMeasurePage(self, page, tab, results): | |
58 if not _DidTestSucceed(tab): | |
59 raise page_test.Failure('Page indicated a failure') | |
60 | |
61 if not tab.screenshot_supported: | |
62 raise page_test.Failure('Browser does not support screenshot capture') | |
63 | |
64 screenshot = tab.Screenshot(5) | |
65 | |
66 if screenshot is None: | |
67 raise page_test.Failure('Could not capture screenshot') | |
68 | |
69 dpr = tab.EvaluateJavaScript('window.devicePixelRatio') | |
70 | |
71 if hasattr(page, 'test_rect'): | |
72 screenshot = image_util.Crop( | |
73 screenshot, page.test_rect[0] * dpr, page.test_rect[1] * dpr, | |
74 page.test_rect[2] * dpr, page.test_rect[3] * dpr) | |
75 | |
76 if hasattr(page, 'expected_colors'): | |
77 # Use expected pixels instead of ref images for validation. | |
78 expected_colors_file = os.path.abspath(os.path.join( | |
79 os.path.dirname(__file__), page.expected_colors)) | |
80 expected_colors = self._ReadPixelExpectations(expected_colors_file) | |
81 self._ValidateScreenshotSamples( | |
82 tab, page.display_name, screenshot, expected_colors, dpr) | |
83 return | |
84 | |
85 image_name = self._UrlToImageName(page.display_name) | |
86 | |
87 if self.options.upload_refimg_to_cloud_storage: | |
88 if self._ConditionallyUploadToCloudStorage(image_name, page, tab, | |
89 screenshot): | |
90 # This is the new reference image; there's nothing to compare against. | |
91 ref_png = screenshot | |
92 else: | |
93 # There was a preexisting reference image, so we might as well | |
94 # compare against it. | |
95 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) | |
96 elif self.options.download_refimg_from_cloud_storage: | |
97 # This bot doesn't have the ability to properly generate a | |
98 # reference image, so download it from cloud storage. | |
99 try: | |
100 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) | |
101 except cloud_storage.NotFoundError: | |
102 # There is no reference image yet in cloud storage. This | |
103 # happens when the revision of the test is incremented or when | |
104 # a new test is added, because the trybots are not allowed to | |
105 # produce reference images, only the bots on the main | |
106 # waterfalls. Report this as a failure so the developer has to | |
107 # take action by explicitly suppressing the failure and | |
108 # removing the suppression once the reference images have been | |
109 # generated. Otherwise silent failures could happen for long | |
110 # periods of time. | |
111 raise page_test.Failure('Could not find image %s in cloud storage' % | |
112 image_name) | |
113 else: | |
114 # Legacy path using on-disk results. | |
115 ref_png = self._GetReferenceImage(self.options.reference_dir, | |
116 image_name, page.revision, screenshot) | |
117 | |
118 # Test new snapshot against existing reference image | |
119 if not image_util.AreEqual(ref_png, screenshot, tolerance=page.tolerance): | |
120 if self.options.test_machine_name: | |
121 self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png) | |
122 else: | |
123 self._WriteErrorImages(self.options.generated_dir, image_name, | |
124 screenshot, ref_png) | |
125 raise page_test.Failure('Reference image did not match captured screen') | |
126 | |
127 def _DeleteOldReferenceImages(self, ref_image_path, cur_revision): | |
128 if not cur_revision: | |
129 return | |
130 | |
131 old_revisions = glob.glob(ref_image_path + "_*.png") | |
132 for rev_path in old_revisions: | |
133 m = re.match(r'^.*_(\d+)\.png$', rev_path) | |
134 if m and int(m.group(1)) < cur_revision: | |
135 print 'Found deprecated reference image. Deleting rev ' + m.group(1) | |
136 os.remove(rev_path) | |
137 | |
138 def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot): | |
139 if not cur_revision: | |
140 cur_revision = 0 | |
141 | |
142 image_path = os.path.join(img_dir, img_name) | |
143 | |
144 self._DeleteOldReferenceImages(image_path, cur_revision) | |
145 | |
146 image_path = image_path + '_' + str(cur_revision) + '.png' | |
147 | |
148 try: | |
149 ref_png = image_util.FromPngFile(image_path) | |
150 # This can raise a couple of exceptions including IOError and ValueError. | |
151 except Exception: | |
152 ref_png = None | |
153 | |
154 if ref_png is not None: | |
155 return ref_png | |
156 | |
157 print ('Reference image not found. Writing tab contents as reference to: ' + | |
158 image_path) | |
159 | |
160 self._WriteImage(image_path, screenshot) | |
161 return screenshot | |
162 | |
163 def _ReadPixelExpectations(self, json_file_path): | |
164 with open(json_file_path, 'r') as f: | |
165 json_contents = json.load(f) | |
166 return json_contents | |
167 | |
168 class Pixel(cloud_storage_test_base.CloudStorageTestBase): | |
169 test = PixelValidator | |
170 | |
171 @classmethod | |
172 def Name(cls): | |
173 return 'pixel' | |
174 | |
175 @classmethod | |
176 def AddBenchmarkCommandLineArgs(cls, group): | |
177 super(Pixel, cls).AddBenchmarkCommandLineArgs(group) | |
178 group.add_option('--reference-dir', | |
179 help='Overrides the default on-disk location for reference images ' | |
180 '(only used for local testing without a cloud storage account)', | |
181 default=default_reference_image_dir) | |
182 | |
183 def CreateStorySet(self, options): | |
184 story_set = page_sets.PixelTestsStorySet(self.GetExpectations(), | |
185 try_es3=True) | |
186 for page in story_set: | |
187 page.script_to_evaluate_on_commit = test_harness_script | |
188 return story_set | |
189 | |
190 def _CreateExpectations(self): | |
191 return pixel_expectations.PixelExpectations() | |
OLD | NEW |