| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 """Base classes for a test and validator which upload results | |
| 6 (reference images, error images) to cloud storage.""" | |
| 7 | |
| 8 import logging | |
| 9 import os | |
| 10 import re | |
| 11 import tempfile | |
| 12 | |
| 13 from py_utils import cloud_storage | |
| 14 from telemetry.page import legacy_page_test | |
| 15 from telemetry.util import image_util | |
| 16 from telemetry.util import rgba_color | |
| 17 | |
| 18 from gpu_tests import gpu_test_base | |
| 19 | |
| 20 test_data_dir = os.path.abspath(os.path.join( | |
| 21 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) | |
| 22 | |
| 23 default_generated_data_dir = os.path.join(test_data_dir, 'generated') | |
| 24 | |
| 25 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' | |
| 26 | |
| 27 def _CompareScreenshotSamples(tab, screenshot, expectations, device_pixel_ratio, | |
| 28 test_machine_name): | |
| 29 # First scan through the expectations and see if there are any scale | |
| 30 # factor overrides that would preempt the device pixel ratio. This | |
| 31 # is mainly a workaround for complex tests like the Maps test. | |
| 32 for expectation in expectations: | |
| 33 if 'scale_factor_overrides' in expectation: | |
| 34 for override in expectation['scale_factor_overrides']: | |
| 35 # Require exact matches to avoid confusion, because some | |
| 36 # machine models and names might be subsets of others | |
| 37 # (e.g. Nexus 5 vs Nexus 5X). | |
| 38 if ('device_type' in override and | |
| 39 (tab.browser.platform.GetDeviceTypeName() == | |
| 40 override['device_type'])): | |
| 41 logging.warning('Overriding device_pixel_ratio ' + | |
| 42 str(device_pixel_ratio) + ' with scale factor ' + | |
| 43 str(override['scale_factor']) + ' for device type ' + | |
| 44 override['device_type']) | |
| 45 device_pixel_ratio = override['scale_factor'] | |
| 46 break | |
| 47 if (test_machine_name and 'machine_name' in override and | |
| 48 override["machine_name"] == test_machine_name): | |
| 49 logging.warning('Overriding device_pixel_ratio ' + | |
| 50 str(device_pixel_ratio) + ' with scale factor ' + | |
| 51 str(override['scale_factor']) + ' for machine name ' + | |
| 52 test_machine_name) | |
| 53 device_pixel_ratio = override['scale_factor'] | |
| 54 break | |
| 55 # Only support one "scale_factor_overrides" in the expectation format. | |
| 56 break | |
| 57 for expectation in expectations: | |
| 58 if "scale_factor_overrides" in expectation: | |
| 59 continue | |
| 60 location = expectation["location"] | |
| 61 size = expectation["size"] | |
| 62 x0 = int(location[0] * device_pixel_ratio) | |
| 63 x1 = int((location[0] + size[0]) * device_pixel_ratio) | |
| 64 y0 = int(location[1] * device_pixel_ratio) | |
| 65 y1 = int((location[1] + size[1]) * device_pixel_ratio) | |
| 66 for x in range(x0, x1): | |
| 67 for y in range(y0, y1): | |
| 68 if (x < 0 or y < 0 or x >= image_util.Width(screenshot) or | |
| 69 y >= image_util.Height(screenshot)): | |
| 70 raise legacy_page_test.Failure( | |
| 71 ('Expected pixel location [%d, %d] is out of range on ' + | |
| 72 '[%d, %d] image') % | |
| 73 (x, y, image_util.Width(screenshot), | |
| 74 image_util.Height(screenshot))) | |
| 75 | |
| 76 actual_color = image_util.GetPixelColor(screenshot, x, y) | |
| 77 expected_color = rgba_color.RgbaColor( | |
| 78 expectation["color"][0], | |
| 79 expectation["color"][1], | |
| 80 expectation["color"][2]) | |
| 81 if not actual_color.IsEqual(expected_color, expectation["tolerance"]): | |
| 82 raise legacy_page_test.Failure('Expected pixel at ' + str(location) + | |
| 83 ' (actual pixel (' + str(x) + ', ' + str(y) + ')) ' + | |
| 84 ' to be ' + | |
| 85 str(expectation["color"]) + " but got [" + | |
| 86 str(actual_color.r) + ", " + | |
| 87 str(actual_color.g) + ", " + | |
| 88 str(actual_color.b) + "]") | |
| 89 | |
| 90 class ValidatorBase(gpu_test_base.ValidatorBase): | |
| 91 def __init__(self): | |
| 92 super(ValidatorBase, self).__init__() | |
| 93 # Parameters for cloud storage reference images. | |
| 94 self.vendor_id = None | |
| 95 self.device_id = None | |
| 96 self.vendor_string = None | |
| 97 self.device_string = None | |
| 98 self.msaa = False | |
| 99 self.model_name = None | |
| 100 | |
| 101 ### | |
| 102 ### Routines working with the local disk (only used for local | |
| 103 ### testing without a cloud storage account -- the bots do not use | |
| 104 ### this code path). | |
| 105 ### | |
| 106 | |
| 107 def _UrlToImageName(self, url): | |
| 108 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) | |
| 109 image_name = re.sub(r'\.\./', '', image_name) | |
| 110 image_name = re.sub(r'(\.|/|-)', '_', image_name) | |
| 111 return image_name | |
| 112 | |
| 113 def _WriteImage(self, image_path, png_image): | |
| 114 output_dir = os.path.dirname(image_path) | |
| 115 if not os.path.exists(output_dir): | |
| 116 os.makedirs(output_dir) | |
| 117 image_util.WritePngFile(png_image, image_path) | |
| 118 | |
| 119 def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): | |
| 120 full_image_name = img_name + '_' + str(self.options.build_revision) | |
| 121 full_image_name = full_image_name + '.png' | |
| 122 | |
| 123 # Always write the failing image. | |
| 124 self._WriteImage( | |
| 125 os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) | |
| 126 | |
| 127 if ref_png is not None: | |
| 128 # Save the reference image. | |
| 129 # This ensures that we get the right revision number. | |
| 130 self._WriteImage( | |
| 131 os.path.join(img_dir, full_image_name), ref_png) | |
| 132 | |
| 133 # Save the difference image. | |
| 134 diff_png = image_util.Diff(screenshot, ref_png) | |
| 135 self._WriteImage( | |
| 136 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) | |
| 137 | |
| 138 ### | |
| 139 ### Cloud storage code path -- the bots use this. | |
| 140 ### | |
| 141 | |
| 142 def _ComputeGpuInfo(self, tab): | |
| 143 if ((self.vendor_id and self.device_id) or | |
| 144 (self.vendor_string and self.device_string)): | |
| 145 return | |
| 146 browser = tab.browser | |
| 147 if not browser.supports_system_info: | |
| 148 raise Exception('System info must be supported by the browser') | |
| 149 system_info = browser.GetSystemInfo() | |
| 150 if not system_info.gpu: | |
| 151 raise Exception('GPU information was absent') | |
| 152 device = system_info.gpu.devices[0] | |
| 153 if device.vendor_id and device.device_id: | |
| 154 self.vendor_id = device.vendor_id | |
| 155 self.device_id = device.device_id | |
| 156 elif device.vendor_string and device.device_string: | |
| 157 self.vendor_string = device.vendor_string | |
| 158 self.device_string = device.device_string | |
| 159 else: | |
| 160 raise Exception('GPU device information was incomplete') | |
| 161 # TODO(senorblanco): This should probably be checking | |
| 162 # for the presence of the extensions in system_info.gpu_aux_attributes | |
| 163 # in order to check for MSAA, rather than sniffing the blacklist. | |
| 164 self.msaa = not ( | |
| 165 ('disable_chromium_framebuffer_multisample' in | |
| 166 system_info.gpu.driver_bug_workarounds) or | |
| 167 ('disable_multisample_render_to_texture' in | |
| 168 system_info.gpu.driver_bug_workarounds)) | |
| 169 self.model_name = system_info.model_name | |
| 170 | |
| 171 def _FormatGpuInfo(self, tab): | |
| 172 self._ComputeGpuInfo(tab) | |
| 173 msaa_string = '_msaa' if self.msaa else '_non_msaa' | |
| 174 if self.vendor_id: | |
| 175 return '%s_%04x_%04x%s' % ( | |
| 176 self.options.os_type, self.vendor_id, self.device_id, msaa_string) | |
| 177 else: | |
| 178 # This is the code path for Android devices. Include the model | |
| 179 # name (e.g. "Nexus 9") in the GPU string to disambiguate | |
| 180 # multiple devices on the waterfall which might have the same | |
| 181 # device string ("NVIDIA Tegra") but different screen | |
| 182 # resolutions and device pixel ratios. | |
| 183 return '%s_%s_%s_%s%s' % ( | |
| 184 self.options.os_type, self.vendor_string, self.device_string, | |
| 185 self.model_name, msaa_string) | |
| 186 | |
| 187 def _FormatReferenceImageName(self, img_name, page, tab): | |
| 188 return '%s_v%s_%s.png' % ( | |
| 189 img_name, | |
| 190 page.revision, | |
| 191 self._FormatGpuInfo(tab)) | |
| 192 | |
| 193 def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): | |
| 194 # This sequence of steps works on all platforms to write a temporary | |
| 195 # PNG to disk, following the pattern in bitmap_unittest.py. The key to | |
| 196 # avoiding PermissionErrors seems to be to not actually try to write to | |
| 197 # the temporary file object, but to re-open its name for all operations. | |
| 198 temp_file = tempfile.NamedTemporaryFile(suffix='.png').name | |
| 199 image_util.WritePngFile(bitmap, temp_file) | |
| 200 cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) | |
| 201 | |
| 202 def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): | |
| 203 """Uploads the screenshot to cloud storage as the reference image | |
| 204 for this test, unless it already exists. Returns True if the | |
| 205 upload was actually performed.""" | |
| 206 if not self.options.refimg_cloud_storage_bucket: | |
| 207 raise Exception('--refimg-cloud-storage-bucket argument is required') | |
| 208 cloud_name = self._FormatReferenceImageName(img_name, page, tab) | |
| 209 if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, | |
| 210 cloud_name): | |
| 211 self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, | |
| 212 cloud_name, | |
| 213 screenshot) | |
| 214 return True | |
| 215 return False | |
| 216 | |
| 217 def _DownloadFromCloudStorage(self, img_name, page, tab): | |
| 218 """Downloads the reference image for the given test from cloud | |
| 219 storage, returning it as a Telemetry Bitmap object.""" | |
| 220 # TODO(kbr): there's a race condition between the deletion of the | |
| 221 # temporary file and gsutil's overwriting it. | |
| 222 if not self.options.refimg_cloud_storage_bucket: | |
| 223 raise Exception('--refimg-cloud-storage-bucket argument is required') | |
| 224 temp_file = tempfile.NamedTemporaryFile(suffix='.png').name | |
| 225 cloud_storage.Get(self.options.refimg_cloud_storage_bucket, | |
| 226 self._FormatReferenceImageName(img_name, page, tab), | |
| 227 temp_file) | |
| 228 return image_util.FromPngFile(temp_file) | |
| 229 | |
| 230 def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): | |
| 231 """For a failing run, uploads the failing image, reference image (if | |
| 232 supplied), and diff image (if reference image was supplied) to cloud | |
| 233 storage. This subsumes the functionality of the | |
| 234 archive_gpu_pixel_test_results.py script.""" | |
| 235 machine_name = re.sub(r'\W+', '_', self.options.test_machine_name) | |
| 236 upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) | |
| 237 base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) | |
| 238 image_name_with_revision = '%s_%s.png' % ( | |
| 239 image_name, self.options.build_revision) | |
| 240 self._UploadBitmapToCloudStorage( | |
| 241 base_bucket + '/gen', image_name_with_revision, screenshot, | |
| 242 public=True) | |
| 243 if ref_img is not None: | |
| 244 self._UploadBitmapToCloudStorage( | |
| 245 base_bucket + '/ref', image_name_with_revision, ref_img, public=True) | |
| 246 diff_img = image_util.Diff(screenshot, ref_img) | |
| 247 self._UploadBitmapToCloudStorage( | |
| 248 base_bucket + '/diff', image_name_with_revision, diff_img, | |
| 249 public=True) | |
| 250 print ('See http://%s.commondatastorage.googleapis.com/' | |
| 251 'view_test_results.html?%s for this run\'s test results') % ( | |
| 252 error_image_cloud_storage_bucket, upload_dir) | |
| 253 | |
| 254 def _ValidateScreenshotSamples(self, tab, url, | |
| 255 screenshot, expectations, device_pixel_ratio): | |
| 256 """Samples the given screenshot and verifies pixel color values. | |
| 257 The sample locations and expected color values are given in expectations. | |
| 258 In case any of the samples do not match the expected color, it raises | |
| 259 a Failure and dumps the screenshot locally or cloud storage depending on | |
| 260 what machine the test is being run.""" | |
| 261 try: | |
| 262 _CompareScreenshotSamples(tab, screenshot, expectations, | |
| 263 device_pixel_ratio, | |
| 264 self.options.test_machine_name) | |
| 265 except legacy_page_test.Failure: | |
| 266 image_name = self._UrlToImageName(url) | |
| 267 if self.options.test_machine_name: | |
| 268 self._UploadErrorImagesToCloudStorage(image_name, screenshot, None) | |
| 269 else: | |
| 270 self._WriteErrorImages(self.options.generated_dir, image_name, | |
| 271 screenshot, None) | |
| 272 raise | |
| 273 | |
| 274 | |
| 275 class CloudStorageTestBase(gpu_test_base.TestBase): | |
| 276 @classmethod | |
| 277 def AddBenchmarkCommandLineArgs(cls, group): | |
| 278 group.add_option('--build-revision', | |
| 279 help='Chrome revision being tested.', | |
| 280 default="unknownrev") | |
| 281 group.add_option('--upload-refimg-to-cloud-storage', | |
| 282 dest='upload_refimg_to_cloud_storage', | |
| 283 action='store_true', default=False, | |
| 284 help='Upload resulting images to cloud storage as reference images') | |
| 285 group.add_option('--download-refimg-from-cloud-storage', | |
| 286 dest='download_refimg_from_cloud_storage', | |
| 287 action='store_true', default=False, | |
| 288 help='Download reference images from cloud storage') | |
| 289 group.add_option('--refimg-cloud-storage-bucket', | |
| 290 help='Name of the cloud storage bucket to use for reference images; ' | |
| 291 'required with --upload-refimg-to-cloud-storage and ' | |
| 292 '--download-refimg-from-cloud-storage. Example: ' | |
| 293 '"chromium-gpu-archive/reference-images"') | |
| 294 group.add_option('--os-type', | |
| 295 help='Type of operating system on which the pixel test is being run, ' | |
| 296 'used only to distinguish different operating systems with the same ' | |
| 297 'graphics card. Any value is acceptable, but canonical values are ' | |
| 298 '"win", "mac", and "linux", and probably, eventually, "chromeos" ' | |
| 299 'and "android").', | |
| 300 default='') | |
| 301 group.add_option('--test-machine-name', | |
| 302 help='Name of the test machine. Specifying this argument causes this ' | |
| 303 'script to upload failure images and diffs to cloud storage directly, ' | |
| 304 'instead of relying on the archive_gpu_pixel_test_results.py script.', | |
| 305 default='') | |
| 306 group.add_option('--generated-dir', | |
| 307 help='Overrides the default on-disk location for generated test images ' | |
| 308 '(only used for local testing without a cloud storage account)', | |
| 309 default=default_generated_data_dir) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2618983004:
Port Maps test to browser_test_runner harness. (Closed)
