| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Base classes for a test and validator which upload results | 5 """Base classes for a test and validator which upload results |
| 6 (reference images, error images) to cloud storage.""" | 6 (reference images, error images) to cloud storage.""" |
| 7 | 7 |
| 8 import os | 8 import os |
| 9 import re | 9 import re |
| 10 import tempfile | 10 import tempfile |
| 11 | 11 |
| 12 from telemetry import benchmark | 12 from telemetry import benchmark |
| 13 from telemetry.core import bitmap | 13 from telemetry.image_processing import image_util |
| 14 from telemetry.image_processing import rgba_color |
| 14 from telemetry.page import page_test | 15 from telemetry.page import page_test |
| 15 from telemetry.util import cloud_storage | 16 from telemetry.util import cloud_storage |
| 16 | 17 |
| 17 | 18 |
| 18 test_data_dir = os.path.abspath(os.path.join( | 19 test_data_dir = os.path.abspath(os.path.join( |
| 19 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) | 20 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) |
| 20 | 21 |
| 21 default_generated_data_dir = os.path.join(test_data_dir, 'generated') | 22 default_generated_data_dir = os.path.join(test_data_dir, 'generated') |
| 22 | 23 |
| 23 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' | 24 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' |
| 24 | 25 |
| 25 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio): | 26 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio): |
| 26 for expectation in expectations: | 27 for expectation in expectations: |
| 27 location = expectation["location"] | 28 location = expectation["location"] |
| 28 x = int(location[0] * device_pixel_ratio) | 29 x = int(location[0] * device_pixel_ratio) |
| 29 y = int(location[1] * device_pixel_ratio) | 30 y = int(location[1] * device_pixel_ratio) |
| 30 | 31 |
| 31 if x < 0 or y < 0 or x > screenshot.width or y > screenshot.height: | 32 if (x < 0 or y < 0 or x > image_util.Width(screenshot) or |
| 33 y > image_util.Height(screenshot)): |
| 32 raise page_test.Failure( | 34 raise page_test.Failure( |
| 33 'Expected pixel location [%d, %d] is out of range on [%d, %d] image' % | 35 'Expected pixel location [%d, %d] is out of range on [%d, %d] image' % |
| 34 (x, y, screenshot.width, screenshot.height)) | 36 (x, y, image_util.Width(screenshot), image_util.Height(screenshot))) |
| 35 | 37 |
| 36 actual_color = screenshot.GetPixelColor(x, y) | 38 actual_color = image_util.GetPixelColor(screenshot, x, y) |
| 37 expected_color = bitmap.RgbaColor( | 39 expected_color = rgba_color.RgbaColor( |
| 38 expectation["color"][0], | 40 expectation["color"][0], |
| 39 expectation["color"][1], | 41 expectation["color"][1], |
| 40 expectation["color"][2]) | 42 expectation["color"][2]) |
| 41 if not actual_color.IsEqual(expected_color, expectation["tolerance"]): | 43 if not actual_color.IsEqual(expected_color, expectation["tolerance"]): |
| 42 raise page_test.Failure('Expected pixel at ' + str(location) + | 44 raise page_test.Failure('Expected pixel at ' + str(location) + |
| 43 ' to be ' + | 45 ' to be ' + |
| 44 str(expectation["color"]) + " but got [" + | 46 str(expectation["color"]) + " but got [" + |
| 45 str(actual_color.r) + ", " + | 47 str(actual_color.r) + ", " + |
| 46 str(actual_color.g) + ", " + | 48 str(actual_color.g) + ", " + |
| 47 str(actual_color.b) + "]") | 49 str(actual_color.b) + "]") |
| (...skipping 17 matching lines...) Expand all Loading... |
| 65 def _UrlToImageName(self, url): | 67 def _UrlToImageName(self, url): |
| 66 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) | 68 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) |
| 67 image_name = re.sub(r'\.\./', '', image_name) | 69 image_name = re.sub(r'\.\./', '', image_name) |
| 68 image_name = re.sub(r'(\.|/|-)', '_', image_name) | 70 image_name = re.sub(r'(\.|/|-)', '_', image_name) |
| 69 return image_name | 71 return image_name |
| 70 | 72 |
| 71 def _WriteImage(self, image_path, png_image): | 73 def _WriteImage(self, image_path, png_image): |
| 72 output_dir = os.path.dirname(image_path) | 74 output_dir = os.path.dirname(image_path) |
| 73 if not os.path.exists(output_dir): | 75 if not os.path.exists(output_dir): |
| 74 os.makedirs(output_dir) | 76 os.makedirs(output_dir) |
| 75 png_image.WritePngFile(image_path) | 77 image_util.WritePngFile(png_image, image_path) |
| 76 | 78 |
| 77 def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): | 79 def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): |
| 78 full_image_name = img_name + '_' + str(self.options.build_revision) | 80 full_image_name = img_name + '_' + str(self.options.build_revision) |
| 79 full_image_name = full_image_name + '.png' | 81 full_image_name = full_image_name + '.png' |
| 80 | 82 |
| 81 # Always write the failing image. | 83 # Always write the failing image. |
| 82 self._WriteImage( | 84 self._WriteImage( |
| 83 os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) | 85 os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) |
| 84 | 86 |
| 85 if ref_png: | 87 if ref_png is not None: |
| 86 # Save the reference image. | 88 # Save the reference image. |
| 87 # This ensures that we get the right revision number. | 89 # This ensures that we get the right revision number. |
| 88 self._WriteImage( | 90 self._WriteImage( |
| 89 os.path.join(img_dir, full_image_name), ref_png) | 91 os.path.join(img_dir, full_image_name), ref_png) |
| 90 | 92 |
| 91 # Save the difference image. | 93 # Save the difference image. |
| 92 diff_png = screenshot.Diff(ref_png) | 94 diff_png = image_util.Diff(screenshot, ref_png) |
| 93 self._WriteImage( | 95 self._WriteImage( |
| 94 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) | 96 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) |
| 95 | 97 |
| 96 ### | 98 ### |
| 97 ### Cloud storage code path -- the bots use this. | 99 ### Cloud storage code path -- the bots use this. |
| 98 ### | 100 ### |
| 99 | 101 |
| 100 def _ComputeGpuInfo(self, tab): | 102 def _ComputeGpuInfo(self, tab): |
| 101 if ((self.vendor_id and self.device_id) or | 103 if ((self.vendor_id and self.device_id) or |
| 102 (self.vendor_string and self.device_string)): | 104 (self.vendor_string and self.device_string)): |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 135 img_name, | 137 img_name, |
| 136 page.revision, | 138 page.revision, |
| 137 self._FormatGpuInfo(tab)) | 139 self._FormatGpuInfo(tab)) |
| 138 | 140 |
| 139 def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): | 141 def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): |
| 140 # This sequence of steps works on all platforms to write a temporary | 142 # This sequence of steps works on all platforms to write a temporary |
| 141 # PNG to disk, following the pattern in bitmap_unittest.py. The key to | 143 # PNG to disk, following the pattern in bitmap_unittest.py. The key to |
| 142 # avoiding PermissionErrors seems to be to not actually try to write to | 144 # avoiding PermissionErrors seems to be to not actually try to write to |
| 143 # the temporary file object, but to re-open its name for all operations. | 145 # the temporary file object, but to re-open its name for all operations. |
| 144 temp_file = tempfile.NamedTemporaryFile().name | 146 temp_file = tempfile.NamedTemporaryFile().name |
| 145 bitmap.WritePngFile(temp_file) | 147 image_util.WritePngFile(bitmap, temp_file) |
| 146 cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) | 148 cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) |
| 147 | 149 |
| 148 def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): | 150 def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): |
| 149 """Uploads the screenshot to cloud storage as the reference image | 151 """Uploads the screenshot to cloud storage as the reference image |
| 150 for this test, unless it already exists. Returns True if the | 152 for this test, unless it already exists. Returns True if the |
| 151 upload was actually performed.""" | 153 upload was actually performed.""" |
| 152 if not self.options.refimg_cloud_storage_bucket: | 154 if not self.options.refimg_cloud_storage_bucket: |
| 153 raise Exception('--refimg-cloud-storage-bucket argument is required') | 155 raise Exception('--refimg-cloud-storage-bucket argument is required') |
| 154 cloud_name = self._FormatReferenceImageName(img_name, page, tab) | 156 cloud_name = self._FormatReferenceImageName(img_name, page, tab) |
| 155 if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, | 157 if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, |
| 156 cloud_name): | 158 cloud_name): |
| 157 self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, | 159 self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, |
| 158 cloud_name, | 160 cloud_name, |
| 159 screenshot) | 161 screenshot) |
| 160 return True | 162 return True |
| 161 return False | 163 return False |
| 162 | 164 |
| 163 def _DownloadFromCloudStorage(self, img_name, page, tab): | 165 def _DownloadFromCloudStorage(self, img_name, page, tab): |
| 164 """Downloads the reference image for the given test from cloud | 166 """Downloads the reference image for the given test from cloud |
| 165 storage, returning it as a Telemetry Bitmap object.""" | 167 storage, returning it as a Telemetry Bitmap object.""" |
| 166 # TODO(kbr): there's a race condition between the deletion of the | 168 # TODO(kbr): there's a race condition between the deletion of the |
| 167 # temporary file and gsutil's overwriting it. | 169 # temporary file and gsutil's overwriting it. |
| 168 if not self.options.refimg_cloud_storage_bucket: | 170 if not self.options.refimg_cloud_storage_bucket: |
| 169 raise Exception('--refimg-cloud-storage-bucket argument is required') | 171 raise Exception('--refimg-cloud-storage-bucket argument is required') |
| 170 temp_file = tempfile.NamedTemporaryFile().name | 172 temp_file = tempfile.NamedTemporaryFile().name |
| 171 cloud_storage.Get(self.options.refimg_cloud_storage_bucket, | 173 cloud_storage.Get(self.options.refimg_cloud_storage_bucket, |
| 172 self._FormatReferenceImageName(img_name, page, tab), | 174 self._FormatReferenceImageName(img_name, page, tab), |
| 173 temp_file) | 175 temp_file) |
| 174 return bitmap.Bitmap.FromPngFile(temp_file) | 176 return image_util.FromPngFile(temp_file) |
| 175 | 177 |
| 176 def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): | 178 def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): |
| 177 """For a failing run, uploads the failing image, reference image (if | 179 """For a failing run, uploads the failing image, reference image (if |
| 178 supplied), and diff image (if reference image was supplied) to cloud | 180 supplied), and diff image (if reference image was supplied) to cloud |
| 179 storage. This subsumes the functionality of the | 181 storage. This subsumes the functionality of the |
| 180 archive_gpu_pixel_test_results.py script.""" | 182 archive_gpu_pixel_test_results.py script.""" |
| 181 machine_name = re.sub('\W+', '_', self.options.test_machine_name) | 183 machine_name = re.sub('\W+', '_', self.options.test_machine_name) |
| 182 upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) | 184 upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) |
| 183 base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) | 185 base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) |
| 184 image_name_with_revision = '%s_%s.png' % ( | 186 image_name_with_revision = '%s_%s.png' % ( |
| 185 image_name, self.options.build_revision) | 187 image_name, self.options.build_revision) |
| 186 self._UploadBitmapToCloudStorage( | 188 self._UploadBitmapToCloudStorage( |
| 187 base_bucket + '/gen', image_name_with_revision, screenshot, | 189 base_bucket + '/gen', image_name_with_revision, screenshot, |
| 188 public=True) | 190 public=True) |
| 189 if ref_img: | 191 if ref_img is not None: |
| 190 self._UploadBitmapToCloudStorage( | 192 self._UploadBitmapToCloudStorage( |
| 191 base_bucket + '/ref', image_name_with_revision, ref_img, public=True) | 193 base_bucket + '/ref', image_name_with_revision, ref_img, public=True) |
| 192 diff_img = screenshot.Diff(ref_img) | 194 diff_img = image_util.Diff(screenshot, ref_img) |
| 193 self._UploadBitmapToCloudStorage( | 195 self._UploadBitmapToCloudStorage( |
| 194 base_bucket + '/diff', image_name_with_revision, diff_img, | 196 base_bucket + '/diff', image_name_with_revision, diff_img, |
| 195 public=True) | 197 public=True) |
| 196 print ('See http://%s.commondatastorage.googleapis.com/' | 198 print ('See http://%s.commondatastorage.googleapis.com/' |
| 197 'view_test_results.html?%s for this run\'s test results') % ( | 199 'view_test_results.html?%s for this run\'s test results') % ( |
| 198 error_image_cloud_storage_bucket, upload_dir) | 200 error_image_cloud_storage_bucket, upload_dir) |
| 199 | 201 |
| 200 def _ValidateScreenshotSamples(self, url, | 202 def _ValidateScreenshotSamples(self, url, |
| 201 screenshot, expectations, device_pixel_ratio): | 203 screenshot, expectations, device_pixel_ratio): |
| 202 """Samples the given screenshot and verifies pixel color values. | 204 """Samples the given screenshot and verifies pixel color values. |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 244 default='') | 246 default='') |
| 245 group.add_option('--test-machine-name', | 247 group.add_option('--test-machine-name', |
| 246 help='Name of the test machine. Specifying this argument causes this ' | 248 help='Name of the test machine. Specifying this argument causes this ' |
| 247 'script to upload failure images and diffs to cloud storage directly, ' | 249 'script to upload failure images and diffs to cloud storage directly, ' |
| 248 'instead of relying on the archive_gpu_pixel_test_results.py script.', | 250 'instead of relying on the archive_gpu_pixel_test_results.py script.', |
| 249 default='') | 251 default='') |
| 250 group.add_option('--generated-dir', | 252 group.add_option('--generated-dir', |
| 251 help='Overrides the default on-disk location for generated test images ' | 253 help='Overrides the default on-disk location for generated test images ' |
| 252 '(only used for local testing without a cloud storage account)', | 254 '(only used for local testing without a cloud storage account)', |
| 253 default=default_generated_data_dir) | 255 default=default_generated_data_dir) |
| OLD | NEW |