| OLD | NEW | 
|    1 # Copyright 2013 The Chromium Authors. All rights reserved. |    1 # Copyright 2013 The Chromium Authors. All rights reserved. | 
|    2 # Use of this source code is governed by a BSD-style license that can be |    2 # Use of this source code is governed by a BSD-style license that can be | 
|    3 # found in the LICENSE file. |    3 # found in the LICENSE file. | 
|    4  |    4  | 
|    5 """Base classes for a test and validator which upload results |    5 """Base classes for a test and validator which upload results | 
|    6 (reference images, error images) to cloud storage.""" |    6 (reference images, error images) to cloud storage.""" | 
|    7  |    7  | 
|    8 import os |    8 import os | 
|    9 import re |    9 import re | 
|   10 import tempfile |   10 import tempfile | 
|   11  |   11  | 
|   12 from telemetry import benchmark |   12 from telemetry import benchmark | 
|   13 from telemetry.core import bitmap |   13 from telemetry.image_processing import image_util | 
 |   14 from telemetry.image_processing import rgba_color | 
|   14 from telemetry.page import page_test |   15 from telemetry.page import page_test | 
|   15 from telemetry.util import cloud_storage |   16 from telemetry.util import cloud_storage | 
|   16  |   17  | 
|   17  |   18  | 
|   18 test_data_dir = os.path.abspath(os.path.join( |   19 test_data_dir = os.path.abspath(os.path.join( | 
|   19     os.path.dirname(__file__), '..', '..', 'data', 'gpu')) |   20     os.path.dirname(__file__), '..', '..', 'data', 'gpu')) | 
|   20  |   21  | 
|   21 default_generated_data_dir = os.path.join(test_data_dir, 'generated') |   22 default_generated_data_dir = os.path.join(test_data_dir, 'generated') | 
|   22  |   23  | 
|   23 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' |   24 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' | 
|   24  |   25  | 
|   25 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio): |   26 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio): | 
|   26   for expectation in expectations: |   27   for expectation in expectations: | 
|   27     location = expectation["location"] |   28     location = expectation["location"] | 
|   28     x = int(location[0] * device_pixel_ratio) |   29     x = int(location[0] * device_pixel_ratio) | 
|   29     y = int(location[1] * device_pixel_ratio) |   30     y = int(location[1] * device_pixel_ratio) | 
|   30  |   31  | 
|   31     if x < 0 or y < 0 or x > screenshot.width or y > screenshot.height: |   32     if (x < 0 or y < 0 or x > image_util.Width(screenshot) or | 
 |   33         y > image_util.Height(screenshot)): | 
|   32       raise page_test.Failure( |   34       raise page_test.Failure( | 
|   33           'Expected pixel location [%d, %d] is out of range on [%d, %d] image' % |   35           'Expected pixel location [%d, %d] is out of range on [%d, %d] image' % | 
|   34           (x, y, screenshot.width, screenshot.height)) |   36           (x, y, image_util.Width(screenshot), image_util.Height(screenshot))) | 
|   35  |   37  | 
|   36     actual_color = screenshot.GetPixelColor(x, y) |   38     actual_color = image_util.GetPixelColor(screenshot, x, y) | 
|   37     expected_color = bitmap.RgbaColor( |   39     expected_color = rgba_color.RgbaColor( | 
|   38         expectation["color"][0], |   40         expectation["color"][0], | 
|   39         expectation["color"][1], |   41         expectation["color"][1], | 
|   40         expectation["color"][2]) |   42         expectation["color"][2]) | 
|   41     if not actual_color.IsEqual(expected_color, expectation["tolerance"]): |   43     if not actual_color.IsEqual(expected_color, expectation["tolerance"]): | 
|   42       raise page_test.Failure('Expected pixel at ' + str(location) + |   44       raise page_test.Failure('Expected pixel at ' + str(location) + | 
|   43           ' to be ' + |   45           ' to be ' + | 
|   44           str(expectation["color"]) + " but got [" + |   46           str(expectation["color"]) + " but got [" + | 
|   45           str(actual_color.r) + ", " + |   47           str(actual_color.r) + ", " + | 
|   46           str(actual_color.g) + ", " + |   48           str(actual_color.g) + ", " + | 
|   47           str(actual_color.b) + "]") |   49           str(actual_color.b) + "]") | 
| (...skipping 17 matching lines...) Expand all  Loading... | 
|   65   def _UrlToImageName(self, url): |   67   def _UrlToImageName(self, url): | 
|   66     image_name = re.sub(r'^(http|https|file)://(/*)', '', url) |   68     image_name = re.sub(r'^(http|https|file)://(/*)', '', url) | 
|   67     image_name = re.sub(r'\.\./', '', image_name) |   69     image_name = re.sub(r'\.\./', '', image_name) | 
|   68     image_name = re.sub(r'(\.|/|-)', '_', image_name) |   70     image_name = re.sub(r'(\.|/|-)', '_', image_name) | 
|   69     return image_name |   71     return image_name | 
|   70  |   72  | 
|   71   def _WriteImage(self, image_path, png_image): |   73   def _WriteImage(self, image_path, png_image): | 
|   72     output_dir = os.path.dirname(image_path) |   74     output_dir = os.path.dirname(image_path) | 
|   73     if not os.path.exists(output_dir): |   75     if not os.path.exists(output_dir): | 
|   74       os.makedirs(output_dir) |   76       os.makedirs(output_dir) | 
|   75     png_image.WritePngFile(image_path) |   77     image_util.WritePngFile(png_image, image_path) | 
|   76  |   78  | 
|   77   def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): |   79   def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): | 
|   78     full_image_name = img_name + '_' + str(self.options.build_revision) |   80     full_image_name = img_name + '_' + str(self.options.build_revision) | 
|   79     full_image_name = full_image_name + '.png' |   81     full_image_name = full_image_name + '.png' | 
|   80  |   82  | 
|   81     # Always write the failing image. |   83     # Always write the failing image. | 
|   82     self._WriteImage( |   84     self._WriteImage( | 
|   83         os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) |   85         os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) | 
|   84  |   86  | 
|   85     if ref_png: |   87     if ref_png is not None: | 
|   86       # Save the reference image. |   88       # Save the reference image. | 
|   87       # This ensures that we get the right revision number. |   89       # This ensures that we get the right revision number. | 
|   88       self._WriteImage( |   90       self._WriteImage( | 
|   89           os.path.join(img_dir, full_image_name), ref_png) |   91           os.path.join(img_dir, full_image_name), ref_png) | 
|   90  |   92  | 
|   91       # Save the difference image. |   93       # Save the difference image. | 
|   92       diff_png = screenshot.Diff(ref_png) |   94       diff_png = image_util.Diff(screenshot, ref_png) | 
|   93       self._WriteImage( |   95       self._WriteImage( | 
|   94           os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) |   96           os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) | 
|   95  |   97  | 
|   96   ### |   98   ### | 
|   97   ### Cloud storage code path -- the bots use this. |   99   ### Cloud storage code path -- the bots use this. | 
|   98   ### |  100   ### | 
|   99  |  101  | 
|  100   def _ComputeGpuInfo(self, tab): |  102   def _ComputeGpuInfo(self, tab): | 
|  101     if ((self.vendor_id and self.device_id) or |  103     if ((self.vendor_id and self.device_id) or | 
|  102         (self.vendor_string and self.device_string)): |  104         (self.vendor_string and self.device_string)): | 
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  135       img_name, |  137       img_name, | 
|  136       page.revision, |  138       page.revision, | 
|  137       self._FormatGpuInfo(tab)) |  139       self._FormatGpuInfo(tab)) | 
|  138  |  140  | 
|  139   def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): |  141   def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): | 
|  140     # This sequence of steps works on all platforms to write a temporary |  142     # This sequence of steps works on all platforms to write a temporary | 
|  141     # PNG to disk, following the pattern in bitmap_unittest.py. The key to |  143     # PNG to disk, following the pattern in bitmap_unittest.py. The key to | 
|  142     # avoiding PermissionErrors seems to be to not actually try to write to |  144     # avoiding PermissionErrors seems to be to not actually try to write to | 
|  143     # the temporary file object, but to re-open its name for all operations. |  145     # the temporary file object, but to re-open its name for all operations. | 
|  144     temp_file = tempfile.NamedTemporaryFile().name |  146     temp_file = tempfile.NamedTemporaryFile().name | 
|  145     bitmap.WritePngFile(temp_file) |  147     image_util.WritePngFile(bitmap, temp_file) | 
|  146     cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) |  148     cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) | 
|  147  |  149  | 
|  148   def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): |  150   def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): | 
|  149     """Uploads the screenshot to cloud storage as the reference image |  151     """Uploads the screenshot to cloud storage as the reference image | 
|  150     for this test, unless it already exists. Returns True if the |  152     for this test, unless it already exists. Returns True if the | 
|  151     upload was actually performed.""" |  153     upload was actually performed.""" | 
|  152     if not self.options.refimg_cloud_storage_bucket: |  154     if not self.options.refimg_cloud_storage_bucket: | 
|  153       raise Exception('--refimg-cloud-storage-bucket argument is required') |  155       raise Exception('--refimg-cloud-storage-bucket argument is required') | 
|  154     cloud_name = self._FormatReferenceImageName(img_name, page, tab) |  156     cloud_name = self._FormatReferenceImageName(img_name, page, tab) | 
|  155     if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, |  157     if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, | 
|  156                                 cloud_name): |  158                                 cloud_name): | 
|  157       self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, |  159       self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, | 
|  158                                        cloud_name, |  160                                        cloud_name, | 
|  159                                        screenshot) |  161                                        screenshot) | 
|  160       return True |  162       return True | 
|  161     return False |  163     return False | 
|  162  |  164  | 
|  163   def _DownloadFromCloudStorage(self, img_name, page, tab): |  165   def _DownloadFromCloudStorage(self, img_name, page, tab): | 
|  164     """Downloads the reference image for the given test from cloud |  166     """Downloads the reference image for the given test from cloud | 
|  165     storage, returning it as a Telemetry Bitmap object.""" |  167     storage, returning it as a Telemetry Bitmap object.""" | 
|  166     # TODO(kbr): there's a race condition between the deletion of the |  168     # TODO(kbr): there's a race condition between the deletion of the | 
|  167     # temporary file and gsutil's overwriting it. |  169     # temporary file and gsutil's overwriting it. | 
|  168     if not self.options.refimg_cloud_storage_bucket: |  170     if not self.options.refimg_cloud_storage_bucket: | 
|  169       raise Exception('--refimg-cloud-storage-bucket argument is required') |  171       raise Exception('--refimg-cloud-storage-bucket argument is required') | 
|  170     temp_file = tempfile.NamedTemporaryFile().name |  172     temp_file = tempfile.NamedTemporaryFile().name | 
|  171     cloud_storage.Get(self.options.refimg_cloud_storage_bucket, |  173     cloud_storage.Get(self.options.refimg_cloud_storage_bucket, | 
|  172                       self._FormatReferenceImageName(img_name, page, tab), |  174                       self._FormatReferenceImageName(img_name, page, tab), | 
|  173                       temp_file) |  175                       temp_file) | 
|  174     return bitmap.Bitmap.FromPngFile(temp_file) |  176     return image_util.FromPngFile(temp_file) | 
|  175  |  177  | 
|  176   def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): |  178   def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): | 
|  177     """For a failing run, uploads the failing image, reference image (if |  179     """For a failing run, uploads the failing image, reference image (if | 
|  178     supplied), and diff image (if reference image was supplied) to cloud |  180     supplied), and diff image (if reference image was supplied) to cloud | 
|  179     storage. This subsumes the functionality of the |  181     storage. This subsumes the functionality of the | 
|  180     archive_gpu_pixel_test_results.py script.""" |  182     archive_gpu_pixel_test_results.py script.""" | 
|  181     machine_name = re.sub('\W+', '_', self.options.test_machine_name) |  183     machine_name = re.sub('\W+', '_', self.options.test_machine_name) | 
|  182     upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) |  184     upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) | 
|  183     base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) |  185     base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) | 
|  184     image_name_with_revision = '%s_%s.png' % ( |  186     image_name_with_revision = '%s_%s.png' % ( | 
|  185       image_name, self.options.build_revision) |  187       image_name, self.options.build_revision) | 
|  186     self._UploadBitmapToCloudStorage( |  188     self._UploadBitmapToCloudStorage( | 
|  187       base_bucket + '/gen', image_name_with_revision, screenshot, |  189       base_bucket + '/gen', image_name_with_revision, screenshot, | 
|  188       public=True) |  190       public=True) | 
|  189     if ref_img: |  191     if ref_img is not None: | 
|  190       self._UploadBitmapToCloudStorage( |  192       self._UploadBitmapToCloudStorage( | 
|  191         base_bucket + '/ref', image_name_with_revision, ref_img, public=True) |  193         base_bucket + '/ref', image_name_with_revision, ref_img, public=True) | 
|  192       diff_img = screenshot.Diff(ref_img) |  194       diff_img = image_util.Diff(screenshot, ref_img) | 
|  193       self._UploadBitmapToCloudStorage( |  195       self._UploadBitmapToCloudStorage( | 
|  194         base_bucket + '/diff', image_name_with_revision, diff_img, |  196         base_bucket + '/diff', image_name_with_revision, diff_img, | 
|  195         public=True) |  197         public=True) | 
|  196     print ('See http://%s.commondatastorage.googleapis.com/' |  198     print ('See http://%s.commondatastorage.googleapis.com/' | 
|  197            'view_test_results.html?%s for this run\'s test results') % ( |  199            'view_test_results.html?%s for this run\'s test results') % ( | 
|  198       error_image_cloud_storage_bucket, upload_dir) |  200       error_image_cloud_storage_bucket, upload_dir) | 
|  199  |  201  | 
|  200   def _ValidateScreenshotSamples(self, url, |  202   def _ValidateScreenshotSamples(self, url, | 
|  201                                  screenshot, expectations, device_pixel_ratio): |  203                                  screenshot, expectations, device_pixel_ratio): | 
|  202     """Samples the given screenshot and verifies pixel color values. |  204     """Samples the given screenshot and verifies pixel color values. | 
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  244         default='') |  246         default='') | 
|  245     group.add_option('--test-machine-name', |  247     group.add_option('--test-machine-name', | 
|  246         help='Name of the test machine. Specifying this argument causes this ' |  248         help='Name of the test machine. Specifying this argument causes this ' | 
|  247         'script to upload failure images and diffs to cloud storage directly, ' |  249         'script to upload failure images and diffs to cloud storage directly, ' | 
|  248         'instead of relying on the archive_gpu_pixel_test_results.py script.', |  250         'instead of relying on the archive_gpu_pixel_test_results.py script.', | 
|  249         default='') |  251         default='') | 
|  250     group.add_option('--generated-dir', |  252     group.add_option('--generated-dir', | 
|  251         help='Overrides the default on-disk location for generated test images ' |  253         help='Overrides the default on-disk location for generated test images ' | 
|  252         '(only used for local testing without a cloud storage account)', |  254         '(only used for local testing without a cloud storage account)', | 
|  253         default=default_generated_data_dir) |  255         default=default_generated_data_dir) | 
| OLD | NEW |