Index: content/test/gpu/gpu_tests/cloud_storage_integration_test_base.py |
diff --git a/content/test/gpu/gpu_tests/cloud_storage_integration_test_base.py b/content/test/gpu/gpu_tests/cloud_storage_integration_test_base.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..56230326be9bfc36c0396294131f3728cf210065 |
--- /dev/null |
+++ b/content/test/gpu/gpu_tests/cloud_storage_integration_test_base.py |
@@ -0,0 +1,362 @@ |
+# Copyright 2016 The Chromium Authors. All rights reserved. |
+# Use of this source code is governed by a BSD-style license that can be |
+# found in the LICENSE file. |
+ |
+"""Base classes for a test which uploads results (reference images, |
+error images) to cloud storage.""" |
+ |
+import logging |
+import os |
+import re |
+import tempfile |
+ |
+from py_utils import cloud_storage |
+from telemetry.util import image_util |
+from telemetry.util import rgba_color |
+ |
+from gpu_tests import gpu_integration_test |
+ |
+test_data_dir = os.path.abspath(os.path.join( |
+ os.path.dirname(__file__), '..', '..', 'data', 'gpu')) |
+ |
+default_generated_data_dir = os.path.join(test_data_dir, 'generated') |
+ |
+error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' |
+ |
+ |
+class _ReferenceImageParameters(object): |
+ def __init__(self): |
+ # Parameters for cloud storage reference images. |
+ self.vendor_id = None |
+ self.device_id = None |
+ self.vendor_string = None |
+ self.device_string = None |
+ self.msaa = False |
+ self.model_name = None |
+ |
+ |
+class CloudStorageIntegrationTestBase(gpu_integration_test.GpuIntegrationTest): |
+ # This class is abstract; don't warn about the superclass's abstract |
+ # methods that aren't overridden. |
+ # pylint: disable=abstract-method |
+ |
+ # This information is class-scoped, so that it can be shared across |
+ # invocations of tests; but it's zapped every time the browser is |
+ # restarted with different command line arguments. |
+ _reference_image_parameters = None |
+ |
+ # The command line options (which are passed to subclasses' |
+ # GenerateGpuTests) *must* be configured here, via a call to |
+ # SetParsedCommandLineOptions. If they are not, an error will be |
+ # raised when running the tests. |
+ _parsed_command_line_options = None |
+ |
+ @classmethod |
+ def SetParsedCommandLineOptions(cls, options): |
+ cls._parsed_command_line_options = options |
+ |
+ @classmethod |
+ def GetParsedCommandLineOptions(cls): |
+ return cls._parsed_command_line_options |
+ |
+ @classmethod |
+ def AddCommandlineArgs(cls, parser): |
+ parser.add_option( |
+ '--build-revision', |
+ help='Chrome revision being tested.', |
+ default="unknownrev") |
+ parser.add_option( |
+ '--upload-refimg-to-cloud-storage', |
+ dest='upload_refimg_to_cloud_storage', |
+ action='store_true', default=False, |
+ help='Upload resulting images to cloud storage as reference images') |
+ parser.add_option( |
+ '--download-refimg-from-cloud-storage', |
+ dest='download_refimg_from_cloud_storage', |
+ action='store_true', default=False, |
+ help='Download reference images from cloud storage') |
+ parser.add_option( |
+ '--refimg-cloud-storage-bucket', |
+ help='Name of the cloud storage bucket to use for reference images; ' |
+ 'required with --upload-refimg-to-cloud-storage and ' |
+ '--download-refimg-from-cloud-storage. Example: ' |
+ '"chromium-gpu-archive/reference-images"') |
+ parser.add_option( |
+ '--os-type', |
+ help='Type of operating system on which the pixel test is being run, ' |
+ 'used only to distinguish different operating systems with the same ' |
+ 'graphics card. Any value is acceptable, but canonical values are ' |
+ '"win", "mac", and "linux", and probably, eventually, "chromeos" ' |
+ 'and "android").', |
+ default='') |
+ parser.add_option( |
+ '--test-machine-name', |
+ help='Name of the test machine. Specifying this argument causes this ' |
+ 'script to upload failure images and diffs to cloud storage directly, ' |
+ 'instead of relying on the archive_gpu_pixel_test_results.py script.', |
+ default='') |
+ parser.add_option( |
+ '--generated-dir', |
+ help='Overrides the default on-disk location for generated test images ' |
+ '(only used for local testing without a cloud storage account)', |
+ default=default_generated_data_dir) |
+ |
+ def _CompareScreenshotSamples(self, tab, screenshot, expected_colors, |
+ device_pixel_ratio, test_machine_name): |
+ # First scan through the expected_colors and see if there are any scale |
+ # factor overrides that would preempt the device pixel ratio. This |
+ # is mainly a workaround for complex tests like the Maps test. |
+ for expectation in expected_colors: |
+ if 'scale_factor_overrides' in expectation: |
+ for override in expectation['scale_factor_overrides']: |
+ # Require exact matches to avoid confusion, because some |
+ # machine models and names might be subsets of others |
+ # (e.g. Nexus 5 vs Nexus 5X). |
+ if ('device_type' in override and |
+ (tab.browser.platform.GetDeviceTypeName() == |
+ override['device_type'])): |
+ logging.warning( |
+ 'Overriding device_pixel_ratio ' + str(device_pixel_ratio) + |
+ ' with scale factor ' + str(override['scale_factor']) + |
+ ' for device type ' + override['device_type']) |
+ device_pixel_ratio = override['scale_factor'] |
+ break |
+ if (test_machine_name and 'machine_name' in override and |
+ override["machine_name"] == test_machine_name): |
+ logging.warning( |
+ 'Overriding device_pixel_ratio ' + str(device_pixel_ratio) + |
+ ' with scale factor ' + str(override['scale_factor']) + |
+ ' for machine name ' + test_machine_name) |
+ device_pixel_ratio = override['scale_factor'] |
+ break |
+ # Only support one "scale_factor_overrides" in the expectation format. |
+ break |
+ for expectation in expected_colors: |
+ if "scale_factor_overrides" in expectation: |
+ continue |
+ location = expectation["location"] |
+ size = expectation["size"] |
+ x0 = int(location[0] * device_pixel_ratio) |
+ x1 = int((location[0] + size[0]) * device_pixel_ratio) |
+ y0 = int(location[1] * device_pixel_ratio) |
+ y1 = int((location[1] + size[1]) * device_pixel_ratio) |
+ for x in range(x0, x1): |
+ for y in range(y0, y1): |
+ if (x < 0 or y < 0 or x >= image_util.Width(screenshot) or |
+ y >= image_util.Height(screenshot)): |
+ self.fail( |
+ ('Expected pixel location [%d, %d] is out of range on ' + |
+ '[%d, %d] image') % |
+ (x, y, image_util.Width(screenshot), |
+ image_util.Height(screenshot))) |
+ |
+ actual_color = image_util.GetPixelColor(screenshot, x, y) |
+ expected_color = rgba_color.RgbaColor( |
+ expectation["color"][0], |
+ expectation["color"][1], |
+ expectation["color"][2]) |
+ if not actual_color.IsEqual(expected_color, expectation["tolerance"]): |
+ self.fail('Expected pixel at ' + str(location) + |
+ ' (actual pixel (' + str(x) + ', ' + str(y) + ')) ' + |
+ ' to be ' + |
+ str(expectation["color"]) + " but got [" + |
+ str(actual_color.r) + ", " + |
+ str(actual_color.g) + ", " + |
+ str(actual_color.b) + "]") |
+ |
+ ### |
+ ### Routines working with the local disk (only used for local |
+ ### testing without a cloud storage account -- the bots do not use |
+ ### this code path). |
+ ### |
+ |
+ def _UrlToImageName(self, url): |
+ image_name = re.sub(r'^(http|https|file)://(/*)', '', url) |
+ image_name = re.sub(r'\.\./', '', image_name) |
+ image_name = re.sub(r'(\.|/|-)', '_', image_name) |
+ return image_name |
+ |
+ def _WriteImage(self, image_path, png_image): |
+ output_dir = os.path.dirname(image_path) |
+ if not os.path.exists(output_dir): |
+ os.makedirs(output_dir) |
+ image_util.WritePngFile(png_image, image_path) |
+ |
+ def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): |
+ full_image_name = img_name + '_' + str( |
+ self.GetParsedCommandLineOptions().build_revision) |
+ full_image_name = full_image_name + '.png' |
+ |
+ # Always write the failing image. |
+ self._WriteImage( |
+ os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) |
+ |
+ if ref_png is not None: |
+ # Save the reference image. |
+ # This ensures that we get the right revision number. |
+ self._WriteImage( |
+ os.path.join(img_dir, full_image_name), ref_png) |
+ |
+ # Save the difference image. |
+ diff_png = image_util.Diff(screenshot, ref_png) |
+ self._WriteImage( |
+ os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) |
+ |
+ ### |
+ ### Cloud storage code path -- the bots use this. |
+ ### |
+ |
+ @classmethod |
+ def ResetGpuInfo(cls): |
+ cls._reference_image_parameters = None |
+ |
+ @classmethod |
+ def _ComputeGpuInfo(cls, tab): |
+ if cls._reference_image_parameters: |
+ return |
+ browser = cls.browser |
+ if not browser.supports_system_info: |
+ raise Exception('System info must be supported by the browser') |
+ system_info = browser.GetSystemInfo() |
+ if not system_info.gpu: |
+ raise Exception('GPU information was absent') |
+ device = system_info.gpu.devices[0] |
+ cls._reference_image_parameters = _ReferenceImageParameters() |
+ params = cls._reference_image_parameters |
+ if device.vendor_id and device.device_id: |
+ params.vendor_id = device.vendor_id |
+ params.device_id = device.device_id |
+ elif device.vendor_string and device.device_string: |
+ params.vendor_string = device.vendor_string |
+ params.device_string = device.device_string |
+ else: |
+ raise Exception('GPU device information was incomplete') |
+ # TODO(senorblanco): This should probably be checking |
+ # for the presence of the extensions in system_info.gpu_aux_attributes |
+ # in order to check for MSAA, rather than sniffing the blacklist. |
+ params.msaa = not ( |
+ ('disable_chromium_framebuffer_multisample' in |
+ system_info.gpu.driver_bug_workarounds) or |
+ ('disable_multisample_render_to_texture' in |
+ system_info.gpu.driver_bug_workarounds)) |
+ params.model_name = system_info.model_name |
+ |
+ @classmethod |
+ def _FormatGpuInfo(cls, tab): |
+ cls._ComputeGpuInfo(tab) |
+ params = cls._reference_image_parameters |
+ msaa_string = '_msaa' if params.msaa else '_non_msaa' |
+ if params.vendor_id: |
+ return '%s_%04x_%04x%s' % ( |
+ cls.GetParsedCommandLineOptions().os_type, params.vendor_id, |
+ params.device_id, msaa_string) |
+ else: |
+ # This is the code path for Android devices. Include the model |
+ # name (e.g. "Nexus 9") in the GPU string to disambiguate |
+ # multiple devices on the waterfall which might have the same |
+ # device string ("NVIDIA Tegra") but different screen |
+ # resolutions and device pixel ratios. |
+ return '%s_%s_%s_%s%s' % ( |
+ cls.GetParsedCommandLineOptions().os_type, |
+ params.vendor_string, params.device_string, |
+ params.model_name, msaa_string) |
+ |
+ @classmethod |
+ def _FormatReferenceImageName(cls, img_name, page, tab): |
+ return '%s_v%s_%s.png' % ( |
+ img_name, |
+ page.revision, |
+ cls._FormatGpuInfo(tab)) |
+ |
+ @classmethod |
+ def _UploadBitmapToCloudStorage(cls, bucket, name, bitmap, public=False): |
+ # This sequence of steps works on all platforms to write a temporary |
+ # PNG to disk, following the pattern in bitmap_unittest.py. The key to |
+ # avoiding PermissionErrors seems to be to not actually try to write to |
+ # the temporary file object, but to re-open its name for all operations. |
+ temp_file = tempfile.NamedTemporaryFile(suffix='.png').name |
+ image_util.WritePngFile(bitmap, temp_file) |
+ cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) |
+ |
+ @classmethod |
+ def _ConditionallyUploadToCloudStorage(cls, img_name, page, tab, screenshot): |
+ """Uploads the screenshot to cloud storage as the reference image |
+ for this test, unless it already exists. Returns True if the |
+ upload was actually performed.""" |
+ if not cls.GetParsedCommandLineOptions().refimg_cloud_storage_bucket: |
+ raise Exception('--refimg-cloud-storage-bucket argument is required') |
+ cloud_name = cls._FormatReferenceImageName(img_name, page, tab) |
+ if not cloud_storage.Exists( |
+ cls.GetParsedCommandLineOptions().refimg_cloud_storage_bucket, |
+ cloud_name): |
+ cls._UploadBitmapToCloudStorage( |
+ cls.GetParsedCommandLineOptions().refimg_cloud_storage_bucket, |
+ cloud_name, |
+ screenshot) |
+ return True |
+ return False |
+ |
+ @classmethod |
+ def _DownloadFromCloudStorage(cls, img_name, page, tab): |
+ """Downloads the reference image for the given test from cloud |
+ storage, returning it as a Telemetry Bitmap object.""" |
+ # TODO(kbr): there's a race condition between the deletion of the |
+ # temporary file and gsutil's overwriting it. |
+ if not cls.GetParsedCommandLineOptions().refimg_cloud_storage_bucket: |
+ raise Exception('--refimg-cloud-storage-bucket argument is required') |
+ temp_file = tempfile.NamedTemporaryFile(suffix='.png').name |
+ cloud_storage.Get( |
+ cls.GetParsedCommandLineOptions().refimg_cloud_storage_bucket, |
+ cls._FormatReferenceImageName(img_name, page, tab), |
+ temp_file) |
+ return image_util.FromPngFile(temp_file) |
+ |
+ @classmethod |
+ def _UploadErrorImagesToCloudStorage(cls, image_name, screenshot, ref_img): |
+ """For a failing run, uploads the failing image, reference image (if |
+ supplied), and diff image (if reference image was supplied) to cloud |
+ storage. This subsumes the functionality of the |
+ archive_gpu_pixel_test_results.py script.""" |
+ machine_name = re.sub(r'\W+', '_', |
+ cls.GetParsedCommandLineOptions().test_machine_name) |
+ upload_dir = '%s_%s_telemetry' % ( |
+ cls.GetParsedCommandLineOptions().build_revision, machine_name) |
+ base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) |
+ image_name_with_revision = '%s_%s.png' % ( |
+ image_name, cls.GetParsedCommandLineOptions().build_revision) |
+ cls._UploadBitmapToCloudStorage( |
+ base_bucket + '/gen', image_name_with_revision, screenshot, |
+ public=True) |
+ if ref_img is not None: |
+ cls._UploadBitmapToCloudStorage( |
+ base_bucket + '/ref', image_name_with_revision, ref_img, public=True) |
+ diff_img = image_util.Diff(screenshot, ref_img) |
+ cls._UploadBitmapToCloudStorage( |
+ base_bucket + '/diff', image_name_with_revision, diff_img, |
+ public=True) |
+ print ('See http://%s.commondatastorage.googleapis.com/' |
+ 'view_test_results.html?%s for this run\'s test results') % ( |
+ error_image_cloud_storage_bucket, upload_dir) |
+ |
+ def _ValidateScreenshotSamples(self, tab, url, |
+ screenshot, expectations, device_pixel_ratio): |
+ """Samples the given screenshot and verifies pixel color values. |
+ The sample locations and expected color values are given in expectations. |
+ In case any of the samples do not match the expected color, it raises |
+ a Failure and dumps the screenshot locally or cloud storage depending on |
+ what machine the test is being run.""" |
+ try: |
+ self._CompareScreenshotSamples( |
+ tab, screenshot, expectations, |
+ device_pixel_ratio, |
+ self.GetParsedCommandLineOptions().test_machine_name) |
+ except Exception: |
+ # An exception raised from self.fail() indicates a failure. |
+ image_name = self._UrlToImageName(url) |
+ if self.GetParsedCommandLineOptions().test_machine_name: |
+ self._UploadErrorImagesToCloudStorage(image_name, screenshot, None) |
+ else: |
+ self._WriteErrorImages( |
+ self.GetParsedCommandLineOptions().generated_dir, image_name, |
+ screenshot, None) |
+ raise |