| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2016 The Chromium Authors. All rights reserved. | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 import argparse | |
| 8 import collections | |
| 9 import logging | |
| 10 import os | |
| 11 import posixpath | |
| 12 import re | |
| 13 import shutil | |
| 14 import sys | |
| 15 import tempfile | |
| 16 import zipfile | |
| 17 | |
| 18 sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir)) | |
| 19 import devil_chromium | |
| 20 from devil.android import device_utils | |
| 21 from devil.utils import cmd_helper | |
| 22 from pylib.constants import host_paths | |
| 23 | |
| 24 sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build')) | |
| 25 import find_depot_tools # pylint: disable=import-error | |
| 26 | |
| 27 sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party')) | |
| 28 import jinja2 # pylint: disable=import-error | |
| 29 | |
| 30 try: | |
| 31 from PIL import Image # pylint: disable=import-error | |
| 32 from PIL import ImageChops # pylint: disable=import-error | |
| 33 can_compute_diffs = True | |
| 34 except ImportError: | |
| 35 can_compute_diffs = False | |
| 36 logging.exception('Error importing PIL library. Image diffs will not be ' | |
| 37 'displayed properly unless PIL module is installed.') | |
| 38 | |
| 39 _RE_IMAGE_NAME = re.compile( | |
| 40 r'(?P<test_class>\w+)\.' | |
| 41 r'(?P<description>\w+)\.' | |
| 42 r'(?P<device_model>\w+)\.' | |
| 43 r'(?P<orientation>port|land)\.png') | |
| 44 | |
| 45 _RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/' | |
| 46 _RENDER_TEST_BUCKET = 'gs://chromium-render-tests/' | |
| 47 | |
| 48 _JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 49 _JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2' | |
| 50 | |
| 51 | |
| 52 def _UploadFiles(upload_dir, files): | |
| 53 """Upload files to the render tests GS bucket.""" | |
| 54 if files: | |
| 55 google_storage_upload_dir = os.path.join(_RENDER_TEST_BUCKET, upload_dir) | |
| 56 cmd = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'), | |
| 57 '-m', 'cp'] | |
| 58 cmd.extend(files) | |
| 59 cmd.append(google_storage_upload_dir) | |
| 60 cmd_helper.RunCmd(cmd) | |
| 61 | |
| 62 | |
| 63 def _GoogleStorageUrl(upload_dir, filename): | |
| 64 return os.path.join( | |
| 65 _RENDER_TEST_BASE_URL, upload_dir, os.path.basename(filename)) | |
| 66 | |
| 67 | |
| 68 def _ComputeImageDiff(failure_image, golden_image): | |
| 69 """Compute mask showing which pixels are different between two images.""" | |
| 70 return (ImageChops.difference(failure_image, golden_image) | |
| 71 .convert('L') | |
| 72 .point(lambda i: 255 if i else 0)) | |
| 73 | |
| 74 | |
| 75 def ProcessRenderTestResults(devices, render_results_dir, | |
| 76 upload_dir, html_file): | |
| 77 """Grabs render results from device and generates webpage displaying results. | |
| 78 | |
| 79 Args: | |
| 80 devices: List of DeviceUtils objects to grab results from. | |
| 81 render_results_path: Path where render test results are storage. | |
| 82 Will look for failures render test results on the device in | |
| 83 /sdcard/chromium_tests_root/<render_results_path>/failures/ | |
| 84 and will look for golden images at Chromium src/<render_results_path>/. | |
| 85 upload_dir: Directory to upload the render test results to. | |
| 86 html_file: File to write the test results to. | |
| 87 """ | |
| 88 results_dict = collections.defaultdict(lambda: collections.defaultdict(list)) | |
| 89 | |
| 90 diff_upload_dir = os.path.join(upload_dir, 'diffs') | |
| 91 failure_upload_dir = os.path.join(upload_dir, 'failures') | |
| 92 golden_upload_dir = os.path.join(upload_dir, 'goldens') | |
| 93 | |
| 94 diff_images = [] | |
| 95 failure_images = [] | |
| 96 golden_images = [] | |
| 97 | |
| 98 temp_dir = None | |
| 99 try: | |
| 100 temp_dir = tempfile.mkdtemp() | |
| 101 | |
| 102 for device in devices: | |
| 103 failures_device_dir = posixpath.join( | |
| 104 device.GetExternalStoragePath(), | |
| 105 'chromium_tests_root', render_results_dir, 'failures') | |
| 106 device.PullFile(failures_device_dir, temp_dir) | |
| 107 | |
| 108 for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')): | |
| 109 m = _RE_IMAGE_NAME.match(failure_filename) | |
| 110 if not m: | |
| 111 logging.warning( | |
| 112 'Unexpected file in render test failures, %s', failure_filename) | |
| 113 continue | |
| 114 failure_file = os.path.join(temp_dir, 'failures', failure_filename) | |
| 115 | |
| 116 # Check to make sure we have golden image for this failure. | |
| 117 golden_file = os.path.join( | |
| 118 host_paths.DIR_SOURCE_ROOT, render_results_dir, failure_filename) | |
| 119 if not os.path.exists(golden_file): | |
| 120 logging.error('Cannot find golden image for %s', failure_filename) | |
| 121 continue | |
| 122 | |
| 123 # Compute image diff between failure and golden. | |
| 124 if can_compute_diffs: | |
| 125 diff_image = _ComputeImageDiff( | |
| 126 Image.open(failure_file), Image.open(golden_file)) | |
| 127 diff_filename = '_diff'.join( | |
| 128 os.path.splitext(os.path.basename(failure_file))) | |
| 129 diff_file = os.path.join(temp_dir, diff_filename) | |
| 130 diff_image.save(diff_file) | |
| 131 diff_images.append(diff_file) | |
| 132 | |
| 133 failure_images.append(failure_file) | |
| 134 golden_images.append(golden_file) | |
| 135 | |
| 136 test_class = m.group('test_class') | |
| 137 device_model = m.group('device_model') | |
| 138 | |
| 139 results_entry = { | |
| 140 'description': m.group('description'), | |
| 141 'orientation': m.group('orientation'), | |
| 142 'failure_image': _GoogleStorageUrl(failure_upload_dir, failure_file), | |
| 143 'golden_image': _GoogleStorageUrl(golden_upload_dir, golden_file), | |
| 144 } | |
| 145 if can_compute_diffs: | |
| 146 results_entry.update( | |
| 147 {'diff_image': _GoogleStorageUrl(diff_upload_dir, diff_file)}) | |
| 148 results_dict[test_class][device_model].append(results_entry) | |
| 149 | |
| 150 if can_compute_diffs: | |
| 151 _UploadFiles(diff_upload_dir, diff_images) | |
| 152 _UploadFiles(failure_upload_dir, failure_images) | |
| 153 _UploadFiles(golden_upload_dir, golden_images) | |
| 154 | |
| 155 if failure_images: | |
| 156 failures_zipfile = os.path.join(temp_dir, 'failures.zip') | |
| 157 with zipfile.ZipFile(failures_zipfile, mode='w') as zf: | |
| 158 for failure_file in failure_images: | |
| 159 zf.write(failure_file, os.path.join( | |
| 160 render_results_dir, os.path.basename(failure_file))) | |
| 161 failure_zip_url = _GoogleStorageUrl(upload_dir, failures_zipfile) | |
| 162 _UploadFiles(upload_dir, [failures_zipfile]) | |
| 163 else: | |
| 164 failure_zip_url = None | |
| 165 | |
| 166 jinja2_env = jinja2.Environment( | |
| 167 loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR), | |
| 168 trim_blocks=True) | |
| 169 template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME) | |
| 170 # pylint: disable=no-member | |
| 171 processed_template_output = template.render( | |
| 172 full_results=dict(results_dict), | |
| 173 failure_zip_url=failure_zip_url, show_diffs=can_compute_diffs) | |
| 174 # pylint: enable=no-member | |
| 175 with open(html_file, 'wb') as f: | |
| 176 f.write(processed_template_output) | |
| 177 finally: | |
| 178 if temp_dir: | |
| 179 shutil.rmtree(temp_dir) | |
| 180 | |
| 181 | |
| 182 def main(): | |
| 183 parser = argparse.ArgumentParser() | |
| 184 | |
| 185 parser.add_argument('--render-results-dir', | |
| 186 required=True, | |
| 187 help='Path on device to look for render test images') | |
| 188 parser.add_argument('--output-html-file', | |
| 189 required=True, | |
| 190 help='File to output the results webpage.') | |
| 191 parser.add_argument('-d', '--device', dest='devices', action='append', | |
| 192 default=[], | |
| 193 help='Device to look for render test results on. ' | |
| 194 'Default is to look on all connected devices.') | |
| 195 parser.add_argument('--adb-path', type=os.path.abspath, | |
| 196 help='Absolute path to the adb binary to use.') | |
| 197 parser.add_argument('--buildername', type=str, required=True, | |
| 198 help='Bot buildername. Used to generate path to upload ' | |
| 199 'render test results') | |
| 200 parser.add_argument('--build-number', type=str, required=True, | |
| 201 help='Bot build number. Used to generate path to upload ' | |
| 202 'render test results') | |
| 203 | |
| 204 args = parser.parse_args() | |
| 205 devil_chromium.Initialize(adb_path=args.adb_path) | |
| 206 devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices) | |
| 207 | |
| 208 upload_dir = os.path.join(args.buildername, args.build_number) | |
| 209 ProcessRenderTestResults( | |
| 210 devices, args.render_results_dir, upload_dir, args.output_html_file) | |
| 211 | |
| 212 | |
| 213 if __name__ == '__main__': | |
| 214 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2866103002:
Add render test results to the results_details webpage. (Closed)
