OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # -*- coding: utf-8 -*- | 2 # -*- coding: utf-8 -*- |
3 | 3 |
4 from __future__ import print_function | 4 from __future__ import print_function |
5 import argparse | 5 import argparse |
6 import BaseHTTPServer | 6 import BaseHTTPServer |
7 import json | 7 import json |
8 import os | 8 import os |
9 import os.path | 9 import os.path |
10 import re | 10 import re |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
92 writer.write(reader.read()) | 92 writer.write(reader.read()) |
93 | 93 |
94 | 94 |
95 def download_gm_image(image_name, image_path, hash_val): | 95 def download_gm_image(image_name, image_path, hash_val): |
96 """Download the gm result into the given path. | 96 """Download the gm result into the given path. |
97 | 97 |
98 @param image_name The GM file name, for example imageblur_gpu.png. | 98 @param image_name The GM file name, for example imageblur_gpu.png. |
99 @param image_path Path to place the image. | 99 @param image_path Path to place the image. |
100 @param hash_val The hash value of the image. | 100 @param hash_val The hash value of the image. |
101 """ | 101 """ |
| 102 if hash_val is None: |
| 103 return |
102 | 104 |
103 # Separate the test name from a image name | 105 # Separate the test name from a image name |
104 image_match = IMAGE_FILENAME_RE.match(image_name) | 106 image_match = IMAGE_FILENAME_RE.match(image_name) |
105 test_name = image_match.group(1) | 107 test_name = image_match.group(1) |
106 | 108 |
107 # Calculate the URL of the requested image | 109 # Calculate the URL of the requested image |
108 image_url = gm_json.CreateGmActualUrl( | 110 image_url = gm_json.CreateGmActualUrl( |
109 test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val) | 111 test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val) |
110 | 112 |
111 # Download the image as requested | 113 # Download the image as requested |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
167 git_version_content, _ = git_show_proc.communicate() | 169 git_version_content, _ = git_show_proc.communicate() |
168 return git_version_content | 170 return git_version_content |
169 | 171 |
170 | 172 |
171 class GMInstance: | 173 class GMInstance: |
172 """Information about a GM test result on a specific device: | 174 """Information about a GM test result on a specific device: |
173 - device_name = the name of the device that rendered it | 175 - device_name = the name of the device that rendered it |
174 - image_name = the GM test name and config | 176 - image_name = the GM test name and config |
175 - expected_hash = the current expected hash value | 177 - expected_hash = the current expected hash value |
176 - actual_hash = the actual hash value | 178 - actual_hash = the actual hash value |
| 179 - is_rebaselined = True if actual_hash is what is currently in the expected |
| 180 results file, False otherwise. |
177 """ | 181 """ |
178 def __init__(self, | 182 def __init__(self, |
179 device_name, image_name, | 183 device_name, image_name, |
180 expected_hash, actual_hash): | 184 expected_hash, actual_hash, |
| 185 is_rebaselined): |
181 self.device_name = device_name | 186 self.device_name = device_name |
182 self.image_name = image_name | 187 self.image_name = image_name |
183 self.expected_hash = expected_hash | 188 self.expected_hash = expected_hash |
184 self.actual_hash = actual_hash | 189 self.actual_hash = actual_hash |
| 190 self.is_rebaselined = is_rebaselined |
185 | 191 |
186 | 192 |
187 class ExpectationsManager: | 193 class ExpectationsManager: |
188 def __init__(self, expectations_dir, expected_name, updated_name, | 194 def __init__(self, expectations_dir, expected_name, updated_name, |
189 skpdiff_path): | 195 skpdiff_path): |
190 """ | 196 """ |
191 @param expectations_dir The directory to traverse for results files. | 197 @param expectations_dir The directory to traverse for results files. |
192 This should resemble expectations/gm in the Skia trunk. | 198 This should resemble expectations/gm in the Skia trunk. |
193 @param expected_name The name of the expected result files. These | 199 @param expected_name The name of the expected result files. These |
194 are in the format of expected-results.json. | 200 are in the format of expected-results.json. |
(...skipping 26 matching lines...) Expand all Loading... |
221 actual_image_dir = os.path.join(image_output_dir, 'actual') | 227 actual_image_dir = os.path.join(image_output_dir, 'actual') |
222 os.mkdir(expected_image_dir) | 228 os.mkdir(expected_image_dir) |
223 os.mkdir(actual_image_dir) | 229 os.mkdir(actual_image_dir) |
224 | 230 |
225 # Download expected and actual images that differed into the temporary | 231 # Download expected and actual images that differed into the temporary |
226 # file tree. | 232 # file tree. |
227 self._download_expectation_images(expected_image_dir, actual_image_dir) | 233 self._download_expectation_images(expected_image_dir, actual_image_dir) |
228 | 234 |
229 # Invoke skpdiff with our downloaded images and place its results in the | 235 # Invoke skpdiff with our downloaded images and place its results in the |
230 # temporary directory. | 236 # temporary directory. |
231 self.skpdiff_output_path = os.path.join(image_output_dir, | 237 self._skpdiff_output_path = os.path.join(image_output_dir, |
232 'skpdiff_output.json') | 238 'skpdiff_output.json') |
233 skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path, | 239 skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path, |
234 self.skpdiff_output_path, | 240 self._skpdiff_output_path, |
235 expected_image_dir, | 241 expected_image_dir, |
236 actual_image_dir) | 242 actual_image_dir) |
237 os.system(skpdiff_cmd) | 243 os.system(skpdiff_cmd) |
| 244 self._load_skpdiff_output() |
238 | 245 |
239 | 246 |
240 def _get_expectations(self): | 247 def _get_expectations(self): |
241 """Fills self._expectations with GMInstance objects for each test whose | 248 """Fills self._expectations with GMInstance objects for each test whose |
242 expectation is different between the following two files: | 249 expectation is different between the following two files: |
243 - the local filesystem's updated results file | 250 - the local filesystem's updated results file |
244 - git's head version of the expected results file | 251 - git's head version of the expected results file |
245 """ | 252 """ |
246 differ = jsondiff.GMDiffer() | 253 differ = jsondiff.GMDiffer() |
247 self._expectations = [] | 254 self._expectations = [] |
(...skipping 12 matching lines...) Expand all Loading... |
260 if not os.path.isfile(updated_file_path): | 267 if not os.path.isfile(updated_file_path): |
261 continue | 268 continue |
262 | 269 |
263 # Always get the expected results from git because we may have | 270 # Always get the expected results from git because we may have |
264 # changed them in a previous instance of the server. | 271 # changed them in a previous instance of the server. |
265 expected_contents = get_head_version(expected_file_path) | 272 expected_contents = get_head_version(expected_file_path) |
266 updated_contents = None | 273 updated_contents = None |
267 with open(updated_file_path, 'rb') as updated_file: | 274 with open(updated_file_path, 'rb') as updated_file: |
268 updated_contents = updated_file.read() | 275 updated_contents = updated_file.read() |
269 | 276 |
| 277 # Read the expected results on disk to determine what we've |
| 278 # already rebaselined. |
| 279 commited_contents = None |
| 280 with open(expected_file_path, 'rb') as expected_file: |
| 281 commited_contents = expected_file.read() |
| 282 |
270 # Find all expectations that did not match. | 283 # Find all expectations that did not match. |
271 expected_diff = differ.GenerateDiffDictFromStrings( | 284 expected_diff = differ.GenerateDiffDictFromStrings( |
272 expected_contents, | 285 expected_contents, |
273 updated_contents) | 286 updated_contents) |
274 | 287 |
| 288 # Generate a set of images that have already been rebaselined |
| 289 # onto disk. |
| 290 rebaselined_diff = differ.GenerateDiffDictFromStrings( |
| 291 expected_contents, |
| 292 commited_contents) |
| 293 |
| 294 rebaselined_set = set(rebaselined_diff.keys()) |
| 295 |
275 # The name of the device corresponds to the name of the folder | 296 # The name of the device corresponds to the name of the folder |
276 # we are in. | 297 # we are in. |
277 device_name = os.path.basename(root) | 298 device_name = os.path.basename(root) |
278 | 299 |
279 # Store old and new versions of the expectation for each GM | 300 # Store old and new versions of the expectation for each GM |
280 for image_name, hashes in expected_diff.iteritems(): | 301 for image_name, hashes in expected_diff.iteritems(): |
281 self._expectations.append( | 302 self._expectations.append( |
282 GMInstance(device_name, image_name, | 303 GMInstance(device_name, image_name, |
283 hashes['old'], hashes['new'])) | 304 hashes['old'], hashes['new'], |
| 305 image_name in rebaselined_set)) |
| 306 |
| 307 def _load_skpdiff_output(self): |
| 308 """Loads the results of skpdiff and annotates them with whether they |
| 309 have already been rebaselined or not. The resulting data is store in |
| 310 self.skpdiff_records.""" |
| 311 self.skpdiff_records = None |
| 312 with open(self._skpdiff_output_path, 'rb') as skpdiff_output_file: |
| 313 self.skpdiff_records = json.load(skpdiff_output_file)['records'] |
| 314 for record in self.skpdiff_records: |
| 315 record['isRebaselined'] = self.image_map[record['baselinePath']]
[1].is_rebaselined |
284 | 316 |
285 | 317 |
286 def _download_expectation_images(self, expected_image_dir, actual_image_dir)
: | 318 def _download_expectation_images(self, expected_image_dir, actual_image_dir)
: |
287 """Download the expected and actual images for the _expectations array. | 319 """Download the expected and actual images for the _expectations array. |
288 | 320 |
289 @param expected_image_dir The directory to download expected images | 321 @param expected_image_dir The directory to download expected images |
290 into. | 322 into. |
291 @param actual_image_dir The directory to download actual images into. | 323 @param actual_image_dir The directory to download actual images into. |
292 """ | 324 """ |
293 image_map = {} | 325 image_map = {} |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
339 json_path = os.path.join(self._expectations_dir, device_name, | 371 json_path = os.path.join(self._expectations_dir, device_name, |
340 self._expected_name) | 372 self._expected_name) |
341 expectations = gm_json.LoadFromFile(json_path) | 373 expectations = gm_json.LoadFromFile(json_path) |
342 | 374 |
343 # Set the specified hash. | 375 # Set the specified hash. |
344 set_expected_hash_in_json(expectations, image_name, hash_value) | 376 set_expected_hash_in_json(expectations, image_name, hash_value) |
345 | 377 |
346 # Write it out to disk using gm_json to keep the formatting consistent. | 378 # Write it out to disk using gm_json to keep the formatting consistent. |
347 gm_json.WriteToFile(expectations, json_path) | 379 gm_json.WriteToFile(expectations, json_path) |
348 | 380 |
349 def use_hash_of(self, image_path): | 381 def commit_rebaselines(self, rebaselines): |
350 """Determine the hash of the image at the path using the records, and | 382 """Sets the expected results file to use the hashes of the images in |
351 set it as the expected hash for its device and image config. | 383 the rebaselines list. If a expected result image is not in rebaselines |
| 384 at all, the old hash will be used. |
352 | 385 |
353 @param image_path The path of the image as it was stored in the output | 386 @param rebaselines A list of image paths to use the hash of. |
354 of skpdiff_path | |
355 """ | 387 """ |
| 388 # Reset all expectations to their old hashes because some of them may |
| 389 # have been set to the new hash by a previous call to this function. |
| 390 for expectation in self._expectations: |
| 391 expectation.is_rebaselined = False |
| 392 self._set_expected_hash(expectation.device_name, |
| 393 expectation.image_name, |
| 394 expectation.expected_hash) |
356 | 395 |
357 # Get the metadata about the image at the path. | 396 # Take all the images to rebaseline |
358 is_actual, expectation = self.image_map[image_path] | 397 for image_path in rebaselines: |
| 398 # Get the metadata about the image at the path. |
| 399 is_actual, expectation = self.image_map[image_path] |
359 | 400 |
360 expectation_hash = expectation.actual_hash if is_actual else\ | 401 expectation.is_rebaselined = is_actual |
361 expectation.expected_hash | 402 expectation_hash = expectation.actual_hash if is_actual else\ |
| 403 expectation.expected_hash |
362 | 404 |
363 # Write out that image's hash directly to the expected results file. | 405 # Write out that image's hash directly to the expected results file. |
364 self._set_expected_hash(expectation.device_name, expectation.image_name, | 406 self._set_expected_hash(expectation.device_name, |
365 expectation_hash) | 407 expectation.image_name, |
| 408 expectation_hash) |
| 409 |
| 410 self._load_skpdiff_output() |
366 | 411 |
367 | 412 |
368 class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler): | 413 class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler): |
369 def send_file(self, file_path): | 414 def send_file(self, file_path): |
370 # Grab the extension if there is one | 415 # Grab the extension if there is one |
371 extension = os.path.splitext(file_path)[1] | 416 extension = os.path.splitext(file_path)[1] |
372 if len(extension) >= 1: | 417 if len(extension) >= 1: |
373 extension = extension[1:] | 418 extension = extension[1:] |
374 | 419 |
375 # Determine the MIME type of the file from its extension | 420 # Determine the MIME type of the file from its extension |
(...skipping 27 matching lines...) Expand all Loading... |
403 | 448 |
404 # The [1:] chops off the leading '/' | 449 # The [1:] chops off the leading '/' |
405 file_path = self.path[1:] | 450 file_path = self.path[1:] |
406 | 451 |
407 # Handle skpdiff_output.json manually because it is was processed by the | 452 # Handle skpdiff_output.json manually because it is was processed by the |
408 # server when it was started and does not exist as a file. | 453 # server when it was started and does not exist as a file. |
409 if file_path == 'skpdiff_output.json': | 454 if file_path == 'skpdiff_output.json': |
410 self.send_response(200) | 455 self.send_response(200) |
411 self.send_header('Content-type', MIME_TYPE_MAP['json']) | 456 self.send_header('Content-type', MIME_TYPE_MAP['json']) |
412 self.end_headers() | 457 self.end_headers() |
413 self.wfile.write(self.server.skpdiff_output_json) | 458 |
| 459 # Add JSONP padding to the JSON because the web page expects it. It |
| 460 # expects it because it was designed to run with or without a web |
| 461 # server. Without a web server, the only way to load JSON is with |
| 462 # JSONP. |
| 463 skpdiff_records = self.server.expectations_manager.skpdiff_records |
| 464 self.wfile.write('var SkPDiffRecords = ') |
| 465 json.dump({'records': skpdiff_records}, self.wfile) |
| 466 self.wfile.write(';') |
414 return | 467 return |
415 | 468 |
416 # Attempt to send static asset files first. | 469 # Attempt to send static asset files first. |
417 if self.serve_if_in_dir(SCRIPT_DIR, file_path): | 470 if self.serve_if_in_dir(SCRIPT_DIR, file_path): |
418 return | 471 return |
419 | 472 |
420 # WARNING: Serving any file the user wants is incredibly insecure. Its | 473 # WARNING: Serving any file the user wants is incredibly insecure. Its |
421 # redeeming quality is that we only serve gm files on a white list. | 474 # redeeming quality is that we only serve gm files on a white list. |
422 if self.path in self.server.image_set: | 475 if self.path in self.server.image_set: |
423 self.send_file(self.path) | 476 self.send_file(self.path) |
424 return | 477 return |
425 | 478 |
426 # If no file to send was found, just give the standard 404 | 479 # If no file to send was found, just give the standard 404 |
427 self.send_error(404) | 480 self.send_error(404) |
428 | 481 |
429 def do_POST(self): | 482 def do_POST(self): |
430 if self.path == '/set_hash': | 483 if self.path == '/commit_rebaselines': |
431 content_length = int(self.headers['Content-length']) | 484 content_length = int(self.headers['Content-length']) |
432 request_data = json.loads(self.rfile.read(content_length)) | 485 request_data = json.loads(self.rfile.read(content_length)) |
433 self.server.expectations_manager.use_hash_of(request_data['path']) | 486 rebaselines = request_data['rebaselines'] |
| 487 self.server.expectations_manager.commit_rebaselines(rebaselines) |
434 self.send_response(200) | 488 self.send_response(200) |
435 self.send_header('Content-type', 'application/json') | 489 self.send_header('Content-type', 'application/json') |
436 self.end_headers() | 490 self.end_headers() |
437 self.wfile.write('{"success":true}') | 491 self.wfile.write('{"success":true}') |
438 return | 492 return |
439 | 493 |
440 # If the we have no handler for this path, give em' the 404 | 494 # If the we have no handler for this path, give em' the 404 |
441 self.send_error(404) | 495 self.send_error(404) |
442 | 496 |
443 | 497 |
444 def run_server(expectations_manager, port=8080): | 498 def run_server(expectations_manager, port=8080): |
445 # Preload the skpdiff results file. This is so we can perform some | |
446 # processing on it. | |
447 skpdiff_output_json = '' | |
448 with open(expectations_manager.skpdiff_output_path, 'rb') as records_file: | |
449 skpdiff_output_json = records_file.read() | |
450 | |
451 # It's important to parse the results file so that we can make a set of | 499 # It's important to parse the results file so that we can make a set of |
452 # images that the web page might request. | 500 # images that the web page might request. |
453 skpdiff_records = json.loads(skpdiff_output_json)['records'] | 501 skpdiff_records = expectations_manager.skpdiff_records |
454 image_set = get_image_set_from_skpdiff(skpdiff_records) | 502 image_set = get_image_set_from_skpdiff(skpdiff_records) |
455 | 503 |
456 # Add JSONP padding to the JSON because the web page expects it. It expects | |
457 # it because it was designed to run with or without a web server. Without a | |
458 # web server, the only way to load JSON is with JSONP. | |
459 skpdiff_output_json = ('var SkPDiffRecords = ' + | |
460 json.dumps({'records': skpdiff_records}) + ';') | |
461 | |
462 # Do not bind to interfaces other than localhost because the server will | 504 # Do not bind to interfaces other than localhost because the server will |
463 # attempt to serve files relative to the root directory as a last resort | 505 # attempt to serve files relative to the root directory as a last resort |
464 # before 404ing. This means all of your files can be accessed from this | 506 # before 404ing. This means all of your files can be accessed from this |
465 # server, so DO NOT let this server listen to anything but localhost. | 507 # server, so DO NOT let this server listen to anything but localhost. |
466 server_address = ('127.0.0.1', port) | 508 server_address = ('127.0.0.1', port) |
467 http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler) | 509 http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler) |
468 http_server.image_set = image_set | 510 http_server.image_set = image_set |
469 http_server.skpdiff_output_json = skpdiff_output_json | |
470 http_server.expectations_manager = expectations_manager | 511 http_server.expectations_manager = expectations_manager |
471 print('Navigate thine browser to: http://{}:{}/'.format(*server_address)) | 512 print('Navigate thine browser to: http://{}:{}/'.format(*server_address)) |
472 http_server.serve_forever() | 513 http_server.serve_forever() |
473 | 514 |
474 | 515 |
475 def main(): | 516 def main(): |
476 parser = argparse.ArgumentParser() | 517 parser = argparse.ArgumentParser() |
477 parser.add_argument('--port', '-p', metavar='PORT', | 518 parser.add_argument('--port', '-p', metavar='PORT', |
478 type=int, | 519 type=int, |
479 default=8080, | 520 default=8080, |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
523 | 564 |
524 expectations_manager = ExpectationsManager(args['expectations_dir'], | 565 expectations_manager = ExpectationsManager(args['expectations_dir'], |
525 args['expected'], | 566 args['expected'], |
526 args['updated'], | 567 args['updated'], |
527 skpdiff_path) | 568 skpdiff_path) |
528 | 569 |
529 run_server(expectations_manager, port=args['port']) | 570 run_server(expectations_manager, port=args['port']) |
530 | 571 |
531 if __name__ == '__main__': | 572 if __name__ == '__main__': |
532 main() | 573 main() |
OLD | NEW |