Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(535)

Side by Side Diff: tools/skpdiff/skpdiff_server.py

Issue 1502173003: When was SkPDiff last used? (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: Created 5 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/skpdiff/skpdiff_main.cpp ('k') | tools/skpdiff/skpdiff_util.h » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import print_function
5 import argparse
6 import BaseHTTPServer
7 import json
8 import os
9 import os.path
10 import re
11 import subprocess
12 import sys
13 import tempfile
14 import urllib2
15
16 # Grab the script path because that is where all the static assets are
17 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
18
19 # Find the tools directory for python imports
20 TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
21
22 # Find the root of the skia trunk for finding skpdiff binary
23 SKIA_ROOT_DIR = os.path.dirname(TOOLS_DIR)
24
25 # Find the default location of gm expectations
26 DEFAULT_GM_EXPECTATIONS_DIR = os.path.join(SKIA_ROOT_DIR, 'expectations', 'gm')
27
28 # Imports from within Skia
29 if TOOLS_DIR not in sys.path:
30 sys.path.append(TOOLS_DIR)
31 GM_DIR = os.path.join(SKIA_ROOT_DIR, 'gm')
32 if GM_DIR not in sys.path:
33 sys.path.append(GM_DIR)
34 import gm_json
35 import jsondiff
36
37 # A simple dictionary of file name extensions to MIME types. The empty string
38 # entry is used as the default when no extension was given or if the extension
39 # has no entry in this dictionary.
40 MIME_TYPE_MAP = {'': 'application/octet-stream',
41 'html': 'text/html',
42 'css': 'text/css',
43 'png': 'image/png',
44 'js': 'application/javascript',
45 'json': 'application/json'
46 }
47
48
49 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
50
51 SKPDIFF_INVOKE_FORMAT = '{} --jsonp=false -o {} -f {} {}'
52
53
54 def get_skpdiff_path(user_path=None):
55 """Find the skpdiff binary.
56
57 @param user_path If none, searches in Release and Debug out directories of
58 the skia root. If set, checks that the path is a real file and
59 returns it.
60 """
61 skpdiff_path = None
62 possible_paths = []
63
64 # Use the user given path, or try out some good default paths.
65 if user_path:
66 possible_paths.append(user_path)
67 else:
68 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
69 'Release', 'skpdiff'))
70 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
71 'Release', 'skpdiff.exe'))
72 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
73 'Debug', 'skpdiff'))
74 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
75 'Debug', 'skpdiff.exe'))
76 # Use the first path that actually points to the binary
77 for possible_path in possible_paths:
78 if os.path.isfile(possible_path):
79 skpdiff_path = possible_path
80 break
81
82 # If skpdiff was not found, print out diagnostic info for the user.
83 if skpdiff_path is None:
84 print('Could not find skpdiff binary. Either build it into the ' +
85 'default directory, or specify the path on the command line.')
86 print('skpdiff paths tried:')
87 for possible_path in possible_paths:
88 print(' ', possible_path)
89 return skpdiff_path
90
91
92 def download_file(url, output_path):
93 """Download the file at url and place it in output_path"""
94 reader = urllib2.urlopen(url)
95 with open(output_path, 'wb') as writer:
96 writer.write(reader.read())
97
98
99 def download_gm_image(image_name, image_path, hash_val):
100 """Download the gm result into the given path.
101
102 @param image_name The GM file name, for example imageblur_gpu.png.
103 @param image_path Path to place the image.
104 @param hash_val The hash value of the image.
105 """
106 if hash_val is None:
107 return
108
109 # Separate the test name from a image name
110 image_match = IMAGE_FILENAME_RE.match(image_name)
111 test_name = image_match.group(1)
112
113 # Calculate the URL of the requested image
114 image_url = gm_json.CreateGmActualUrl(
115 test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val)
116
117 # Download the image as requested
118 download_file(image_url, image_path)
119
120
121 def get_image_set_from_skpdiff(skpdiff_records):
122 """Get the set of all images references in the given records.
123
124 @param skpdiff_records An array of records, which are dictionary objects.
125 """
126 expected_set = frozenset([r['baselinePath'] for r in skpdiff_records])
127 actual_set = frozenset([r['testPath'] for r in skpdiff_records])
128 return expected_set | actual_set
129
130
131 def set_expected_hash_in_json(expected_results_json, image_name, hash_value):
132 """Set the expected hash for the object extracted from
133 expected-results.json. Note that this only work with bitmap-64bitMD5 hash
134 types.
135
136 @param expected_results_json The Python dictionary with the results to
137 modify.
138 @param image_name The name of the image to set the hash of.
139 @param hash_value The hash to set for the image.
140 """
141 expected_results = expected_results_json[gm_json.JSONKEY_EXPECTEDRESULTS]
142
143 if image_name in expected_results:
144 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGE STS][0][1] = hash_value
145 else:
146 expected_results[image_name] = {
147 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS:
148 [
149 [
150 gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
151 hash_value
152 ]
153 ]
154 }
155
156
157 def get_head_version(path):
158 """Get the version of the file at the given path stored inside the HEAD of
159 the git repository. It is returned as a string.
160
161 @param path The path of the file whose HEAD is returned. It is assumed the
162 path is inside a git repo rooted at SKIA_ROOT_DIR.
163 """
164
165 # git-show will not work with absolute paths. This ensures we give it a path
166 # relative to the skia root. This path also has to use forward slashes, even
167 # on windows.
168 git_path = os.path.relpath(path, SKIA_ROOT_DIR).replace('\\', '/')
169 git_show_proc = subprocess.Popen(['git', 'show', 'HEAD:' + git_path],
170 stdout=subprocess.PIPE)
171
172 # When invoked outside a shell, git will output the last committed version
173 # of the file directly to stdout.
174 git_version_content, _ = git_show_proc.communicate()
175 return git_version_content
176
177
178 class GMInstance:
179 """Information about a GM test result on a specific device:
180 - device_name = the name of the device that rendered it
181 - image_name = the GM test name and config
182 - expected_hash = the current expected hash value
183 - actual_hash = the actual hash value
184 - is_rebaselined = True if actual_hash is what is currently in the expected
185 results file, False otherwise.
186 """
187 def __init__(self,
188 device_name, image_name,
189 expected_hash, actual_hash,
190 is_rebaselined):
191 self.device_name = device_name
192 self.image_name = image_name
193 self.expected_hash = expected_hash
194 self.actual_hash = actual_hash
195 self.is_rebaselined = is_rebaselined
196
197
198 class ExpectationsManager:
199 def __init__(self, expectations_dir, expected_name, updated_name,
200 skpdiff_path):
201 """
202 @param expectations_dir The directory to traverse for results files.
203 This should resemble expectations/gm in the Skia trunk.
204 @param expected_name The name of the expected result files. These
205 are in the format of expected-results.json.
206 @param updated_name The name of the updated expected result files.
207 Normally this matches --expectations-filename-output for the
208 rebaseline.py tool.
209 @param skpdiff_path The path used to execute the skpdiff command.
210 """
211 self._expectations_dir = expectations_dir
212 self._expected_name = expected_name
213 self._updated_name = updated_name
214 self._skpdiff_path = skpdiff_path
215 self._generate_gm_comparison()
216
217 def _generate_gm_comparison(self):
218 """Generate all the data needed to compare GMs:
219 - determine which GMs changed
220 - download the changed images
221 - compare them with skpdiff
222 """
223
224 # Get the expectations and compare them with actual hashes
225 self._get_expectations()
226
227
228 # Create a temporary file tree that makes sense for skpdiff to operate
229 # on. We take the realpath of the new temp directory because some OSs
230 # (*cough* osx) put the temp directory behind a symlink that gets
231 # resolved later down the pipeline and breaks the image map.
232 image_output_dir = os.path.realpath(tempfile.mkdtemp('skpdiff'))
233 expected_image_dir = os.path.join(image_output_dir, 'expected')
234 actual_image_dir = os.path.join(image_output_dir, 'actual')
235 os.mkdir(expected_image_dir)
236 os.mkdir(actual_image_dir)
237
238 # Download expected and actual images that differed into the temporary
239 # file tree.
240 self._download_expectation_images(expected_image_dir, actual_image_dir)
241
242 # Invoke skpdiff with our downloaded images and place its results in the
243 # temporary directory.
244 self._skpdiff_output_path = os.path.join(image_output_dir,
245 'skpdiff_output.json')
246 skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path,
247 self._skpdiff_output_path,
248 expected_image_dir,
249 actual_image_dir)
250 os.system(skpdiff_cmd)
251 self._load_skpdiff_output()
252
253
254 def _get_expectations(self):
255 """Fills self._expectations with GMInstance objects for each test whose
256 expectation is different between the following two files:
257 - the local filesystem's updated results file
258 - git's head version of the expected results file
259 """
260 differ = jsondiff.GMDiffer()
261 self._expectations = []
262 for root, dirs, files in os.walk(self._expectations_dir):
263 for expectation_file in files:
264 # There are many files in the expectations directory. We only
265 # care about expected results.
266 if expectation_file != self._expected_name:
267 continue
268
269 # Get the name of the results file, and be sure there is an
270 # updated result to compare against. If there is not, there is
271 # no point in diffing this device.
272 expected_file_path = os.path.join(root, self._expected_name)
273 updated_file_path = os.path.join(root, self._updated_name)
274 if not os.path.isfile(updated_file_path):
275 continue
276
277 # Always get the expected results from git because we may have
278 # changed them in a previous instance of the server.
279 expected_contents = get_head_version(expected_file_path)
280 updated_contents = None
281 with open(updated_file_path, 'rb') as updated_file:
282 updated_contents = updated_file.read()
283
284 # Read the expected results on disk to determine what we've
285 # already rebaselined.
286 commited_contents = None
287 with open(expected_file_path, 'rb') as expected_file:
288 commited_contents = expected_file.read()
289
290 # Find all expectations that did not match.
291 expected_diff = differ.GenerateDiffDictFromStrings(
292 expected_contents,
293 updated_contents)
294
295 # Generate a set of images that have already been rebaselined
296 # onto disk.
297 rebaselined_diff = differ.GenerateDiffDictFromStrings(
298 expected_contents,
299 commited_contents)
300
301 rebaselined_set = set(rebaselined_diff.keys())
302
303 # The name of the device corresponds to the name of the folder
304 # we are in.
305 device_name = os.path.basename(root)
306
307 # Store old and new versions of the expectation for each GM
308 for image_name, hashes in expected_diff.iteritems():
309 self._expectations.append(
310 GMInstance(device_name, image_name,
311 hashes['old'], hashes['new'],
312 image_name in rebaselined_set))
313
314 def _load_skpdiff_output(self):
315 """Loads the results of skpdiff and annotates them with whether they
316 have already been rebaselined or not. The resulting data is store in
317 self.skpdiff_records."""
318 self.skpdiff_records = None
319 with open(self._skpdiff_output_path, 'rb') as skpdiff_output_file:
320 self.skpdiff_records = json.load(skpdiff_output_file)['records']
321 for record in self.skpdiff_records:
322 record['isRebaselined'] = self.image_map[record['baselinePath']] [1].is_rebaselined
323
324
325 def _download_expectation_images(self, expected_image_dir, actual_image_dir) :
326 """Download the expected and actual images for the _expectations array.
327
328 @param expected_image_dir The directory to download expected images
329 into.
330 @param actual_image_dir The directory to download actual images into.
331 """
332 image_map = {}
333
334 # Look through expectations and download their images.
335 for expectation in self._expectations:
336 # Build appropriate paths to download the images into.
337 expected_image_path = os.path.join(expected_image_dir,
338 expectation.device_name + '-' +
339 expectation.image_name)
340
341 actual_image_path = os.path.join(actual_image_dir,
342 expectation.device_name + '-' +
343 expectation.image_name)
344
345 print('Downloading %s for device %s' % (
346 expectation.image_name, expectation.device_name))
347
348 # Download images
349 download_gm_image(expectation.image_name,
350 expected_image_path,
351 expectation.expected_hash)
352
353 download_gm_image(expectation.image_name,
354 actual_image_path,
355 expectation.actual_hash)
356
357 # Annotate the expectations with where the images were downloaded
358 # to.
359 expectation.expected_image_path = expected_image_path
360 expectation.actual_image_path = actual_image_path
361
362 # Map the image paths back to the expectations.
363 image_map[expected_image_path] = (False, expectation)
364 image_map[actual_image_path] = (True, expectation)
365
366 self.image_map = image_map
367
368 def _set_expected_hash(self, device_name, image_name, hash_value):
369 """Set the expected hash for the image of the given device. This always
370 writes directly to the expected results file of the given device
371
372 @param device_name The name of the device to write the hash to.
373 @param image_name The name of the image whose hash to set.
374 @param hash_value The value of the hash to set.
375 """
376
377 # Retrieve the expected results file as it is in the working tree
378 json_path = os.path.join(self._expectations_dir, device_name,
379 self._expected_name)
380 expectations = gm_json.LoadFromFile(json_path)
381
382 # Set the specified hash.
383 set_expected_hash_in_json(expectations, image_name, hash_value)
384
385 # Write it out to disk using gm_json to keep the formatting consistent.
386 gm_json.WriteToFile(expectations, json_path)
387
388 def commit_rebaselines(self, rebaselines):
389 """Sets the expected results file to use the hashes of the images in
390 the rebaselines list. If a expected result image is not in rebaselines
391 at all, the old hash will be used.
392
393 @param rebaselines A list of image paths to use the hash of.
394 """
395 # Reset all expectations to their old hashes because some of them may
396 # have been set to the new hash by a previous call to this function.
397 for expectation in self._expectations:
398 expectation.is_rebaselined = False
399 self._set_expected_hash(expectation.device_name,
400 expectation.image_name,
401 expectation.expected_hash)
402
403 # Take all the images to rebaseline
404 for image_path in rebaselines:
405 # Get the metadata about the image at the path.
406 is_actual, expectation = self.image_map[image_path]
407
408 expectation.is_rebaselined = is_actual
409 expectation_hash = expectation.actual_hash if is_actual else\
410 expectation.expected_hash
411
412 # Write out that image's hash directly to the expected results file.
413 self._set_expected_hash(expectation.device_name,
414 expectation.image_name,
415 expectation_hash)
416
417 self._load_skpdiff_output()
418
419
420 class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler):
421 def send_file(self, file_path):
422 # Grab the extension if there is one
423 extension = os.path.splitext(file_path)[1]
424 if len(extension) >= 1:
425 extension = extension[1:]
426
427 # Determine the MIME type of the file from its extension
428 mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP[''])
429
430 # Open the file and send it over HTTP
431 if os.path.isfile(file_path):
432 with open(file_path, 'rb') as sending_file:
433 self.send_response(200)
434 self.send_header('Content-type', mime_type)
435 self.end_headers()
436 self.wfile.write(sending_file.read())
437 else:
438 self.send_error(404)
439
440 def serve_if_in_dir(self, dir_path, file_path):
441 # Determine if the file exists relative to the given dir_path AND exists
442 # under the dir_path. This is to prevent accidentally serving files
443 # outside the directory intended using symlinks, or '../'.
444 real_path = os.path.normpath(os.path.join(dir_path, file_path))
445 if os.path.commonprefix([real_path, dir_path]) == dir_path:
446 if os.path.isfile(real_path):
447 self.send_file(real_path)
448 return True
449 return False
450
451 def do_GET(self):
452 # Simple rewrite rule of the root path to 'viewer.html'
453 if self.path == '' or self.path == '/':
454 self.path = '/viewer.html'
455
456 # The [1:] chops off the leading '/'
457 file_path = self.path[1:]
458
459 # Handle skpdiff_output.json manually because it is was processed by the
460 # server when it was started and does not exist as a file.
461 if file_path == 'skpdiff_output.json':
462 self.send_response(200)
463 self.send_header('Content-type', MIME_TYPE_MAP['json'])
464 self.end_headers()
465
466 # Add JSONP padding to the JSON because the web page expects it. It
467 # expects it because it was designed to run with or without a web
468 # server. Without a web server, the only way to load JSON is with
469 # JSONP.
470 skpdiff_records = self.server.expectations_manager.skpdiff_records
471 self.wfile.write('var SkPDiffRecords = ')
472 json.dump({'records': skpdiff_records}, self.wfile)
473 self.wfile.write(';')
474 return
475
476 # Attempt to send static asset files first.
477 if self.serve_if_in_dir(SCRIPT_DIR, file_path):
478 return
479
480 # WARNING: Serving any file the user wants is incredibly insecure. Its
481 # redeeming quality is that we only serve gm files on a white list.
482 if self.path in self.server.image_set:
483 self.send_file(self.path)
484 return
485
486 # If no file to send was found, just give the standard 404
487 self.send_error(404)
488
489 def do_POST(self):
490 if self.path == '/commit_rebaselines':
491 content_length = int(self.headers['Content-length'])
492 request_data = json.loads(self.rfile.read(content_length))
493 rebaselines = request_data['rebaselines']
494 self.server.expectations_manager.commit_rebaselines(rebaselines)
495 self.send_response(200)
496 self.send_header('Content-type', 'application/json')
497 self.end_headers()
498 self.wfile.write('{"success":true}')
499 return
500
501 # If the we have no handler for this path, give em' the 404
502 self.send_error(404)
503
504
505 def run_server(expectations_manager, port=8080):
506 # It's important to parse the results file so that we can make a set of
507 # images that the web page might request.
508 skpdiff_records = expectations_manager.skpdiff_records
509 image_set = get_image_set_from_skpdiff(skpdiff_records)
510
511 # Do not bind to interfaces other than localhost because the server will
512 # attempt to serve files relative to the root directory as a last resort
513 # before 404ing. This means all of your files can be accessed from this
514 # server, so DO NOT let this server listen to anything but localhost.
515 server_address = ('127.0.0.1', port)
516 http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler)
517 http_server.image_set = image_set
518 http_server.expectations_manager = expectations_manager
519 print('Navigate thine browser to: http://{}:{}/'.format(*server_address))
520 http_server.serve_forever()
521
522
523 def main():
524 parser = argparse.ArgumentParser()
525 parser.add_argument('--port', '-p', metavar='PORT',
526 type=int,
527 default=8080,
528 help='port to bind the server to; ' +
529 'defaults to %(default)s',
530 )
531
532 parser.add_argument('--expectations-dir', metavar='EXPECTATIONS_DIR',
533 default=DEFAULT_GM_EXPECTATIONS_DIR,
534 help='path to the gm expectations; ' +
535 'defaults to %(default)s'
536 )
537
538 parser.add_argument('--expected',
539 metavar='EXPECTATIONS_FILE_NAME',
540 default='expected-results.json',
541 help='the file name of the expectations JSON; ' +
542 'defaults to %(default)s'
543 )
544
545 parser.add_argument('--updated',
546 metavar='UPDATED_FILE_NAME',
547 default='updated-results.json',
548 help='the file name of the updated expectations JSON;' +
549 ' defaults to %(default)s'
550 )
551
552 parser.add_argument('--skpdiff-path', metavar='SKPDIFF_PATH',
553 default=None,
554 help='the path to the skpdiff binary to use; ' +
555 'defaults to out/Release/skpdiff or out/Default/skpdiff'
556 )
557
558 args = vars(parser.parse_args()) # Convert args into a python dict
559
560 # Make sure we have access to an skpdiff binary
561 skpdiff_path = get_skpdiff_path(args['skpdiff_path'])
562 if skpdiff_path is None:
563 sys.exit(1)
564
565 # Print out the paths of things for easier debugging
566 print('script dir :', SCRIPT_DIR)
567 print('tools dir :', TOOLS_DIR)
568 print('root dir :', SKIA_ROOT_DIR)
569 print('expectations dir :', args['expectations_dir'])
570 print('skpdiff path :', skpdiff_path)
571
572 expectations_manager = ExpectationsManager(args['expectations_dir'],
573 args['expected'],
574 args['updated'],
575 skpdiff_path)
576
577 run_server(expectations_manager, port=args['port'])
578
579 if __name__ == '__main__':
580 main()
OLDNEW
« no previous file with comments | « tools/skpdiff/skpdiff_main.cpp ('k') | tools/skpdiff/skpdiff_util.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698