OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 ''' | |
3 Copyright 2012 Google Inc. | |
4 | |
5 Use of this source code is governed by a BSD-style license that can be | |
6 found in the LICENSE file. | |
7 ''' | |
8 | |
9 ''' | |
10 Generates a visual diff of all pending changes in the local SVN (or git!) | |
11 checkout. | |
12 | |
13 Launch with --help to see more information. | |
14 | |
15 TODO(epoger): Now that this tool supports either git or svn, rename it. | |
16 TODO(epoger): Fix indentation in this file (2-space indents, not 4-space). | |
17 ''' | |
18 | |
19 # common Python modules | |
20 import optparse | |
21 import os | |
22 import posixpath | |
23 import re | |
24 import shutil | |
25 import subprocess | |
26 import sys | |
27 import tempfile | |
28 import urllib2 | |
29 | |
30 # Imports from within Skia | |
31 # | |
32 # We need to add the 'gm' directory, so that we can import gm_json.py within | |
33 # that directory. That script allows us to parse the actual-results.json file | |
34 # written out by the GM tool. | |
35 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | |
36 # so any dirs that are already in the PYTHONPATH will be preferred. | |
37 # | |
38 # This assumes that the 'gm' directory has been checked out as a sibling of | |
39 # the 'tools' directory containing this script, which will be the case if | |
40 # 'trunk' was checked out as a single unit. | |
41 GM_DIRECTORY = os.path.realpath( | |
42 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) | |
43 if GM_DIRECTORY not in sys.path: | |
44 sys.path.append(GM_DIRECTORY) | |
45 import gm_json | |
46 import jsondiff | |
47 import svn | |
48 | |
49 CHECKOUT_ROOT = os.path.realpath( | |
50 os.path.join(os.path.dirname(__file__), os.pardir)) | |
51 sys.path.append(CHECKOUT_ROOT) | |
52 from common.py.utils import git_utils | |
53 | |
54 USAGE_STRING = 'Usage: %s [options]' | |
55 HELP_STRING = ''' | |
56 | |
57 Generates a visual diff of all pending changes in the local SVN/git checkout. | |
58 | |
59 This includes a list of all files that have been added, deleted, or modified | |
60 (as far as SVN/git knows about). For any image modifications, pixel diffs will | |
61 be generated. | |
62 | |
63 ''' | |
64 | |
65 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | |
66 | |
67 TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir) | |
68 | |
69 OPTION_DEST_DIR = '--dest-dir' | |
70 OPTION_PATH_TO_SKDIFF = '--path-to-skdiff' | |
71 OPTION_SOURCE_DIR = '--source-dir' | |
72 | |
73 def RunCommand(command): | |
74 """Run a command, raising an exception if it fails. | |
75 | |
76 @param command the command as a single string | |
77 """ | |
78 print 'running command [%s]...' % command | |
79 retval = os.system(command) | |
80 if retval is not 0: | |
81 raise Exception('command [%s] failed' % command) | |
82 | |
83 def FindPathToSkDiff(user_set_path=None): | |
84 """Return path to an existing skdiff binary, or raise an exception if we | |
85 cannot find one. | |
86 | |
87 @param user_set_path if None, the user did not specify a path, so look in | |
88 some likely places; otherwise, only check at this path | |
89 """ | |
90 if user_set_path is not None: | |
91 if os.path.isfile(user_set_path): | |
92 return user_set_path | |
93 raise Exception('unable to find skdiff at user-set path %s' % | |
94 user_set_path) | |
95 trunk_path = os.path.join(os.path.dirname(__file__), os.pardir) | |
96 | |
97 extension = '' | |
98 if os.name is 'nt': | |
99 extension = '.exe' | |
100 | |
101 possible_paths = [os.path.join(trunk_path, 'out', 'Release', | |
102 'skdiff' + extension), | |
103 os.path.join(trunk_path, 'out', 'Debug', | |
104 'skdiff' + extension)] | |
105 for try_path in possible_paths: | |
106 if os.path.isfile(try_path): | |
107 return try_path | |
108 raise Exception('cannot find skdiff in paths %s; maybe you need to ' | |
109 'specify the %s option or build skdiff?' % ( | |
110 possible_paths, OPTION_PATH_TO_SKDIFF)) | |
111 | |
112 def _DownloadUrlToFile(source_url, dest_path): | |
113 """Download source_url, and save its contents to dest_path. | |
114 Raises an exception if there were any problems.""" | |
115 try: | |
116 reader = urllib2.urlopen(source_url) | |
117 writer = open(dest_path, 'wb') | |
118 writer.write(reader.read()) | |
119 writer.close() | |
120 except BaseException as e: | |
121 raise Exception( | |
122 '%s: unable to download source_url %s to dest_path %s' % ( | |
123 e, source_url, dest_path)) | |
124 | |
125 def _CreateGSUrl(imagename, hash_type, hash_digest): | |
126 """Return the HTTP URL we can use to download this particular version of | |
127 the actually-generated GM image with this imagename. | |
128 | |
129 imagename: name of the test image, e.g. 'perlinnoise_msaa4.png' | |
130 hash_type: string indicating the hash type used to generate hash_digest, | |
131 e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 | |
132 hash_digest: the hash digest of the image to retrieve | |
133 """ | |
134 return gm_json.CreateGmActualUrl( | |
135 test_name=IMAGE_FILENAME_RE.match(imagename).group(1), | |
136 hash_type=hash_type, | |
137 hash_digest=hash_digest) | |
138 | |
139 def _CallJsonDiff(old_json_path, new_json_path, | |
140 old_flattened_dir, new_flattened_dir, | |
141 filename_prefix): | |
142 """Using jsondiff.py, write the images that differ between two GM | |
143 expectations summary files (old and new) into old_flattened_dir and | |
144 new_flattened_dir. | |
145 | |
146 filename_prefix: prefix to prepend to filenames of all images we write | |
147 into the flattened directories | |
148 """ | |
149 json_differ = jsondiff.GMDiffer() | |
150 diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path, | |
151 newfile=new_json_path) | |
152 print 'Downloading %d before-and-after image pairs...' % len(diff_dict) | |
153 for (imagename, results) in diff_dict.iteritems(): | |
154 # TODO(epoger): Currently, this assumes that all images have been | |
155 # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 | |
156 | |
157 old_checksum = results['old'] | |
158 if old_checksum: | |
159 old_image_url = _CreateGSUrl( | |
160 imagename=imagename, | |
161 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, | |
162 hash_digest=old_checksum) | |
163 _DownloadUrlToFile( | |
164 source_url=old_image_url, | |
165 dest_path=os.path.join(old_flattened_dir, | |
166 filename_prefix + imagename)) | |
167 | |
168 new_checksum = results['new'] | |
169 if new_checksum: | |
170 new_image_url = _CreateGSUrl( | |
171 imagename=imagename, | |
172 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, | |
173 hash_digest=new_checksum) | |
174 _DownloadUrlToFile( | |
175 source_url=new_image_url, | |
176 dest_path=os.path.join(new_flattened_dir, | |
177 filename_prefix + imagename)) | |
178 | |
179 def _RunCommand(args): | |
180 """Run a command (from self._directory) and return stdout as a single | |
181 string. | |
182 | |
183 @param args a list of arguments | |
184 """ | |
185 proc = subprocess.Popen(args, | |
186 stdout=subprocess.PIPE, | |
187 stderr=subprocess.PIPE) | |
188 (stdout, stderr) = proc.communicate() | |
189 if proc.returncode is not 0: | |
190 raise Exception('command "%s" failed: %s' % (args, stderr)) | |
191 return stdout | |
192 | |
193 | |
194 def _GitExportBaseVersionOfFile(file_within_repo, dest_path): | |
195 """Retrieves a copy of the base version of a file within the repository. | |
196 | |
197 @param file_within_repo path to the file within the repo whose base | |
198 version you wish to obtain | |
199 @param dest_path destination to which to write the base content | |
200 | |
201 TODO(epoger): Move this into a git utility package? | |
202 """ | |
203 # TODO(epoger): Replace use of "git show" command with lower-level git | |
204 # commands? senorblanco points out that "git show" is a "porcelain" | |
205 # command, intended for human use, as opposed to the "plumbing" commands | |
206 # generally more suitable for scripting. (See | |
207 # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain ) | |
208 # | |
209 # For now, though, "git show" is the most straightforward implementation | |
210 # I could come up with. I tried using "git cat-file", but I had trouble | |
211 # getting it to work as desired. | |
212 # Note that git expects / rather than \ as a path separator even on | |
213 # windows. | |
214 args = ['git', 'show', posixpath.join('HEAD:.', file_within_repo)] | |
215 with open(dest_path, 'wb') as outfile: | |
216 proc = subprocess.Popen(args, stdout=outfile) | |
217 proc.communicate() | |
218 if proc.returncode is not 0: | |
219 raise Exception('command "%s" failed' % args) | |
220 | |
221 def SvnDiff(path_to_skdiff, dest_dir, source_dir): | |
222 """Generates a visual diff of all pending changes in source_dir. | |
223 | |
224 @param path_to_skdiff | |
225 @param dest_dir existing directory within which to write results | |
226 @param source_dir | |
227 """ | |
228 # Validate parameters, filling in default values if necessary and possible. | |
229 path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff)) | |
230 if not dest_dir: | |
231 dest_dir = tempfile.mkdtemp() | |
232 dest_dir = os.path.abspath(dest_dir) | |
233 | |
234 os.chdir(source_dir) | |
235 svn_repo = svn.Svn('.') | |
236 using_svn = True | |
237 try: | |
238 svn_repo.GetInfo() | |
239 except: | |
240 using_svn = False | |
241 | |
242 # Prepare temporary directories. | |
243 modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened') | |
244 original_flattened_dir = os.path.join(dest_dir, 'original_flattened') | |
245 diff_dir = os.path.join(dest_dir, 'diffs') | |
246 for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] : | |
247 shutil.rmtree(dir, ignore_errors=True) | |
248 os.mkdir(dir) | |
249 | |
250 # Get a list of all locally modified (including added/deleted) files, | |
251 # descending subdirectories. | |
252 if using_svn: | |
253 modified_file_paths = svn_repo.GetFilesWithStatus( | |
254 svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED) | |
255 else: | |
256 modified_file_paths = git_utils.GetModifiedFiles() | |
257 | |
258 # For each modified file: | |
259 # 1. copy its current contents into modified_flattened_dir | |
260 # 2. copy its original contents into original_flattened_dir | |
261 for modified_file_path in modified_file_paths: | |
262 if modified_file_path.endswith('.json'): | |
263 # Special handling for JSON files, in the hopes that they | |
264 # contain GM result summaries. | |
265 original_file = tempfile.NamedTemporaryFile(delete = False) | |
266 original_file.close() | |
267 if using_svn: | |
268 svn_repo.ExportBaseVersionOfFile( | |
269 modified_file_path, original_file.name) | |
270 else: | |
271 _GitExportBaseVersionOfFile( | |
272 modified_file_path, original_file.name) | |
273 modified_dir = os.path.dirname(modified_file_path) | |
274 platform_prefix = (re.sub(re.escape(os.sep), '__', | |
275 os.path.splitdrive(modified_dir)[1]) | |
276 + '__') | |
277 _CallJsonDiff(old_json_path=original_file.name, | |
278 new_json_path=modified_file_path, | |
279 old_flattened_dir=original_flattened_dir, | |
280 new_flattened_dir=modified_flattened_dir, | |
281 filename_prefix=platform_prefix) | |
282 os.remove(original_file.name) | |
283 else: | |
284 dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path) | |
285 # If the file had STATUS_DELETED, it won't exist anymore... | |
286 if os.path.isfile(modified_file_path): | |
287 shutil.copyfile(modified_file_path, | |
288 os.path.join(modified_flattened_dir, | |
289 dest_filename)) | |
290 if using_svn: | |
291 svn_repo.ExportBaseVersionOfFile( | |
292 modified_file_path, | |
293 os.path.join(original_flattened_dir, dest_filename)) | |
294 else: | |
295 _GitExportBaseVersionOfFile( | |
296 modified_file_path, | |
297 os.path.join(original_flattened_dir, dest_filename)) | |
298 | |
299 # Run skdiff: compare original_flattened_dir against modified_flattened_dir | |
300 RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir, | |
301 modified_flattened_dir, diff_dir)) | |
302 print '\nskdiff results are ready in file://%s/index.html' % diff_dir | |
303 | |
304 def RaiseUsageException(): | |
305 raise Exception('%s\nRun with --help for more detail.' % ( | |
306 USAGE_STRING % __file__)) | |
307 | |
308 def Main(options, args): | |
309 """Allow other scripts to call this script with fake command-line args. | |
310 """ | |
311 num_args = len(args) | |
312 if num_args != 0: | |
313 RaiseUsageException() | |
314 SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir, | |
315 source_dir=options.source_dir) | |
316 | |
317 if __name__ == '__main__': | |
318 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) | |
319 parser.add_option(OPTION_DEST_DIR, | |
320 action='store', type='string', default=None, | |
321 help='existing directory within which to write results; ' | |
322 'if not set, will create a temporary directory which ' | |
323 'will remain in place after this script completes') | |
324 parser.add_option(OPTION_PATH_TO_SKDIFF, | |
325 action='store', type='string', default=None, | |
326 help='path to already-built skdiff tool; if not set, ' | |
327 'will search for it in typical directories near this ' | |
328 'script') | |
329 parser.add_option(OPTION_SOURCE_DIR, | |
330 action='store', type='string', | |
331 default=os.path.join('expectations', 'gm'), | |
332 help='root directory within which to compare all ' + | |
333 'files; defaults to "%default"') | |
334 (options, args) = parser.parse_args() | |
335 Main(options, args) | |
OLD | NEW |