OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 | |
3 # Copyright 2014 The Chromium Authors. All rights reserved. | |
4 # Use of this source code is governed by a BSD-style license that can be | |
5 # found in the LICENSE file. | |
6 | |
7 """Given the output of -t commands from a ninja build for a gyp and GN generated | |
8 build, report on differences between the command lines. | |
9 | |
10 This script is best used interactively. Here's a recommended setup: | |
11 $ PYTHONPATH=tools/gn/bin python | |
12 >>> import sys | |
13 >>> import pprint | |
14 >>> sys.displayhook = pprint.pprint | |
15 >>> import gyp_flag_compare as fc | |
16 >>> fc.ConfigureBuild(['gyp_define=1', 'define=2'], ['gn_arg=1', 'arg=2']) | |
17 >>> chrome = fc.Comparison('chrome') | |
18 | |
19 The above starts interactive Python, sets up the output to be pretty-printed | |
20 (useful for making lists, dicts, and sets readable), configures the build with | |
21 GN arguments and GYP defines, and then generates a comparison for that build | |
22 configuration for the "chrome" target. | |
23 | |
24 After that, the |chrome| object can be used to investigate differences in the | |
25 build. | |
26 | |
27 To configure an official build, use this configuration. Disabling NaCl produces | |
28 a more meaningful comparison, as certain files need to get compiled twice | |
29 for the IRT build, which uses different flags: | |
30 >>> fc.ConfigureBuild( | |
31 ['disable_nacl=1', 'buildtype=Official', 'branding=Chrome'], | |
32 ['enable_nacl=false', 'is_official_build=true', | |
33 'is_chrome_branded=true']) | |
34 """ | |
35 | |
36 | |
37 import os | |
38 import shlex | |
39 import subprocess | |
40 import sys | |
41 | |
42 # Must be in src/. | |
43 os.chdir(os.path.join(os.path.dirname(__file__), '..', '..', '..')) | |
44 | |
45 _GN_OUT_DIR = 'out/gn_flags' | |
46 _GYP_OUT_DIR = 'out_gyp_flags/Release' | |
47 | |
48 | |
49 def ConfigureBuild(gyp_args=[], gn_args=[]): | |
50 print >> sys.stderr, 'Regenerating in %s...' % _GN_OUT_DIR | |
51 # Currently only Release, non-component. | |
52 Run('gn gen %s --args="is_debug=false is_component_build=false %s"' % \ | |
53 (_GN_OUT_DIR, ' '.join(gn_args))) | |
54 | |
55 os.environ.pop('GYP_DEFINES', None) | |
56 # Remove environment variables required by gn but conflicting with GYP. | |
57 # Relevant if Windows toolchain isn't provided by depot_tools. | |
58 os.environ.pop('GYP_MSVS_OVERRIDE_PATH', None) | |
59 os.environ.pop('WINDOWSSDKDIR', None) | |
60 | |
61 gyp_defines = '' | |
62 if len(gyp_args) > 0: | |
63 gyp_defines = '-D' + ' -D'.join(gyp_args) | |
64 | |
65 print >> sys.stderr, 'Regenerating in %s...' % _GYP_OUT_DIR | |
66 Run('python build/gyp_chromium -Goutput_dir=out_gyp_flags ' + \ | |
67 '-Gconfig=Release %s' % gyp_defines) | |
68 | |
69 | |
70 def Counts(dict_of_list): | |
71 """Given a dictionary whose value are lists, returns a dictionary whose values | |
72 are the length of the list. This can be used to summarize a dictionary. | |
73 """ | |
74 return {k: len(v) for k, v in dict_of_list.iteritems()} | |
75 | |
76 | |
77 def CountsByDirname(dict_of_list): | |
78 """Given a list of files, returns a dict of dirname to counts in that dir.""" | |
79 r = {} | |
80 for path in dict_of_list: | |
81 dirname = os.path.dirname(path) | |
82 r.setdefault(dirname, 0) | |
83 r[dirname] += 1 | |
84 return r | |
85 | |
86 | |
87 class Comparison(object): | |
88 """A comparison of the currently-configured build for a target.""" | |
89 | |
90 def __init__(self, gyp_target, gn_target=None): | |
91 """Creates a comparison of a GN and GYP target. If the target names differ | |
92 between the two build systems, then two names may be passed. | |
93 """ | |
94 if gn_target is None: | |
95 gn_target = gyp_target | |
96 self._gyp_target = gyp_target | |
97 self._gn_target = gn_target | |
98 | |
99 self._skipped = [] | |
100 | |
101 self._total_diffs = 0 | |
102 | |
103 self._missing_gyp_flags = {} | |
104 self._missing_gn_flags = {} | |
105 | |
106 self._missing_gyp_files = {} | |
107 self._missing_gn_files = {} | |
108 | |
109 self._CompareFiles() | |
110 | |
111 @property | |
112 def gyp_files(self): | |
113 """Returns the set of files that are in the GYP target.""" | |
114 return set(self._gyp_flags.keys()) | |
115 | |
116 @property | |
117 def gn_files(self): | |
118 """Returns the set of files that are in the GN target.""" | |
119 return set(self._gn_flags.keys()) | |
120 | |
121 @property | |
122 def skipped(self): | |
123 """Returns the list of compiler commands that were not processed during the | |
124 comparison. | |
125 """ | |
126 return self._skipped | |
127 | |
128 @property | |
129 def total_differences(self): | |
130 """Returns the total number of differences detected.""" | |
131 return self._total_diffs | |
132 | |
133 @property | |
134 def missing_in_gyp(self): | |
135 """Differences that are only in GYP build but not in GN, indexed by the | |
136 difference.""" | |
137 return self._missing_gyp_flags | |
138 | |
139 @property | |
140 def missing_in_gn(self): | |
141 """Differences that are only in the GN build but not in GYP, indexed by | |
142 the difference.""" | |
143 return self._missing_gn_flags | |
144 | |
145 @property | |
146 def missing_in_gyp_by_file(self): | |
147 """Differences that are only in the GYP build but not in GN, indexed by | |
148 file. | |
149 """ | |
150 return self._missing_gyp_files | |
151 | |
152 @property | |
153 def missing_in_gn_by_file(self): | |
154 """Differences that are only in the GYP build but not in GN, indexed by | |
155 file. | |
156 """ | |
157 return self._missing_gn_files | |
158 | |
159 def _CompareFiles(self): | |
160 """Performs the actual target comparison.""" | |
161 if sys.platform == 'win32': | |
162 # On Windows flags are stored in .rsp files which are created by building. | |
163 print >> sys.stderr, 'Building in %s...' % _GN_OUT_DIR | |
164 Run('ninja -C %s -d keeprsp %s' % (_GN_OUT_DIR, self._gn_target)) | |
165 print >> sys.stderr, 'Building in %s...' % _GYP_OUT_DIR | |
166 Run('ninja -C %s -d keeprsp %s' % (_GYP_OUT_DIR, self._gn_target)) | |
167 | |
168 gn = Run('ninja -C %s -t commands %s' % (_GN_OUT_DIR, self._gn_target)) | |
169 gyp = Run('ninja -C %s -t commands %s' % (_GYP_OUT_DIR, self._gyp_target)) | |
170 | |
171 self._gn_flags = self._GetFlags(gn.splitlines(), | |
172 os.path.join(os.getcwd(), _GN_OUT_DIR)) | |
173 self._gyp_flags = self._GetFlags(gyp.splitlines(), | |
174 os.path.join(os.getcwd(), _GYP_OUT_DIR)) | |
175 | |
176 all_files = sorted(self.gn_files & self.gyp_files) | |
177 for filename in all_files: | |
178 gyp_flags = self._gyp_flags[filename] | |
179 gn_flags = self._gn_flags[filename] | |
180 self._CompareLists(filename, gyp_flags, gn_flags, 'dash_f') | |
181 self._CompareLists(filename, gyp_flags, gn_flags, 'defines') | |
182 self._CompareLists(filename, gyp_flags, gn_flags, 'include_dirs') | |
183 self._CompareLists(filename, gyp_flags, gn_flags, 'warnings', | |
184 # More conservative warnings in GN we consider to be OK. | |
185 dont_care_gyp=[ | |
186 '/wd4091', # 'keyword' : ignored on left of 'type' when no variable | |
187 # is declared. | |
188 '/wd4456', # Declaration hides previous local declaration. | |
189 '/wd4457', # Declaration hides function parameter. | |
190 '/wd4458', # Declaration hides class member. | |
191 '/wd4459', # Declaration hides global declaration. | |
192 '/wd4702', # Unreachable code. | |
193 '/wd4800', # Forcing value to bool 'true' or 'false'. | |
194 '/wd4838', # Conversion from 'type' to 'type' requires a narrowing | |
195 # conversion. | |
196 ] if sys.platform == 'win32' else None, | |
197 dont_care_gn=[ | |
198 '-Wendif-labels', | |
199 '-Wextra', | |
200 '-Wsign-compare', | |
201 ] if not sys.platform == 'win32' else None) | |
202 self._CompareLists(filename, gyp_flags, gn_flags, 'other') | |
203 | |
204 def _CompareLists(self, filename, gyp, gn, name, | |
205 dont_care_gyp=None, dont_care_gn=None): | |
206 """Return a report of any differences between gyp and gn lists, ignoring | |
207 anything in |dont_care_{gyp|gn}| respectively.""" | |
208 if gyp[name] == gn[name]: | |
209 return | |
210 if not dont_care_gyp: | |
211 dont_care_gyp = [] | |
212 if not dont_care_gn: | |
213 dont_care_gn = [] | |
214 gyp_set = set(gyp[name]) | |
215 gn_set = set(gn[name]) | |
216 missing_in_gyp = gyp_set - gn_set | |
217 missing_in_gn = gn_set - gyp_set | |
218 missing_in_gyp -= set(dont_care_gyp) | |
219 missing_in_gn -= set(dont_care_gn) | |
220 | |
221 for m in missing_in_gyp: | |
222 self._missing_gyp_flags.setdefault(name, {}) \ | |
223 .setdefault(m, []).append(filename) | |
224 self._total_diffs += 1 | |
225 self._missing_gyp_files.setdefault(filename, {}) \ | |
226 .setdefault(name, set()).update(missing_in_gyp) | |
227 | |
228 for m in missing_in_gn: | |
229 self._missing_gn_flags.setdefault(name, {}) \ | |
230 .setdefault(m, []).append(filename) | |
231 self._total_diffs += 1 | |
232 self._missing_gn_files.setdefault(filename, {}) \ | |
233 .setdefault(name, set()).update(missing_in_gn) | |
234 | |
235 def _GetFlags(self, lines, build_dir): | |
236 """Turn a list of command lines into a semi-structured dict.""" | |
237 is_win = sys.platform == 'win32' | |
238 flags_by_output = {} | |
239 for line in lines: | |
240 command_line = shlex.split(line.strip(), posix=not is_win)[1:] | |
241 | |
242 output_name = _FindAndRemoveArgWithValue(command_line, '-o') | |
243 dep_name = _FindAndRemoveArgWithValue(command_line, '-MF') | |
244 | |
245 command_line = _MergeSpacedArgs(command_line, '-Xclang') | |
246 | |
247 cc_file = [x for x in command_line if x.endswith('.cc') or | |
248 x.endswith('.c') or | |
249 x.endswith('.cpp') or | |
250 x.endswith('.mm') or | |
251 x.endswith('.m')] | |
252 if len(cc_file) != 1: | |
253 self._skipped.append(command_line) | |
254 continue | |
255 assert len(cc_file) == 1 | |
256 | |
257 if is_win: | |
258 rsp_file = [x for x in command_line if x.endswith('.rsp')] | |
259 assert len(rsp_file) <= 1 | |
260 if rsp_file: | |
261 rsp_file = os.path.join(build_dir, rsp_file[0][1:]) | |
262 with open(rsp_file, "r") as open_rsp_file: | |
263 command_line = shlex.split(open_rsp_file, posix=False) | |
264 | |
265 defines = [x for x in command_line if x.startswith('-D')] | |
266 include_dirs = [x for x in command_line if x.startswith('-I')] | |
267 dash_f = [x for x in command_line if x.startswith('-f')] | |
268 warnings = \ | |
269 [x for x in command_line if x.startswith('/wd' if is_win else '-W')] | |
270 others = [x for x in command_line if x not in defines and \ | |
271 x not in include_dirs and \ | |
272 x not in dash_f and \ | |
273 x not in warnings and \ | |
274 x not in cc_file] | |
275 | |
276 for index, value in enumerate(include_dirs): | |
277 if value == '-Igen': | |
278 continue | |
279 path = value[2:] | |
280 if not os.path.isabs(path): | |
281 path = os.path.join(build_dir, path) | |
282 include_dirs[index] = '-I' + os.path.normpath(path) | |
283 | |
284 # GYP supports paths above the source root like <(DEPTH)/../foo while such | |
285 # paths are unsupported by gn. But gn allows to use system-absolute paths | |
286 # instead (paths that start with single '/'). Normalize all paths. | |
287 cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))] | |
288 | |
289 # Filter for libFindBadConstructs.so having a relative path in one and | |
290 # absolute path in the other. | |
291 others_filtered = [] | |
292 for x in others: | |
293 if x.startswith('-Xclang ') and \ | |
294 (x.endswith('libFindBadConstructs.so') or \ | |
295 x.endswith('libFindBadConstructs.dylib')): | |
296 others_filtered.append( | |
297 '-Xclang ' + | |
298 os.path.join(os.getcwd(), os.path.normpath( | |
299 os.path.join('out/gn_flags', x.split(' ', 1)[1])))) | |
300 elif x.startswith('-B'): | |
301 others_filtered.append( | |
302 '-B' + | |
303 os.path.join(os.getcwd(), os.path.normpath( | |
304 os.path.join('out/gn_flags', x[2:])))) | |
305 else: | |
306 others_filtered.append(x) | |
307 others = others_filtered | |
308 | |
309 flags_by_output[cc_file[0]] = { | |
310 'output': output_name, | |
311 'depname': dep_name, | |
312 'defines': sorted(defines), | |
313 'include_dirs': sorted(include_dirs), # TODO(scottmg): This is wrong. | |
314 'dash_f': sorted(dash_f), | |
315 'warnings': sorted(warnings), | |
316 'other': sorted(others), | |
317 } | |
318 return flags_by_output | |
319 | |
320 | |
321 def _FindAndRemoveArgWithValue(command_line, argname): | |
322 """Given a command line as a list, remove and return the value of an option | |
323 that takes a value as a separate entry. | |
324 | |
325 Modifies |command_line| in place. | |
326 """ | |
327 if argname not in command_line: | |
328 return '' | |
329 location = command_line.index(argname) | |
330 value = command_line[location + 1] | |
331 command_line[location:location + 2] = [] | |
332 return value | |
333 | |
334 | |
335 def _MergeSpacedArgs(command_line, argname): | |
336 """Combine all arguments |argname| with their values, separated by a space.""" | |
337 i = 0 | |
338 result = [] | |
339 while i < len(command_line): | |
340 arg = command_line[i] | |
341 if arg == argname: | |
342 result.append(arg + ' ' + command_line[i + 1]) | |
343 i += 1 | |
344 else: | |
345 result.append(arg) | |
346 i += 1 | |
347 return result | |
348 | |
349 | |
350 def Run(command_line): | |
351 """Run |command_line| as a subprocess and return stdout. Raises on error.""" | |
352 print >> sys.stderr, command_line | |
353 return subprocess.check_output(command_line, shell=True) | |
354 | |
355 | |
356 def main(): | |
357 if len(sys.argv) != 2 and len(sys.argv) != 3: | |
358 print 'usage: %s gyp_target gn_target' % __file__ | |
359 print ' or: %s target' % __file__ | |
360 return 1 | |
361 | |
362 ConfigureBuild() | |
363 | |
364 gyp_target = sys.argv[1] | |
365 if len(sys.argv) == 2: | |
366 gn_target = gyp_target | |
367 else: | |
368 gn_target = sys.argv[2] | |
369 comparison = Comparison(gyp_target, gn_target) | |
370 | |
371 gyp_files = comparison.gyp_files | |
372 gn_files = comparison.gn_files | |
373 different_source_list = comparison.gyp_files != comparison.gn_files | |
374 if different_source_list: | |
375 print 'Different set of sources files:' | |
376 print ' In gyp, not in GN:\n %s' % '\n '.join( | |
377 sorted(gyp_files - gn_files)) | |
378 print ' In GN, not in gyp:\n %s' % '\n '.join( | |
379 sorted(gn_files - gyp_files)) | |
380 print '\nNote that flags will only be compared for files in both sets.\n' | |
381 | |
382 differing_files = set(comparison.missing_in_gn_by_file.keys()) & \ | |
383 set(comparison.missing_in_gyp_by_file.keys()) | |
384 files_with_given_differences = {} | |
385 for filename in differing_files: | |
386 output = '' | |
387 missing_in_gyp = comparison.missing_in_gyp_by_file.get(filename, {}) | |
388 missing_in_gn = comparison.missing_in_gn_by_file.get(filename, {}) | |
389 difference_types = sorted(set(missing_in_gyp.keys() + missing_in_gn.keys())) | |
390 for difference_type in difference_types: | |
391 output += ' %s differ:\n' % difference_type | |
392 if difference_type in missing_in_gyp: | |
393 output += ' In gyp, but not in GN:\n %s' % '\n '.join( | |
394 sorted(missing_in_gyp[difference_type])) + '\n' | |
395 if difference_type in missing_in_gn: | |
396 output += ' In GN, but not in gyp:\n %s' % '\n '.join( | |
397 sorted(missing_in_gn[difference_type])) + '\n' | |
398 if output: | |
399 files_with_given_differences.setdefault(output, []).append(filename) | |
400 | |
401 for diff, files in files_with_given_differences.iteritems(): | |
402 print '\n'.join(sorted(files)) | |
403 print diff | |
404 | |
405 print 'Total differences:', comparison.total_differences | |
406 # TODO(scottmg): Return failure on difference once we're closer to identical. | |
407 return 0 | |
408 | |
409 | |
410 if __name__ == '__main__': | |
411 sys.exit(main()) | |
OLD | NEW |