Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1)

Side by Side Diff: tools/compare_build_artifacts/compare_build_artifacts.py

Issue 2232723002: Import compare_build_artifacts.py from isolate recipe module. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: updated Android. Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | tools/compare_build_artifacts/deterministic_build_blacklist.json » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Compare the artifacts from two builds."""
7
8 import difflib
9 import json
10 import optparse
11 import os
12 import struct
13 import sys
14 import time
15
16 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
17
18
19 # List of files that are known to be non deterministic. This is a "temporary"
20 # workaround to find regression on the deterministic builders.
21 #
22 # PNaCl general bug: https://crbug.com/429358
23 #
24 # TODO(sebmarchand): Remove this once all the files are deterministic.
25 WHITELIST = {
26 # https://crbug.com/383340
27 'android': {
28 'flatc',
29 },
30
31 # https://crbug.com/330263
32 'linux': {
33 # Completed.
34 },
35
36 # https://crbug.com/330262
37 'mac': {
38 'accelerated_widget_mac_unittests',
39 'accessibility_unittests',
40 'angle_end2end_tests',
41 'angle_unittests',
42 'App Shell',
43 'app_shell_unittests',
44 'ar_sample_test_driver',
45 'audio_unittests',
46 'base_unittests',
47 'battor_agent_unittests',
48 'blink_heap_unittests',
49 'blink_platform_perftests',
50 'blink_platform_unittests',
51 'bluetooth_metrics_hash',
52 'boringssl_unittests',
53 'browser_tests',
54 'cacheinvalidation_unittests',
55 'capture_unittests',
56 'cast_benchmarks',
57 'cast_receiver_app',
58 'cast_sender_app',
59 'cast_simulator',
60 'cast_unittests',
61 'cc_blink_unittests',
62 'cc_perftests',
63 'cc_unittests',
64 'chrome_app_unittests',
65 'chromedriver',
66 'chromedriver_tests',
67 'chromedriver_unittests',
68 'chromoting_test_driver',
69 'command_buffer_gles2_tests',
70 'components_browsertests',
71 'components_perftests',
72 'components_unittests',
73 'compositor_unittests',
74 'content_browsertests',
75 'content_perftests',
76 'content_unittests',
77 'courgette_unittests',
78 'crashpad_handler',
79 'crypto_unittests',
80 'device_unittests',
81 'display_compositor_benchmark',
82 'display_compositor_gl_tests',
83 'display_unittests',
84 'events_unittests',
85 'extensions_browsertests',
86 'extensions_unittests',
87 'ffmpeg_regression_tests',
88 'filesystem_service_unittests',
89 'filter_fuzz_stub',
90 'flatc',
91 'gcm_unit_tests',
92 'generate_barcode_video',
93 'generate_timecode_audio',
94 'gfx_unittests',
95 'gin_unittests',
96 'gles2_conform_support',
97 'gles2_conform_test',
98 'gl_tests',
99 'gl_unittests',
100 'gn_unittests',
101 'google_apis_unittests',
102 'gpu_ipc_service_unittests',
103 'gpu_perftests',
104 'gpu_unittests',
105 'interactive_ui_tests',
106 'ipc_fuzzer',
107 'ipc_fuzzer_replay',
108 'ipc_message_dump.so',
109 'ipc_message_list',
110 'ipc_message_util',
111 'ipc_tests',
112 'it2me_standalone_host_main',
113 'jingle_unittests',
114 'khronos_glcts_test',
115 'leveldb_service_unittests',
116 'libaddressinput_unittests',
117 'libapp_shell_framework.dylib',
118 'libcommand_buffer_gles2.dylib',
119 'libmedia_library.dylib',
120 'libphonenumber_unittests',
121 'mac_installer_unittests',
122 'macviews_interactive_ui_tests',
123 'media_blink_unittests',
124 'media_mojo_shell_unittests',
125 'media_mojo_unittests',
126 'media_perftests',
127 'media_pipeline_integration_unittests',
128 'media_unittests',
129 'message_center_unittests',
130 'midi_unittests',
131 'mojo_common_unittests',
132 'mojo_js_integration_tests',
133 'mojo_js_unittests',
134 'mojo_public_bindings_unittests',
135 'mojo_public_system_unittests',
136 'mojo_runner_host_unittests',
137 'mojo_shell_unittests',
138 'mojo_system_unittests',
139 'nacl_loader_unittests',
140 'native_theme_unittests',
141 'net_unittests',
142 'osmesa.so',
143 'performance_browser_tests',
144 'ppapi_perftests',
145 'ppapi_unittests',
146 'printing_unittests',
147 'proximity_auth_unittests',
148 'remoting_perftests',
149 'remoting_start_host',
150 'remoting_unittests',
151 'sandbox_mac_unittests',
152 'shell_dialogs_unittests',
153 'skia_unittests',
154 'snapshot_unittests',
155 'sql_unittests',
156 'sync_client',
157 'sync_integration_tests',
158 'sync_listen_notifications',
159 'sync_performance_tests',
160 'sync_unit_tests',
161 'udp_proxy',
162 'ui_base_unittests',
163 'ui_struct_traits_unittests',
164 'ui_touch_selection_unittests',
165 'unit_tests',
166 'url_ipc_unittests',
167 'url_unittests',
168 'video_encode_accelerator_unittest',
169 'views_examples_exe',
170 'views_examples_with_content_exe',
171 'views_unittests',
172 'webkit_unit_tests',
173 'wtf_unittests',
174 },
175
176 # https://crbug.com/330260
177 'win': {
178 'accessibility_unittests.exe',
179 'angle_end2end_tests.exe',
180 'angle_perftests.exe',
181 'angle_unittests.exe',
182 'app_driver_library.dll',
183 'app_list_demo.exe',
184 'app_list_presenter_unittests.exe',
185 'app_list_unittests.exe',
186 'app_shell.exe',
187 'app_shell_unittests.exe',
188 'ar_sample_test_driver.exe',
189 'ash_library.dll',
190 'ash_shell_with_content.exe',
191 'ash_sysui_library.dll',
192 'ash_unittests.exe',
193 'audio_unittests.exe',
194 'aura_demo.exe',
195 'aura_unittests.exe',
196 'base_i18n_perftests.exe',
197 'base_perftests.exe',
198 'base_unittests.exe',
199 'blink_converters_unittests.exe',
200 'blink_deprecated_test_plugin.dll',
201 'blink_heap_unittests.exe',
202 'blink_platform_perftests.exe',
203 'blink_platform_unittests.exe',
204 'blink_test_plugin.dll',
205 'bluetooth_metrics_hash.exe',
206 'browser_library.dll',
207 'browser_tests.exe',
208 'capture_unittests.exe',
209 'cast_benchmarks.exe',
210 'cast_receiver_app.exe',
211 'cast_sender_app.exe',
212 'cast_simulator.exe',
213 'cast_unittests.exe',
214 'catalog_viewer_library.dll',
215 'cc_blink_unittests.exe',
216 'cc_perftests.exe',
217 'cctest.exe',
218 'cc_unittests.exe',
219 'ced_unittests.exe',
220 'cert_verify_tool.exe',
221 'chrome_app_unittests.exe',
222 'chrome_child.dll',
223 'chrome.dll',
224 'chromedriver.exe',
225 'chromedriver_tests.exe',
226 'chromedriver_unittests.exe',
227 'chrome_elf_unittests.exe',
228 'chrome.exe',
229 'chromoting_test_driver.exe',
230 'command_buffer_gles2.dll',
231 'components_browsertests.exe',
232 'components_perftests.exe',
233 'components_unittests.exe',
234 'compositor_unittests.exe',
235 'content_browsertests.exe',
236 'content_perftests.exe',
237 'content_shell.exe',
238 'content_unittests.exe',
239 'courgette64.exe',
240 'crypto_unittests.exe',
241 'd8.exe',
242 'device_unittests.exe',
243 'display_compositor_benchmark.exe',
244 'display_compositor_gl_tests.exe',
245 'display_unittests.exe',
246 'events_unittests.exe',
247 'extensions_browsertests.exe',
248 'extensions_unittests.exe',
249 'ffmpeg_regression_tests.exe',
250 'filesystem_service_unittests.exe',
251 'filter_fuzz_stub.exe',
252 'force_mic_volume_max.exe',
253 'gcapi_test.exe',
254 'gcm_unit_tests.exe',
255 'generate_barcode_video.exe',
256 'generate-bytecode-expectations.exe',
257 'generate_timecode_audio.exe',
258 'get_server_time.exe',
259 'gfx_unittests.exe',
260 'gin_shell.exe',
261 'gin_unittests.exe',
262 'gles2_conform_support.exe',
263 'gles2_conform_test.exe',
264 'gl_tests.exe',
265 'gl_unittests.exe',
266 'google_apis_unittests.exe',
267 'gpu_ipc_service_unittests.exe',
268 'gpu_perftests.exe',
269 'gpu_unittests.exe',
270 'image_operations_bench.exe',
271 'input_device_unittests.exe',
272 'interactive_ui_tests.exe',
273 'ipc_perftests.exe',
274 'ipc_tests.exe',
275 'it2me_standalone_host_main.exe',
276 'jingle_unittests.exe',
277 'keyboard_unittests.exe',
278 'khronos_glcts_test.exe',
279 'leveldb_service_unittests.exe',
280 'libaddressinput_unittests.exe',
281 'login_library.dll',
282 'mash_init_library.dll',
283 'mash_unittests.exe',
284 'media_blink_unittests.exe',
285 'media_library.dll',
286 'media_mojo_shell_unittests.exe',
287 'media_mojo_unittests.exe',
288 'media_perftests.exe',
289 'media_pipeline_integration_unittests.exe',
290 'media_unittests.exe',
291 'message_center_unittests.exe',
292 'midi_unittests.exe',
293 'mini_installer.exe',
294 'mksnapshot.exe',
295 'mojo_js_integration_tests.exe',
296 'mojo_js_unittests.exe',
297 'mojo_message_pipe_perftests.exe',
298 'mojo_public_bindings_perftests.exe',
299 'mojo_public_bindings_unittests.exe',
300 'mojo_public_system_perftests.exe',
301 'mojo_public_system_unittests.exe',
302 'mojo_system_unittests.exe',
303 'mus_clipboard_unittests.exe',
304 'mus_common_unittests.exe',
305 'mus_demo_library.dll',
306 'mus_demo_unittests.exe',
307 'mus_gpu_unittests.exe',
308 'mus_ime_unittests.exe',
309 'mus_public_unittests.exe',
310 'mus_ws_unittests.exe',
311 'nacl_irt_x86_32.nexe',
312 'nacl_irt_x86_64.nexe',
313 'nacl_loader_unittests.exe',
314 'native_theme_unittests.exe',
315 'navigation.exe',
316 'navigation_unittests.exe',
317 'net_perftests.exe',
318 'net_unittests.exe',
319 'next_version_mini_installer.exe',
320 'pdfium_embeddertests.exe',
321 'pdfium_test.exe',
322 'performance_browser_tests.exe',
323 'power_saver_test_plugin.dll',
324 'ppapi_nacl_tests_newlib_x86_32.nexe',
325 'ppapi_nacl_tests_newlib_x86_64.nexe',
326 'ppapi_nacl_tests_pnacl_newlib_x32.nexe',
327 'ppapi_nacl_tests_pnacl_newlib_x32_nonsfi.nexe',
328 'ppapi_nacl_tests_pnacl_newlib_x64.nexe',
329 'ppapi_perftests.exe',
330 'ppapi_tests.dll',
331 'ppapi_unittests.exe',
332 'printing_unittests.exe',
333 'proximity_auth_unittests.exe',
334 'quic_client.exe',
335 'quick_launch_library.dll',
336 'quic_packet_printer.exe',
337 'quic_server.exe',
338 'remoting_breakpad_tester.exe',
339 'remoting_core.dll',
340 'remoting_perftests.exe',
341 'remoting_unittests.exe',
342 'sbox_unittests.exe',
343 'screenlock_library.dll',
344 'shell_dialogs_unittests.exe',
345 'skia_unittests.exe',
346 'snapshot_unittests.exe',
347 'sql_unittests.exe',
348 'sync_client.exe',
349 'sync_integration_tests.exe',
350 'sync_listen_notifications.exe',
351 'sync_performance_tests.exe',
352 'sync_unit_tests.exe',
353 'task_viewer_library.dll',
354 'test_ime_driver_library.dll',
355 'test_wm_library.dll',
356 'touch_hud_library.dll',
357 'udp_proxy.exe',
358 'ui_base_unittests.exe',
359 'ui_library.dll',
360 'ui_struct_traits_unittests.exe',
361 'ui_touch_selection_unittests.exe',
362 'unit_tests.exe',
363 'unittests.exe',
364 'url_unittests.exe',
365 'v8_hello_world.exe',
366 'v8_parser_shell.exe',
367 'v8_sample_process.exe',
368 'v8_shell.exe',
369 'v8_simple_json_fuzzer.exe',
370 'v8_simple_parser_fuzzer.exe',
371 'v8_simple_regexp_fuzzer.exe',
372 'v8_simple_wasm_asmjs_fuzzer.exe',
373 'v8_simple_wasm_fuzzer.exe',
374 'video_decode_accelerator_unittest.exe',
375 'video_encode_accelerator_unittest.exe',
376 'views_examples_exe.exe',
377 'views_examples_library.dll',
378 'views_examples_with_content_exe.exe',
379 'views_mus_interactive_ui_tests.exe',
380 'views_mus_unittests.exe',
381 'views_unittests.exe',
382 'webkit_unit_tests.exe',
383 'webtest_library.dll',
384 'window_type_launcher_library.dll',
385 'wm_unittests.exe',
386 'wtf_unittests.exe',
387 },
388 }
389
390 def get_files_to_compare(build_dir, recursive=False):
391 """Get the list of files to compare."""
392 # TODO(maruel): Add '.pdb'.
393 allowed = frozenset(
394 ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so'))
395 non_x_ok_exts = frozenset(('.apk', '.isolated'))
396 def check(f):
397 if not os.path.isfile(f):
398 return False
399 if os.path.basename(f).startswith('.'):
400 return False
401 ext = os.path.splitext(f)[1]
402 if ext in non_x_ok_exts:
403 return True
404 return ext in allowed and os.access(f, os.X_OK)
405
406 ret_files = set()
407 for root, dirs, files in os.walk(build_dir):
408 if not recursive:
409 dirs[:] = [d for d in dirs if d.endswith('_apk')]
410 for f in (f for f in files if check(os.path.join(root, f))):
411 ret_files.add(os.path.relpath(os.path.join(root, f), build_dir))
412 return ret_files
413
414
415 def diff_dict(a, b):
416 """Returns a yaml-like textural diff of two dict.
417
418 It is currently optimized for the .isolated format.
419 """
420 out = ''
421 for key in set(a) | set(b):
422 va = a.get(key)
423 vb = b.get(key)
424 if va.__class__ != vb.__class__:
425 out += '- %s: %r != %r\n' % (key, va, vb)
426 elif isinstance(va, dict):
427 c = diff_dict(va, vb)
428 if c:
429 out += '- %s:\n%s\n' % (
430 key, '\n'.join(' ' + l for l in c.splitlines()))
431 elif va != vb:
432 out += '- %s: %s != %s\n' % (key, va, vb)
433 return out.rstrip()
434
435
436 def diff_binary(first_filepath, second_filepath, file_len):
437 """Returns a compact binary diff if the diff is small enough."""
438 CHUNK_SIZE = 32
439 MAX_STREAMS = 10
440 diffs = 0
441 streams = []
442 offset = 0
443 with open(first_filepath, 'rb') as lhs:
444 with open(second_filepath, 'rb') as rhs:
445 while True:
446 lhs_data = lhs.read(CHUNK_SIZE)
447 rhs_data = rhs.read(CHUNK_SIZE)
448 if not lhs_data:
449 break
450 if lhs_data != rhs_data:
451 diffs += sum(l != r for l, r in zip(lhs_data, rhs_data))
452 if streams is not None:
453 if len(streams) < MAX_STREAMS:
454 streams.append((offset, lhs_data, rhs_data))
455 else:
456 streams = None
457 offset += len(lhs_data)
458 del lhs_data
459 del rhs_data
460 if not diffs:
461 return None
462 result = '%d out of %d bytes are different (%.2f%%)' % (
463 diffs, file_len, 100.0 * diffs / file_len)
464 if streams:
465 encode = lambda text: ''.join(i if 31 < ord(i) < 128 else '.' for i in text)
466 for offset, lhs_data, rhs_data in streams:
467 lhs_line = '%s \'%s\'' % (lhs_data.encode('hex'), encode(lhs_data))
468 rhs_line = '%s \'%s\'' % (rhs_data.encode('hex'), encode(rhs_data))
469 diff = list(difflib.Differ().compare([lhs_line], [rhs_line]))[-1][2:-1]
470 result += '\n 0x%-8x: %s\n %s\n %s' % (
471 offset, lhs_line, rhs_line, diff)
472 return result
473
474
475 def compare_files(first_filepath, second_filepath):
476 """Compares two binaries and return the number of differences between them.
477
478 Returns None if the files are equal, a string otherwise.
479 """
480 if first_filepath.endswith('.isolated'):
481 with open(first_filepath, 'rb') as f:
482 lhs = json.load(f)
483 with open(second_filepath, 'rb') as f:
484 rhs = json.load(f)
485 diff = diff_dict(lhs, rhs)
486 if diff:
487 return '\n' + '\n'.join(' ' + line for line in diff.splitlines())
488 # else, falls through binary comparison, it must be binary equal too.
489
490 file_len = os.stat(first_filepath).st_size
491 if file_len != os.stat(second_filepath).st_size:
492 return 'different size: %d != %d' % (
493 file_len, os.stat(second_filepath).st_size)
494
495 return diff_binary(first_filepath, second_filepath, file_len)
496
497
498 def compare_build_artifacts(first_dir, second_dir, target_platform,
499 recursive=False):
500 """Compares the artifacts from two distinct builds."""
501 if not os.path.isdir(first_dir):
502 print >> sys.stderr, '%s isn\'t a valid directory.' % first_dir
503 return 1
504 if not os.path.isdir(second_dir):
505 print >> sys.stderr, '%s isn\'t a valid directory.' % second_dir
506 return 1
507
508 epoch_hex = struct.pack('<I', int(time.time())).encode('hex')
509 print('Epoch: %s' %
510 ' '.join(epoch_hex[i:i+2] for i in xrange(0, len(epoch_hex), 2)))
511
512 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f:
513 blacklist = frozenset(json.load(f))
514 whitelist = WHITELIST[target_platform]
515
516 # The two directories.
517 first_list = get_files_to_compare(first_dir, recursive) - blacklist
518 second_list = get_files_to_compare(second_dir, recursive) - blacklist
519
520 equals = []
521 expected_diffs = []
522 unexpected_diffs = []
523 unexpected_equals = []
524 all_files = sorted(first_list & second_list)
525 missing_files = sorted(first_list.symmetric_difference(second_list))
526 if missing_files:
527 print >> sys.stderr, 'Different list of files in both directories:'
528 print >> sys.stderr, '\n'.join(' ' + i for i in missing_files)
529 unexpected_diffs.extend(missing_files)
530
531 max_filepath_len = max(len(n) for n in all_files)
532 for f in all_files:
533 first_file = os.path.join(first_dir, f)
534 second_file = os.path.join(second_dir, f)
535 result = compare_files(first_file, second_file)
536 if not result:
537 tag = 'equal'
538 equals.append(f)
539 if f in whitelist:
540 unexpected_equals.append(f)
541 else:
542 if f in whitelist:
543 expected_diffs.append(f)
544 tag = 'expected'
545 else:
546 unexpected_diffs.append(f)
547 tag = 'unexpected'
548 result = 'DIFFERENT (%s): %s' % (tag, result)
549 print('%-*s: %s' % (max_filepath_len, f, result))
550 unexpected_diffs.sort()
551
552 print('Equals: %d' % len(equals))
553 print('Expected diffs: %d' % len(expected_diffs))
554 print('Unexpected diffs: %d' % len(unexpected_diffs))
555 if unexpected_diffs:
556 print('Unexpected files with diffs:\n')
557 for u in unexpected_diffs:
558 print(' %s' % u)
559 if unexpected_equals:
560 print('Unexpected files with no diffs:\n')
561 for u in unexpected_equals:
562 print(' %s' % u)
563
564 return int(bool(unexpected_diffs))
565
566
567 def main():
568 parser = optparse.OptionParser(usage='%prog [options]')
569 parser.add_option(
570 '-f', '--first-build-dir', help='The first build directory.')
571 parser.add_option(
572 '-s', '--second-build-dir', help='The second build directory.')
573 parser.add_option('-r', '--recursive', action='store_true', default=False,
574 help='Indicates if the comparison should be recursive.')
575 target = {
576 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win'
577 }.get(sys.platform, sys.platform)
578 parser.add_option('-t', '--target-platform', help='The target platform.'
579 default=target, choices=('android', 'mac', 'linux', 'win'))
580 options, _ = parser.parse_args()
581
582 if not options.first_build_dir:
583 parser.error('--first-build-dir is required')
584 if not options.second_build_dir:
585 parser.error('--second-build-dir is required')
586 if not options.target_platform:
587 parser.error('--target-platform is required')
588
589 return compare_build_artifacts(os.path.abspath(options.first_build_dir),
590 os.path.abspath(options.second_build_dir),
591 options.target_platform,
592 options.recursive)
593
594
595 if __name__ == '__main__':
596 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | tools/compare_build_artifacts/deterministic_build_blacklist.json » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698