OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """Compare the artifacts from two builds.""" | |
7 | |
8 import difflib | |
9 import json | |
10 import optparse | |
11 import os | |
12 import struct | |
13 import sys | |
14 import time | |
15 | |
16 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
17 | |
18 | |
19 # List of files that are known to be non deterministic. This is a "temporary" | |
20 # workaround to find regression on the deterministic builders. | |
21 # | |
22 # PNaCl general bug: https://crbug.com/429358 | |
23 # | |
24 # TODO(sebmarchand): Remove this once all the files are deterministic. | |
25 WHITELIST = { | |
26 # https://crbug.com/383340 | |
27 'android': { | |
28 'flatc', | |
29 }, | |
30 | |
31 # https://crbug.com/330263 | |
32 'linux': { | |
33 # Completed. | |
34 }, | |
35 | |
36 # https://crbug.com/330262 | |
37 'mac': { | |
38 'accelerated_widget_mac_unittests', | |
39 'accessibility_unittests', | |
40 'angle_end2end_tests', | |
41 'angle_unittests', | |
42 'App Shell', | |
43 'app_shell_unittests', | |
44 'ar_sample_test_driver', | |
45 'audio_unittests', | |
46 'base_unittests', | |
47 'battor_agent_unittests', | |
48 'blink_heap_unittests', | |
49 'blink_platform_perftests', | |
50 'blink_platform_unittests', | |
51 'bluetooth_metrics_hash', | |
52 'boringssl_unittests', | |
53 'browser_tests', | |
54 'cacheinvalidation_unittests', | |
55 'capture_unittests', | |
56 'cast_benchmarks', | |
57 'cast_receiver_app', | |
58 'cast_sender_app', | |
59 'cast_simulator', | |
60 'cast_unittests', | |
61 'cc_blink_unittests', | |
62 'cc_perftests', | |
63 'cc_unittests', | |
64 'chrome_app_unittests', | |
65 'chromedriver', | |
66 'chromedriver_tests', | |
67 'chromedriver_unittests', | |
68 'chromoting_test_driver', | |
69 'command_buffer_gles2_tests', | |
70 'components_browsertests', | |
71 'components_perftests', | |
72 'components_unittests', | |
73 'compositor_unittests', | |
74 'content_browsertests', | |
75 'content_perftests', | |
76 'content_unittests', | |
77 'courgette_unittests', | |
78 'crashpad_handler', | |
79 'crypto_unittests', | |
80 'device_unittests', | |
81 'display_compositor_benchmark', | |
82 'display_compositor_gl_tests', | |
83 'display_unittests', | |
84 'events_unittests', | |
85 'extensions_browsertests', | |
86 'extensions_unittests', | |
87 'ffmpeg_regression_tests', | |
88 'filesystem_service_unittests', | |
89 'filter_fuzz_stub', | |
90 'flatc', | |
91 'gcm_unit_tests', | |
92 'generate_barcode_video', | |
93 'generate_timecode_audio', | |
94 'gfx_unittests', | |
95 'gin_unittests', | |
96 'gles2_conform_support', | |
97 'gles2_conform_test', | |
98 'gl_tests', | |
99 'gl_unittests', | |
100 'gn_unittests', | |
101 'google_apis_unittests', | |
102 'gpu_ipc_service_unittests', | |
103 'gpu_perftests', | |
104 'gpu_unittests', | |
105 'interactive_ui_tests', | |
106 'ipc_fuzzer', | |
107 'ipc_fuzzer_replay', | |
108 'ipc_message_dump.so', | |
109 'ipc_message_list', | |
110 'ipc_message_util', | |
111 'ipc_tests', | |
112 'it2me_standalone_host_main', | |
113 'jingle_unittests', | |
114 'khronos_glcts_test', | |
115 'leveldb_service_unittests', | |
116 'libaddressinput_unittests', | |
117 'libapp_shell_framework.dylib', | |
118 'libcommand_buffer_gles2.dylib', | |
119 'libmedia_library.dylib', | |
120 'libphonenumber_unittests', | |
121 'mac_installer_unittests', | |
122 'macviews_interactive_ui_tests', | |
123 'media_blink_unittests', | |
124 'media_mojo_shell_unittests', | |
125 'media_mojo_unittests', | |
126 'media_perftests', | |
127 'media_pipeline_integration_unittests', | |
128 'media_unittests', | |
129 'message_center_unittests', | |
130 'midi_unittests', | |
131 'mojo_common_unittests', | |
132 'mojo_js_integration_tests', | |
133 'mojo_js_unittests', | |
134 'mojo_public_bindings_unittests', | |
135 'mojo_public_system_unittests', | |
136 'mojo_runner_host_unittests', | |
137 'mojo_shell_unittests', | |
138 'mojo_system_unittests', | |
139 'nacl_loader_unittests', | |
140 'native_theme_unittests', | |
141 'net_unittests', | |
142 'osmesa.so', | |
143 'performance_browser_tests', | |
144 'ppapi_perftests', | |
145 'ppapi_unittests', | |
146 'printing_unittests', | |
147 'proximity_auth_unittests', | |
148 'remoting_perftests', | |
149 'remoting_start_host', | |
150 'remoting_unittests', | |
151 'sandbox_mac_unittests', | |
152 'shell_dialogs_unittests', | |
153 'skia_unittests', | |
154 'snapshot_unittests', | |
155 'sql_unittests', | |
156 'sync_client', | |
157 'sync_integration_tests', | |
158 'sync_listen_notifications', | |
159 'sync_performance_tests', | |
160 'sync_unit_tests', | |
161 'udp_proxy', | |
162 'ui_base_unittests', | |
163 'ui_struct_traits_unittests', | |
164 'ui_touch_selection_unittests', | |
165 'unit_tests', | |
166 'url_ipc_unittests', | |
167 'url_unittests', | |
168 'video_encode_accelerator_unittest', | |
169 'views_examples_exe', | |
170 'views_examples_with_content_exe', | |
171 'views_unittests', | |
172 'webkit_unit_tests', | |
173 'wtf_unittests', | |
174 }, | |
175 | |
176 # https://crbug.com/330260 | |
177 'win': { | |
178 'accessibility_unittests.exe', | |
179 'angle_end2end_tests.exe', | |
180 'angle_perftests.exe', | |
181 'angle_unittests.exe', | |
182 'app_driver_library.dll', | |
183 'app_list_demo.exe', | |
184 'app_list_presenter_unittests.exe', | |
185 'app_list_unittests.exe', | |
186 'app_shell.exe', | |
187 'app_shell_unittests.exe', | |
188 'ar_sample_test_driver.exe', | |
189 'ash_library.dll', | |
190 'ash_shell_with_content.exe', | |
191 'ash_sysui_library.dll', | |
192 'ash_unittests.exe', | |
193 'audio_unittests.exe', | |
194 'aura_demo.exe', | |
195 'aura_unittests.exe', | |
196 'base_i18n_perftests.exe', | |
197 'base_perftests.exe', | |
198 'base_unittests.exe', | |
199 'blink_converters_unittests.exe', | |
200 'blink_deprecated_test_plugin.dll', | |
201 'blink_heap_unittests.exe', | |
202 'blink_platform_perftests.exe', | |
203 'blink_platform_unittests.exe', | |
204 'blink_test_plugin.dll', | |
205 'bluetooth_metrics_hash.exe', | |
206 'browser_library.dll', | |
207 'browser_tests.exe', | |
208 'capture_unittests.exe', | |
209 'cast_benchmarks.exe', | |
210 'cast_receiver_app.exe', | |
211 'cast_sender_app.exe', | |
212 'cast_simulator.exe', | |
213 'cast_unittests.exe', | |
214 'catalog_viewer_library.dll', | |
215 'cc_blink_unittests.exe', | |
216 'cc_perftests.exe', | |
217 'cctest.exe', | |
218 'cc_unittests.exe', | |
219 'ced_unittests.exe', | |
220 'cert_verify_tool.exe', | |
221 'chrome_app_unittests.exe', | |
222 'chrome_child.dll', | |
223 'chrome.dll', | |
224 'chromedriver.exe', | |
225 'chromedriver_tests.exe', | |
226 'chromedriver_unittests.exe', | |
227 'chrome_elf_unittests.exe', | |
228 'chrome.exe', | |
229 'chromoting_test_driver.exe', | |
230 'command_buffer_gles2.dll', | |
231 'components_browsertests.exe', | |
232 'components_perftests.exe', | |
233 'components_unittests.exe', | |
234 'compositor_unittests.exe', | |
235 'content_browsertests.exe', | |
236 'content_perftests.exe', | |
237 'content_shell.exe', | |
238 'content_unittests.exe', | |
239 'courgette64.exe', | |
240 'crypto_unittests.exe', | |
241 'd8.exe', | |
242 'device_unittests.exe', | |
243 'display_compositor_benchmark.exe', | |
244 'display_compositor_gl_tests.exe', | |
245 'display_unittests.exe', | |
246 'events_unittests.exe', | |
247 'extensions_browsertests.exe', | |
248 'extensions_unittests.exe', | |
249 'ffmpeg_regression_tests.exe', | |
250 'filesystem_service_unittests.exe', | |
251 'filter_fuzz_stub.exe', | |
252 'force_mic_volume_max.exe', | |
253 'gcapi_test.exe', | |
254 'gcm_unit_tests.exe', | |
255 'generate_barcode_video.exe', | |
256 'generate-bytecode-expectations.exe', | |
257 'generate_timecode_audio.exe', | |
258 'get_server_time.exe', | |
259 'gfx_unittests.exe', | |
260 'gin_shell.exe', | |
261 'gin_unittests.exe', | |
262 'gles2_conform_support.exe', | |
263 'gles2_conform_test.exe', | |
264 'gl_tests.exe', | |
265 'gl_unittests.exe', | |
266 'google_apis_unittests.exe', | |
267 'gpu_ipc_service_unittests.exe', | |
268 'gpu_perftests.exe', | |
269 'gpu_unittests.exe', | |
270 'image_operations_bench.exe', | |
271 'input_device_unittests.exe', | |
272 'interactive_ui_tests.exe', | |
273 'ipc_perftests.exe', | |
274 'ipc_tests.exe', | |
275 'it2me_standalone_host_main.exe', | |
276 'jingle_unittests.exe', | |
277 'keyboard_unittests.exe', | |
278 'khronos_glcts_test.exe', | |
279 'leveldb_service_unittests.exe', | |
280 'libaddressinput_unittests.exe', | |
281 'login_library.dll', | |
282 'mash_init_library.dll', | |
283 'mash_unittests.exe', | |
284 'media_blink_unittests.exe', | |
285 'media_library.dll', | |
286 'media_mojo_shell_unittests.exe', | |
287 'media_mojo_unittests.exe', | |
288 'media_perftests.exe', | |
289 'media_pipeline_integration_unittests.exe', | |
290 'media_unittests.exe', | |
291 'message_center_unittests.exe', | |
292 'midi_unittests.exe', | |
293 'mini_installer.exe', | |
294 'mksnapshot.exe', | |
295 'mojo_js_integration_tests.exe', | |
296 'mojo_js_unittests.exe', | |
297 'mojo_message_pipe_perftests.exe', | |
298 'mojo_public_bindings_perftests.exe', | |
299 'mojo_public_bindings_unittests.exe', | |
300 'mojo_public_system_perftests.exe', | |
301 'mojo_public_system_unittests.exe', | |
302 'mojo_system_unittests.exe', | |
303 'mus_clipboard_unittests.exe', | |
304 'mus_common_unittests.exe', | |
305 'mus_demo_library.dll', | |
306 'mus_demo_unittests.exe', | |
307 'mus_gpu_unittests.exe', | |
308 'mus_ime_unittests.exe', | |
309 'mus_public_unittests.exe', | |
310 'mus_ws_unittests.exe', | |
311 'nacl_irt_x86_32.nexe', | |
312 'nacl_irt_x86_64.nexe', | |
313 'nacl_loader_unittests.exe', | |
314 'native_theme_unittests.exe', | |
315 'navigation.exe', | |
316 'navigation_unittests.exe', | |
317 'net_perftests.exe', | |
318 'net_unittests.exe', | |
319 'next_version_mini_installer.exe', | |
320 'pdfium_embeddertests.exe', | |
321 'pdfium_test.exe', | |
322 'performance_browser_tests.exe', | |
323 'power_saver_test_plugin.dll', | |
324 'ppapi_nacl_tests_newlib_x86_32.nexe', | |
325 'ppapi_nacl_tests_newlib_x86_64.nexe', | |
326 'ppapi_nacl_tests_pnacl_newlib_x32.nexe', | |
327 'ppapi_nacl_tests_pnacl_newlib_x32_nonsfi.nexe', | |
328 'ppapi_nacl_tests_pnacl_newlib_x64.nexe', | |
329 'ppapi_perftests.exe', | |
330 'ppapi_tests.dll', | |
331 'ppapi_unittests.exe', | |
332 'printing_unittests.exe', | |
333 'proximity_auth_unittests.exe', | |
334 'quic_client.exe', | |
335 'quick_launch_library.dll', | |
336 'quic_packet_printer.exe', | |
337 'quic_server.exe', | |
338 'remoting_breakpad_tester.exe', | |
339 'remoting_core.dll', | |
340 'remoting_perftests.exe', | |
341 'remoting_unittests.exe', | |
342 'sbox_unittests.exe', | |
343 'screenlock_library.dll', | |
344 'shell_dialogs_unittests.exe', | |
345 'skia_unittests.exe', | |
346 'snapshot_unittests.exe', | |
347 'sql_unittests.exe', | |
348 'sync_client.exe', | |
349 'sync_integration_tests.exe', | |
350 'sync_listen_notifications.exe', | |
351 'sync_performance_tests.exe', | |
352 'sync_unit_tests.exe', | |
353 'task_viewer_library.dll', | |
354 'test_ime_driver_library.dll', | |
355 'test_wm_library.dll', | |
356 'touch_hud_library.dll', | |
357 'udp_proxy.exe', | |
358 'ui_base_unittests.exe', | |
359 'ui_library.dll', | |
360 'ui_struct_traits_unittests.exe', | |
361 'ui_touch_selection_unittests.exe', | |
362 'unit_tests.exe', | |
363 'unittests.exe', | |
364 'url_unittests.exe', | |
365 'v8_hello_world.exe', | |
366 'v8_parser_shell.exe', | |
367 'v8_sample_process.exe', | |
368 'v8_shell.exe', | |
369 'v8_simple_json_fuzzer.exe', | |
370 'v8_simple_parser_fuzzer.exe', | |
371 'v8_simple_regexp_fuzzer.exe', | |
372 'v8_simple_wasm_asmjs_fuzzer.exe', | |
373 'v8_simple_wasm_fuzzer.exe', | |
374 'video_decode_accelerator_unittest.exe', | |
375 'video_encode_accelerator_unittest.exe', | |
376 'views_examples_exe.exe', | |
377 'views_examples_library.dll', | |
378 'views_examples_with_content_exe.exe', | |
379 'views_mus_interactive_ui_tests.exe', | |
380 'views_mus_unittests.exe', | |
381 'views_unittests.exe', | |
382 'webkit_unit_tests.exe', | |
383 'webtest_library.dll', | |
384 'window_type_launcher_library.dll', | |
385 'wm_unittests.exe', | |
386 'wtf_unittests.exe', | |
387 }, | |
388 } | |
389 | |
390 def get_files_to_compare(build_dir, recursive=False): | |
391 """Get the list of files to compare.""" | |
392 # TODO(maruel): Add '.pdb'. | |
393 allowed = frozenset( | |
394 ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so')) | |
395 non_x_ok_exts = frozenset(('.apk', '.isolated')) | |
396 def check(f): | |
397 if not os.path.isfile(f): | |
398 return False | |
399 if os.path.basename(f).startswith('.'): | |
400 return False | |
401 ext = os.path.splitext(f)[1] | |
402 if ext in non_x_ok_exts: | |
403 return True | |
404 return ext in allowed and os.access(f, os.X_OK) | |
405 | |
406 ret_files = set() | |
407 for root, dirs, files in os.walk(build_dir): | |
408 if not recursive: | |
409 dirs[:] = [d for d in dirs if d.endswith('_apk')] | |
410 for f in (f for f in files if check(os.path.join(root, f))): | |
411 ret_files.add(os.path.relpath(os.path.join(root, f), build_dir)) | |
412 return ret_files | |
413 | |
414 | |
415 def diff_dict(a, b): | |
416 """Returns a yaml-like textural diff of two dict. | |
417 | |
418 It is currently optimized for the .isolated format. | |
419 """ | |
420 out = '' | |
421 for key in set(a) | set(b): | |
422 va = a.get(key) | |
423 vb = b.get(key) | |
424 if va.__class__ != vb.__class__: | |
425 out += '- %s: %r != %r\n' % (key, va, vb) | |
426 elif isinstance(va, dict): | |
427 c = diff_dict(va, vb) | |
428 if c: | |
429 out += '- %s:\n%s\n' % ( | |
430 key, '\n'.join(' ' + l for l in c.splitlines())) | |
431 elif va != vb: | |
432 out += '- %s: %s != %s\n' % (key, va, vb) | |
433 return out.rstrip() | |
434 | |
435 | |
436 def diff_binary(first_filepath, second_filepath, file_len): | |
437 """Returns a compact binary diff if the diff is small enough.""" | |
438 CHUNK_SIZE = 32 | |
439 MAX_STREAMS = 10 | |
440 diffs = 0 | |
441 streams = [] | |
442 offset = 0 | |
443 with open(first_filepath, 'rb') as lhs: | |
444 with open(second_filepath, 'rb') as rhs: | |
445 while True: | |
446 lhs_data = lhs.read(CHUNK_SIZE) | |
447 rhs_data = rhs.read(CHUNK_SIZE) | |
448 if not lhs_data: | |
449 break | |
450 if lhs_data != rhs_data: | |
451 diffs += sum(l != r for l, r in zip(lhs_data, rhs_data)) | |
452 if streams is not None: | |
453 if len(streams) < MAX_STREAMS: | |
454 streams.append((offset, lhs_data, rhs_data)) | |
455 else: | |
456 streams = None | |
457 offset += len(lhs_data) | |
458 del lhs_data | |
459 del rhs_data | |
460 if not diffs: | |
461 return None | |
462 result = '%d out of %d bytes are different (%.2f%%)' % ( | |
463 diffs, file_len, 100.0 * diffs / file_len) | |
464 if streams: | |
465 encode = lambda text: ''.join(i if 31 < ord(i) < 128 else '.' for i in text) | |
466 for offset, lhs_data, rhs_data in streams: | |
467 lhs_line = '%s \'%s\'' % (lhs_data.encode('hex'), encode(lhs_data)) | |
468 rhs_line = '%s \'%s\'' % (rhs_data.encode('hex'), encode(rhs_data)) | |
469 diff = list(difflib.Differ().compare([lhs_line], [rhs_line]))[-1][2:-1] | |
470 result += '\n 0x%-8x: %s\n %s\n %s' % ( | |
471 offset, lhs_line, rhs_line, diff) | |
472 return result | |
473 | |
474 | |
475 def compare_files(first_filepath, second_filepath): | |
476 """Compares two binaries and return the number of differences between them. | |
477 | |
478 Returns None if the files are equal, a string otherwise. | |
479 """ | |
480 if first_filepath.endswith('.isolated'): | |
481 with open(first_filepath, 'rb') as f: | |
482 lhs = json.load(f) | |
483 with open(second_filepath, 'rb') as f: | |
484 rhs = json.load(f) | |
485 diff = diff_dict(lhs, rhs) | |
486 if diff: | |
487 return '\n' + '\n'.join(' ' + line for line in diff.splitlines()) | |
488 # else, falls through binary comparison, it must be binary equal too. | |
489 | |
490 file_len = os.stat(first_filepath).st_size | |
491 if file_len != os.stat(second_filepath).st_size: | |
492 return 'different size: %d != %d' % ( | |
493 file_len, os.stat(second_filepath).st_size) | |
494 | |
495 return diff_binary(first_filepath, second_filepath, file_len) | |
496 | |
497 | |
498 def compare_build_artifacts(first_dir, second_dir, target_platform, | |
499 recursive=False): | |
500 """Compares the artifacts from two distinct builds.""" | |
501 if not os.path.isdir(first_dir): | |
502 print >> sys.stderr, '%s isn\'t a valid directory.' % first_dir | |
503 return 1 | |
504 if not os.path.isdir(second_dir): | |
505 print >> sys.stderr, '%s isn\'t a valid directory.' % second_dir | |
506 return 1 | |
507 | |
508 epoch_hex = struct.pack('<I', int(time.time())).encode('hex') | |
509 print('Epoch: %s' % | |
510 ' '.join(epoch_hex[i:i+2] for i in xrange(0, len(epoch_hex), 2))) | |
511 | |
512 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f: | |
513 blacklist = frozenset(json.load(f)) | |
514 whitelist = WHITELIST[target_platform] | |
515 | |
516 # The two directories. | |
517 first_list = get_files_to_compare(first_dir, recursive) - blacklist | |
518 second_list = get_files_to_compare(second_dir, recursive) - blacklist | |
519 | |
520 equals = [] | |
521 expected_diffs = [] | |
522 unexpected_diffs = [] | |
523 unexpected_equals = [] | |
524 all_files = sorted(first_list & second_list) | |
525 missing_files = sorted(first_list.symmetric_difference(second_list)) | |
526 if missing_files: | |
527 print >> sys.stderr, 'Different list of files in both directories:' | |
528 print >> sys.stderr, '\n'.join(' ' + i for i in missing_files) | |
529 unexpected_diffs.extend(missing_files) | |
530 | |
531 max_filepath_len = max(len(n) for n in all_files) | |
532 for f in all_files: | |
533 first_file = os.path.join(first_dir, f) | |
534 second_file = os.path.join(second_dir, f) | |
535 result = compare_files(first_file, second_file) | |
536 if not result: | |
537 tag = 'equal' | |
538 equals.append(f) | |
539 if f in whitelist: | |
540 unexpected_equals.append(f) | |
541 else: | |
542 if f in whitelist: | |
543 expected_diffs.append(f) | |
544 tag = 'expected' | |
545 else: | |
546 unexpected_diffs.append(f) | |
547 tag = 'unexpected' | |
548 result = 'DIFFERENT (%s): %s' % (tag, result) | |
549 print('%-*s: %s' % (max_filepath_len, f, result)) | |
550 unexpected_diffs.sort() | |
551 | |
552 print('Equals: %d' % len(equals)) | |
553 print('Expected diffs: %d' % len(expected_diffs)) | |
554 print('Unexpected diffs: %d' % len(unexpected_diffs)) | |
555 if unexpected_diffs: | |
556 print('Unexpected files with diffs:\n') | |
557 for u in unexpected_diffs: | |
558 print(' %s' % u) | |
559 if unexpected_equals: | |
560 print('Unexpected files with no diffs:\n') | |
561 for u in unexpected_equals: | |
562 print(' %s' % u) | |
563 | |
564 return int(bool(unexpected_diffs)) | |
565 | |
566 | |
567 def main(): | |
568 parser = optparse.OptionParser(usage='%prog [options]') | |
569 parser.add_option( | |
570 '-f', '--first-build-dir', help='The first build directory.') | |
571 parser.add_option( | |
572 '-s', '--second-build-dir', help='The second build directory.') | |
573 parser.add_option('-r', '--recursive', action='store_true', default=False, | |
574 help='Indicates if the comparison should be recursive.') | |
575 target = { | |
576 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win' | |
577 }.get(sys.platform, sys.platform) | |
578 parser.add_option('-t', '--target-platform', help='The target platform.' | |
579 default=target, choices=('android', 'mac', 'linux', 'win')) | |
580 options, _ = parser.parse_args() | |
581 | |
582 if not options.first_build_dir: | |
583 parser.error('--first-build-dir is required') | |
584 if not options.second_build_dir: | |
585 parser.error('--second-build-dir is required') | |
586 if not options.target_platform: | |
587 parser.error('--target-platform is required') | |
588 | |
589 return compare_build_artifacts(os.path.abspath(options.first_build_dir), | |
590 os.path.abspath(options.second_build_dir), | |
591 options.target_platform, | |
592 options.recursive) | |
593 | |
594 | |
595 if __name__ == '__main__': | |
596 sys.exit(main()) | |
OLD | NEW |