OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/env python | |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 """Remove the build metadata embedded in the artifacts of a build.""" | |
6 | |
7 import json | |
8 import multiprocessing | |
9 import optparse | |
10 import os | |
11 import Queue | |
12 import shutil | |
13 import subprocess | |
14 import sys | |
15 import tempfile | |
16 import threading | |
17 import zipfile | |
18 | |
19 | |
20 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
21 | |
22 | |
23 def get_files_to_clean(build_dir, recursive=False): | |
24 """Get the list of files to clean.""" | |
25 allowed = frozenset( | |
26 ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so')) | |
27 non_x_ok_exts = frozenset(('.apk', '.isolated')) | |
28 min_timestamp = 0 | |
29 if os.path.exists(os.path.join(build_dir, 'build.ninja')): | |
30 min_timestamp = os.path.getmtime(os.path.join(build_dir, 'build.ninja')) | |
31 | |
32 def check(f): | |
33 if not os.path.isfile(f) or os.path.basename(f).startswith('.'): | |
34 return False | |
35 if os.path.getmtime(os.path.join(build_dir, f)) < min_timestamp: | |
36 return False | |
37 ext = os.path.splitext(f)[1] | |
38 return (ext in non_x_ok_exts) or (ext in allowed and os.access(f, os.X_OK)) | |
39 | |
40 ret_files = set() | |
41 for root, dirs, files in os.walk(build_dir): | |
42 if not recursive: | |
43 dirs[:] = [d for d in dirs if d.endswith('_apk')] | |
44 for f in (f for f in files if check(os.path.join(root, f))): | |
45 ret_files.add(os.path.relpath(os.path.join(root, f), build_dir)) | |
46 return ret_files | |
47 | |
48 | |
49 def run_zap_timestamp(src_dir, filepath): | |
50 """Run zap_timestamp.exe on a PE binary.""" | |
51 assert sys.platform == 'win32' | |
52 syzygy_dir = os.path.join( | |
53 src_dir, 'third_party', 'syzygy', 'binaries', 'exe') | |
54 zap_timestamp_exe = os.path.join(syzygy_dir, 'zap_timestamp.exe') | |
55 sys.stdout.write('Processing: %s\n' % os.path.basename(filepath)) | |
56 proc = subprocess.Popen( | |
57 [zap_timestamp_exe, '--input-image=%s' % filepath, '--overwrite'], | |
58 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
59 log, _ = proc.communicate() | |
60 if proc.returncode != 0: | |
61 sys.stderr.write('%s failed:\n%s\n' % (os.path.basename(filepath), log)) | |
62 return proc.returncode | |
63 | |
64 | |
65 def remove_pe_metadata(filename, src_dir): | |
66 """Remove the build metadata from a PE file.""" | |
67 # Only run zap_timestamp on the PE files for which we have a PDB. | |
68 ret = 0 | |
69 if os.path.exists(filename + '.pdb'): | |
70 ret = run_zap_timestamp(src_dir, filename) | |
71 return ret | |
72 | |
73 | |
74 def remove_apk_timestamps(filename): | |
75 """Remove the timestamps embedded in an apk archive.""" | |
76 sys.stdout.write('Processing: %s\n' % os.path.basename(filename)) | |
77 with zipfile.ZipFile(filename, 'r') as zf: | |
78 # Creates a temporary file. | |
79 out_file, out_filename = tempfile.mkstemp(prefix='remote_apk_timestamp') | |
80 os.close(out_file) | |
81 try: | |
82 with zipfile.ZipFile(out_filename, 'w') as zf_o: | |
83 # Copy the data from the original file to the new one. | |
84 for info in zf.infolist(): | |
85 # Overwrite the timestamp with a constant value. | |
86 info.date_time = (1980, 1, 1, 0, 0, 0) | |
87 zf_o.writestr(info, zf.read(info.filename)) | |
88 # Remove the original file and replace it by the modified one. | |
89 os.remove(filename) | |
90 shutil.move(out_filename, filename) | |
91 finally: | |
92 if os.path.isfile(out_filename): | |
93 os.remove(out_filename) | |
94 | |
95 | |
96 def remove_metadata_worker(file_queue, failed_queue, build_dir, src_dir): | |
97 """Worker thread for the remove_metadata function.""" | |
98 while True: | |
99 f = file_queue.get() | |
100 if f.endswith(('.dll', '.exe')): | |
101 if remove_pe_metadata(os.path.join(build_dir, f), src_dir): | |
102 ret = 1 | |
103 failed_queue.put(f) | |
104 elif f.endswith('.apk'): | |
105 remove_apk_timestamps(os.path.join(build_dir, f)) | |
106 file_queue.task_done() | |
107 | |
108 | |
109 def remove_metadata(build_dir, src_dir, recursive): | |
110 """Remove the build metadata from the artifacts of a build.""" | |
111 tool_dir = os.path.join(src_dir, 'tools', 'determinism') | |
112 with open(os.path.join(tool_dir, 'deterministic_build_blacklist.json')) as f: | |
113 blacklist = frozenset(json.load(f)) | |
114 files = Queue.Queue() | |
115 for f in get_files_to_clean(build_dir, recursive) - blacklist: | |
116 files.put(f) | |
117 failed_files = Queue.Queue() | |
118 | |
119 for _ in xrange(multiprocessing.cpu_count()): | |
120 worker = threading.Thread(target=remove_metadata_worker, | |
121 args=(files, | |
122 failed_files, | |
123 build_dir, | |
124 src_dir)) | |
125 worker.daemon = True | |
126 worker.start() | |
127 | |
128 files.join() | |
129 if not failed_files.empty(): | |
130 print >> sys.stderr, 'Failed for the following files:' | |
131 failed_files_list = [] | |
132 while not failed_files.empty(): | |
133 failed_files_list.append(failed_files.get()) | |
134 print >> sys.stderr, '\n'.join(' ' + i for i in sorted(failed_files_list)) | |
135 return 1 | |
136 | |
137 return 0 | |
138 | |
139 | |
140 def main(): | |
141 parser = optparse.OptionParser(usage='%prog [options]') | |
142 # TODO(sebmarchand): Add support for reading the list of artifact from a | |
143 # .isolated file. | |
144 parser.add_option('--build-dir', help='The build directory.') | |
145 parser.add_option('--src-dir', help='The source directory.') | |
M-A Ruel
2016/08/24 02:15:45
you can remove this flag and make it src_dir == os
Yoshisato Yanagisawa
2016/08/24 03:02:03
Done.
| |
146 parser.add_option('-r', '--recursive', action='store_true', default=False, | |
147 help='Indicates if the script should be recursive.') | |
148 options, _ = parser.parse_args() | |
149 | |
150 if not options.build_dir: | |
151 parser.error('--build-dir is required') | |
152 | |
153 if not options.src_dir: | |
154 parser.error('--src-dir is required') | |
155 | |
156 return remove_metadata(options.build_dir, options.src_dir, options.recursive) | |
157 | |
158 | |
159 if __name__ == '__main__': | |
160 sys.exit(main()) | |
OLD | NEW |