OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 """Remove the build metadata embedded in the artifacts of a build.""" | 5 """Remove the build metadata embedded in the artifacts of a build.""" |
6 | 6 |
7 import json | 7 import json |
8 import optparse | 8 import optparse |
9 import os | 9 import os |
10 import shutil | |
10 import subprocess | 11 import subprocess |
11 import sys | 12 import sys |
13 import tempfile | |
14 import zipfile | |
12 | 15 |
13 | 16 |
14 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | 17 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) |
15 | 18 |
16 | 19 |
17 def RunZapTimestamp(src_dir, filepath): | 20 def get_files_to_clean(build_dir, recursive=False): |
21 """Get the list of files to clean.""" | |
22 allowed = frozenset( | |
23 ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so')) | |
24 non_x_ok_exts = frozenset(('.apk', '.isolated')) | |
25 def check(f): | |
26 if not os.path.isfile(f): | |
M-A Ruel
2014/11/06 18:38:39
if not os.path.isfile(f) or os.path.basename(f).st
Sébastien Marchand
2014/11/06 19:37:31
Done.
| |
27 return False | |
28 if os.path.basename(f).startswith('.'): | |
29 return False | |
30 ext = os.path.splitext(f)[1] | |
31 if ext in non_x_ok_exts: | |
M-A Ruel
2014/11/06 18:38:39
return (ext in non_x_ok_exts) or (ext in allowed a
Sébastien Marchand
2014/11/06 19:37:32
Done.
| |
32 return True | |
33 return ext in allowed and os.access(f, os.X_OK) | |
34 | |
35 ret_files = set() | |
36 for root, dirs, files in os.walk(build_dir): | |
37 if not recursive: | |
38 dirs[:] = [d for d in dirs if d.endswith('_apk')] | |
39 for f in (f for f in files if check(os.path.join(root, f))): | |
40 ret_files.add(os.path.relpath(os.path.join(root, f), build_dir)) | |
41 return ret_files | |
42 | |
43 | |
44 def run_zap_timestamp(src_dir, filepath): | |
45 """Run zap_timestamp.exe on a PE binary.""" | |
46 assert sys.platform == 'win32' | |
18 syzygy_dir = os.path.join( | 47 syzygy_dir = os.path.join( |
19 src_dir, 'third_party', 'syzygy', 'binaries', 'exe') | 48 src_dir, 'third_party', 'syzygy', 'binaries', 'exe') |
20 zap_timestamp_exe = os.path.join(syzygy_dir, 'zap_timestamp.exe') | 49 zap_timestamp_exe = os.path.join(syzygy_dir, 'zap_timestamp.exe') |
21 print('Processing: %s' % os.path.basename(filepath)) | 50 print('Processing: %s' % os.path.basename(filepath)) |
22 proc = subprocess.Popen( | 51 proc = subprocess.Popen( |
23 [zap_timestamp_exe, '--input-image=%s' % filepath, '--overwrite'], | 52 [zap_timestamp_exe, '--input-image=%s' % filepath, '--overwrite'], |
24 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | 53 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
25 log, _ = proc.communicate() | 54 log, _ = proc.communicate() |
26 if proc.returncode != 0: | 55 if proc.returncode != 0: |
27 print >> sys.stderr, log | 56 print >> sys.stderr, log |
28 return proc.returncode | 57 return proc.returncode |
29 | 58 |
30 | 59 |
31 def RemovePEMetadata(build_dir, src_dir): | 60 def remove_pe_metadata(filename, src_dir): |
32 """Remove the build metadata from a PE file.""" | 61 """Remove the build metadata from a PE file.""" |
33 files = (i for i in os.listdir(build_dir) if i.endswith(('.dll', '.exe'))) | 62 # Only run zap_timestamp on the PE files for which we have a PDB. |
63 ret = 0 | |
64 if os.path.exists(filename + '.pdb'): | |
M-A Ruel
2014/11/06 18:38:39
is this check still necessary? e.g. zap_timestamp
Sébastien Marchand
2014/11/06 19:37:32
Yes it is, sorry, I'll remove it once we update za
| |
65 ret = run_zap_timestamp(src_dir, filename) | |
66 return ret | |
34 | 67 |
68 | |
69 def remove_archive_timestamps(filename): | |
M-A Ruel
2014/11/06 18:38:39
remove_apk_medata so it fits with remote_pe_metada
Sébastien Marchand
2014/11/06 19:37:31
Done, but it also works for the zip files :)
| |
70 """Remove the timestamps embedded in a zip archive.""" | |
71 print('Processing: %s' % os.path.basename(filename)) | |
72 zf = zipfile.ZipFile(filename, 'r') | |
M-A Ruel
2014/11/06 18:38:39
with zipfile.ZipFile(filename, 'r') as zf:
Sébastien Marchand
2014/11/06 19:37:32
Done.
| |
73 # Creates a temporary file. | |
74 out_file, out_filename = tempfile.mkstemp() | |
M-A Ruel
2014/11/06 18:38:38
prefix='remote_apk_timestamp'
put in try/finally
Sébastien Marchand
2014/11/06 19:37:31
Done.
| |
75 os.close(out_file) | |
76 zf_o = zipfile.ZipFile(out_filename, 'w') | |
M-A Ruel
2014/11/06 18:38:39
with ..
Sébastien Marchand
2014/11/06 19:37:32
Done.
| |
77 # Copy the data from the original file to the new one. | |
78 for info in zf.infolist(): | |
79 # Overwrite the timestamp with a constant value. | |
80 info.date_time = (2008, 9, 2, 8, 0 , 0) | |
M-A Ruel
2014/11/06 18:38:39
I'd prefer 0 than an arbitrary value.
Sébastien Marchand
2014/11/06 19:37:31
Doesn't work with 0... It looks like the minimal a
| |
81 zf_o.writestr(info, zf.read(info.filename)) | |
82 zf.close() | |
83 zf_o.close() | |
84 # Remove the original file and replace it by the modified one. | |
85 os.remove(filename) | |
86 shutil.move(out_filename, filename) | |
87 | |
88 | |
89 def remove_metadata(build_dir, src_dir, recursive): | |
90 """Remove the build metadata from the artifacts of a build.""" | |
35 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f: | 91 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f: |
36 blacklist = frozenset(json.load(f)) | 92 blacklist = frozenset(json.load(f)) |
93 files = get_files_to_clean(build_dir, recursive) - blacklist | |
94 failed_files = [] | |
95 ret = 0 | |
96 for f in files: | |
97 if f.endswith(('.dll', '.exe')): | |
98 if remove_pe_metadata(os.path.join(build_dir, f), src_dir): | |
99 ret = 1 | |
100 failed_files.append(f) | |
101 elif f.endswith('.apk'): | |
102 remove_archive_timestamps(os.path.join(build_dir, f)) | |
37 | 103 |
38 failed = [] | 104 if failed_files: |
39 for filename in files: | 105 print >> sys.stderr, 'Failed for the following files:' |
40 # Ignore the blacklisted files. | 106 print >> sys.stderr, '\n'.join(' ' + i for i in sorted(failed_files)) |
41 if filename in blacklist: | |
42 print('Ignored: %s' % filename) | |
43 continue | |
44 # Only run zap_timestamp on the PE files for which we have a PDB. | |
45 if os.path.exists(os.path.join(build_dir, filename + '.pdb')): | |
46 ret = RunZapTimestamp(src_dir, os.path.join(build_dir, filename)) | |
47 if ret != 0: | |
48 failed.append(filename) | |
49 | |
50 if failed: | |
51 print >> sys.stderr, 'zap_timestamp.exe failed for the following files:' | |
52 print >> sys.stderr, '\n'.join(' ' + i for i in sorted(failed)) | |
53 return 1 | 107 return 1 |
54 | 108 |
55 return 0 | 109 return ret |
56 | 110 |
57 | 111 |
58 def main(): | 112 def main(): |
59 parser = optparse.OptionParser(usage='%prog [options]') | 113 parser = optparse.OptionParser(usage='%prog [options]') |
60 # TODO(sebmarchand): Add support for reading the list of artifact from a | 114 # TODO(sebmarchand): Add support for reading the list of artifact from a |
61 # .isolated file. | 115 # .isolated file. |
62 parser.add_option('--build-dir', help='The build directory.') | 116 parser.add_option('--build-dir', help='The build directory.') |
63 parser.add_option('--src-dir', help='The source directory.') | 117 parser.add_option('--src-dir', help='The source directory.') |
118 parser.add_option('-r', '--recursive', action='store_true', default=False, | |
119 help='Indicates if the script should be recursive.') | |
64 options, _ = parser.parse_args() | 120 options, _ = parser.parse_args() |
65 | 121 |
66 if not options.build_dir: | 122 if not options.build_dir: |
67 parser.error('--build-dir is required') | 123 parser.error('--build-dir is required') |
68 if not options.src_dir: | 124 |
125 if sys.platform == 'win32' and not options.src_dir: | |
69 parser.error('--src-dir is required') | 126 parser.error('--src-dir is required') |
70 | 127 |
71 # There's nothing to do for the non-Windows platform yet. | 128 return remove_metadata(options.build_dir, options.src_dir, options.recursive) |
72 if sys.platform == 'win32': | |
73 return RemovePEMetadata(options.build_dir, options.src_dir) | |
74 | 129 |
75 | 130 |
76 if __name__ == '__main__': | 131 if __name__ == '__main__': |
77 sys.exit(main()) | 132 sys.exit(main()) |
OLD | NEW |