Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(6)

Side by Side Diff: tools/determinism/remove_build_metadata.py

Issue 2930483002: swarming: Don't normalize .apk files (breaks alignment) (Closed)
Patch Set: Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 """Remove the build metadata embedded in the artifacts of a build.""" 5 """Remove the build metadata embedded in the artifacts of a build."""
6 6
7 import json 7 import json
8 import multiprocessing 8 import multiprocessing
9 import optparse 9 import optparse
10 import os 10 import os
(...skipping 10 matching lines...) Expand all
21 SRC_DIR = os.path.dirname(os.path.dirname(BASE_DIR)) 21 SRC_DIR = os.path.dirname(os.path.dirname(BASE_DIR))
22 22
23 # Files that can't be processed by zap_timestamp.exe. 23 # Files that can't be processed by zap_timestamp.exe.
24 _ZAP_TIMESTAMP_BLACKLIST = { 24 _ZAP_TIMESTAMP_BLACKLIST = {
25 'mini_installer.exe', 25 'mini_installer.exe',
26 } 26 }
27 27
28 def get_files_to_clean(build_dir, recursive=False): 28 def get_files_to_clean(build_dir, recursive=False):
29 """Get the list of files to clean.""" 29 """Get the list of files to clean."""
30 allowed = frozenset( 30 allowed = frozenset(
31 ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so')) 31 ('', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so'))
32 non_x_ok_exts = frozenset(('.apk', '.isolated', '.jar')) 32 non_x_ok_exts = frozenset(('.isolated', '.jar'))
33 min_timestamp = 0 33 min_timestamp = 0
34 if os.path.exists(os.path.join(build_dir, 'build.ninja')): 34 if os.path.exists(os.path.join(build_dir, 'build.ninja')):
35 min_timestamp = os.path.getmtime(os.path.join(build_dir, 'build.ninja')) 35 min_timestamp = os.path.getmtime(os.path.join(build_dir, 'build.ninja'))
36 36
37 def check(f): 37 def check(f):
38 if not os.path.isfile(f) or os.path.basename(f).startswith('.'): 38 if not os.path.isfile(f) or os.path.basename(f).startswith('.'):
39 return False 39 return False
40 if os.path.getmtime(os.path.join(build_dir, f)) < min_timestamp: 40 if os.path.getmtime(os.path.join(build_dir, f)) < min_timestamp:
41 return False 41 return False
42 ext = os.path.splitext(f)[1] 42 ext = os.path.splitext(f)[1]
(...skipping 28 matching lines...) Expand all
71 """Remove the build metadata from a PE file.""" 71 """Remove the build metadata from a PE file."""
72 # Only run zap_timestamp on the PE files for which we have a PDB. 72 # Only run zap_timestamp on the PE files for which we have a PDB.
73 ret = 0 73 ret = 0
74 if ((not os.path.basename(filename) in _ZAP_TIMESTAMP_BLACKLIST) and 74 if ((not os.path.basename(filename) in _ZAP_TIMESTAMP_BLACKLIST) and
75 os.path.exists(filename + '.pdb')): 75 os.path.exists(filename + '.pdb')):
76 ret = run_zap_timestamp(filename) 76 ret = run_zap_timestamp(filename)
77 return ret 77 return ret
78 78
79 79
80 def remove_zip_timestamps(filename): 80 def remove_zip_timestamps(filename):
81 """Remove the timestamps embedded in an apk archive.""" 81 """Remove the timestamps embedded in a zip archive."""
82 sys.stdout.write('Processing: %s\n' % os.path.basename(filename)) 82 sys.stdout.write('Processing: %s\n' % os.path.basename(filename))
83 with zipfile.ZipFile(filename, 'r') as zf: 83 with zipfile.ZipFile(filename, 'r') as zf:
84 # Creates a temporary file. 84 # Creates a temporary file.
85 out_file, out_filename = tempfile.mkstemp(prefix='remote_apk_timestamp') 85 out_file, out_filename = tempfile.mkstemp(prefix='remote_apk_timestamp')
86 os.close(out_file) 86 os.close(out_file)
87 try: 87 try:
88 with zipfile.ZipFile(out_filename, 'w') as zf_o: 88 with zipfile.ZipFile(out_filename, 'w') as zf_o:
89 # Copy the data from the original file to the new one. 89 # Copy the data from the original file to the new one.
90 for info in zf.infolist(): 90 for info in zf.infolist():
91 # Overwrite the timestamp with a constant value. 91 # Overwrite the timestamp with a constant value.
92 info.date_time = (1980, 1, 1, 0, 0, 0) 92 info.date_time = (1980, 1, 1, 0, 0, 0)
93 zf_o.writestr(info, zf.read(info.filename)) 93 zf_o.writestr(info, zf.read(info.filename))
94 # Remove the original file and replace it by the modified one. 94 # Remove the original file and replace it by the modified one.
95 os.remove(filename) 95 os.remove(filename)
96 shutil.move(out_filename, filename) 96 shutil.move(out_filename, filename)
97 finally: 97 finally:
98 if os.path.isfile(out_filename): 98 if os.path.isfile(out_filename):
99 os.remove(out_filename) 99 os.remove(out_filename)
100 100
101 101
102 def remove_metadata_worker(file_queue, failed_queue, build_dir): 102 def remove_metadata_worker(file_queue, failed_queue, build_dir):
103 """Worker thread for the remove_metadata function.""" 103 """Worker thread for the remove_metadata function."""
104 while True: 104 while True:
105 f = file_queue.get() 105 f = file_queue.get()
106 if f.endswith(('.dll', '.exe')): 106 if f.endswith(('.dll', '.exe')):
107 if remove_pe_metadata(os.path.join(build_dir, f)): 107 if remove_pe_metadata(os.path.join(build_dir, f)):
108 failed_queue.put(f) 108 failed_queue.put(f)
109 elif f.endswith(('.apk', '.jar')): 109 elif f.endswith('.jar'):
110 remove_zip_timestamps(os.path.join(build_dir, f)) 110 remove_zip_timestamps(os.path.join(build_dir, f))
111 file_queue.task_done() 111 file_queue.task_done()
112 112
113 113
114 def remove_metadata(build_dir, recursive): 114 def remove_metadata(build_dir, recursive):
115 """Remove the build metadata from the artifacts of a build.""" 115 """Remove the build metadata from the artifacts of a build."""
116 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f: 116 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f:
117 blacklist = frozenset(json.load(f)) 117 blacklist = frozenset(json.load(f))
118 files = Queue.Queue() 118 files = Queue.Queue()
119 for f in get_files_to_clean(build_dir, recursive) - blacklist: 119 for f in get_files_to_clean(build_dir, recursive) - blacklist:
(...skipping 30 matching lines...) Expand all
150 options, _ = parser.parse_args() 150 options, _ = parser.parse_args()
151 151
152 if not options.build_dir: 152 if not options.build_dir:
153 parser.error('--build-dir is required') 153 parser.error('--build-dir is required')
154 154
155 return remove_metadata(options.build_dir, options.recursive) 155 return remove_metadata(options.build_dir, options.recursive)
156 156
157 157
158 if __name__ == '__main__': 158 if __name__ == '__main__':
159 sys.exit(main()) 159 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698