OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 """Remove the build metadata embedded in the artifacts of a build.""" |
| 6 |
| 7 import json |
| 8 import multiprocessing |
| 9 import optparse |
| 10 import os |
| 11 import Queue |
| 12 import shutil |
| 13 import subprocess |
| 14 import sys |
| 15 import tempfile |
| 16 import threading |
| 17 import zipfile |
| 18 |
| 19 |
| 20 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 21 SRC_DIR = os.path.dirname(os.path.dirname(BASE_DIR)) |
| 22 |
| 23 |
| 24 def get_files_to_clean(build_dir, recursive=False): |
| 25 """Get the list of files to clean.""" |
| 26 allowed = frozenset( |
| 27 ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so')) |
| 28 non_x_ok_exts = frozenset(('.apk', '.isolated')) |
| 29 min_timestamp = 0 |
| 30 if os.path.exists(os.path.join(build_dir, 'build.ninja')): |
| 31 min_timestamp = os.path.getmtime(os.path.join(build_dir, 'build.ninja')) |
| 32 |
| 33 def check(f): |
| 34 if not os.path.isfile(f) or os.path.basename(f).startswith('.'): |
| 35 return False |
| 36 if os.path.getmtime(os.path.join(build_dir, f)) < min_timestamp: |
| 37 return False |
| 38 ext = os.path.splitext(f)[1] |
| 39 return (ext in non_x_ok_exts) or (ext in allowed and os.access(f, os.X_OK)) |
| 40 |
| 41 ret_files = set() |
| 42 for root, dirs, files in os.walk(build_dir): |
| 43 if not recursive: |
| 44 dirs[:] = [d for d in dirs if d.endswith('_apk')] |
| 45 for f in (f for f in files if check(os.path.join(root, f))): |
| 46 ret_files.add(os.path.relpath(os.path.join(root, f), build_dir)) |
| 47 return ret_files |
| 48 |
| 49 |
| 50 def run_zap_timestamp(filepath): |
| 51 """Run zap_timestamp.exe on a PE binary.""" |
| 52 assert sys.platform == 'win32' |
| 53 syzygy_dir = os.path.join( |
| 54 SRC_DIR, 'third_party', 'syzygy', 'binaries', 'exe') |
| 55 zap_timestamp_exe = os.path.join(syzygy_dir, 'zap_timestamp.exe') |
| 56 sys.stdout.write('Processing: %s\n' % os.path.basename(filepath)) |
| 57 proc = subprocess.Popen( |
| 58 [zap_timestamp_exe, '--input-image=%s' % filepath, '--overwrite'], |
| 59 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| 60 log, _ = proc.communicate() |
| 61 if proc.returncode != 0: |
| 62 sys.stderr.write('%s failed:\n%s\n' % (os.path.basename(filepath), log)) |
| 63 return proc.returncode |
| 64 |
| 65 |
| 66 def remove_pe_metadata(filename): |
| 67 """Remove the build metadata from a PE file.""" |
| 68 # Only run zap_timestamp on the PE files for which we have a PDB. |
| 69 ret = 0 |
| 70 if os.path.exists(filename + '.pdb'): |
| 71 ret = run_zap_timestamp(filename) |
| 72 return ret |
| 73 |
| 74 |
| 75 def remove_apk_timestamps(filename): |
| 76 """Remove the timestamps embedded in an apk archive.""" |
| 77 sys.stdout.write('Processing: %s\n' % os.path.basename(filename)) |
| 78 with zipfile.ZipFile(filename, 'r') as zf: |
| 79 # Creates a temporary file. |
| 80 out_file, out_filename = tempfile.mkstemp(prefix='remote_apk_timestamp') |
| 81 os.close(out_file) |
| 82 try: |
| 83 with zipfile.ZipFile(out_filename, 'w') as zf_o: |
| 84 # Copy the data from the original file to the new one. |
| 85 for info in zf.infolist(): |
| 86 # Overwrite the timestamp with a constant value. |
| 87 info.date_time = (1980, 1, 1, 0, 0, 0) |
| 88 zf_o.writestr(info, zf.read(info.filename)) |
| 89 # Remove the original file and replace it by the modified one. |
| 90 os.remove(filename) |
| 91 shutil.move(out_filename, filename) |
| 92 finally: |
| 93 if os.path.isfile(out_filename): |
| 94 os.remove(out_filename) |
| 95 |
| 96 |
| 97 def remove_metadata_worker(file_queue, failed_queue, build_dir): |
| 98 """Worker thread for the remove_metadata function.""" |
| 99 while True: |
| 100 f = file_queue.get() |
| 101 if f.endswith(('.dll', '.exe')): |
| 102 if remove_pe_metadata(os.path.join(build_dir, f)): |
| 103 failed_queue.put(f) |
| 104 elif f.endswith('.apk'): |
| 105 remove_apk_timestamps(os.path.join(build_dir, f)) |
| 106 file_queue.task_done() |
| 107 |
| 108 |
| 109 def remove_metadata(build_dir, recursive): |
| 110 """Remove the build metadata from the artifacts of a build.""" |
| 111 with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f: |
| 112 blacklist = frozenset(json.load(f)) |
| 113 files = Queue.Queue() |
| 114 for f in get_files_to_clean(build_dir, recursive) - blacklist: |
| 115 files.put(f) |
| 116 failed_files = Queue.Queue() |
| 117 |
| 118 for _ in xrange(multiprocessing.cpu_count()): |
| 119 worker = threading.Thread(target=remove_metadata_worker, |
| 120 args=(files, |
| 121 failed_files, |
| 122 build_dir)) |
| 123 worker.daemon = True |
| 124 worker.start() |
| 125 |
| 126 files.join() |
| 127 if not failed_files.empty(): |
| 128 print >> sys.stderr, 'Failed for the following files:' |
| 129 failed_files_list = [] |
| 130 while not failed_files.empty(): |
| 131 failed_files_list.append(failed_files.get()) |
| 132 print >> sys.stderr, '\n'.join(' ' + i for i in sorted(failed_files_list)) |
| 133 return 1 |
| 134 |
| 135 return 0 |
| 136 |
| 137 |
| 138 def main(): |
| 139 parser = optparse.OptionParser(usage='%prog [options]') |
| 140 # TODO(sebmarchand): Add support for reading the list of artifact from a |
| 141 # .isolated file. |
| 142 parser.add_option('--build-dir', help='The build directory.') |
| 143 parser.add_option('-r', '--recursive', action='store_true', default=False, |
| 144 help='Indicates if the script should be recursive.') |
| 145 options, _ = parser.parse_args() |
| 146 |
| 147 if not options.build_dir: |
| 148 parser.error('--build-dir is required') |
| 149 |
| 150 return remove_metadata(options.build_dir, options.recursive) |
| 151 |
| 152 |
| 153 if __name__ == '__main__': |
| 154 sys.exit(main()) |
OLD | NEW |