OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Adaptor script called through build/isolate.gypi. | 6 """Adaptor script called through build/isolate.gypi. |
7 | 7 |
8 Creates a wrapping .isolate which 'includes' the original one, that can be | 8 Creates a wrapping .isolate which 'includes' the original one, that can be |
9 consumed by tools/swarming_client/isolate.py. Path variables are determined | 9 consumed by tools/swarming_client/isolate.py. Path variables are determined |
10 based on the current working directory. The relative_cwd in the .isolated file | 10 based on the current working directory. The relative_cwd in the .isolated file |
11 is determined based on the .isolate file that declare the 'command' variable to | 11 is determined based on the .isolate file that declare the 'command' variable to |
12 be used so the wrapping .isolate doesn't affect this value. | 12 be used so the wrapping .isolate doesn't affect this value. |
13 | 13 |
14 This script loads build.ninja and processes it to determine all the executables | 14 This script loads build.ninja and processes it to determine all the executables |
15 referenced by the isolated target. It adds them in the wrapping .isolate file. | 15 referenced by the isolated target. It adds them in the wrapping .isolate file. |
16 | 16 |
17 WARNING: The target to use for build.ninja analysis is the base name of the | 17 WARNING: The target to use for build.ninja analysis is the base name of the |
18 .isolate file plus '_run'. For example, 'foo_test.isolate' would have the target | 18 .isolate file plus '_run'. For example, 'foo_test.isolate' would have the target |
19 'foo_test_run' analysed. | 19 'foo_test_run' analysed. |
20 """ | 20 """ |
21 | 21 |
22 import glob | 22 import glob |
23 import json | 23 import json |
24 import logging | 24 import logging |
25 import os | 25 import os |
26 import posixpath | 26 import posixpath |
27 import re | |
28 import StringIO | 27 import StringIO |
29 import subprocess | 28 import subprocess |
30 import sys | 29 import sys |
31 import time | 30 import time |
32 | 31 |
33 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) | 32 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) |
34 SWARMING_CLIENT_DIR = os.path.join(TOOLS_DIR, 'swarming_client') | 33 SWARMING_CLIENT_DIR = os.path.join(TOOLS_DIR, 'swarming_client') |
35 SRC_DIR = os.path.dirname(TOOLS_DIR) | 34 SRC_DIR = os.path.dirname(TOOLS_DIR) |
36 | 35 |
37 sys.path.insert(0, SWARMING_CLIENT_DIR) | 36 sys.path.insert(0, SWARMING_CLIENT_DIR) |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
93 # TODO(maruel): Skip the files known to not be needed. It saves an aweful | 92 # TODO(maruel): Skip the files known to not be needed. It saves an aweful |
94 # lot of processing time. | 93 # lot of processing time. |
95 total += load_ninja_recursively(build_dir, rel_path, build_steps) | 94 total += load_ninja_recursively(build_dir, rel_path, build_steps) |
96 except IOError: | 95 except IOError: |
97 print >> sys.stderr, '... as referenced by %s' % ninja_path | 96 print >> sys.stderr, '... as referenced by %s' % ninja_path |
98 raise | 97 raise |
99 return total | 98 return total |
100 | 99 |
101 | 100 |
102 def load_ninja(build_dir): | 101 def load_ninja(build_dir): |
103 """Loads the tree of .ninja files in build_dir. | 102 """Loads the tree of .ninja files in build_dir.""" |
104 | |
105 Returns: | |
106 dict(target: list of dependencies). | |
107 """ | |
108 build_steps = {} | 103 build_steps = {} |
109 total = load_ninja_recursively(build_dir, 'build.ninja', build_steps) | 104 total = load_ninja_recursively(build_dir, 'build.ninja', build_steps) |
110 logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps)) | 105 logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps)) |
111 return build_steps | 106 return build_steps |
112 | 107 |
113 | 108 |
114 def using_blacklist(item): | 109 def using_blacklist(item): |
115 """Returns True if an item should be analyzed. | 110 """Returns True if an item should be analyzed. |
116 | 111 |
117 Ignores many rules that are assumed to not depend on a dynamic library. If | 112 Ignores many rules that are assumed to not depend on a dynamic library. If |
118 the assumption doesn't hold true anymore for a file format, remove it from | 113 the assumption doesn't hold true anymore for a file format, remove it from |
119 this list. This is simply an optimization. | 114 this list. This is simply an optimization. |
120 """ | 115 """ |
121 # *.json is ignored below, *.isolated.gen.json is an exception, it is produced | 116 # *.json is ignored below, *.isolated.gen.json is an exception, it is produced |
122 # by isolate_driver.py in 'test_isolation_mode==prepare'. | 117 # by isolate_driver.py in 'test_isolation_mode==prepare'. |
123 if item.endswith('.isolated.gen.json'): | 118 if item.endswith('.isolated.gen.json'): |
124 return True | 119 return True |
125 IGNORED = ( | 120 IGNORED = ( |
126 '.a', '.cc', '.css', '.dat', '.def', '.frag', '.h', '.html', '.isolate', | 121 '.a', '.cc', '.css', '.dat', '.def', '.frag', '.h', '.html', '.isolate', |
127 '.js', '.json', '.manifest', '.o', '.obj', '.pak', '.png', '.pdb', '.py', | 122 '.js', '.json', '.manifest', '.o', '.obj', '.pak', '.png', '.pdb', '.py', |
128 '.strings', '.test', '.txt', '.vert', | 123 '.strings', '.test', '.txt', '.vert', |
129 ) | 124 ) |
130 # ninja files use native path format. | 125 # ninja files use native path format. |
131 ext = os.path.splitext(item)[1] | 126 ext = os.path.splitext(item)[1] |
132 if ext in IGNORED: | 127 if ext in IGNORED: |
133 return False | 128 return False |
134 # Special case Windows, keep .dll.lib but discard .lib. | 129 # Special case Windows, keep .dll.lib but discard .lib. |
135 if sys.platform == 'win32': | 130 if item.endswith('.dll.lib'): |
136 if item.endswith('.dll.lib'): | 131 return True |
137 return True | 132 if ext == '.lib': |
138 if ext == '.lib': | 133 return False |
139 return False | |
140 return item not in ('', '|', '||') | 134 return item not in ('', '|', '||') |
141 | 135 |
142 # This is a whitelist of known ninja native rules. | |
143 KNOWN_TOOLS = frozenset( | |
144 ( | |
145 'copy', | |
146 'copy_infoplist', | |
147 'cxx', | |
148 'idl', | |
149 'link', | |
150 'link_embed', | |
151 'mac_tool', | |
152 'package_framework', | |
153 'phony', | |
154 'rc', | |
155 'solink', | |
156 'solink_embed', | |
157 'solink_module', | |
158 'solink_module_embed', | |
159 'solink_module_notoc', | |
160 'solink_notoc', | |
161 'stamp', | |
162 )) | |
163 | |
164 | 136 |
165 def raw_build_to_deps(item): | 137 def raw_build_to_deps(item): |
166 """Converts a raw ninja build statement into the list of interesting | 138 """Converts a raw ninja build statement into the list of interesting |
167 dependencies. | 139 dependencies. |
168 """ | 140 """ |
169 items = filter(None, item.split(' ')) | 141 # TODO(maruel): Use a whitelist instead? .stamp, .so.TOC, .dylib.TOC, |
170 for i in xrange(len(items) - 2, 0, -1): | 142 # .dll.lib, .exe and empty. |
171 # Merge back '$ ' escaping. | 143 # The first item is the build rule, e.g. 'link', 'cxx', 'phony', etc. |
172 # OMG please delete this code as soon as possible. | 144 return filter(using_blacklist, item.split(' ')[1:]) |
173 if items[i].endswith('$'): | |
174 items[i] = items[i][:-1] + ' ' + items[i+1] | |
175 items.pop(i+1) | |
176 | |
177 # Always skip the first item; it is the build rule type, e.g. , etc. | |
178 if items[0] not in KNOWN_TOOLS: | |
179 # Check for phony ninja rules. | |
180 assert re.match(r'^[^.]+_[0-9a-f]{32}$', items[0]), items | |
181 | |
182 return filter(using_blacklist, items[1:]) | |
183 | 145 |
184 | 146 |
185 def collect_deps(target, build_steps, dependencies_added, rules_seen): | 147 def collect_deps(target, build_steps, dependencies_added, rules_seen): |
186 """Recursively adds all the interesting dependencies for |target| | 148 """Recursively adds all the interesting dependencies for |target| |
187 into |dependencies_added|. | 149 into |dependencies_added|. |
188 """ | 150 """ |
189 if rules_seen is None: | 151 if rules_seen is None: |
190 rules_seen = set() | 152 rules_seen = set() |
191 if target in rules_seen: | 153 if target in rules_seen: |
192 # TODO(maruel): Figure out how it happens. | 154 # TODO(maruel): Figure out how it happens. |
193 logging.warning('Circular dependency for %s!', target) | 155 logging.warning('Circular dependency for %s!', target) |
194 return | 156 return |
195 rules_seen.add(target) | 157 rules_seen.add(target) |
196 try: | 158 try: |
197 dependencies = raw_build_to_deps(build_steps[target]) | 159 dependencies = raw_build_to_deps(build_steps[target]) |
198 except KeyError: | 160 except KeyError: |
199 logging.info('Failed to find a build step to generate: %s', target) | 161 logging.info('Failed to find a build step to generate: %s', target) |
200 return | 162 return |
201 logging.debug('collect_deps(%s) -> %s', target, dependencies) | 163 logging.debug('collect_deps(%s) -> %s', target, dependencies) |
202 for dependency in dependencies: | 164 for dependency in dependencies: |
203 dependencies_added.add(dependency) | 165 dependencies_added.add(dependency) |
204 collect_deps(dependency, build_steps, dependencies_added, rules_seen) | 166 collect_deps(dependency, build_steps, dependencies_added, rules_seen) |
205 | 167 |
206 | 168 |
207 def post_process_deps(build_dir, dependencies): | 169 def post_process_deps(build_dir, dependencies): |
208 """Processes the dependency list with OS specific rules. | 170 """Processes the dependency list with OS specific rules.""" |
| 171 def filter_item(i): |
| 172 if i.endswith('.so.TOC'): |
| 173 # Remove only the suffix .TOC, not the .so! |
| 174 return i[:-4] |
| 175 if i.endswith('.dylib.TOC'): |
| 176 # Remove only the suffix .TOC, not the .dylib! |
| 177 return i[:-4] |
| 178 if i.endswith('.dll.lib'): |
| 179 # Remove only the suffix .lib, not the .dll! |
| 180 return i[:-4] |
| 181 return i |
209 | 182 |
210 Returns: | 183 def is_exe(i): |
211 list of dependencies to add. | 184 # This script is only for adding new binaries that are created as part of |
212 """ | 185 # the component build. |
213 out = [] | 186 ext = os.path.splitext(i)[1] |
214 for i in dependencies: | 187 # On POSIX, executables have no extension. |
| 188 if ext not in ('', '.dll', '.dylib', '.exe', '.nexe', '.so'): |
| 189 return False |
215 if os.path.isabs(i): | 190 if os.path.isabs(i): |
216 # In some rare case, there's dependency set explicitly on files outside | 191 # In some rare case, there's dependency set explicitly on files outside |
217 # the checkout. In practice, it was observed on /usr/bin/eu-strip only on | 192 # the checkout. |
218 # official Chrome build. | 193 return False |
219 continue | 194 |
220 if os.path.isdir(os.path.join(build_dir, i)): | 195 # Check for execute access and strip directories. This gets rid of all the |
221 if sys.platform == 'darwin': | 196 # phony rules. |
222 # This is an application. | 197 p = os.path.join(build_dir, i) |
223 out.append(i + '/') | 198 return os.access(p, os.X_OK) and not os.path.isdir(p) |
224 elif i.endswith('.so.TOC'): | 199 |
225 out.append(i[:-4]) | 200 return filter(is_exe, map(filter_item, dependencies)) |
226 elif i.endswith('.dylib.TOC'): | |
227 i = i[:-4] | |
228 out.append(i) | |
229 # Debug symbols may not be present. | |
230 i += '.dSym' | |
231 if os.path.isdir(os.path.join(build_dir, i)): | |
232 out.append(i + '/') | |
233 elif i.endswith('.dylib'): | |
234 out.append(i) | |
235 # Debug symbols may not be present. | |
236 i += '.dSym' | |
237 if os.path.isdir(os.path.join(build_dir, i)): | |
238 out.append(i + '/') | |
239 elif i.endswith('.dll.lib'): | |
240 i = i[:-4] | |
241 out.append(i) | |
242 # Naming is inconsistent. | |
243 if os.path.isfile(os.path.join(build_dir, i + '.pdb')): | |
244 out.append(i + '.pdb') | |
245 if os.path.isfile(os.path.join(build_dir, i[:-4] + '.pdb')): | |
246 out.append(i[:-4] + '.pdb') | |
247 elif i.endswith('.exe'): | |
248 out.append(i) | |
249 # Naming is inconsistent. | |
250 if os.path.isfile(os.path.join(build_dir, i + '.pdb')): | |
251 out.append(i + '.pdb') | |
252 if os.path.isfile(os.path.join(build_dir, i[:-4] + '.pdb')): | |
253 out.append(i[:-4] + '.pdb') | |
254 elif i.endswith('.nexe'): | |
255 out.append(i) | |
256 i += '.debug' | |
257 if os.path.isfile(os.path.join(build_dir, i)): | |
258 out.append(i) | |
259 elif sys.platform != 'win32': | |
260 # On POSIX, executables have no extension. | |
261 if not os.path.splitext(i)[1]: | |
262 out.append(i) | |
263 return out | |
264 | 201 |
265 | 202 |
266 def create_wrapper(args, isolate_index, isolated_index): | 203 def create_wrapper(args, isolate_index, isolated_index): |
267 """Creates a wrapper .isolate that add dynamic libs. | 204 """Creates a wrapper .isolate that add dynamic libs. |
268 | 205 |
269 The original .isolate is not modified. | 206 The original .isolate is not modified. |
270 """ | 207 """ |
271 cwd = os.getcwd() | 208 cwd = os.getcwd() |
272 isolate = args[isolate_index] | 209 isolate = args[isolate_index] |
273 # The code assumes the .isolate file is always specified path-less in cwd. Fix | 210 # The code assumes the .isolate file is always specified path-less in cwd. Fix |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
367 | 304 |
368 swarming_client = os.path.join(SRC_DIR, 'tools', 'swarming_client') | 305 swarming_client = os.path.join(SRC_DIR, 'tools', 'swarming_client') |
369 sys.stdout.flush() | 306 sys.stdout.flush() |
370 result = subprocess.call( | 307 result = subprocess.call( |
371 [sys.executable, os.path.join(swarming_client, 'isolate.py')] + args) | 308 [sys.executable, os.path.join(swarming_client, 'isolate.py')] + args) |
372 return result | 309 return result |
373 | 310 |
374 | 311 |
375 if __name__ == '__main__': | 312 if __name__ == '__main__': |
376 sys.exit(main()) | 313 sys.exit(main()) |
OLD | NEW |