OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Adaptor script called through build/isolate.gypi. | 6 """Adaptor script called through build/isolate.gypi. |
7 | 7 |
8 Creates a wrapping .isolate which 'includes' the original one, that can be | 8 Creates a wrapping .isolate which 'includes' the original one, that can be |
9 consumed by tools/swarming_client/isolate.py. Path variables are determined | 9 consumed by tools/swarming_client/isolate.py. Path variables are determined |
10 based on the current working directory. The relative_cwd in the .isolated file | 10 based on the current working directory. The relative_cwd in the .isolated file |
11 is determined based on the .isolate file that declare the 'command' variable to | 11 is determined based on the .isolate file that declare the 'command' variable to |
12 be used so the wrapping .isolate doesn't affect this value. | 12 be used so the wrapping .isolate doesn't affect this value. |
13 | 13 |
14 This script loads build.ninja and processes it to determine all the executables | 14 This script loads build.ninja and processes it to determine all the executables |
15 referenced by the isolated target. It adds them in the wrapping .isolate file. | 15 referenced by the isolated target. It adds them in the wrapping .isolate file. |
16 | 16 |
17 WARNING: The target to use for build.ninja analysis is the base name of the | 17 WARNING: The target to use for build.ninja analysis is the base name of the |
18 .isolate file plus '_run'. For example, 'foo_test.isolate' would have the target | 18 .isolate file plus '_run'. For example, 'foo_test.isolate' would have the target |
19 'foo_test_run' analysed. | 19 'foo_test_run' analysed. |
20 """ | 20 """ |
21 | 21 |
22 import glob | 22 import glob |
23 import json | 23 import json |
24 import logging | 24 import logging |
25 import os | 25 import os |
26 import posixpath | 26 import posixpath |
| 27 import re |
27 import StringIO | 28 import StringIO |
28 import subprocess | 29 import subprocess |
29 import sys | 30 import sys |
30 import time | 31 import time |
31 | 32 |
32 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) | 33 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) |
33 SWARMING_CLIENT_DIR = os.path.join(TOOLS_DIR, 'swarming_client') | 34 SWARMING_CLIENT_DIR = os.path.join(TOOLS_DIR, 'swarming_client') |
34 SRC_DIR = os.path.dirname(TOOLS_DIR) | 35 SRC_DIR = os.path.dirname(TOOLS_DIR) |
35 | 36 |
36 sys.path.insert(0, SWARMING_CLIENT_DIR) | 37 sys.path.insert(0, SWARMING_CLIENT_DIR) |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
92 # TODO(maruel): Skip the files known to not be needed. It saves an aweful | 93 # TODO(maruel): Skip the files known to not be needed. It saves an aweful |
93 # lot of processing time. | 94 # lot of processing time. |
94 total += load_ninja_recursively(build_dir, rel_path, build_steps) | 95 total += load_ninja_recursively(build_dir, rel_path, build_steps) |
95 except IOError: | 96 except IOError: |
96 print >> sys.stderr, '... as referenced by %s' % ninja_path | 97 print >> sys.stderr, '... as referenced by %s' % ninja_path |
97 raise | 98 raise |
98 return total | 99 return total |
99 | 100 |
100 | 101 |
101 def load_ninja(build_dir): | 102 def load_ninja(build_dir): |
102 """Loads the tree of .ninja files in build_dir.""" | 103 """Loads the tree of .ninja files in build_dir. |
| 104 |
| 105 Returns: |
| 106 dict(target: list of dependencies). |
| 107 """ |
103 build_steps = {} | 108 build_steps = {} |
104 total = load_ninja_recursively(build_dir, 'build.ninja', build_steps) | 109 total = load_ninja_recursively(build_dir, 'build.ninja', build_steps) |
105 logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps)) | 110 logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps)) |
106 return build_steps | 111 return build_steps |
107 | 112 |
108 | 113 |
109 def using_blacklist(item): | 114 def using_blacklist(item): |
110 """Returns True if an item should be analyzed. | 115 """Returns True if an item should be analyzed. |
111 | 116 |
112 Ignores many rules that are assumed to not depend on a dynamic library. If | 117 Ignores many rules that are assumed to not depend on a dynamic library. If |
113 the assumption doesn't hold true anymore for a file format, remove it from | 118 the assumption doesn't hold true anymore for a file format, remove it from |
114 this list. This is simply an optimization. | 119 this list. This is simply an optimization. |
115 """ | 120 """ |
116 # *.json is ignored below, *.isolated.gen.json is an exception, it is produced | 121 # *.json is ignored below, *.isolated.gen.json is an exception, it is produced |
117 # by isolate_driver.py in 'test_isolation_mode==prepare'. | 122 # by isolate_driver.py in 'test_isolation_mode==prepare'. |
118 if item.endswith('.isolated.gen.json'): | 123 if item.endswith('.isolated.gen.json'): |
119 return True | 124 return True |
120 IGNORED = ( | 125 IGNORED = ( |
121 '.a', '.cc', '.css', '.dat', '.def', '.frag', '.h', '.html', '.isolate', | 126 '.a', '.cc', '.css', '.dat', '.def', '.frag', '.h', '.html', '.isolate', |
122 '.js', '.json', '.manifest', '.o', '.obj', '.pak', '.png', '.pdb', '.py', | 127 '.js', '.json', '.manifest', '.o', '.obj', '.pak', '.png', '.pdb', '.py', |
123 '.strings', '.test', '.txt', '.vert', | 128 '.strings', '.test', '.txt', '.vert', |
124 ) | 129 ) |
125 # ninja files use native path format. | 130 # ninja files use native path format. |
126 ext = os.path.splitext(item)[1] | 131 ext = os.path.splitext(item)[1] |
127 if ext in IGNORED: | 132 if ext in IGNORED: |
128 return False | 133 return False |
129 # Special case Windows, keep .dll.lib but discard .lib. | 134 # Special case Windows, keep .dll.lib but discard .lib. |
130 if item.endswith('.dll.lib'): | 135 if sys.platform == 'win32': |
131 return True | 136 if item.endswith('.dll.lib'): |
132 if ext == '.lib': | 137 return True |
133 return False | 138 if ext == '.lib': |
| 139 return False |
134 return item not in ('', '|', '||') | 140 return item not in ('', '|', '||') |
135 | 141 |
| 142 # This is a whitelist of known ninja native rules. |
| 143 KNOWN_TOOLS = frozenset( |
| 144 ( |
| 145 'copy', |
| 146 'copy_infoplist', |
| 147 'cxx', |
| 148 'idl', |
| 149 'link', |
| 150 'link_embed', |
| 151 'mac_tool', |
| 152 'package_framework', |
| 153 'phony', |
| 154 'rc', |
| 155 'solink', |
| 156 'solink_embed', |
| 157 'solink_module', |
| 158 'solink_module_embed', |
| 159 'solink_module_notoc', |
| 160 'solink_notoc', |
| 161 'stamp', |
| 162 )) |
| 163 |
136 | 164 |
137 def raw_build_to_deps(item): | 165 def raw_build_to_deps(item): |
138 """Converts a raw ninja build statement into the list of interesting | 166 """Converts a raw ninja build statement into the list of interesting |
139 dependencies. | 167 dependencies. |
140 """ | 168 """ |
141 # TODO(maruel): Use a whitelist instead? .stamp, .so.TOC, .dylib.TOC, | 169 items = filter(None, item.split(' ')) |
142 # .dll.lib, .exe and empty. | 170 for i in xrange(len(items) - 2, 0, -1): |
143 # The first item is the build rule, e.g. 'link', 'cxx', 'phony', etc. | 171 # Merge back '$ ' escaping. |
144 return filter(using_blacklist, item.split(' ')[1:]) | 172 # OMG please delete this code as soon as possible. |
| 173 if items[i].endswith('$'): |
| 174 items[i] = items[i][:-1] + ' ' + items[i+1] |
| 175 items.pop(i+1) |
| 176 |
| 177 # Always skip the first item; it is the build rule type, e.g. , etc. |
| 178 if items[0] not in KNOWN_TOOLS: |
| 179 # Check for phony ninja rules. |
| 180 assert re.match(r'^[^.]+_[0-9a-f]{32}$', items[0]), items |
| 181 |
| 182 return filter(using_blacklist, items[1:]) |
145 | 183 |
146 | 184 |
147 def collect_deps(target, build_steps, dependencies_added, rules_seen): | 185 def collect_deps(target, build_steps, dependencies_added, rules_seen): |
148 """Recursively adds all the interesting dependencies for |target| | 186 """Recursively adds all the interesting dependencies for |target| |
149 into |dependencies_added|. | 187 into |dependencies_added|. |
150 """ | 188 """ |
151 if rules_seen is None: | 189 if rules_seen is None: |
152 rules_seen = set() | 190 rules_seen = set() |
153 if target in rules_seen: | 191 if target in rules_seen: |
154 # TODO(maruel): Figure out how it happens. | 192 # TODO(maruel): Figure out how it happens. |
155 logging.warning('Circular dependency for %s!', target) | 193 logging.warning('Circular dependency for %s!', target) |
156 return | 194 return |
157 rules_seen.add(target) | 195 rules_seen.add(target) |
158 try: | 196 try: |
159 dependencies = raw_build_to_deps(build_steps[target]) | 197 dependencies = raw_build_to_deps(build_steps[target]) |
160 except KeyError: | 198 except KeyError: |
161 logging.info('Failed to find a build step to generate: %s', target) | 199 logging.info('Failed to find a build step to generate: %s', target) |
162 return | 200 return |
163 logging.debug('collect_deps(%s) -> %s', target, dependencies) | 201 logging.debug('collect_deps(%s) -> %s', target, dependencies) |
164 for dependency in dependencies: | 202 for dependency in dependencies: |
165 dependencies_added.add(dependency) | 203 dependencies_added.add(dependency) |
166 collect_deps(dependency, build_steps, dependencies_added, rules_seen) | 204 collect_deps(dependency, build_steps, dependencies_added, rules_seen) |
167 | 205 |
168 | 206 |
169 def post_process_deps(build_dir, dependencies): | 207 def post_process_deps(build_dir, dependencies): |
170 """Processes the dependency list with OS specific rules.""" | 208 """Processes the dependency list with OS specific rules. |
171 def filter_item(i): | |
172 if i.endswith('.so.TOC'): | |
173 # Remove only the suffix .TOC, not the .so! | |
174 return i[:-4] | |
175 if i.endswith('.dylib.TOC'): | |
176 # Remove only the suffix .TOC, not the .dylib! | |
177 return i[:-4] | |
178 if i.endswith('.dll.lib'): | |
179 # Remove only the suffix .lib, not the .dll! | |
180 return i[:-4] | |
181 return i | |
182 | 209 |
183 def is_exe(i): | 210 Returns: |
184 # This script is only for adding new binaries that are created as part of | 211 list of dependencies to add. |
185 # the component build. | 212 """ |
186 ext = os.path.splitext(i)[1] | 213 out = [] |
187 # On POSIX, executables have no extension. | 214 for i in dependencies: |
188 if ext not in ('', '.dll', '.dylib', '.exe', '.nexe', '.so'): | |
189 return False | |
190 if os.path.isabs(i): | 215 if os.path.isabs(i): |
191 # In some rare case, there's dependency set explicitly on files outside | 216 # In some rare case, there's dependency set explicitly on files outside |
192 # the checkout. | 217 # the checkout. In practice, it was observed on /usr/bin/eu-strip only on |
193 return False | 218 # official Chrome build. |
194 | 219 continue |
195 # Check for execute access and strip directories. This gets rid of all the | 220 if os.path.isdir(os.path.join(build_dir, i)): |
196 # phony rules. | 221 if sys.platform == 'darwin': |
197 p = os.path.join(build_dir, i) | 222 # This is an application. |
198 return os.access(p, os.X_OK) and not os.path.isdir(p) | 223 out.append(i + '/') |
199 | 224 elif i.endswith('.so.TOC'): |
200 return filter(is_exe, map(filter_item, dependencies)) | 225 out.append(i[:-4]) |
| 226 elif i.endswith('.dylib.TOC'): |
| 227 i = i[:-4] |
| 228 out.append(i) |
| 229 # Debug symbols may not be present. |
| 230 i += '.dSym' |
| 231 if os.path.isdir(os.path.join(build_dir, i)): |
| 232 out.append(i + '/') |
| 233 elif i.endswith('.dylib'): |
| 234 out.append(i) |
| 235 # Debug symbols may not be present. |
| 236 i += '.dSym' |
| 237 if os.path.isdir(os.path.join(build_dir, i)): |
| 238 out.append(i + '/') |
| 239 elif i.endswith('.dll.lib'): |
| 240 i = i[:-4] |
| 241 out.append(i) |
| 242 # Naming is inconsistent. |
| 243 if os.path.isfile(os.path.join(build_dir, i + '.pdb')): |
| 244 out.append(i + '.pdb') |
| 245 if os.path.isfile(os.path.join(build_dir, i[:-4] + '.pdb')): |
| 246 out.append(i[:-4] + '.pdb') |
| 247 elif i.endswith('.exe'): |
| 248 out.append(i) |
| 249 # Naming is inconsistent. |
| 250 if os.path.isfile(os.path.join(build_dir, i + '.pdb')): |
| 251 out.append(i + '.pdb') |
| 252 if os.path.isfile(os.path.join(build_dir, i[:-4] + '.pdb')): |
| 253 out.append(i[:-4] + '.pdb') |
| 254 elif i.endswith('.nexe'): |
| 255 out.append(i) |
| 256 i += '.debug' |
| 257 if os.path.isfile(os.path.join(build_dir, i)): |
| 258 out.append(i) |
| 259 elif sys.platform != 'win32': |
| 260 # On POSIX, executables have no extension. |
| 261 if not os.path.splitext(i)[1]: |
| 262 out.append(i) |
| 263 return out |
201 | 264 |
202 | 265 |
203 def create_wrapper(args, isolate_index, isolated_index): | 266 def create_wrapper(args, isolate_index, isolated_index): |
204 """Creates a wrapper .isolate that add dynamic libs. | 267 """Creates a wrapper .isolate that add dynamic libs. |
205 | 268 |
206 The original .isolate is not modified. | 269 The original .isolate is not modified. |
207 """ | 270 """ |
208 cwd = os.getcwd() | 271 cwd = os.getcwd() |
209 isolate = args[isolate_index] | 272 isolate = args[isolate_index] |
210 # The code assumes the .isolate file is always specified path-less in cwd. Fix | 273 # The code assumes the .isolate file is always specified path-less in cwd. Fix |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
304 | 367 |
305 swarming_client = os.path.join(SRC_DIR, 'tools', 'swarming_client') | 368 swarming_client = os.path.join(SRC_DIR, 'tools', 'swarming_client') |
306 sys.stdout.flush() | 369 sys.stdout.flush() |
307 result = subprocess.call( | 370 result = subprocess.call( |
308 [sys.executable, os.path.join(swarming_client, 'isolate.py')] + args) | 371 [sys.executable, os.path.join(swarming_client, 'isolate.py')] + args) |
309 return result | 372 return result |
310 | 373 |
311 | 374 |
312 if __name__ == '__main__': | 375 if __name__ == '__main__': |
313 sys.exit(main()) | 376 sys.exit(main()) |
OLD | NEW |