OLD | NEW |
1 # Copyright 2017 The Chromium Authors. All rights reserved. | 1 # Copyright 2017 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Main Python API for analyzing binary size.""" | 5 """Main Python API for analyzing binary size.""" |
6 | 6 |
7 import argparse | 7 import argparse |
8 import calendar | 8 import calendar |
9 import collections | 9 import collections |
10 import datetime | 10 import datetime |
11 import gzip | 11 import gzip |
12 import logging | 12 import logging |
13 import os | 13 import os |
14 import posixpath | 14 import posixpath |
15 import re | 15 import re |
16 import subprocess | 16 import subprocess |
17 import sys | 17 import sys |
18 import tempfile | 18 import tempfile |
19 import zipfile | 19 import zipfile |
20 | 20 |
| 21 import concurrent |
21 import describe | 22 import describe |
22 import file_format | 23 import file_format |
23 import function_signature | 24 import function_signature |
24 import helpers | |
25 import linker_map_parser | 25 import linker_map_parser |
26 import models | 26 import models |
27 import ninja_parser | 27 import ninja_parser |
| 28 import nm |
28 import paths | 29 import paths |
29 | 30 |
30 | 31 |
31 def _OpenMaybeGz(path, mode=None): | 32 def _OpenMaybeGz(path, mode=None): |
32 """Calls `gzip.open()` if |path| ends in ".gz", otherwise calls `open()`.""" | 33 """Calls `gzip.open()` if |path| ends in ".gz", otherwise calls `open()`.""" |
33 if path.endswith('.gz'): | 34 if path.endswith('.gz'): |
34 if mode and 'w' in mode: | 35 if mode and 'w' in mode: |
35 return gzip.GzipFile(path, mode, 1) | 36 return gzip.GzipFile(path, mode, 1) |
36 return gzip.open(path, mode) | 37 return gzip.open(path, mode) |
37 return open(path, mode or 'r') | 38 return open(path, mode or 'r') |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
120 symbol.full_name = symbol.name | 121 symbol.full_name = symbol.name |
121 symbol.name = re.sub(r'\(.*\)', '', symbol.full_name) | 122 symbol.name = re.sub(r'\(.*\)', '', symbol.full_name) |
122 | 123 |
123 # Don't bother storing both if they are the same. | 124 # Don't bother storing both if they are the same. |
124 if symbol.full_name == symbol.name: | 125 if symbol.full_name == symbol.name: |
125 symbol.full_name = '' | 126 symbol.full_name = '' |
126 | 127 |
127 logging.debug('Found name prefixes of: %r', found_prefixes) | 128 logging.debug('Found name prefixes of: %r', found_prefixes) |
128 | 129 |
129 | 130 |
130 def _NormalizeObjectPaths(symbols): | 131 def _NormalizeObjectPath(path): |
131 """Ensures that all paths are formatted in a useful way.""" | 132 if path.startswith('obj/'): |
132 for symbol in symbols: | 133 # Convert obj/third_party/... -> third_party/... |
133 path = symbol.object_path | 134 path = path[4:] |
134 if path.startswith('obj/'): | 135 elif path.startswith('../../'): |
135 # Convert obj/third_party/... -> third_party/... | 136 # Convert ../../third_party/... -> third_party/... |
136 path = path[4:] | 137 path = path[6:] |
137 elif path.startswith('../../'): | 138 if path.endswith(')'): |
138 # Convert ../../third_party/... -> third_party/... | 139 # Convert foo/bar.a(baz.o) -> foo/bar.a/baz.o |
139 path = path[6:] | 140 start_idx = path.index('(') |
140 if path.endswith(')'): | 141 path = os.path.join(path[:start_idx], path[start_idx + 1:-1]) |
141 # Convert foo/bar.a(baz.o) -> foo/bar.a/baz.o | 142 return path |
142 start_idx = path.index('(') | |
143 path = os.path.join(path[:start_idx], path[start_idx + 1:-1]) | |
144 symbol.object_path = path | |
145 | 143 |
146 | 144 |
147 def _NormalizeSourcePath(path): | 145 def _NormalizeSourcePath(path): |
148 if path.startswith('gen/'): | 146 if path.startswith('gen/'): |
149 # Convert gen/third_party/... -> third_party/... | 147 # Convert gen/third_party/... -> third_party/... |
150 return path[4:] | 148 return path[4:] |
151 if path.startswith('../../'): | 149 if path.startswith('../../'): |
152 # Convert ../../third_party/... -> third_party/... | 150 # Convert ../../third_party/... -> third_party/... |
153 return path[6:] | 151 return path[6:] |
154 return path | 152 return path |
155 | 153 |
156 | 154 |
| 155 def _SourcePathForObjectPath(object_path, source_mapper): |
| 156 # We don't have source info for prebuilt .a files. |
| 157 if not os.path.isabs(object_path) and not object_path.startswith('..'): |
| 158 source_path = source_mapper.FindSourceForPath(object_path) |
| 159 if source_path: |
| 160 return _NormalizeSourcePath(source_path) |
| 161 return '' |
| 162 |
| 163 |
157 def _ExtractSourcePaths(symbols, source_mapper): | 164 def _ExtractSourcePaths(symbols, source_mapper): |
158 """Fills in the .source_path attribute of all symbols.""" | 165 """Fills in the |source_path| attribute.""" |
159 logging.debug('Parsed %d .ninja files.', source_mapper.parsed_file_count) | 166 logging.debug('Parsed %d .ninja files.', source_mapper.parsed_file_count) |
160 | |
161 for symbol in symbols: | 167 for symbol in symbols: |
162 object_path = symbol.object_path | 168 object_path = symbol.object_path |
163 if symbol.source_path or not object_path: | 169 if object_path and not symbol.source_path: |
| 170 symbol.source_path = _SourcePathForObjectPath(object_path, source_mapper) |
| 171 |
| 172 |
| 173 def _ComputeAnscestorPath(path_list): |
| 174 """Returns the common anscestor of the given paths.""" |
| 175 # Ignore missing paths. |
| 176 path_list = [p for p in path_list if p] |
| 177 prefix = os.path.commonprefix(path_list) |
| 178 # Put the path count as a subdirectory to allow for better grouping when |
| 179 # path-based breakdowns. |
| 180 if not prefix: |
| 181 if len(path_list) < 2: |
| 182 return '' |
| 183 return os.path.join('{shared}', str(len(path_list))) |
| 184 if prefix == path_list[0]: |
| 185 return prefix |
| 186 assert len(path_list) > 1, 'path_list: ' + repr(path_list) |
| 187 return os.path.join(os.path.dirname(prefix), '{shared}', str(len(path_list))) |
| 188 |
| 189 |
| 190 # This must normalize object paths at the same time because normalization |
| 191 # needs to occur before finding common ancestor. |
| 192 def _ComputeAnscestorPathsAndNormalizeObjectPaths( |
| 193 symbols, object_paths_by_name, source_mapper): |
| 194 num_found_paths = 0 |
| 195 num_unknown_names = 0 |
| 196 num_path_mismatches = 0 |
| 197 num_unmatched_aliases = 0 |
| 198 for symbol in symbols: |
| 199 name = symbol.name |
| 200 if (symbol.IsBss() or |
| 201 not name or |
| 202 name[0] in '*.' or # e.g. ** merge symbols, .Lswitch.table |
| 203 name == 'startup'): |
| 204 symbol.object_path = _NormalizeObjectPath(symbol.object_path) |
164 continue | 205 continue |
165 # We don't have source info for prebuilt .a files. | 206 |
166 if not os.path.isabs(object_path) and not object_path.startswith('..'): | 207 object_paths = object_paths_by_name.get(name) |
167 source_path = source_mapper.FindSourceForPath(object_path) | 208 if object_paths: |
168 if source_path: | 209 num_found_paths += 1 |
169 symbol.source_path = _NormalizeSourcePath(source_path) | 210 else: |
| 211 if not symbol.object_path and symbol.aliases: |
| 212 # Happens when aliases are from object files where all symbols were |
| 213 # pruned or de-duped as aliases. Since we are only scanning .o files |
| 214 # referenced by included symbols, such files are missed. |
| 215 # TODO(agrieve): This could be fixed by retrieving linker inputs from |
| 216 # build.ninja, or by looking for paths within the .map file's |
| 217 # discarded sections. |
| 218 num_unmatched_aliases += 1 |
| 219 continue |
| 220 if num_unknown_names < 10: |
| 221 logging.warning('Symbol not found in any .o files: %r', symbol) |
| 222 num_unknown_names += 1 |
| 223 symbol.object_path = _NormalizeObjectPath(symbol.object_path) |
| 224 continue |
| 225 |
| 226 if symbol.object_path and symbol.object_path not in object_paths: |
| 227 if num_path_mismatches < 10: |
| 228 logging.warning('Symbol path reported by .map not found by nm.') |
| 229 logging.warning('sym=%r', symbol) |
| 230 logging.warning('paths=%r', object_paths) |
| 231 num_path_mismatches += 1 |
| 232 |
| 233 if source_mapper: |
| 234 source_paths = [ |
| 235 _SourcePathForObjectPath(p, source_mapper) for p in object_paths] |
| 236 symbol.source_path = _ComputeAnscestorPath(source_paths) |
| 237 |
| 238 object_paths = [_NormalizeObjectPath(p) for p in object_paths] |
| 239 symbol.object_path = _ComputeAnscestorPath(object_paths) |
| 240 |
| 241 logging.debug('Cross-referenced %d symbols with nm output. ' |
| 242 'num_unknown_names=%d num_path_mismatches=%d ' |
| 243 'num_unused_aliases=%d', num_found_paths, num_unknown_names, |
| 244 num_path_mismatches, num_unmatched_aliases) |
| 245 |
| 246 |
| 247 def _DiscoverMissedObjectPaths(symbols, elf_object_paths): |
| 248 # Missing object paths are caused by .a files added by -l flags, which are not |
| 249 # listed as explicit inputs within .ninja rules. |
| 250 parsed_inputs = set(elf_object_paths) |
| 251 missed_inputs = set() |
| 252 for symbol in symbols: |
| 253 path = symbol.object_path |
| 254 if path.endswith(')'): |
| 255 # Convert foo/bar.a(baz.o) -> foo/bar.a |
| 256 path = path[:path.index('(')] |
| 257 if path and path not in parsed_inputs: |
| 258 missed_inputs.add(path) |
| 259 return missed_inputs |
170 | 260 |
171 | 261 |
172 def _CalculatePadding(symbols): | 262 def _CalculatePadding(symbols): |
173 """Populates the |padding| field based on symbol addresses. | 263 """Populates the |padding| field based on symbol addresses. |
174 | 264 |
175 Symbols must already be sorted by |address|. | 265 Symbols must already be sorted by |address|. |
176 """ | 266 """ |
177 seen_sections = [] | 267 seen_sections = [] |
178 for i, symbol in enumerate(symbols[1:]): | 268 for i, symbol in enumerate(symbols[1:]): |
179 prev_symbol = symbols[i] | 269 prev_symbol = symbols[i] |
180 if prev_symbol.section_name != symbol.section_name: | 270 if prev_symbol.section_name != symbol.section_name: |
181 assert symbol.section_name not in seen_sections, ( | 271 assert symbol.section_name not in seen_sections, ( |
182 'Input symbols must be sorted by section, then address.') | 272 'Input symbols must be sorted by section, then address.') |
183 seen_sections.append(symbol.section_name) | 273 seen_sections.append(symbol.section_name) |
184 continue | 274 continue |
185 if symbol.address <= 0 or prev_symbol.address <= 0: | 275 if symbol.address <= 0 or prev_symbol.address <= 0: |
186 continue | 276 continue |
187 # Padding-only symbols happen for ** symbol gaps. | 277 |
188 prev_is_padding_only = prev_symbol.size_without_padding == 0 | 278 if symbol.address == prev_symbol.address: |
189 if symbol.address == prev_symbol.address and not prev_is_padding_only: | 279 if symbol.aliases and symbol.aliases is prev_symbol.aliases: |
190 assert False, 'Found duplicate symbols:\n%r\n%r' % (prev_symbol, symbol) | 280 symbol.padding = prev_symbol.padding |
191 # Even with symbols at the same address removed, overlaps can still | 281 symbol.size = prev_symbol.size |
192 # happen. In this case, padding will be negative (and this is fine). | 282 continue |
| 283 # Padding-only symbols happen for ** symbol gaps. |
| 284 assert prev_symbol.size_without_padding == 0, ( |
| 285 'Found duplicate symbols:\n%r\n%r' % (prev_symbol, symbol)) |
| 286 |
193 padding = symbol.address - prev_symbol.end_address | 287 padding = symbol.address - prev_symbol.end_address |
194 # These thresholds were found by manually auditing arm32 Chrome. | 288 # These thresholds were found by experimenting with arm32 Chrome. |
195 # E.g.: Set them to 0 and see what warnings get logged. | 289 # E.g.: Set them to 0 and see what warnings get logged, then take max value. |
196 # TODO(agrieve): See if these thresholds make sense for architectures | 290 # TODO(agrieve): See if these thresholds make sense for architectures |
197 # other than arm32. | 291 # other than arm32. |
198 if not symbol.name.startswith('*') and ( | 292 if not symbol.name.startswith('*') and ( |
199 symbol.section in 'rd' and padding >= 256 or | 293 symbol.section in 'rd' and padding >= 256 or |
200 symbol.section in 't' and padding >= 64): | 294 symbol.section in 't' and padding >= 64): |
201 # For nm data, this is caused by data that has no associated symbol. | 295 # Should not happen. |
202 # The linker map file lists them with no name, but with a file. | 296 logging.warning('Large padding of %d between:\n A) %r\n B) %r' % ( |
203 # Example: | 297 padding, prev_symbol, symbol)) |
204 # .data 0x02d42764 0x120 .../V8SharedWorkerGlobalScope.o | |
205 # Where as most look like: | |
206 # .data.MANGLED_NAME... | |
207 logging.debug('Large padding of %d between:\n A) %r\n B) %r' % ( | |
208 padding, prev_symbol, symbol)) | |
209 continue | |
210 symbol.padding = padding | 298 symbol.padding = padding |
211 symbol.size += padding | 299 symbol.size += padding |
212 assert symbol.size >= 0, ( | 300 assert symbol.size >= 0, ( |
213 'Symbol has negative size (likely not sorted propertly): ' | 301 'Symbol has negative size (likely not sorted propertly): ' |
214 '%r\nprev symbol: %r' % (symbol, prev_symbol)) | 302 '%r\nprev symbol: %r' % (symbol, prev_symbol)) |
215 | 303 |
216 | 304 |
217 def _ClusterSymbols(symbols): | 305 def _ClusterSymbols(symbols): |
218 """Returns a new list of symbols with some symbols moved into groups. | 306 """Returns a new list of symbols with some symbols moved into groups. |
219 | 307 |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
310 group_symbols, name=name_tup[0], full_name=name_tup[1], | 398 group_symbols, name=name_tup[0], full_name=name_tup[1], |
311 section_name=group_symbols[0].section_name) | 399 section_name=group_symbols[0].section_name) |
312 dest_index += 1 | 400 dest_index += 1 |
313 | 401 |
314 assert len(grouped_symbols[dest_index:None]) == len(symbols[src_index:None]) | 402 assert len(grouped_symbols[dest_index:None]) == len(symbols[src_index:None]) |
315 grouped_symbols[dest_index:None] = symbols[src_index:None] | 403 grouped_symbols[dest_index:None] = symbols[src_index:None] |
316 logging.debug('Finished making groups.') | 404 logging.debug('Finished making groups.') |
317 return grouped_symbols | 405 return grouped_symbols |
318 | 406 |
319 | 407 |
| 408 def _AddSymbolAliases(symbols, aliases_by_address): |
| 409 # Step 1: Create list of (index_of_symbol, name_list). |
| 410 logging.debug('Creating alias list') |
| 411 replacements = [] |
| 412 num_new_symbols = 0 |
| 413 for i, s in enumerate(symbols): |
| 414 # Don't alias padding-only symbols (e.g. ** symbol gap) |
| 415 if s.size_without_padding == 0: |
| 416 continue |
| 417 name_list = aliases_by_address.get(s.address) |
| 418 if name_list: |
| 419 if s.name not in name_list: |
| 420 logging.warning('Name missing from aliases: %s %s', s.name, name_list) |
| 421 continue |
| 422 replacements.append((i, name_list)) |
| 423 num_new_symbols += len(name_list) - 1 |
| 424 |
| 425 # Step 2: Create new symbols as siblings to each existing one. |
| 426 logging.debug('Creating %d aliases', num_new_symbols) |
| 427 src_cursor_end = len(symbols) |
| 428 symbols += [None] * num_new_symbols |
| 429 dst_cursor_end = len(symbols) |
| 430 for src_index, name_list in reversed(replacements): |
| 431 # Copy over symbols that come after the current one. |
| 432 chunk_size = src_cursor_end - src_index - 1 |
| 433 dst_cursor_end -= chunk_size |
| 434 src_cursor_end -= chunk_size |
| 435 symbols[dst_cursor_end:dst_cursor_end + chunk_size] = ( |
| 436 symbols[src_cursor_end:src_cursor_end + chunk_size]) |
| 437 sym = symbols[src_index] |
| 438 src_cursor_end -= 1 |
| 439 |
| 440 # Create aliases (does not bother reusing the existing symbol). |
| 441 aliases = [None] * len(name_list) |
| 442 for i, name in enumerate(name_list): |
| 443 aliases[i] = models.Symbol( |
| 444 sym.section_name, sym.size, address=sym.address, name=name, |
| 445 aliases=aliases) |
| 446 |
| 447 dst_cursor_end -= len(aliases) |
| 448 symbols[dst_cursor_end:dst_cursor_end + len(aliases)] = aliases |
| 449 |
| 450 assert dst_cursor_end == src_cursor_end |
| 451 |
| 452 |
320 def LoadAndPostProcessSizeInfo(path): | 453 def LoadAndPostProcessSizeInfo(path): |
321 """Returns a SizeInfo for the given |path|.""" | 454 """Returns a SizeInfo for the given |path|.""" |
322 logging.debug('Loading results from: %s', path) | 455 logging.debug('Loading results from: %s', path) |
323 size_info = file_format.LoadSizeInfo(path) | 456 size_info = file_format.LoadSizeInfo(path) |
324 _PostProcessSizeInfo(size_info) | 457 _PostProcessSizeInfo(size_info) |
325 return size_info | 458 return size_info |
326 | 459 |
327 | 460 |
328 def _PostProcessSizeInfo(size_info): | 461 def _PostProcessSizeInfo(size_info): |
329 logging.info('Normalizing symbol names') | 462 logging.info('Normalizing symbol names') |
330 _NormalizeNames(size_info.raw_symbols) | 463 _NormalizeNames(size_info.raw_symbols) |
331 logging.info('Calculating padding') | 464 logging.info('Calculating padding') |
332 _CalculatePadding(size_info.raw_symbols) | 465 _CalculatePadding(size_info.raw_symbols) |
333 logging.info('Grouping decomposed functions') | 466 logging.info('Grouping decomposed functions') |
334 size_info.symbols = models.SymbolGroup( | 467 size_info.symbols = models.SymbolGroup( |
335 _ClusterSymbols(size_info.raw_symbols)) | 468 _ClusterSymbols(size_info.raw_symbols)) |
336 logging.info('Processed %d symbols', len(size_info.raw_symbols)) | 469 logging.info('Processed %d symbols', len(size_info.raw_symbols)) |
337 | 470 |
338 | 471 |
339 def CreateSizeInfo(map_path, lazy_paths=None, no_source_paths=False, | 472 def CreateMetadata(map_path, elf_path, apk_path, tool_prefix, output_directory): |
| 473 metadata = None |
| 474 if elf_path: |
| 475 logging.debug('Constructing metadata') |
| 476 git_rev = _DetectGitRevision(os.path.dirname(elf_path)) |
| 477 architecture = _ArchFromElf(elf_path, tool_prefix) |
| 478 build_id = BuildIdFromElf(elf_path, tool_prefix) |
| 479 timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime( |
| 480 elf_path)) |
| 481 timestamp = calendar.timegm(timestamp_obj.timetuple()) |
| 482 |
| 483 metadata = { |
| 484 models.METADATA_GIT_REVISION: git_rev, |
| 485 models.METADATA_ELF_ARCHITECTURE: architecture, |
| 486 models.METADATA_ELF_MTIME: timestamp, |
| 487 models.METADATA_ELF_BUILD_ID: build_id, |
| 488 } |
| 489 |
| 490 if output_directory: |
| 491 relative_to_out = lambda path: os.path.relpath(path, output_directory) |
| 492 gn_args = _ParseGnArgs(os.path.join(output_directory, 'args.gn')) |
| 493 metadata[models.METADATA_MAP_FILENAME] = relative_to_out(map_path) |
| 494 metadata[models.METADATA_ELF_FILENAME] = relative_to_out(elf_path) |
| 495 metadata[models.METADATA_GN_ARGS] = gn_args |
| 496 |
| 497 if apk_path: |
| 498 metadata[models.METADATA_APK_FILENAME] = relative_to_out(apk_path) |
| 499 return metadata |
| 500 |
| 501 |
| 502 def CreateSizeInfo(map_path, elf_path, tool_prefix, output_directory, |
340 raw_only=False): | 503 raw_only=False): |
341 """Creates a SizeInfo from the given map file.""" | 504 """Creates a SizeInfo. |
342 # tool_prefix needed for c++filt. | |
343 lazy_paths.VerifyToolPrefix() | |
344 | 505 |
345 if not no_source_paths: | 506 Args: |
346 # Parse .ninja files at the same time as parsing the .map file. | 507 map_path: Path to the linker .map(.gz) file to parse. |
347 source_mapper_result = helpers.ForkAndCall( | 508 elf_path: Path to the corresponding unstripped ELF file. Used to find symbol |
348 ninja_parser.Parse, lazy_paths.VerifyOutputDirectory()) | 509 aliases and inlined functions. Can be None. |
| 510 tool_prefix: Prefix for c++filt & nm (required). |
| 511 output_directory: Build output directory. If None, source_paths and symbol |
| 512 alias information will not be recorded. |
| 513 raw_only: Fill in just the information required for creating a .size file. |
| 514 """ |
| 515 source_mapper = None |
| 516 if output_directory: |
| 517 # Start by finding the elf_object_paths, so that nm can run on them while |
| 518 # the linker .map is being parsed. |
| 519 logging.info('Parsing ninja files.') |
| 520 source_mapper, elf_object_paths = ninja_parser.Parse( |
| 521 output_directory, elf_path) |
| 522 assert not elf_path or elf_object_paths, ( |
| 523 'Failed to find link command in ninja files for ' + |
| 524 os.path.relpath(elf_path, output_directory)) |
| 525 |
| 526 if elf_path: |
| 527 # Run nm on the elf file to retrieve the list of symbol names per-address. |
| 528 # This list is required because the .map file contains only a single name |
| 529 # for each address, yet multiple symbols are often coalesced when they are |
| 530 # identical. This coalescing happens mainly for small symbols and for C++ |
| 531 # templates. Such symbols make up ~500kb of libchrome.so on Android. |
| 532 elf_nm_result = nm.CollectAliasesByAddressAsync(elf_path, tool_prefix) |
| 533 |
| 534 # Run nm on all .o/.a files to retrieve the symbol names within them. |
| 535 # The list is used to detect when mutiple .o files contain the same symbol |
| 536 # (e.g. inline functions), and to update the object_path / source_path |
| 537 # fields accordingly. |
| 538 # Looking in object files is required because the .map file choses a |
| 539 # single path for these symbols. |
| 540 # Rather than record all paths for each symbol, set the paths to be the |
| 541 # common ancestor of all paths. |
| 542 if output_directory: |
| 543 bulk_analyzer = nm.BulkObjectFileAnalyzer(tool_prefix, output_directory) |
| 544 bulk_analyzer.AnalyzePaths(elf_object_paths) |
349 | 545 |
350 with _OpenMaybeGz(map_path) as map_file: | 546 with _OpenMaybeGz(map_path) as map_file: |
351 section_sizes, raw_symbols = ( | 547 section_sizes, raw_symbols = ( |
352 linker_map_parser.MapFileParser().Parse(map_file)) | 548 linker_map_parser.MapFileParser().Parse(map_file)) |
353 | 549 |
354 if not no_source_paths: | 550 if elf_path: |
355 logging.info('Extracting source paths from .ninja files') | 551 logging.debug('Validating section sizes') |
356 source_mapper = source_mapper_result.get() | 552 elf_section_sizes = _SectionSizesFromElf(elf_path, tool_prefix) |
| 553 for k, v in elf_section_sizes.iteritems(): |
| 554 if v != section_sizes.get(k): |
| 555 logging.error('ELF file and .map file do not agree on section sizes.') |
| 556 logging.error('.map file: %r', section_sizes) |
| 557 logging.error('readelf: %r', elf_section_sizes) |
| 558 sys.exit(1) |
| 559 |
| 560 if elf_path and output_directory: |
| 561 missed_object_paths = _DiscoverMissedObjectPaths( |
| 562 raw_symbols, elf_object_paths) |
| 563 bulk_analyzer.AnalyzePaths(missed_object_paths) |
| 564 bulk_analyzer.Close() |
| 565 |
| 566 if source_mapper: |
| 567 logging.info('Looking up source paths from ninja files') |
357 _ExtractSourcePaths(raw_symbols, source_mapper) | 568 _ExtractSourcePaths(raw_symbols, source_mapper) |
358 assert source_mapper.unmatched_paths_count == 0, ( | 569 assert source_mapper.unmatched_paths_count == 0, ( |
359 'One or more source file paths could not be found. Likely caused by ' | 570 'One or more source file paths could not be found. Likely caused by ' |
360 '.ninja files being generated at a different time than the .map file.') | 571 '.ninja files being generated at a different time than the .map file.') |
361 | 572 |
362 logging.info('Stripping linker prefixes from symbol names') | 573 logging.info('Stripping linker prefixes from symbol names') |
363 _StripLinkerAddedSymbolPrefixes(raw_symbols) | 574 _StripLinkerAddedSymbolPrefixes(raw_symbols) |
364 # Map file for some reason doesn't unmangle all names. | 575 # Map file for some reason doesn't unmangle all names. |
365 # Unmangle prints its own log statement. | 576 # Unmangle prints its own log statement. |
366 _UnmangleRemainingSymbols(raw_symbols, lazy_paths.tool_prefix) | 577 _UnmangleRemainingSymbols(raw_symbols, tool_prefix) |
367 logging.info('Normalizing object paths') | 578 |
368 _NormalizeObjectPaths(raw_symbols) | 579 if elf_path: |
| 580 logging.info('Adding aliased symbols, as reported by nm') |
| 581 # This normally does not block (it's finished by this time). |
| 582 aliases_by_address = elf_nm_result.get() |
| 583 _AddSymbolAliases(raw_symbols, aliases_by_address) |
| 584 |
| 585 if output_directory: |
| 586 # For aliases, this provides path information where there wasn't any. |
| 587 logging.info('Computing ancestor paths for inline functions and ' |
| 588 'normalizing object paths') |
| 589 |
| 590 object_paths_by_name = bulk_analyzer.Get() |
| 591 logging.debug('Fetched path information for %d symbols from %d files', |
| 592 len(object_paths_by_name), |
| 593 len(elf_object_paths) + len(missed_object_paths)) |
| 594 _ComputeAnscestorPathsAndNormalizeObjectPaths( |
| 595 raw_symbols, object_paths_by_name, source_mapper) |
| 596 |
| 597 if not elf_path or not output_directory: |
| 598 logging.info('Normalizing object paths.') |
| 599 for symbol in raw_symbols: |
| 600 symbol.object_path = _NormalizeObjectPath(symbol.object_path) |
| 601 |
369 size_info = models.SizeInfo(section_sizes, raw_symbols) | 602 size_info = models.SizeInfo(section_sizes, raw_symbols) |
370 | 603 |
371 # Name normalization not strictly required, but makes for smaller files. | 604 # Name normalization not strictly required, but makes for smaller files. |
372 if raw_only: | 605 if raw_only: |
373 logging.info('Normalizing symbol names') | 606 logging.info('Normalizing symbol names') |
374 _NormalizeNames(size_info.raw_symbols) | 607 _NormalizeNames(size_info.raw_symbols) |
375 else: | 608 else: |
376 _PostProcessSizeInfo(size_info) | 609 _PostProcessSizeInfo(size_info) |
377 | 610 |
378 if logging.getLogger().isEnabledFor(logging.DEBUG): | 611 if logging.getLogger().isEnabledFor(logging.DEBUG): |
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
481 any_path_within_output_directory=any_input) | 714 any_path_within_output_directory=any_input) |
482 if apk_path: | 715 if apk_path: |
483 with zipfile.ZipFile(apk_path) as z: | 716 with zipfile.ZipFile(apk_path) as z: |
484 lib_infos = [f for f in z.infolist() | 717 lib_infos = [f for f in z.infolist() |
485 if f.filename.endswith('.so') and f.file_size > 0] | 718 if f.filename.endswith('.so') and f.file_size > 0] |
486 assert lib_infos, 'APK has no .so files.' | 719 assert lib_infos, 'APK has no .so files.' |
487 # TODO(agrieve): Add support for multiple .so files, and take into account | 720 # TODO(agrieve): Add support for multiple .so files, and take into account |
488 # secondary architectures. | 721 # secondary architectures. |
489 apk_so_path = max(lib_infos, key=lambda x:x.file_size).filename | 722 apk_so_path = max(lib_infos, key=lambda x:x.file_size).filename |
490 logging.debug('Sub-apk path=%s', apk_so_path) | 723 logging.debug('Sub-apk path=%s', apk_so_path) |
491 if not elf_path: | 724 if not elf_path and lazy_paths.output_directory: |
492 elf_path = os.path.join( | 725 elf_path = os.path.join( |
493 lazy_paths.output_directory, 'lib.unstripped', | 726 lazy_paths.output_directory, 'lib.unstripped', |
494 os.path.basename(apk_so_path.replace('crazy.', ''))) | 727 os.path.basename(apk_so_path.replace('crazy.', ''))) |
495 logging.debug('Detected --elf-file=%s', elf_path) | 728 logging.debug('Detected --elf-file=%s', elf_path) |
496 | 729 |
497 if map_path: | 730 if map_path: |
498 if not map_path.endswith('.map') and not map_path.endswith('.map.gz'): | 731 if not map_path.endswith('.map') and not map_path.endswith('.map.gz'): |
499 parser.error('Expected --map-file to end with .map or .map.gz') | 732 parser.error('Expected --map-file to end with .map or .map.gz') |
500 else: | 733 else: |
501 map_path = elf_path + '.map' | 734 map_path = elf_path + '.map' |
502 if not os.path.exists(map_path): | 735 if not os.path.exists(map_path): |
503 map_path += '.gz' | 736 map_path += '.gz' |
504 if not os.path.exists(map_path): | 737 if not os.path.exists(map_path): |
505 parser.error('Could not find .map(.gz)? file. Use --map-file.') | 738 parser.error('Could not find .map(.gz)? file. Use --map-file.') |
506 | 739 |
507 metadata = None | 740 tool_prefix = lazy_paths.VerifyToolPrefix() |
508 if elf_path: | 741 output_directory = None |
509 logging.debug('Constructing metadata') | 742 if not args.no_source_paths: |
510 git_rev = _DetectGitRevision(os.path.dirname(elf_path)) | 743 output_directory = lazy_paths.VerifyOutputDirectory() |
511 architecture = _ArchFromElf(elf_path, lazy_paths.tool_prefix) | |
512 build_id = BuildIdFromElf(elf_path, lazy_paths.tool_prefix) | |
513 timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime( | |
514 elf_path)) | |
515 timestamp = calendar.timegm(timestamp_obj.timetuple()) | |
516 gn_args = _ParseGnArgs(os.path.join(lazy_paths.output_directory, 'args.gn')) | |
517 | 744 |
518 def relative_to_out(path): | 745 metadata = CreateMetadata(map_path, elf_path, apk_path, tool_prefix, |
519 return os.path.relpath(path, lazy_paths.VerifyOutputDirectory()) | 746 output_directory) |
520 | 747 if apk_path and elf_path: |
521 metadata = { | 748 # Extraction takes around 1 second, so do it in parallel. |
522 models.METADATA_GIT_REVISION: git_rev, | 749 apk_elf_result = concurrent.ForkAndCall( |
523 models.METADATA_MAP_FILENAME: relative_to_out(map_path), | 750 _ElfInfoFromApk, (apk_path, apk_so_path, tool_prefix)) |
524 models.METADATA_ELF_ARCHITECTURE: architecture, | |
525 models.METADATA_ELF_FILENAME: relative_to_out(elf_path), | |
526 models.METADATA_ELF_MTIME: timestamp, | |
527 models.METADATA_ELF_BUILD_ID: build_id, | |
528 models.METADATA_GN_ARGS: gn_args, | |
529 } | |
530 | |
531 if apk_path: | |
532 metadata[models.METADATA_APK_FILENAME] = relative_to_out(apk_path) | |
533 # Extraction takes around 1 second, so do it in parallel. | |
534 apk_elf_result = helpers.ForkAndCall( | |
535 _ElfInfoFromApk, apk_path, apk_so_path, lazy_paths.tool_prefix) | |
536 | 751 |
537 size_info = CreateSizeInfo( | 752 size_info = CreateSizeInfo( |
538 map_path, lazy_paths, no_source_paths=args.no_source_paths, raw_only=True) | 753 map_path, elf_path, tool_prefix, output_directory, raw_only=True) |
539 | 754 |
540 if metadata: | 755 if metadata: |
541 size_info.metadata = metadata | 756 size_info.metadata = metadata |
542 logging.debug('Validating section sizes') | |
543 elf_section_sizes = _SectionSizesFromElf(elf_path, lazy_paths.tool_prefix) | |
544 for k, v in elf_section_sizes.iteritems(): | |
545 assert v == size_info.section_sizes.get(k), ( | |
546 'ELF file and .map file do not match.') | |
547 | 757 |
548 if apk_path: | 758 if apk_path: |
549 logging.debug('Extracting section sizes from .so within .apk') | 759 logging.debug('Extracting section sizes from .so within .apk') |
550 unstripped_section_sizes = size_info.section_sizes | 760 unstripped_section_sizes = size_info.section_sizes |
551 apk_build_id, size_info.section_sizes = apk_elf_result.get() | 761 apk_build_id, size_info.section_sizes = apk_elf_result.get() |
552 assert apk_build_id == build_id, ( | 762 assert apk_build_id == metadata[models.METADATA_ELF_BUILD_ID], ( |
553 'BuildID for %s within %s did not match the one at %s' % | 763 'BuildID for %s within %s did not match the one at %s' % |
554 (apk_so_path, apk_path, elf_path)) | 764 (apk_so_path, apk_path, elf_path)) |
555 | 765 |
556 packed_section_name = None | 766 packed_section_name = None |
| 767 architecture = metadata[models.METADATA_ELF_ARCHITECTURE] |
557 if architecture == 'ARM': | 768 if architecture == 'ARM': |
558 packed_section_name = '.rel.dyn' | 769 packed_section_name = '.rel.dyn' |
559 elif architecture == 'AArch64': | 770 elif architecture == 'AArch64': |
560 packed_section_name = '.rela.dyn' | 771 packed_section_name = '.rela.dyn' |
561 | 772 |
562 if packed_section_name: | 773 if packed_section_name: |
563 logging.debug('Recording size of unpacked relocations') | 774 logging.debug('Recording size of unpacked relocations') |
564 if packed_section_name not in size_info.section_sizes: | 775 if packed_section_name not in size_info.section_sizes: |
565 logging.warning('Packed section not present: %s', packed_section_name) | 776 logging.warning('Packed section not present: %s', packed_section_name) |
566 else: | 777 else: |
567 size_info.section_sizes['%s (unpacked)' % packed_section_name] = ( | 778 size_info.section_sizes['%s (unpacked)' % packed_section_name] = ( |
568 unstripped_section_sizes.get(packed_section_name)) | 779 unstripped_section_sizes.get(packed_section_name)) |
569 | 780 |
570 logging.info('Recording metadata: \n %s', | 781 logging.info('Recording metadata: \n %s', |
571 '\n '.join(describe.DescribeMetadata(size_info.metadata))) | 782 '\n '.join(describe.DescribeMetadata(size_info.metadata))) |
572 logging.info('Saving result to %s', args.size_file) | 783 logging.info('Saving result to %s', args.size_file) |
573 file_format.SaveSizeInfo(size_info, args.size_file) | 784 file_format.SaveSizeInfo(size_info, args.size_file) |
574 logging.info('Done') | 785 logging.info('Done') |
OLD | NEW |