Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(12)

Side by Side Diff: tools/binary_size/libsupersize/archive.py

Issue 2851473003: supersize: Track symbol aliases and shared symbols (Closed)
Patch Set: fix regression in calculate padding introduced in ps3 Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | tools/binary_size/libsupersize/concurrent.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2017 The Chromium Authors. All rights reserved. 1 # Copyright 2017 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Main Python API for analyzing binary size.""" 5 """Main Python API for analyzing binary size."""
6 6
7 import argparse 7 import argparse
8 import calendar 8 import calendar
9 import collections 9 import collections
10 import datetime 10 import datetime
11 import gzip 11 import gzip
12 import logging 12 import logging
13 import os 13 import os
14 import posixpath 14 import posixpath
15 import re 15 import re
16 import subprocess 16 import subprocess
17 import sys 17 import sys
18 import tempfile 18 import tempfile
19 import zipfile 19 import zipfile
20 20
21 import concurrent
21 import describe 22 import describe
22 import file_format 23 import file_format
23 import function_signature 24 import function_signature
24 import helpers
25 import linker_map_parser 25 import linker_map_parser
26 import models 26 import models
27 import ninja_parser 27 import ninja_parser
28 import nm
28 import paths 29 import paths
29 30
30 31
31 def _OpenMaybeGz(path, mode=None): 32 def _OpenMaybeGz(path, mode=None):
32 """Calls `gzip.open()` if |path| ends in ".gz", otherwise calls `open()`.""" 33 """Calls `gzip.open()` if |path| ends in ".gz", otherwise calls `open()`."""
33 if path.endswith('.gz'): 34 if path.endswith('.gz'):
34 if mode and 'w' in mode: 35 if mode and 'w' in mode:
35 return gzip.GzipFile(path, mode, 1) 36 return gzip.GzipFile(path, mode, 1)
36 return gzip.open(path, mode) 37 return gzip.open(path, mode)
37 return open(path, mode or 'r') 38 return open(path, mode or 'r')
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after
120 symbol.full_name = symbol.name 121 symbol.full_name = symbol.name
121 symbol.name = re.sub(r'\(.*\)', '', symbol.full_name) 122 symbol.name = re.sub(r'\(.*\)', '', symbol.full_name)
122 123
123 # Don't bother storing both if they are the same. 124 # Don't bother storing both if they are the same.
124 if symbol.full_name == symbol.name: 125 if symbol.full_name == symbol.name:
125 symbol.full_name = '' 126 symbol.full_name = ''
126 127
127 logging.debug('Found name prefixes of: %r', found_prefixes) 128 logging.debug('Found name prefixes of: %r', found_prefixes)
128 129
129 130
130 def _NormalizeObjectPaths(symbols): 131 def _NormalizeObjectPath(path):
131 """Ensures that all paths are formatted in a useful way.""" 132 if path.startswith('obj/'):
132 for symbol in symbols: 133 # Convert obj/third_party/... -> third_party/...
133 path = symbol.object_path 134 path = path[4:]
134 if path.startswith('obj/'): 135 elif path.startswith('../../'):
135 # Convert obj/third_party/... -> third_party/... 136 # Convert ../../third_party/... -> third_party/...
136 path = path[4:] 137 path = path[6:]
137 elif path.startswith('../../'): 138 if path.endswith(')'):
138 # Convert ../../third_party/... -> third_party/... 139 # Convert foo/bar.a(baz.o) -> foo/bar.a/baz.o
139 path = path[6:] 140 start_idx = path.index('(')
140 if path.endswith(')'): 141 path = os.path.join(path[:start_idx], path[start_idx + 1:-1])
141 # Convert foo/bar.a(baz.o) -> foo/bar.a/baz.o 142 return path
142 start_idx = path.index('(')
143 path = os.path.join(path[:start_idx], path[start_idx + 1:-1])
144 symbol.object_path = path
145 143
146 144
147 def _NormalizeSourcePath(path): 145 def _NormalizeSourcePath(path):
148 if path.startswith('gen/'): 146 if path.startswith('gen/'):
149 # Convert gen/third_party/... -> third_party/... 147 # Convert gen/third_party/... -> third_party/...
150 return path[4:] 148 return path[4:]
151 if path.startswith('../../'): 149 if path.startswith('../../'):
152 # Convert ../../third_party/... -> third_party/... 150 # Convert ../../third_party/... -> third_party/...
153 return path[6:] 151 return path[6:]
154 return path 152 return path
155 153
156 154
155 def _SourcePathForObjectPath(object_path, source_mapper):
156 # We don't have source info for prebuilt .a files.
157 if not os.path.isabs(object_path) and not object_path.startswith('..'):
158 source_path = source_mapper.FindSourceForPath(object_path)
159 if source_path:
160 return _NormalizeSourcePath(source_path)
161 return ''
162
163
157 def _ExtractSourcePaths(symbols, source_mapper): 164 def _ExtractSourcePaths(symbols, source_mapper):
158 """Fills in the .source_path attribute of all symbols.""" 165 """Fills in the |source_path| attribute."""
159 logging.debug('Parsed %d .ninja files.', source_mapper.parsed_file_count) 166 logging.debug('Parsed %d .ninja files.', source_mapper.parsed_file_count)
160
161 for symbol in symbols: 167 for symbol in symbols:
162 object_path = symbol.object_path 168 object_path = symbol.object_path
163 if symbol.source_path or not object_path: 169 if object_path and not symbol.source_path:
170 symbol.source_path = _SourcePathForObjectPath(object_path, source_mapper)
171
172
173 def _ComputeAnscestorPath(path_list):
174 """Returns the common anscestor of the given paths."""
175 # Ignore missing paths.
176 path_list = [p for p in path_list if p]
177 prefix = os.path.commonprefix(path_list)
178 # Put the path count as a subdirectory to allow for better grouping when
179 # path-based breakdowns.
180 if not prefix:
181 if len(path_list) < 2:
182 return ''
183 return os.path.join('{shared}', str(len(path_list)))
184 if prefix == path_list[0]:
185 return prefix
186 assert len(path_list) > 1, 'path_list: ' + repr(path_list)
187 return os.path.join(os.path.dirname(prefix), '{shared}', str(len(path_list)))
188
189
190 # This must normalize object paths at the same time because normalization
191 # needs to occur before finding common ancestor.
192 def _ComputeAnscestorPathsAndNormalizeObjectPaths(
193 symbols, object_paths_by_name, source_mapper):
194 num_found_paths = 0
195 num_unknown_names = 0
196 num_path_mismatches = 0
197 num_unmatched_aliases = 0
198 for symbol in symbols:
199 name = symbol.name
200 if (symbol.IsBss() or
201 not name or
202 name[0] in '*.' or # e.g. ** merge symbols, .Lswitch.table
203 name == 'startup'):
204 symbol.object_path = _NormalizeObjectPath(symbol.object_path)
164 continue 205 continue
165 # We don't have source info for prebuilt .a files. 206
166 if not os.path.isabs(object_path) and not object_path.startswith('..'): 207 object_paths = object_paths_by_name.get(name)
167 source_path = source_mapper.FindSourceForPath(object_path) 208 if object_paths:
168 if source_path: 209 num_found_paths += 1
169 symbol.source_path = _NormalizeSourcePath(source_path) 210 else:
211 if not symbol.object_path and symbol.aliases:
212 # Happens when aliases are from object files where all symbols were
213 # pruned or de-duped as aliases. Since we are only scanning .o files
214 # referenced by included symbols, such files are missed.
215 # TODO(agrieve): This could be fixed by retrieving linker inputs from
216 # build.ninja, or by looking for paths within the .map file's
217 # discarded sections.
218 num_unmatched_aliases += 1
219 continue
220 if num_unknown_names < 10:
221 logging.warning('Symbol not found in any .o files: %r', symbol)
222 num_unknown_names += 1
223 symbol.object_path = _NormalizeObjectPath(symbol.object_path)
224 continue
225
226 if symbol.object_path and symbol.object_path not in object_paths:
227 if num_path_mismatches < 10:
228 logging.warning('Symbol path reported by .map not found by nm.')
229 logging.warning('sym=%r', symbol)
230 logging.warning('paths=%r', object_paths)
231 num_path_mismatches += 1
232
233 if source_mapper:
234 source_paths = [
235 _SourcePathForObjectPath(p, source_mapper) for p in object_paths]
236 symbol.source_path = _ComputeAnscestorPath(source_paths)
237
238 object_paths = [_NormalizeObjectPath(p) for p in object_paths]
239 symbol.object_path = _ComputeAnscestorPath(object_paths)
240
241 logging.debug('Cross-referenced %d symbols with nm output. '
242 'num_unknown_names=%d num_path_mismatches=%d '
243 'num_unused_aliases=%d', num_found_paths, num_unknown_names,
244 num_path_mismatches, num_unmatched_aliases)
245
246
247 def _DiscoverMissedObjectPaths(symbols, elf_object_paths):
248 # Missing object paths are caused by .a files added by -l flags, which are not
249 # listed as explicit inputs within .ninja rules.
250 parsed_inputs = set(elf_object_paths)
251 missed_inputs = set()
252 for symbol in symbols:
253 path = symbol.object_path
254 if path.endswith(')'):
255 # Convert foo/bar.a(baz.o) -> foo/bar.a
256 path = path[:path.index('(')]
257 if path and path not in parsed_inputs:
258 missed_inputs.add(path)
259 return missed_inputs
170 260
171 261
172 def _CalculatePadding(symbols): 262 def _CalculatePadding(symbols):
173 """Populates the |padding| field based on symbol addresses. 263 """Populates the |padding| field based on symbol addresses.
174 264
175 Symbols must already be sorted by |address|. 265 Symbols must already be sorted by |address|.
176 """ 266 """
177 seen_sections = [] 267 seen_sections = []
178 for i, symbol in enumerate(symbols[1:]): 268 for i, symbol in enumerate(symbols[1:]):
179 prev_symbol = symbols[i] 269 prev_symbol = symbols[i]
180 if prev_symbol.section_name != symbol.section_name: 270 if prev_symbol.section_name != symbol.section_name:
181 assert symbol.section_name not in seen_sections, ( 271 assert symbol.section_name not in seen_sections, (
182 'Input symbols must be sorted by section, then address.') 272 'Input symbols must be sorted by section, then address.')
183 seen_sections.append(symbol.section_name) 273 seen_sections.append(symbol.section_name)
184 continue 274 continue
185 if symbol.address <= 0 or prev_symbol.address <= 0: 275 if symbol.address <= 0 or prev_symbol.address <= 0:
186 continue 276 continue
187 # Padding-only symbols happen for ** symbol gaps. 277
188 prev_is_padding_only = prev_symbol.size_without_padding == 0 278 if symbol.address == prev_symbol.address:
189 if symbol.address == prev_symbol.address and not prev_is_padding_only: 279 if symbol.aliases and symbol.aliases is prev_symbol.aliases:
190 assert False, 'Found duplicate symbols:\n%r\n%r' % (prev_symbol, symbol) 280 symbol.padding = prev_symbol.padding
191 # Even with symbols at the same address removed, overlaps can still 281 symbol.size = prev_symbol.size
192 # happen. In this case, padding will be negative (and this is fine). 282 continue
283 # Padding-only symbols happen for ** symbol gaps.
284 assert prev_symbol.size_without_padding == 0, (
285 'Found duplicate symbols:\n%r\n%r' % (prev_symbol, symbol))
286
193 padding = symbol.address - prev_symbol.end_address 287 padding = symbol.address - prev_symbol.end_address
194 # These thresholds were found by manually auditing arm32 Chrome. 288 # These thresholds were found by experimenting with arm32 Chrome.
195 # E.g.: Set them to 0 and see what warnings get logged. 289 # E.g.: Set them to 0 and see what warnings get logged, then take max value.
196 # TODO(agrieve): See if these thresholds make sense for architectures 290 # TODO(agrieve): See if these thresholds make sense for architectures
197 # other than arm32. 291 # other than arm32.
198 if not symbol.name.startswith('*') and ( 292 if not symbol.name.startswith('*') and (
199 symbol.section in 'rd' and padding >= 256 or 293 symbol.section in 'rd' and padding >= 256 or
200 symbol.section in 't' and padding >= 64): 294 symbol.section in 't' and padding >= 64):
201 # For nm data, this is caused by data that has no associated symbol. 295 # Should not happen.
202 # The linker map file lists them with no name, but with a file. 296 logging.warning('Large padding of %d between:\n A) %r\n B) %r' % (
203 # Example: 297 padding, prev_symbol, symbol))
204 # .data 0x02d42764 0x120 .../V8SharedWorkerGlobalScope.o
205 # Where as most look like:
206 # .data.MANGLED_NAME...
207 logging.debug('Large padding of %d between:\n A) %r\n B) %r' % (
208 padding, prev_symbol, symbol))
209 continue
210 symbol.padding = padding 298 symbol.padding = padding
211 symbol.size += padding 299 symbol.size += padding
212 assert symbol.size >= 0, ( 300 assert symbol.size >= 0, (
213 'Symbol has negative size (likely not sorted propertly): ' 301 'Symbol has negative size (likely not sorted propertly): '
214 '%r\nprev symbol: %r' % (symbol, prev_symbol)) 302 '%r\nprev symbol: %r' % (symbol, prev_symbol))
215 303
216 304
217 def _ClusterSymbols(symbols): 305 def _ClusterSymbols(symbols):
218 """Returns a new list of symbols with some symbols moved into groups. 306 """Returns a new list of symbols with some symbols moved into groups.
219 307
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
310 group_symbols, name=name_tup[0], full_name=name_tup[1], 398 group_symbols, name=name_tup[0], full_name=name_tup[1],
311 section_name=group_symbols[0].section_name) 399 section_name=group_symbols[0].section_name)
312 dest_index += 1 400 dest_index += 1
313 401
314 assert len(grouped_symbols[dest_index:None]) == len(symbols[src_index:None]) 402 assert len(grouped_symbols[dest_index:None]) == len(symbols[src_index:None])
315 grouped_symbols[dest_index:None] = symbols[src_index:None] 403 grouped_symbols[dest_index:None] = symbols[src_index:None]
316 logging.debug('Finished making groups.') 404 logging.debug('Finished making groups.')
317 return grouped_symbols 405 return grouped_symbols
318 406
319 407
408 def _AddSymbolAliases(symbols, aliases_by_address):
409 # Step 1: Create list of (index_of_symbol, name_list).
410 logging.debug('Creating alias list')
411 replacements = []
412 num_new_symbols = 0
413 for i, s in enumerate(symbols):
414 # Don't alias padding-only symbols (e.g. ** symbol gap)
415 if s.size_without_padding == 0:
416 continue
417 name_list = aliases_by_address.get(s.address)
418 if name_list:
419 if s.name not in name_list:
420 logging.warning('Name missing from aliases: %s %s', s.name, name_list)
421 continue
422 replacements.append((i, name_list))
423 num_new_symbols += len(name_list) - 1
424
425 # Step 2: Create new symbols as siblings to each existing one.
426 logging.debug('Creating %d aliases', num_new_symbols)
427 src_cursor_end = len(symbols)
428 symbols += [None] * num_new_symbols
429 dst_cursor_end = len(symbols)
430 for src_index, name_list in reversed(replacements):
431 # Copy over symbols that come after the current one.
432 chunk_size = src_cursor_end - src_index - 1
433 dst_cursor_end -= chunk_size
434 src_cursor_end -= chunk_size
435 symbols[dst_cursor_end:dst_cursor_end + chunk_size] = (
436 symbols[src_cursor_end:src_cursor_end + chunk_size])
437 sym = symbols[src_index]
438 src_cursor_end -= 1
439
440 # Create aliases (does not bother reusing the existing symbol).
441 aliases = [None] * len(name_list)
442 for i, name in enumerate(name_list):
443 aliases[i] = models.Symbol(
444 sym.section_name, sym.size, address=sym.address, name=name,
445 aliases=aliases)
446
447 dst_cursor_end -= len(aliases)
448 symbols[dst_cursor_end:dst_cursor_end + len(aliases)] = aliases
449
450 assert dst_cursor_end == src_cursor_end
451
452
320 def LoadAndPostProcessSizeInfo(path): 453 def LoadAndPostProcessSizeInfo(path):
321 """Returns a SizeInfo for the given |path|.""" 454 """Returns a SizeInfo for the given |path|."""
322 logging.debug('Loading results from: %s', path) 455 logging.debug('Loading results from: %s', path)
323 size_info = file_format.LoadSizeInfo(path) 456 size_info = file_format.LoadSizeInfo(path)
324 _PostProcessSizeInfo(size_info) 457 _PostProcessSizeInfo(size_info)
325 return size_info 458 return size_info
326 459
327 460
328 def _PostProcessSizeInfo(size_info): 461 def _PostProcessSizeInfo(size_info):
329 logging.info('Normalizing symbol names') 462 logging.info('Normalizing symbol names')
330 _NormalizeNames(size_info.raw_symbols) 463 _NormalizeNames(size_info.raw_symbols)
331 logging.info('Calculating padding') 464 logging.info('Calculating padding')
332 _CalculatePadding(size_info.raw_symbols) 465 _CalculatePadding(size_info.raw_symbols)
333 logging.info('Grouping decomposed functions') 466 logging.info('Grouping decomposed functions')
334 size_info.symbols = models.SymbolGroup( 467 size_info.symbols = models.SymbolGroup(
335 _ClusterSymbols(size_info.raw_symbols)) 468 _ClusterSymbols(size_info.raw_symbols))
336 logging.info('Processed %d symbols', len(size_info.raw_symbols)) 469 logging.info('Processed %d symbols', len(size_info.raw_symbols))
337 470
338 471
339 def CreateSizeInfo(map_path, lazy_paths=None, no_source_paths=False, 472 def CreateMetadata(map_path, elf_path, apk_path, tool_prefix, output_directory):
473 metadata = None
474 if elf_path:
475 logging.debug('Constructing metadata')
476 git_rev = _DetectGitRevision(os.path.dirname(elf_path))
477 architecture = _ArchFromElf(elf_path, tool_prefix)
478 build_id = BuildIdFromElf(elf_path, tool_prefix)
479 timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime(
480 elf_path))
481 timestamp = calendar.timegm(timestamp_obj.timetuple())
482
483 metadata = {
484 models.METADATA_GIT_REVISION: git_rev,
485 models.METADATA_ELF_ARCHITECTURE: architecture,
486 models.METADATA_ELF_MTIME: timestamp,
487 models.METADATA_ELF_BUILD_ID: build_id,
488 }
489
490 if output_directory:
491 relative_to_out = lambda path: os.path.relpath(path, output_directory)
492 gn_args = _ParseGnArgs(os.path.join(output_directory, 'args.gn'))
493 metadata[models.METADATA_MAP_FILENAME] = relative_to_out(map_path)
494 metadata[models.METADATA_ELF_FILENAME] = relative_to_out(elf_path)
495 metadata[models.METADATA_GN_ARGS] = gn_args
496
497 if apk_path:
498 metadata[models.METADATA_APK_FILENAME] = relative_to_out(apk_path)
499 return metadata
500
501
502 def CreateSizeInfo(map_path, elf_path, tool_prefix, output_directory,
340 raw_only=False): 503 raw_only=False):
341 """Creates a SizeInfo from the given map file.""" 504 """Creates a SizeInfo.
342 # tool_prefix needed for c++filt.
343 lazy_paths.VerifyToolPrefix()
344 505
345 if not no_source_paths: 506 Args:
346 # Parse .ninja files at the same time as parsing the .map file. 507 map_path: Path to the linker .map(.gz) file to parse.
347 source_mapper_result = helpers.ForkAndCall( 508 elf_path: Path to the corresponding unstripped ELF file. Used to find symbol
348 ninja_parser.Parse, lazy_paths.VerifyOutputDirectory()) 509 aliases and inlined functions. Can be None.
510 tool_prefix: Prefix for c++filt & nm (required).
511 output_directory: Build output directory. If None, source_paths and symbol
512 alias information will not be recorded.
513 raw_only: Fill in just the information required for creating a .size file.
514 """
515 source_mapper = None
516 if output_directory:
517 # Start by finding the elf_object_paths, so that nm can run on them while
518 # the linker .map is being parsed.
519 logging.info('Parsing ninja files.')
520 source_mapper, elf_object_paths = ninja_parser.Parse(
521 output_directory, elf_path)
522 assert not elf_path or elf_object_paths, (
523 'Failed to find link command in ninja files for ' +
524 os.path.relpath(elf_path, output_directory))
525
526 if elf_path:
527 # Run nm on the elf file to retrieve the list of symbol names per-address.
528 # This list is required because the .map file contains only a single name
529 # for each address, yet multiple symbols are often coalesced when they are
530 # identical. This coalescing happens mainly for small symbols and for C++
531 # templates. Such symbols make up ~500kb of libchrome.so on Android.
532 elf_nm_result = nm.CollectAliasesByAddressAsync(elf_path, tool_prefix)
533
534 # Run nm on all .o/.a files to retrieve the symbol names within them.
535 # The list is used to detect when mutiple .o files contain the same symbol
536 # (e.g. inline functions), and to update the object_path / source_path
537 # fields accordingly.
538 # Looking in object files is required because the .map file choses a
539 # single path for these symbols.
540 # Rather than record all paths for each symbol, set the paths to be the
541 # common ancestor of all paths.
542 if output_directory:
543 bulk_analyzer = nm.BulkObjectFileAnalyzer(tool_prefix, output_directory)
544 bulk_analyzer.AnalyzePaths(elf_object_paths)
349 545
350 with _OpenMaybeGz(map_path) as map_file: 546 with _OpenMaybeGz(map_path) as map_file:
351 section_sizes, raw_symbols = ( 547 section_sizes, raw_symbols = (
352 linker_map_parser.MapFileParser().Parse(map_file)) 548 linker_map_parser.MapFileParser().Parse(map_file))
353 549
354 if not no_source_paths: 550 if elf_path:
355 logging.info('Extracting source paths from .ninja files') 551 logging.debug('Validating section sizes')
356 source_mapper = source_mapper_result.get() 552 elf_section_sizes = _SectionSizesFromElf(elf_path, tool_prefix)
553 for k, v in elf_section_sizes.iteritems():
554 if v != section_sizes.get(k):
555 logging.error('ELF file and .map file do not agree on section sizes.')
556 logging.error('.map file: %r', section_sizes)
557 logging.error('readelf: %r', elf_section_sizes)
558 sys.exit(1)
559
560 if elf_path and output_directory:
561 missed_object_paths = _DiscoverMissedObjectPaths(
562 raw_symbols, elf_object_paths)
563 bulk_analyzer.AnalyzePaths(missed_object_paths)
564 bulk_analyzer.Close()
565
566 if source_mapper:
567 logging.info('Looking up source paths from ninja files')
357 _ExtractSourcePaths(raw_symbols, source_mapper) 568 _ExtractSourcePaths(raw_symbols, source_mapper)
358 assert source_mapper.unmatched_paths_count == 0, ( 569 assert source_mapper.unmatched_paths_count == 0, (
359 'One or more source file paths could not be found. Likely caused by ' 570 'One or more source file paths could not be found. Likely caused by '
360 '.ninja files being generated at a different time than the .map file.') 571 '.ninja files being generated at a different time than the .map file.')
361 572
362 logging.info('Stripping linker prefixes from symbol names') 573 logging.info('Stripping linker prefixes from symbol names')
363 _StripLinkerAddedSymbolPrefixes(raw_symbols) 574 _StripLinkerAddedSymbolPrefixes(raw_symbols)
364 # Map file for some reason doesn't unmangle all names. 575 # Map file for some reason doesn't unmangle all names.
365 # Unmangle prints its own log statement. 576 # Unmangle prints its own log statement.
366 _UnmangleRemainingSymbols(raw_symbols, lazy_paths.tool_prefix) 577 _UnmangleRemainingSymbols(raw_symbols, tool_prefix)
367 logging.info('Normalizing object paths') 578
368 _NormalizeObjectPaths(raw_symbols) 579 if elf_path:
580 logging.info('Adding aliased symbols, as reported by nm')
581 # This normally does not block (it's finished by this time).
582 aliases_by_address = elf_nm_result.get()
583 _AddSymbolAliases(raw_symbols, aliases_by_address)
584
585 if output_directory:
586 # For aliases, this provides path information where there wasn't any.
587 logging.info('Computing ancestor paths for inline functions and '
588 'normalizing object paths')
589
590 object_paths_by_name = bulk_analyzer.Get()
591 logging.debug('Fetched path information for %d symbols from %d files',
592 len(object_paths_by_name),
593 len(elf_object_paths) + len(missed_object_paths))
594 _ComputeAnscestorPathsAndNormalizeObjectPaths(
595 raw_symbols, object_paths_by_name, source_mapper)
596
597 if not elf_path or not output_directory:
598 logging.info('Normalizing object paths.')
599 for symbol in raw_symbols:
600 symbol.object_path = _NormalizeObjectPath(symbol.object_path)
601
369 size_info = models.SizeInfo(section_sizes, raw_symbols) 602 size_info = models.SizeInfo(section_sizes, raw_symbols)
370 603
371 # Name normalization not strictly required, but makes for smaller files. 604 # Name normalization not strictly required, but makes for smaller files.
372 if raw_only: 605 if raw_only:
373 logging.info('Normalizing symbol names') 606 logging.info('Normalizing symbol names')
374 _NormalizeNames(size_info.raw_symbols) 607 _NormalizeNames(size_info.raw_symbols)
375 else: 608 else:
376 _PostProcessSizeInfo(size_info) 609 _PostProcessSizeInfo(size_info)
377 610
378 if logging.getLogger().isEnabledFor(logging.DEBUG): 611 if logging.getLogger().isEnabledFor(logging.DEBUG):
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after
481 any_path_within_output_directory=any_input) 714 any_path_within_output_directory=any_input)
482 if apk_path: 715 if apk_path:
483 with zipfile.ZipFile(apk_path) as z: 716 with zipfile.ZipFile(apk_path) as z:
484 lib_infos = [f for f in z.infolist() 717 lib_infos = [f for f in z.infolist()
485 if f.filename.endswith('.so') and f.file_size > 0] 718 if f.filename.endswith('.so') and f.file_size > 0]
486 assert lib_infos, 'APK has no .so files.' 719 assert lib_infos, 'APK has no .so files.'
487 # TODO(agrieve): Add support for multiple .so files, and take into account 720 # TODO(agrieve): Add support for multiple .so files, and take into account
488 # secondary architectures. 721 # secondary architectures.
489 apk_so_path = max(lib_infos, key=lambda x:x.file_size).filename 722 apk_so_path = max(lib_infos, key=lambda x:x.file_size).filename
490 logging.debug('Sub-apk path=%s', apk_so_path) 723 logging.debug('Sub-apk path=%s', apk_so_path)
491 if not elf_path: 724 if not elf_path and lazy_paths.output_directory:
492 elf_path = os.path.join( 725 elf_path = os.path.join(
493 lazy_paths.output_directory, 'lib.unstripped', 726 lazy_paths.output_directory, 'lib.unstripped',
494 os.path.basename(apk_so_path.replace('crazy.', ''))) 727 os.path.basename(apk_so_path.replace('crazy.', '')))
495 logging.debug('Detected --elf-file=%s', elf_path) 728 logging.debug('Detected --elf-file=%s', elf_path)
496 729
497 if map_path: 730 if map_path:
498 if not map_path.endswith('.map') and not map_path.endswith('.map.gz'): 731 if not map_path.endswith('.map') and not map_path.endswith('.map.gz'):
499 parser.error('Expected --map-file to end with .map or .map.gz') 732 parser.error('Expected --map-file to end with .map or .map.gz')
500 else: 733 else:
501 map_path = elf_path + '.map' 734 map_path = elf_path + '.map'
502 if not os.path.exists(map_path): 735 if not os.path.exists(map_path):
503 map_path += '.gz' 736 map_path += '.gz'
504 if not os.path.exists(map_path): 737 if not os.path.exists(map_path):
505 parser.error('Could not find .map(.gz)? file. Use --map-file.') 738 parser.error('Could not find .map(.gz)? file. Use --map-file.')
506 739
507 metadata = None 740 tool_prefix = lazy_paths.VerifyToolPrefix()
508 if elf_path: 741 output_directory = None
509 logging.debug('Constructing metadata') 742 if not args.no_source_paths:
510 git_rev = _DetectGitRevision(os.path.dirname(elf_path)) 743 output_directory = lazy_paths.VerifyOutputDirectory()
511 architecture = _ArchFromElf(elf_path, lazy_paths.tool_prefix)
512 build_id = BuildIdFromElf(elf_path, lazy_paths.tool_prefix)
513 timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime(
514 elf_path))
515 timestamp = calendar.timegm(timestamp_obj.timetuple())
516 gn_args = _ParseGnArgs(os.path.join(lazy_paths.output_directory, 'args.gn'))
517 744
518 def relative_to_out(path): 745 metadata = CreateMetadata(map_path, elf_path, apk_path, tool_prefix,
519 return os.path.relpath(path, lazy_paths.VerifyOutputDirectory()) 746 output_directory)
520 747 if apk_path and elf_path:
521 metadata = { 748 # Extraction takes around 1 second, so do it in parallel.
522 models.METADATA_GIT_REVISION: git_rev, 749 apk_elf_result = concurrent.ForkAndCall(
523 models.METADATA_MAP_FILENAME: relative_to_out(map_path), 750 _ElfInfoFromApk, (apk_path, apk_so_path, tool_prefix))
524 models.METADATA_ELF_ARCHITECTURE: architecture,
525 models.METADATA_ELF_FILENAME: relative_to_out(elf_path),
526 models.METADATA_ELF_MTIME: timestamp,
527 models.METADATA_ELF_BUILD_ID: build_id,
528 models.METADATA_GN_ARGS: gn_args,
529 }
530
531 if apk_path:
532 metadata[models.METADATA_APK_FILENAME] = relative_to_out(apk_path)
533 # Extraction takes around 1 second, so do it in parallel.
534 apk_elf_result = helpers.ForkAndCall(
535 _ElfInfoFromApk, apk_path, apk_so_path, lazy_paths.tool_prefix)
536 751
537 size_info = CreateSizeInfo( 752 size_info = CreateSizeInfo(
538 map_path, lazy_paths, no_source_paths=args.no_source_paths, raw_only=True) 753 map_path, elf_path, tool_prefix, output_directory, raw_only=True)
539 754
540 if metadata: 755 if metadata:
541 size_info.metadata = metadata 756 size_info.metadata = metadata
542 logging.debug('Validating section sizes')
543 elf_section_sizes = _SectionSizesFromElf(elf_path, lazy_paths.tool_prefix)
544 for k, v in elf_section_sizes.iteritems():
545 assert v == size_info.section_sizes.get(k), (
546 'ELF file and .map file do not match.')
547 757
548 if apk_path: 758 if apk_path:
549 logging.debug('Extracting section sizes from .so within .apk') 759 logging.debug('Extracting section sizes from .so within .apk')
550 unstripped_section_sizes = size_info.section_sizes 760 unstripped_section_sizes = size_info.section_sizes
551 apk_build_id, size_info.section_sizes = apk_elf_result.get() 761 apk_build_id, size_info.section_sizes = apk_elf_result.get()
552 assert apk_build_id == build_id, ( 762 assert apk_build_id == metadata[models.METADATA_ELF_BUILD_ID], (
553 'BuildID for %s within %s did not match the one at %s' % 763 'BuildID for %s within %s did not match the one at %s' %
554 (apk_so_path, apk_path, elf_path)) 764 (apk_so_path, apk_path, elf_path))
555 765
556 packed_section_name = None 766 packed_section_name = None
767 architecture = metadata[models.METADATA_ELF_ARCHITECTURE]
557 if architecture == 'ARM': 768 if architecture == 'ARM':
558 packed_section_name = '.rel.dyn' 769 packed_section_name = '.rel.dyn'
559 elif architecture == 'AArch64': 770 elif architecture == 'AArch64':
560 packed_section_name = '.rela.dyn' 771 packed_section_name = '.rela.dyn'
561 772
562 if packed_section_name: 773 if packed_section_name:
563 logging.debug('Recording size of unpacked relocations') 774 logging.debug('Recording size of unpacked relocations')
564 if packed_section_name not in size_info.section_sizes: 775 if packed_section_name not in size_info.section_sizes:
565 logging.warning('Packed section not present: %s', packed_section_name) 776 logging.warning('Packed section not present: %s', packed_section_name)
566 else: 777 else:
567 size_info.section_sizes['%s (unpacked)' % packed_section_name] = ( 778 size_info.section_sizes['%s (unpacked)' % packed_section_name] = (
568 unstripped_section_sizes.get(packed_section_name)) 779 unstripped_section_sizes.get(packed_section_name))
569 780
570 logging.info('Recording metadata: \n %s', 781 logging.info('Recording metadata: \n %s',
571 '\n '.join(describe.DescribeMetadata(size_info.metadata))) 782 '\n '.join(describe.DescribeMetadata(size_info.metadata)))
572 logging.info('Saving result to %s', args.size_file) 783 logging.info('Saving result to %s', args.size_file)
573 file_format.SaveSizeInfo(size_info, args.size_file) 784 file_format.SaveSizeInfo(size_info, args.size_file)
574 logging.info('Done') 785 logging.info('Done')
OLDNEW
« no previous file with comments | « no previous file | tools/binary_size/libsupersize/concurrent.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698