OLD | NEW |
---|---|
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """The deep heap profiler script for Chrome.""" | 5 """The deep heap profiler script for Chrome.""" |
6 | 6 |
7 from datetime import datetime | 7 import datetime |
8 import json | 8 import json |
9 import logging | 9 import logging |
10 import optparse | 10 import optparse |
11 import os | 11 import os |
12 import re | 12 import re |
13 import subprocess | 13 import subprocess |
14 import sys | 14 import sys |
15 import tempfile | 15 import tempfile |
16 import zipfile | 16 import zipfile |
17 | 17 |
18 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) | 18 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) |
19 FIND_RUNTIME_SYMBOLS_PATH = os.path.join( | 19 FIND_RUNTIME_SYMBOLS_PATH = os.path.join( |
20 BASE_PATH, os.pardir, 'find_runtime_symbols') | 20 BASE_PATH, os.pardir, 'find_runtime_symbols') |
21 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH) | 21 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH) |
22 | 22 |
23 from find_runtime_symbols import find_runtime_symbols_list | 23 import find_runtime_symbols |
24 from find_runtime_symbols import find_runtime_typeinfo_symbols_list | 24 import prepare_symbol_info |
25 from find_runtime_symbols import RuntimeSymbolsInProcess | 25 |
26 from prepare_symbol_info import prepare_symbol_info | 26 from find_runtime_symbols import FUNCTION_SYMBOLS |
27 from find_runtime_symbols import SOURCEFILE_SYMBOLS | |
28 from find_runtime_symbols import TYPEINFO_SYMBOLS | |
27 | 29 |
28 BUCKET_ID = 5 | 30 BUCKET_ID = 5 |
29 VIRTUAL = 0 | 31 VIRTUAL = 0 |
30 COMMITTED = 1 | 32 COMMITTED = 1 |
31 ALLOC_COUNT = 2 | 33 ALLOC_COUNT = 2 |
32 FREE_COUNT = 3 | 34 FREE_COUNT = 3 |
33 NULL_REGEX = re.compile('') | 35 NULL_REGEX = re.compile('') |
34 | 36 |
35 LOGGER = logging.getLogger('dmprof') | 37 LOGGER = logging.getLogger('dmprof') |
36 POLICIES_JSON_PATH = os.path.join(BASE_PATH, 'policies.json') | 38 POLICIES_JSON_PATH = os.path.join(BASE_PATH, 'policies.json') |
37 FUNCTION_ADDRESS = 'function' | |
38 TYPEINFO_ADDRESS = 'typeinfo' | |
39 | 39 |
40 | 40 |
41 # Heap Profile Dump versions | 41 # Heap Profile Dump versions |
42 | 42 |
43 # DUMP_DEEP_[1-4] are obsolete. | 43 # DUMP_DEEP_[1-4] are obsolete. |
44 # DUMP_DEEP_2+ distinct mmap regions and malloc chunks. | 44 # DUMP_DEEP_2+ distinct mmap regions and malloc chunks. |
45 # DUMP_DEEP_3+ don't include allocation functions in their stack dumps. | 45 # DUMP_DEEP_3+ don't include allocation functions in their stack dumps. |
46 # DUMP_DEEP_4+ support comments with '#' and global stats "nonprofiled-*". | 46 # DUMP_DEEP_4+ support comments with '#' and global stats "nonprofiled-*". |
47 # DUMP_DEEP_[1-2] should be processed by POLICY_DEEP_1. | 47 # DUMP_DEEP_[1-2] should be processed by POLICY_DEEP_1. |
48 # DUMP_DEEP_[3-4] should be processed by POLICY_DEEP_2 or POLICY_DEEP_3. | 48 # DUMP_DEEP_[3-4] should be processed by POLICY_DEEP_2 or POLICY_DEEP_3. |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
148 def prepare(self): | 148 def prepare(self): |
149 """Prepares symbol data sources by extracting mapping from a binary. | 149 """Prepares symbol data sources by extracting mapping from a binary. |
150 | 150 |
151 The prepared symbol data sources are stored in a directory. The directory | 151 The prepared symbol data sources are stored in a directory. The directory |
152 name is stored in |self._prepared_symbol_data_sources_path|. | 152 name is stored in |self._prepared_symbol_data_sources_path|. |
153 | 153 |
154 Returns: | 154 Returns: |
155 True if succeeded. | 155 True if succeeded. |
156 """ | 156 """ |
157 LOGGER.info('Preparing symbol mapping...') | 157 LOGGER.info('Preparing symbol mapping...') |
158 self._prepared_symbol_data_sources_path, used_tempdir = prepare_symbol_info( | 158 self._prepared_symbol_data_sources_path, used_tempdir = ( |
159 self._prefix + '.maps', self._prefix + '.symmap', True) | 159 prepare_symbol_info.prepare_symbol_info( |
160 self._prefix + '.maps', | |
161 output_dir_path=self._prefix + '.symmap', | |
162 use_tempdir=True, | |
163 use_source_file_name=True)) | |
160 if self._prepared_symbol_data_sources_path: | 164 if self._prepared_symbol_data_sources_path: |
161 LOGGER.info(' Prepared symbol mapping.') | 165 LOGGER.info(' Prepared symbol mapping.') |
162 if used_tempdir: | 166 if used_tempdir: |
163 LOGGER.warn(' Using a temporary directory for symbol mapping.') | 167 LOGGER.warn(' Using a temporary directory for symbol mapping.') |
164 LOGGER.warn(' Delete it by yourself.') | 168 LOGGER.warn(' Delete it by yourself.') |
165 LOGGER.warn(' Or, move the directory by yourself to use it later.') | 169 LOGGER.warn(' Or, move the directory by yourself to use it later.') |
166 return True | 170 return True |
167 else: | 171 else: |
168 LOGGER.warn(' Failed to prepare symbol mapping.') | 172 LOGGER.warn(' Failed to prepare symbol mapping.') |
169 return False | 173 return False |
170 | 174 |
171 def get(self): | 175 def get(self): |
172 """Returns the prepared symbol data sources. | 176 """Returns the prepared symbol data sources. |
173 | 177 |
174 Returns: | 178 Returns: |
175 The prepared symbol data sources. None if failed. | 179 The prepared symbol data sources. None if failed. |
176 """ | 180 """ |
177 if not self._prepared_symbol_data_sources_path and not self.prepare(): | 181 if not self._prepared_symbol_data_sources_path and not self.prepare(): |
178 return None | 182 return None |
179 if not self._loaded_symbol_data_sources: | 183 if not self._loaded_symbol_data_sources: |
180 LOGGER.info('Loading symbol mapping...') | 184 LOGGER.info('Loading symbol mapping...') |
181 self._loaded_symbol_data_sources = RuntimeSymbolsInProcess.load( | 185 self._loaded_symbol_data_sources = ( |
182 self._prepared_symbol_data_sources_path) | 186 find_runtime_symbols.RuntimeSymbolsInProcess.load( |
187 self._prepared_symbol_data_sources_path)) | |
183 return self._loaded_symbol_data_sources | 188 return self._loaded_symbol_data_sources |
184 | 189 |
185 def path(self): | 190 def path(self): |
186 """Returns the path of the prepared symbol data sources if possible.""" | 191 """Returns the path of the prepared symbol data sources if possible.""" |
187 if not self._prepared_symbol_data_sources_path and not self.prepare(): | 192 if not self._prepared_symbol_data_sources_path and not self.prepare(): |
188 return None | 193 return None |
189 return self._prepared_symbol_data_sources_path | 194 return self._prepared_symbol_data_sources_path |
190 | 195 |
191 | 196 |
192 class SymbolFinder(object): | 197 class SymbolFinder(object): |
193 """Finds corresponding symbols from addresses. | 198 """Finds corresponding symbols from addresses. |
194 | 199 |
195 This class does only 'find()' symbols from a specified |address_list|. | 200 This class does only 'find()' symbols from a specified |address_list|. |
196 It is introduced to make a finder mockable. | 201 It is introduced to make a finder mockable. |
197 """ | 202 """ |
198 _FIND_RUNTIME_SYMBOLS_FUNCTIONS = { | 203 def __init__(self, symbol_type, symbol_data_sources): |
199 FUNCTION_ADDRESS: find_runtime_symbols_list, | 204 self._symbol_type = symbol_type |
200 TYPEINFO_ADDRESS: find_runtime_typeinfo_symbols_list, | |
201 } | |
202 | |
203 def __init__(self, address_type, symbol_data_sources): | |
204 self._finder_function = self._FIND_RUNTIME_SYMBOLS_FUNCTIONS[address_type] | |
205 self._symbol_data_sources = symbol_data_sources | 205 self._symbol_data_sources = symbol_data_sources |
206 | 206 |
207 def find(self, address_list): | 207 def find(self, address_list): |
208 return self._finder_function(self._symbol_data_sources.get(), address_list) | 208 return find_runtime_symbols.find_runtime_symbols( |
209 self._symbol_type, self._symbol_data_sources.get(), address_list) | |
209 | 210 |
210 | 211 |
211 class SymbolMappingCache(object): | 212 class SymbolMappingCache(object): |
212 """Caches mapping from actually used addresses to symbols. | 213 """Caches mapping from actually used addresses to symbols. |
213 | 214 |
214 'update()' updates the cache from the original symbol data sources via | 215 'update()' updates the cache from the original symbol data sources via |
215 'SymbolFinder'. Symbols can be looked up by the method 'lookup()'. | 216 'SymbolFinder'. Symbols can be looked up by the method 'lookup()'. |
216 """ | 217 """ |
217 def __init__(self): | 218 def __init__(self): |
218 self._symbol_mapping_caches = { | 219 self._symbol_mapping_caches = { |
219 FUNCTION_ADDRESS: {}, | 220 FUNCTION_SYMBOLS: {}, |
220 TYPEINFO_ADDRESS: {}, | 221 SOURCEFILE_SYMBOLS: {}, |
222 TYPEINFO_SYMBOLS: {}, | |
221 } | 223 } |
222 | 224 |
223 def update(self, address_type, bucket_set, symbol_finder, cache_f): | 225 def update(self, symbol_type, bucket_set, symbol_finder, cache_f): |
224 """Updates symbol mapping cache on memory and in a symbol cache file. | 226 """Updates symbol mapping cache on memory and in a symbol cache file. |
225 | 227 |
226 It reads cached symbol mapping from a symbol cache file |cache_f| if it | 228 It reads cached symbol mapping from a symbol cache file |cache_f| if it |
227 exists. Unresolved addresses are then resolved and added to the cache | 229 exists. Unresolved addresses are then resolved and added to the cache |
228 both on memory and in the symbol cache file with using 'SymbolFinder'. | 230 both on memory and in the symbol cache file with using 'SymbolFinder'. |
229 | 231 |
230 A cache file is formatted as follows: | 232 A cache file is formatted as follows: |
231 <Address> <Symbol> | 233 <Address> <Symbol> |
232 <Address> <Symbol> | 234 <Address> <Symbol> |
233 <Address> <Symbol> | 235 <Address> <Symbol> |
234 ... | 236 ... |
235 | 237 |
236 Args: | 238 Args: |
237 address_type: A type of addresses to update. | 239 symbol_type: A type of symbols to update. It should be one of |
238 It should be one of FUNCTION_ADDRESS or TYPEINFO_ADDRESS. | 240 FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS and TYPEINFO_SYMBOLS. |
239 bucket_set: A BucketSet object. | 241 bucket_set: A BucketSet object. |
240 symbol_finder: A SymbolFinder object to find symbols. | 242 symbol_finder: A SymbolFinder object to find symbols. |
241 cache_f: A readable and writable IO object of the symbol cache file. | 243 cache_f: A readable and writable IO object of the symbol cache file. |
242 """ | 244 """ |
243 cache_f.seek(0, os.SEEK_SET) | 245 cache_f.seek(0, os.SEEK_SET) |
244 self._load(cache_f, address_type) | 246 self._load(cache_f, symbol_type) |
245 | 247 |
246 unresolved_addresses = sorted( | 248 unresolved_addresses = sorted( |
247 address for address in bucket_set.iter_addresses(address_type) | 249 address for address in bucket_set.iter_addresses(symbol_type) |
248 if address not in self._symbol_mapping_caches[address_type]) | 250 if address not in self._symbol_mapping_caches[symbol_type]) |
249 | 251 |
250 if not unresolved_addresses: | 252 if not unresolved_addresses: |
251 LOGGER.info('No need to resolve any more addresses.') | 253 LOGGER.info('No need to resolve any more addresses.') |
252 return | 254 return |
253 | 255 |
254 cache_f.seek(0, os.SEEK_END) | 256 cache_f.seek(0, os.SEEK_END) |
255 LOGGER.info('Loading %d unresolved addresses.' % | 257 LOGGER.info('Loading %d unresolved addresses.' % |
256 len(unresolved_addresses)) | 258 len(unresolved_addresses)) |
257 symbol_list = symbol_finder.find(unresolved_addresses) | 259 symbol_dict = symbol_finder.find(unresolved_addresses) |
258 | 260 |
259 for address, symbol in zip(unresolved_addresses, symbol_list): | 261 for address, symbol in symbol_dict.iteritems(): |
260 stripped_symbol = symbol.strip() or '??' | 262 stripped_symbol = symbol.strip() or '?' |
261 self._symbol_mapping_caches[address_type][address] = stripped_symbol | 263 self._symbol_mapping_caches[symbol_type][address] = stripped_symbol |
262 cache_f.write('%x %s\n' % (address, stripped_symbol)) | 264 cache_f.write('%x %s\n' % (address, stripped_symbol)) |
263 | 265 |
264 def lookup(self, address_type, address): | 266 def lookup(self, symbol_type, address): |
265 """Looks up a symbol for a given |address|. | 267 """Looks up a symbol for a given |address|. |
266 | 268 |
267 Args: | 269 Args: |
268 address_type: A type of addresses to lookup. | 270 symbol_type: A type of symbols to update. It should be one of |
269 It should be one of FUNCTION_ADDRESS or TYPEINFO_ADDRESS. | 271 FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS and TYPEINFO_SYMBOLS. |
270 address: An integer that represents an address. | 272 address: An integer that represents an address. |
271 | 273 |
272 Returns: | 274 Returns: |
273 A string that represents a symbol. | 275 A string that represents a symbol. |
274 """ | 276 """ |
275 return self._symbol_mapping_caches[address_type].get(address) | 277 return self._symbol_mapping_caches[symbol_type].get(address) |
276 | 278 |
277 def _load(self, cache_f, address_type): | 279 def _load(self, cache_f, symbol_type): |
278 try: | 280 try: |
279 for line in cache_f: | 281 for line in cache_f: |
280 items = line.rstrip().split(None, 1) | 282 items = line.rstrip().split(None, 1) |
281 if len(items) == 1: | 283 if len(items) == 1: |
282 items.append('??') | 284 items.append('??') |
283 self._symbol_mapping_caches[address_type][int(items[0], 16)] = items[1] | 285 self._symbol_mapping_caches[symbol_type][int(items[0], 16)] = items[1] |
284 LOGGER.info('Loaded %d entries from symbol cache.' % | 286 LOGGER.info('Loaded %d entries from symbol cache.' % |
285 len(self._symbol_mapping_caches[address_type])) | 287 len(self._symbol_mapping_caches[symbol_type])) |
286 except IOError as e: | 288 except IOError as e: |
287 LOGGER.info('The symbol cache file is invalid: %s' % e) | 289 LOGGER.info('The symbol cache file is invalid: %s' % e) |
288 | 290 |
289 | 291 |
290 class Rule(object): | 292 class Rule(object): |
291 """Represents one matching rule in a policy file.""" | 293 """Represents one matching rule in a policy file.""" |
292 | 294 |
293 def __init__(self, name, mmap, stacktrace_pattern, typeinfo_pattern=None): | 295 def __init__(self, |
296 name, | |
297 mmap, | |
298 stackfunction_pattern=None, | |
299 stacksourcefile_pattern=None, | |
300 typeinfo_pattern=None): | |
294 self._name = name | 301 self._name = name |
295 self._mmap = mmap | 302 self._mmap = mmap |
296 self._stacktrace_pattern = re.compile(stacktrace_pattern + r'\Z') | 303 |
304 self._stackfunction_pattern = None | |
305 if stackfunction_pattern: | |
306 self._stackfunction_pattern = re.compile( | |
307 stackfunction_pattern + r'\Z') | |
308 | |
309 self._stacksourcefile_pattern = None | |
310 if stacksourcefile_pattern: | |
311 self._stacksourcefile_pattern = re.compile( | |
312 stacksourcefile_pattern + r'\Z') | |
313 | |
314 self._typeinfo_pattern = None | |
297 if typeinfo_pattern: | 315 if typeinfo_pattern: |
298 self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z') | 316 self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z') |
299 else: | |
300 self._typeinfo_pattern = None | |
301 | 317 |
302 @property | 318 @property |
303 def name(self): | 319 def name(self): |
304 return self._name | 320 return self._name |
305 | 321 |
306 @property | 322 @property |
307 def mmap(self): | 323 def mmap(self): |
308 return self._mmap | 324 return self._mmap |
309 | 325 |
310 @property | 326 @property |
311 def stacktrace_pattern(self): | 327 def stackfunction_pattern(self): |
312 return self._stacktrace_pattern | 328 return self._stackfunction_pattern |
329 | |
330 @property | |
331 def stacksourcefile_pattern(self): | |
332 return self._stacksourcefile_pattern | |
313 | 333 |
314 @property | 334 @property |
315 def typeinfo_pattern(self): | 335 def typeinfo_pattern(self): |
316 return self._typeinfo_pattern | 336 return self._typeinfo_pattern |
317 | 337 |
318 | 338 |
319 class Policy(object): | 339 class Policy(object): |
320 """Represents a policy, a content of a policy file.""" | 340 """Represents a policy, a content of a policy file.""" |
321 | 341 |
322 def __init__(self, rules, version, components): | 342 def __init__(self, rules, version, components): |
(...skipping 20 matching lines...) Expand all Loading... | |
343 bucket: A Bucket object to be searched for. | 363 bucket: A Bucket object to be searched for. |
344 | 364 |
345 Returns: | 365 Returns: |
346 A string representing a component name. | 366 A string representing a component name. |
347 """ | 367 """ |
348 if not bucket: | 368 if not bucket: |
349 return 'no-bucket' | 369 return 'no-bucket' |
350 if bucket.component_cache: | 370 if bucket.component_cache: |
351 return bucket.component_cache | 371 return bucket.component_cache |
352 | 372 |
353 stacktrace = bucket.symbolized_joined_stacktrace | 373 stackfunction = bucket.symbolized_joined_stackfunction |
374 stacksourcefile = bucket.symbolized_joined_stacksourcefile | |
354 typeinfo = bucket.symbolized_typeinfo | 375 typeinfo = bucket.symbolized_typeinfo |
355 if typeinfo.startswith('0x'): | 376 if typeinfo.startswith('0x'): |
356 typeinfo = bucket.typeinfo_name | 377 typeinfo = bucket.typeinfo_name |
357 | 378 |
358 for rule in self._rules: | 379 for rule in self._rules: |
359 if (bucket.mmap == rule.mmap and | 380 if (bucket.mmap == rule.mmap and |
360 rule.stacktrace_pattern.match(stacktrace) and | 381 (not rule.stackfunction_pattern or |
382 rule.stackfunction_pattern.match(stackfunction)) and | |
383 (not rule.stacksourcefile_pattern or | |
384 rule.stacksourcefile_pattern.match(stacksourcefile)) and | |
361 (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))): | 385 (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))): |
362 bucket.component_cache = rule.name | 386 bucket.component_cache = rule.name |
363 return rule.name | 387 return rule.name |
364 | 388 |
365 assert False | 389 assert False |
366 | 390 |
367 @staticmethod | 391 @staticmethod |
368 def load(filename, filetype): | 392 def load(filename, filetype): |
369 """Loads a policy file of |filename| in a |format|. | 393 """Loads a policy file of |filename| in a |format|. |
370 | 394 |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
407 Args: | 431 Args: |
408 policy_f: A File/IO object to read. | 432 policy_f: A File/IO object to read. |
409 | 433 |
410 Returns: | 434 Returns: |
411 A loaded policy object. | 435 A loaded policy object. |
412 """ | 436 """ |
413 policy = json.load(policy_f) | 437 policy = json.load(policy_f) |
414 | 438 |
415 rules = [] | 439 rules = [] |
416 for rule in policy['rules']: | 440 for rule in policy['rules']: |
441 stackfunction = None | |
M-A Ruel
2013/04/09 03:25:17
stackfunction = rule.get('stackfunction') or rule.
Dai Mikurube (NOT FULLTIME)
2013/04/09 04:31:52
Done.
| |
442 if 'stackfunction' in rule: | |
443 stackfunction = rule['stackfunction'] | |
444 elif 'stacktrace' in rule: | |
445 stackfunction = rule['stacktrace'] | |
446 | |
447 stacksourcefile = None | |
M-A Ruel
2013/04/09 03:25:17
same
Dai Mikurube (NOT FULLTIME)
2013/04/09 04:31:52
Done.
| |
448 if 'stacksourcefile' in rule: | |
449 stacksourcefile = rule['stacksourcefile'] | |
450 | |
417 rules.append(Rule( | 451 rules.append(Rule( |
418 rule['name'], | 452 rule['name'], |
419 rule['allocator'] == 'mmap', | 453 rule['allocator'] == 'mmap', |
420 rule['stacktrace'], | 454 stackfunction, |
455 stacksourcefile, | |
421 rule['typeinfo'] if 'typeinfo' in rule else None)) | 456 rule['typeinfo'] if 'typeinfo' in rule else None)) |
422 return Policy(rules, policy['version'], policy['components']) | 457 return Policy(rules, policy['version'], policy['components']) |
423 | 458 |
424 | 459 |
425 class PolicySet(object): | 460 class PolicySet(object): |
426 """Represents a set of policies.""" | 461 """Represents a set of policies.""" |
427 | 462 |
428 def __init__(self, policy_directory): | 463 def __init__(self, policy_directory): |
429 self._policy_directory = policy_directory | 464 self._policy_directory = policy_directory |
430 | 465 |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
486 | 521 |
487 class Bucket(object): | 522 class Bucket(object): |
488 """Represents a bucket, which is a unit of memory block classification.""" | 523 """Represents a bucket, which is a unit of memory block classification.""" |
489 | 524 |
490 def __init__(self, stacktrace, mmap, typeinfo, typeinfo_name): | 525 def __init__(self, stacktrace, mmap, typeinfo, typeinfo_name): |
491 self._stacktrace = stacktrace | 526 self._stacktrace = stacktrace |
492 self._mmap = mmap | 527 self._mmap = mmap |
493 self._typeinfo = typeinfo | 528 self._typeinfo = typeinfo |
494 self._typeinfo_name = typeinfo_name | 529 self._typeinfo_name = typeinfo_name |
495 | 530 |
496 self._symbolized_stacktrace = stacktrace | 531 self._symbolized_stackfunction = stacktrace |
497 self._symbolized_joined_stacktrace = '' | 532 self._symbolized_joined_stackfunction = '' |
533 self._symbolized_stacksourcefile = stacktrace | |
534 self._symbolized_joined_stacksourcefile = '' | |
498 self._symbolized_typeinfo = typeinfo_name | 535 self._symbolized_typeinfo = typeinfo_name |
499 | 536 |
500 self.component_cache = '' | 537 self.component_cache = '' |
501 | 538 |
502 def symbolize(self, symbol_mapping_cache): | 539 def symbolize(self, symbol_mapping_cache): |
503 """Makes a symbolized stacktrace and typeinfo with |symbol_mapping_cache|. | 540 """Makes a symbolized stacktrace and typeinfo with |symbol_mapping_cache|. |
504 | 541 |
505 Args: | 542 Args: |
506 symbol_mapping_cache: A SymbolMappingCache object. | 543 symbol_mapping_cache: A SymbolMappingCache object. |
507 """ | 544 """ |
508 # TODO(dmikurube): Fill explicitly with numbers if symbol not found. | 545 # TODO(dmikurube): Fill explicitly with numbers if symbol not found. |
509 self._symbolized_stacktrace = [ | 546 self._symbolized_stackfunction = [ |
510 symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address) | 547 symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address) |
511 for address in self._stacktrace] | 548 for address in self._stacktrace] |
512 self._symbolized_joined_stacktrace = ' '.join(self._symbolized_stacktrace) | 549 self._symbolized_joined_stackfunction = ' '.join( |
550 self._symbolized_stackfunction) | |
551 self._symbolized_stacksourcefile = [ | |
552 symbol_mapping_cache.lookup(SOURCEFILE_SYMBOLS, address) | |
553 for address in self._stacktrace] | |
554 self._symbolized_joined_stacksourcefile = ' '.join( | |
555 self._symbolized_stacksourcefile) | |
513 if not self._typeinfo: | 556 if not self._typeinfo: |
514 self._symbolized_typeinfo = 'no typeinfo' | 557 self._symbolized_typeinfo = 'no typeinfo' |
515 else: | 558 else: |
516 self._symbolized_typeinfo = symbol_mapping_cache.lookup( | 559 self._symbolized_typeinfo = symbol_mapping_cache.lookup( |
517 TYPEINFO_ADDRESS, self._typeinfo) | 560 TYPEINFO_SYMBOLS, self._typeinfo) |
518 if not self._symbolized_typeinfo: | 561 if not self._symbolized_typeinfo: |
519 self._symbolized_typeinfo = 'no typeinfo' | 562 self._symbolized_typeinfo = 'no typeinfo' |
520 | 563 |
521 def clear_component_cache(self): | 564 def clear_component_cache(self): |
522 self.component_cache = '' | 565 self.component_cache = '' |
523 | 566 |
524 @property | 567 @property |
525 def stacktrace(self): | 568 def stacktrace(self): |
526 return self._stacktrace | 569 return self._stacktrace |
527 | 570 |
528 @property | 571 @property |
529 def mmap(self): | 572 def mmap(self): |
530 return self._mmap | 573 return self._mmap |
531 | 574 |
532 @property | 575 @property |
533 def typeinfo(self): | 576 def typeinfo(self): |
534 return self._typeinfo | 577 return self._typeinfo |
535 | 578 |
536 @property | 579 @property |
537 def typeinfo_name(self): | 580 def typeinfo_name(self): |
538 return self._typeinfo_name | 581 return self._typeinfo_name |
539 | 582 |
540 @property | 583 @property |
541 def symbolized_stacktrace(self): | 584 def symbolized_stackfunction(self): |
542 return self._symbolized_stacktrace | 585 return self._symbolized_stackfunction |
543 | 586 |
544 @property | 587 @property |
545 def symbolized_joined_stacktrace(self): | 588 def symbolized_joined_stackfunction(self): |
546 return self._symbolized_joined_stacktrace | 589 return self._symbolized_joined_stackfunction |
590 | |
591 @property | |
592 def symbolized_stacksourcefile(self): | |
593 return self._symbolized_stacksourcefile | |
594 | |
595 @property | |
596 def symbolized_joined_stacksourcefile(self): | |
597 return self._symbolized_joined_stacksourcefile | |
547 | 598 |
548 @property | 599 @property |
549 def symbolized_typeinfo(self): | 600 def symbolized_typeinfo(self): |
550 return self._symbolized_typeinfo | 601 return self._symbolized_typeinfo |
551 | 602 |
552 | 603 |
553 class BucketSet(object): | 604 class BucketSet(object): |
554 """Represents a set of bucket.""" | 605 """Represents a set of bucket.""" |
555 def __init__(self): | 606 def __init__(self): |
556 self._buckets = {} | 607 self._buckets = {} |
557 self._addresses = { | 608 self._code_addresses = set() |
558 FUNCTION_ADDRESS: set(), | 609 self._typeinfo_addresses = set() |
559 TYPEINFO_ADDRESS: set(), | |
560 } | |
561 | 610 |
562 def load(self, prefix): | 611 def load(self, prefix): |
563 """Loads all related bucket files. | 612 """Loads all related bucket files. |
564 | 613 |
565 Args: | 614 Args: |
566 prefix: A prefix string for bucket file names. | 615 prefix: A prefix string for bucket file names. |
567 """ | 616 """ |
568 LOGGER.info('Loading bucket files.') | 617 LOGGER.info('Loading bucket files.') |
569 | 618 |
570 n = 0 | 619 n = 0 |
(...skipping 13 matching lines...) Expand all Loading... | |
584 for line in bucket_f: | 633 for line in bucket_f: |
585 words = line.split() | 634 words = line.split() |
586 typeinfo = None | 635 typeinfo = None |
587 typeinfo_name = '' | 636 typeinfo_name = '' |
588 stacktrace_begin = 2 | 637 stacktrace_begin = 2 |
589 for index, word in enumerate(words): | 638 for index, word in enumerate(words): |
590 if index < 2: | 639 if index < 2: |
591 continue | 640 continue |
592 if word[0] == 't': | 641 if word[0] == 't': |
593 typeinfo = int(word[1:], 16) | 642 typeinfo = int(word[1:], 16) |
594 self._addresses[TYPEINFO_ADDRESS].add(typeinfo) | 643 self._typeinfo_addresses.add(typeinfo) |
595 elif word[0] == 'n': | 644 elif word[0] == 'n': |
596 typeinfo_name = word[1:] | 645 typeinfo_name = word[1:] |
597 else: | 646 else: |
598 stacktrace_begin = index | 647 stacktrace_begin = index |
599 break | 648 break |
600 stacktrace = [int(address, 16) for address in words[stacktrace_begin:]] | 649 stacktrace = [int(address, 16) for address in words[stacktrace_begin:]] |
601 for frame in stacktrace: | 650 for frame in stacktrace: |
602 self._addresses[FUNCTION_ADDRESS].add(frame) | 651 self._code_addresses.add(frame) |
603 self._buckets[int(words[0])] = Bucket( | 652 self._buckets[int(words[0])] = Bucket( |
604 stacktrace, words[1] == 'mmap', typeinfo, typeinfo_name) | 653 stacktrace, words[1] == 'mmap', typeinfo, typeinfo_name) |
605 | 654 |
606 def __iter__(self): | 655 def __iter__(self): |
607 for bucket_id, bucket_content in self._buckets.iteritems(): | 656 for bucket_id, bucket_content in self._buckets.iteritems(): |
608 yield bucket_id, bucket_content | 657 yield bucket_id, bucket_content |
609 | 658 |
610 def __getitem__(self, bucket_id): | 659 def __getitem__(self, bucket_id): |
611 return self._buckets[bucket_id] | 660 return self._buckets[bucket_id] |
612 | 661 |
613 def get(self, bucket_id): | 662 def get(self, bucket_id): |
614 return self._buckets.get(bucket_id) | 663 return self._buckets.get(bucket_id) |
615 | 664 |
616 def symbolize(self, symbol_mapping_cache): | 665 def symbolize(self, symbol_mapping_cache): |
617 for bucket_content in self._buckets.itervalues(): | 666 for bucket_content in self._buckets.itervalues(): |
618 bucket_content.symbolize(symbol_mapping_cache) | 667 bucket_content.symbolize(symbol_mapping_cache) |
619 | 668 |
620 def clear_component_cache(self): | 669 def clear_component_cache(self): |
621 for bucket_content in self._buckets.itervalues(): | 670 for bucket_content in self._buckets.itervalues(): |
622 bucket_content.clear_component_cache() | 671 bucket_content.clear_component_cache() |
623 | 672 |
624 def iter_addresses(self, address_type): | 673 def iter_addresses(self, symbol_type): |
625 for function in self._addresses[address_type]: | 674 if symbol_type in [FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS]: |
626 yield function | 675 for function in self._code_addresses: |
676 yield function | |
677 else: | |
678 for function in self._typeinfo_addresses: | |
679 yield function | |
627 | 680 |
628 | 681 |
629 class Dump(object): | 682 class Dump(object): |
630 """Represents a heap profile dump.""" | 683 """Represents a heap profile dump.""" |
631 | 684 |
632 def __init__(self, path, time): | 685 def __init__(self, path, time): |
633 self._path = path | 686 self._path = path |
634 self._time = time | 687 self._time = time |
635 self._stacktrace_lines = [] | 688 self._stacktrace_lines = [] |
636 self._global_stats = {} # used only in apply_policy | 689 self._global_stats = {} # used only in apply_policy |
(...skipping 196 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
833 prefix = Command._find_prefix(dump_path) | 886 prefix = Command._find_prefix(dump_path) |
834 symbol_data_sources = SymbolDataSources(prefix) | 887 symbol_data_sources = SymbolDataSources(prefix) |
835 symbol_data_sources.prepare() | 888 symbol_data_sources.prepare() |
836 bucket_set = BucketSet() | 889 bucket_set = BucketSet() |
837 bucket_set.load(prefix) | 890 bucket_set.load(prefix) |
838 if multiple: | 891 if multiple: |
839 dump_list = DumpList.load(Command._find_all_dumps(dump_path)) | 892 dump_list = DumpList.load(Command._find_all_dumps(dump_path)) |
840 else: | 893 else: |
841 dump = Dump.load(dump_path) | 894 dump = Dump.load(dump_path) |
842 symbol_mapping_cache = SymbolMappingCache() | 895 symbol_mapping_cache = SymbolMappingCache() |
843 with open(prefix + '.funcsym', 'a+') as cache_f: | 896 with open(prefix + '.cache.function', 'a+') as cache_f: |
844 symbol_mapping_cache.update( | 897 symbol_mapping_cache.update( |
845 FUNCTION_ADDRESS, bucket_set, | 898 FUNCTION_SYMBOLS, bucket_set, |
846 SymbolFinder(FUNCTION_ADDRESS, symbol_data_sources), cache_f) | 899 SymbolFinder(FUNCTION_SYMBOLS, symbol_data_sources), cache_f) |
847 with open(prefix + '.typesym', 'a+') as cache_f: | 900 with open(prefix + '.cache.typeinfo', 'a+') as cache_f: |
848 symbol_mapping_cache.update( | 901 symbol_mapping_cache.update( |
849 TYPEINFO_ADDRESS, bucket_set, | 902 TYPEINFO_SYMBOLS, bucket_set, |
850 SymbolFinder(TYPEINFO_ADDRESS, symbol_data_sources), cache_f) | 903 SymbolFinder(TYPEINFO_SYMBOLS, symbol_data_sources), cache_f) |
904 with open(prefix + '.cache.sourcefile', 'a+') as cache_f: | |
905 symbol_mapping_cache.update( | |
906 SOURCEFILE_SYMBOLS, bucket_set, | |
907 SymbolFinder(SOURCEFILE_SYMBOLS, symbol_data_sources), cache_f) | |
851 bucket_set.symbolize(symbol_mapping_cache) | 908 bucket_set.symbolize(symbol_mapping_cache) |
852 if multiple: | 909 if multiple: |
853 return (bucket_set, dump_list) | 910 return (bucket_set, dump_list) |
854 else: | 911 else: |
855 return (bucket_set, dump) | 912 return (bucket_set, dump) |
856 | 913 |
857 @staticmethod | 914 @staticmethod |
858 def _find_prefix(path): | 915 def _find_prefix(path): |
859 return re.sub('\.[0-9][0-9][0-9][0-9]\.heap', '', path) | 916 return re.sub('\.[0-9][0-9][0-9][0-9]\.heap', '', path) |
860 | 917 |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
929 bucket_set: A BucketSet object. | 986 bucket_set: A BucketSet object. |
930 out: A file object to output. | 987 out: A file object to output. |
931 """ | 988 """ |
932 for line in dump.iter_stacktrace: | 989 for line in dump.iter_stacktrace: |
933 words = line.split() | 990 words = line.split() |
934 bucket = bucket_set.get(int(words[BUCKET_ID])) | 991 bucket = bucket_set.get(int(words[BUCKET_ID])) |
935 if not bucket: | 992 if not bucket: |
936 continue | 993 continue |
937 for i in range(0, BUCKET_ID - 1): | 994 for i in range(0, BUCKET_ID - 1): |
938 out.write(words[i] + ' ') | 995 out.write(words[i] + ' ') |
939 for frame in bucket.symbolized_stacktrace: | 996 for frame in bucket.symbolized_stackfunction: |
940 out.write(frame + ' ') | 997 out.write(frame + ' ') |
941 out.write('\n') | 998 out.write('\n') |
942 | 999 |
943 | 1000 |
944 class PolicyCommands(Command): | 1001 class PolicyCommands(Command): |
945 def __init__(self, command): | 1002 def __init__(self, command): |
946 super(PolicyCommands, self).__init__( | 1003 super(PolicyCommands, self).__init__( |
947 'Usage: %%prog %s [-p POLICY] <first-dump>' % command) | 1004 'Usage: %%prog %s [-p POLICY] <first-dump>' % command) |
948 self._parser.add_option('-p', '--policy', type='string', dest='policy', | 1005 self._parser.add_option('-p', '--policy', type='string', dest='policy', |
949 help='profile with POLICY', metavar='POLICY') | 1006 help='profile with POLICY', metavar='POLICY') |
(...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1114 json_base['policies'][label] = { | 1171 json_base['policies'][label] = { |
1115 'legends': policy_set[label].components, | 1172 'legends': policy_set[label].components, |
1116 'snapshots': [], | 1173 'snapshots': [], |
1117 } | 1174 } |
1118 | 1175 |
1119 LOGGER.info('Applying a policy %s to...' % label) | 1176 LOGGER.info('Applying a policy %s to...' % label) |
1120 for dump in dumps: | 1177 for dump in dumps: |
1121 component_sizes = PolicyCommands._apply_policy( | 1178 component_sizes = PolicyCommands._apply_policy( |
1122 dump, policy_set[label], bucket_set, dumps[0].time) | 1179 dump, policy_set[label], bucket_set, dumps[0].time) |
1123 component_sizes['dump_path'] = dump.path | 1180 component_sizes['dump_path'] = dump.path |
1124 component_sizes['dump_time'] = datetime.fromtimestamp( | 1181 component_sizes['dump_time'] = datetime.datetime.fromtimestamp( |
1125 dump.time).strftime('%Y-%m-%d %H:%M:%S') | 1182 dump.time).strftime('%Y-%m-%d %H:%M:%S') |
1126 json_base['policies'][label]['snapshots'].append(component_sizes) | 1183 json_base['policies'][label]['snapshots'].append(component_sizes) |
1127 | 1184 |
1128 bucket_set.clear_component_cache() | 1185 bucket_set.clear_component_cache() |
1129 | 1186 |
1130 json.dump(json_base, out, indent=2, sort_keys=True) | 1187 json.dump(json_base, out, indent=2, sort_keys=True) |
1131 | 1188 |
1132 return 0 | 1189 return 0 |
1133 | 1190 |
1134 | 1191 |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1190 out: An IO object to output. | 1247 out: An IO object to output. |
1191 """ | 1248 """ |
1192 sizes = {} | 1249 sizes = {} |
1193 | 1250 |
1194 ExpandCommand._accumulate( | 1251 ExpandCommand._accumulate( |
1195 dump, policy, bucket_set, component_name, depth, sizes) | 1252 dump, policy, bucket_set, component_name, depth, sizes) |
1196 | 1253 |
1197 sorted_sizes_list = sorted( | 1254 sorted_sizes_list = sorted( |
1198 sizes.iteritems(), key=(lambda x: x[1]), reverse=True) | 1255 sizes.iteritems(), key=(lambda x: x[1]), reverse=True) |
1199 total = 0 | 1256 total = 0 |
1257 # TODO(dmikurube): Better formatting. | |
1200 for size_pair in sorted_sizes_list: | 1258 for size_pair in sorted_sizes_list: |
1201 out.write('%10d %s\n' % (size_pair[1], size_pair[0])) | 1259 out.write('%10d %s\n' % (size_pair[1], size_pair[0])) |
1202 total += size_pair[1] | 1260 total += size_pair[1] |
1203 LOGGER.info('total: %d\n' % total) | 1261 LOGGER.info('total: %d\n' % total) |
1204 | 1262 |
1205 @staticmethod | 1263 @staticmethod |
1206 def _accumulate(dump, policy, bucket_set, component_name, depth, sizes): | 1264 def _accumulate(dump, policy, bucket_set, component_name, depth, sizes): |
1207 for line in dump.iter_stacktrace: | 1265 for line in dump.iter_stacktrace: |
1208 words = line.split() | 1266 words = line.split() |
1209 bucket = bucket_set.get(int(words[BUCKET_ID])) | 1267 bucket = bucket_set.get(int(words[BUCKET_ID])) |
1210 component_match = policy.find(bucket) | 1268 component_match = policy.find(bucket) |
1211 if component_match == component_name: | 1269 if component_match == component_name: |
1212 stacktrace_sequence = '' | 1270 stacktrace_sequence = '' |
1213 if bucket.typeinfo: | 1271 if bucket.typeinfo: |
1214 stacktrace_sequence += '(type=%s)' % bucket.symbolized_typeinfo | 1272 stacktrace_sequence += '(type=%s)' % bucket.symbolized_typeinfo |
1215 stacktrace_sequence += ' (type.name=%s) ' % bucket.typeinfo_name | 1273 stacktrace_sequence += ' (type.name=%s) ' % bucket.typeinfo_name |
1216 for stack in bucket.symbolized_stacktrace[ | 1274 for function, sourcefile in zip( |
1217 0 : min(len(bucket.symbolized_stacktrace), 1 + depth)]: | 1275 bucket.symbolized_stackfunction[ |
1218 stacktrace_sequence += stack + ' ' | 1276 0 : min(len(bucket.symbolized_stackfunction), 1 + depth)], |
1277 bucket.symbolized_stacksourcefile[ | |
1278 0 : min(len(bucket.symbolized_stacksourcefile), 1 + depth)]): | |
1279 stacktrace_sequence += '%s(@%s) ' % (function, sourcefile) | |
1219 if not stacktrace_sequence in sizes: | 1280 if not stacktrace_sequence in sizes: |
1220 sizes[stacktrace_sequence] = 0 | 1281 sizes[stacktrace_sequence] = 0 |
1221 sizes[stacktrace_sequence] += int(words[COMMITTED]) | 1282 sizes[stacktrace_sequence] += int(words[COMMITTED]) |
1222 | 1283 |
1223 | 1284 |
1224 class PProfCommand(Command): | 1285 class PProfCommand(Command): |
1225 def __init__(self): | 1286 def __init__(self): |
1226 super(PProfCommand, self).__init__( | 1287 super(PProfCommand, self).__init__( |
1227 'Usage: %prog pprof [-c COMPONENT] <dump> <policy>') | 1288 'Usage: %prog pprof [-c COMPONENT] <dump> <policy>') |
1228 self._parser.add_option('-c', '--component', type='string', | 1289 self._parser.add_option('-c', '--component', type='string', |
(...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1439 errorcode = COMMANDS[action]().do(sys.argv) | 1500 errorcode = COMMANDS[action]().do(sys.argv) |
1440 except ParsingException, e: | 1501 except ParsingException, e: |
1441 errorcode = 1 | 1502 errorcode = 1 |
1442 sys.stderr.write('Exit by parsing error: %s\n' % e) | 1503 sys.stderr.write('Exit by parsing error: %s\n' % e) |
1443 | 1504 |
1444 return errorcode | 1505 return errorcode |
1445 | 1506 |
1446 | 1507 |
1447 if __name__ == '__main__': | 1508 if __name__ == '__main__': |
1448 sys.exit(main()) | 1509 sys.exit(main()) |
OLD | NEW |