Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """The deep heap profiler script for Chrome.""" | 5 """The deep heap profiler script for Chrome.""" |
| 6 | 6 |
| 7 from datetime import datetime | 7 import datetime |
| 8 import json | 8 import json |
| 9 import logging | 9 import logging |
| 10 import optparse | 10 import optparse |
| 11 import os | 11 import os |
| 12 import re | 12 import re |
| 13 import subprocess | 13 import subprocess |
| 14 import sys | 14 import sys |
| 15 import tempfile | 15 import tempfile |
| 16 import zipfile | 16 import zipfile |
| 17 | 17 |
| 18 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) | 18 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) |
| 19 FIND_RUNTIME_SYMBOLS_PATH = os.path.join( | 19 FIND_RUNTIME_SYMBOLS_PATH = os.path.join( |
| 20 BASE_PATH, os.pardir, 'find_runtime_symbols') | 20 BASE_PATH, os.pardir, 'find_runtime_symbols') |
| 21 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH) | 21 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH) |
| 22 | 22 |
| 23 from find_runtime_symbols import find_runtime_symbols_list | 23 import find_runtime_symbols |
| 24 from find_runtime_symbols import find_runtime_typeinfo_symbols_list | 24 import prepare_symbol_info |
| 25 from find_runtime_symbols import RuntimeSymbolsInProcess | 25 |
| 26 from prepare_symbol_info import prepare_symbol_info | 26 from find_runtime_symbols import FUNCTION_SYMBOLS |
| 27 from find_runtime_symbols import SOURCEFILE_SYMBOLS | |
| 28 from find_runtime_symbols import TYPEINFO_SYMBOLS | |
| 27 | 29 |
| 28 BUCKET_ID = 5 | 30 BUCKET_ID = 5 |
| 29 VIRTUAL = 0 | 31 VIRTUAL = 0 |
| 30 COMMITTED = 1 | 32 COMMITTED = 1 |
| 31 ALLOC_COUNT = 2 | 33 ALLOC_COUNT = 2 |
| 32 FREE_COUNT = 3 | 34 FREE_COUNT = 3 |
| 33 NULL_REGEX = re.compile('') | 35 NULL_REGEX = re.compile('') |
| 34 | 36 |
| 35 LOGGER = logging.getLogger('dmprof') | 37 LOGGER = logging.getLogger('dmprof') |
| 36 POLICIES_JSON_PATH = os.path.join(BASE_PATH, 'policies.json') | 38 POLICIES_JSON_PATH = os.path.join(BASE_PATH, 'policies.json') |
| 37 FUNCTION_ADDRESS = 'function' | |
| 38 TYPEINFO_ADDRESS = 'typeinfo' | |
| 39 | 39 |
| 40 | 40 |
| 41 # Heap Profile Dump versions | 41 # Heap Profile Dump versions |
| 42 | 42 |
| 43 # DUMP_DEEP_[1-4] are obsolete. | 43 # DUMP_DEEP_[1-4] are obsolete. |
| 44 # DUMP_DEEP_2+ distinct mmap regions and malloc chunks. | 44 # DUMP_DEEP_2+ distinct mmap regions and malloc chunks. |
| 45 # DUMP_DEEP_3+ don't include allocation functions in their stack dumps. | 45 # DUMP_DEEP_3+ don't include allocation functions in their stack dumps. |
| 46 # DUMP_DEEP_4+ support comments with '#' and global stats "nonprofiled-*". | 46 # DUMP_DEEP_4+ support comments with '#' and global stats "nonprofiled-*". |
| 47 # DUMP_DEEP_[1-2] should be processed by POLICY_DEEP_1. | 47 # DUMP_DEEP_[1-2] should be processed by POLICY_DEEP_1. |
| 48 # DUMP_DEEP_[3-4] should be processed by POLICY_DEEP_2 or POLICY_DEEP_3. | 48 # DUMP_DEEP_[3-4] should be processed by POLICY_DEEP_2 or POLICY_DEEP_3. |
| (...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 148 def prepare(self): | 148 def prepare(self): |
| 149 """Prepares symbol data sources by extracting mapping from a binary. | 149 """Prepares symbol data sources by extracting mapping from a binary. |
| 150 | 150 |
| 151 The prepared symbol data sources are stored in a directory. The directory | 151 The prepared symbol data sources are stored in a directory. The directory |
| 152 name is stored in |self._prepared_symbol_data_sources_path|. | 152 name is stored in |self._prepared_symbol_data_sources_path|. |
| 153 | 153 |
| 154 Returns: | 154 Returns: |
| 155 True if succeeded. | 155 True if succeeded. |
| 156 """ | 156 """ |
| 157 LOGGER.info('Preparing symbol mapping...') | 157 LOGGER.info('Preparing symbol mapping...') |
| 158 self._prepared_symbol_data_sources_path, used_tempdir = prepare_symbol_info( | 158 self._prepared_symbol_data_sources_path, used_tempdir = ( |
| 159 self._prefix + '.maps', self._prefix + '.symmap', True) | 159 prepare_symbol_info.prepare_symbol_info( |
| 160 self._prefix + '.maps', | |
| 161 output_dir_path=self._prefix + '.symmap', | |
| 162 use_tempdir=True, | |
| 163 use_source_file_name=True)) | |
| 160 if self._prepared_symbol_data_sources_path: | 164 if self._prepared_symbol_data_sources_path: |
| 161 LOGGER.info(' Prepared symbol mapping.') | 165 LOGGER.info(' Prepared symbol mapping.') |
| 162 if used_tempdir: | 166 if used_tempdir: |
| 163 LOGGER.warn(' Using a temporary directory for symbol mapping.') | 167 LOGGER.warn(' Using a temporary directory for symbol mapping.') |
| 164 LOGGER.warn(' Delete it by yourself.') | 168 LOGGER.warn(' Delete it by yourself.') |
| 165 LOGGER.warn(' Or, move the directory by yourself to use it later.') | 169 LOGGER.warn(' Or, move the directory by yourself to use it later.') |
| 166 return True | 170 return True |
| 167 else: | 171 else: |
| 168 LOGGER.warn(' Failed to prepare symbol mapping.') | 172 LOGGER.warn(' Failed to prepare symbol mapping.') |
| 169 return False | 173 return False |
| 170 | 174 |
| 171 def get(self): | 175 def get(self): |
| 172 """Returns the prepared symbol data sources. | 176 """Returns the prepared symbol data sources. |
| 173 | 177 |
| 174 Returns: | 178 Returns: |
| 175 The prepared symbol data sources. None if failed. | 179 The prepared symbol data sources. None if failed. |
| 176 """ | 180 """ |
| 177 if not self._prepared_symbol_data_sources_path and not self.prepare(): | 181 if not self._prepared_symbol_data_sources_path and not self.prepare(): |
| 178 return None | 182 return None |
| 179 if not self._loaded_symbol_data_sources: | 183 if not self._loaded_symbol_data_sources: |
| 180 LOGGER.info('Loading symbol mapping...') | 184 LOGGER.info('Loading symbol mapping...') |
| 181 self._loaded_symbol_data_sources = RuntimeSymbolsInProcess.load( | 185 self._loaded_symbol_data_sources = ( |
| 182 self._prepared_symbol_data_sources_path) | 186 find_runtime_symbols.RuntimeSymbolsInProcess.load( |
| 187 self._prepared_symbol_data_sources_path)) | |
| 183 return self._loaded_symbol_data_sources | 188 return self._loaded_symbol_data_sources |
| 184 | 189 |
| 185 def path(self): | 190 def path(self): |
| 186 """Returns the path of the prepared symbol data sources if possible.""" | 191 """Returns the path of the prepared symbol data sources if possible.""" |
| 187 if not self._prepared_symbol_data_sources_path and not self.prepare(): | 192 if not self._prepared_symbol_data_sources_path and not self.prepare(): |
| 188 return None | 193 return None |
| 189 return self._prepared_symbol_data_sources_path | 194 return self._prepared_symbol_data_sources_path |
| 190 | 195 |
| 191 | 196 |
| 192 class SymbolFinder(object): | 197 class SymbolFinder(object): |
| 193 """Finds corresponding symbols from addresses. | 198 """Finds corresponding symbols from addresses. |
| 194 | 199 |
| 195 This class does only 'find()' symbols from a specified |address_list|. | 200 This class does only 'find()' symbols from a specified |address_list|. |
| 196 It is introduced to make a finder mockable. | 201 It is introduced to make a finder mockable. |
| 197 """ | 202 """ |
| 198 _FIND_RUNTIME_SYMBOLS_FUNCTIONS = { | 203 def __init__(self, symbol_type, symbol_data_sources): |
| 199 FUNCTION_ADDRESS: find_runtime_symbols_list, | 204 self._symbol_type = symbol_type |
| 200 TYPEINFO_ADDRESS: find_runtime_typeinfo_symbols_list, | |
| 201 } | |
| 202 | |
| 203 def __init__(self, address_type, symbol_data_sources): | |
| 204 self._finder_function = self._FIND_RUNTIME_SYMBOLS_FUNCTIONS[address_type] | |
| 205 self._symbol_data_sources = symbol_data_sources | 205 self._symbol_data_sources = symbol_data_sources |
| 206 | 206 |
| 207 def find(self, address_list): | 207 def find(self, address_list): |
| 208 return self._finder_function(self._symbol_data_sources.get(), address_list) | 208 return find_runtime_symbols.find_runtime_symbols( |
| 209 self._symbol_type, self._symbol_data_sources.get(), address_list) | |
| 209 | 210 |
| 210 | 211 |
| 211 class SymbolMappingCache(object): | 212 class SymbolMappingCache(object): |
| 212 """Caches mapping from actually used addresses to symbols. | 213 """Caches mapping from actually used addresses to symbols. |
| 213 | 214 |
| 214 'update()' updates the cache from the original symbol data sources via | 215 'update()' updates the cache from the original symbol data sources via |
| 215 'SymbolFinder'. Symbols can be looked up by the method 'lookup()'. | 216 'SymbolFinder'. Symbols can be looked up by the method 'lookup()'. |
| 216 """ | 217 """ |
| 217 def __init__(self): | 218 def __init__(self): |
| 218 self._symbol_mapping_caches = { | 219 self._symbol_mapping_caches = { |
| 219 FUNCTION_ADDRESS: {}, | 220 FUNCTION_SYMBOLS: {}, |
| 220 TYPEINFO_ADDRESS: {}, | 221 SOURCEFILE_SYMBOLS: {}, |
| 222 TYPEINFO_SYMBOLS: {}, | |
| 221 } | 223 } |
| 222 | 224 |
| 223 def update(self, address_type, bucket_set, symbol_finder, cache_f): | 225 def update(self, symbol_type, bucket_set, symbol_finder, cache_f): |
| 224 """Updates symbol mapping cache on memory and in a symbol cache file. | 226 """Updates symbol mapping cache on memory and in a symbol cache file. |
| 225 | 227 |
| 226 It reads cached symbol mapping from a symbol cache file |cache_f| if it | 228 It reads cached symbol mapping from a symbol cache file |cache_f| if it |
| 227 exists. Unresolved addresses are then resolved and added to the cache | 229 exists. Unresolved addresses are then resolved and added to the cache |
| 228 both on memory and in the symbol cache file with using 'SymbolFinder'. | 230 both on memory and in the symbol cache file with using 'SymbolFinder'. |
| 229 | 231 |
| 230 A cache file is formatted as follows: | 232 A cache file is formatted as follows: |
| 231 <Address> <Symbol> | 233 <Address> <Symbol> |
| 232 <Address> <Symbol> | 234 <Address> <Symbol> |
| 233 <Address> <Symbol> | 235 <Address> <Symbol> |
| 234 ... | 236 ... |
| 235 | 237 |
| 236 Args: | 238 Args: |
| 237 address_type: A type of addresses to update. | 239 symbol_type: A type of symbols to update. It should be one of |
| 238 It should be one of FUNCTION_ADDRESS or TYPEINFO_ADDRESS. | 240 FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS and TYPEINFO_SYMBOLS. |
| 239 bucket_set: A BucketSet object. | 241 bucket_set: A BucketSet object. |
| 240 symbol_finder: A SymbolFinder object to find symbols. | 242 symbol_finder: A SymbolFinder object to find symbols. |
| 241 cache_f: A readable and writable IO object of the symbol cache file. | 243 cache_f: A readable and writable IO object of the symbol cache file. |
| 242 """ | 244 """ |
| 243 cache_f.seek(0, os.SEEK_SET) | 245 cache_f.seek(0, os.SEEK_SET) |
| 244 self._load(cache_f, address_type) | 246 self._load(cache_f, symbol_type) |
| 245 | 247 |
| 246 unresolved_addresses = sorted( | 248 unresolved_addresses = sorted( |
| 247 address for address in bucket_set.iter_addresses(address_type) | 249 address for address in bucket_set.iter_addresses(symbol_type) |
| 248 if address not in self._symbol_mapping_caches[address_type]) | 250 if address not in self._symbol_mapping_caches[symbol_type]) |
| 249 | 251 |
| 250 if not unresolved_addresses: | 252 if not unresolved_addresses: |
| 251 LOGGER.info('No need to resolve any more addresses.') | 253 LOGGER.info('No need to resolve any more addresses.') |
| 252 return | 254 return |
| 253 | 255 |
| 254 cache_f.seek(0, os.SEEK_END) | 256 cache_f.seek(0, os.SEEK_END) |
| 255 LOGGER.info('Loading %d unresolved addresses.' % | 257 LOGGER.info('Loading %d unresolved addresses.' % |
| 256 len(unresolved_addresses)) | 258 len(unresolved_addresses)) |
| 257 symbol_list = symbol_finder.find(unresolved_addresses) | 259 symbol_dict = symbol_finder.find(unresolved_addresses) |
| 258 | 260 |
| 259 for address, symbol in zip(unresolved_addresses, symbol_list): | 261 for address, symbol in symbol_dict.iteritems(): |
| 260 stripped_symbol = symbol.strip() or '??' | 262 stripped_symbol = symbol.strip() or '?' |
| 261 self._symbol_mapping_caches[address_type][address] = stripped_symbol | 263 self._symbol_mapping_caches[symbol_type][address] = stripped_symbol |
| 262 cache_f.write('%x %s\n' % (address, stripped_symbol)) | 264 cache_f.write('%x %s\n' % (address, stripped_symbol)) |
| 263 | 265 |
| 264 def lookup(self, address_type, address): | 266 def lookup(self, symbol_type, address): |
| 265 """Looks up a symbol for a given |address|. | 267 """Looks up a symbol for a given |address|. |
| 266 | 268 |
| 267 Args: | 269 Args: |
| 268 address_type: A type of addresses to lookup. | 270 symbol_type: A type of symbols to update. It should be one of |
| 269 It should be one of FUNCTION_ADDRESS or TYPEINFO_ADDRESS. | 271 FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS and TYPEINFO_SYMBOLS. |
| 270 address: An integer that represents an address. | 272 address: An integer that represents an address. |
| 271 | 273 |
| 272 Returns: | 274 Returns: |
| 273 A string that represents a symbol. | 275 A string that represents a symbol. |
| 274 """ | 276 """ |
| 275 return self._symbol_mapping_caches[address_type].get(address) | 277 return self._symbol_mapping_caches[symbol_type].get(address) |
| 276 | 278 |
| 277 def _load(self, cache_f, address_type): | 279 def _load(self, cache_f, symbol_type): |
| 278 try: | 280 try: |
| 279 for line in cache_f: | 281 for line in cache_f: |
| 280 items = line.rstrip().split(None, 1) | 282 items = line.rstrip().split(None, 1) |
| 281 if len(items) == 1: | 283 if len(items) == 1: |
| 282 items.append('??') | 284 items.append('??') |
| 283 self._symbol_mapping_caches[address_type][int(items[0], 16)] = items[1] | 285 self._symbol_mapping_caches[symbol_type][int(items[0], 16)] = items[1] |
| 284 LOGGER.info('Loaded %d entries from symbol cache.' % | 286 LOGGER.info('Loaded %d entries from symbol cache.' % |
| 285 len(self._symbol_mapping_caches[address_type])) | 287 len(self._symbol_mapping_caches[symbol_type])) |
| 286 except IOError as e: | 288 except IOError as e: |
| 287 LOGGER.info('The symbol cache file is invalid: %s' % e) | 289 LOGGER.info('The symbol cache file is invalid: %s' % e) |
| 288 | 290 |
| 289 | 291 |
| 290 class Rule(object): | 292 class Rule(object): |
| 291 """Represents one matching rule in a policy file.""" | 293 """Represents one matching rule in a policy file.""" |
| 292 | 294 |
| 293 def __init__(self, name, mmap, stacktrace_pattern, typeinfo_pattern=None): | 295 def __init__(self, |
| 296 name, | |
| 297 mmap, | |
| 298 stackfunction_pattern=None, | |
| 299 stacksourcefile_pattern=None, | |
| 300 typeinfo_pattern=None): | |
| 294 self._name = name | 301 self._name = name |
| 295 self._mmap = mmap | 302 self._mmap = mmap |
| 296 self._stacktrace_pattern = re.compile(stacktrace_pattern + r'\Z') | 303 |
| 304 self._stackfunction_pattern = None | |
| 305 if stackfunction_pattern: | |
| 306 self._stackfunction_pattern = re.compile( | |
| 307 stackfunction_pattern + r'\Z') | |
| 308 | |
| 309 self._stacksourcefile_pattern = None | |
| 310 if stacksourcefile_pattern: | |
| 311 self._stacksourcefile_pattern = re.compile( | |
| 312 stacksourcefile_pattern + r'\Z') | |
| 313 | |
| 314 self._typeinfo_pattern = None | |
| 297 if typeinfo_pattern: | 315 if typeinfo_pattern: |
| 298 self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z') | 316 self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z') |
| 299 else: | |
| 300 self._typeinfo_pattern = None | |
| 301 | 317 |
| 302 @property | 318 @property |
| 303 def name(self): | 319 def name(self): |
| 304 return self._name | 320 return self._name |
| 305 | 321 |
| 306 @property | 322 @property |
| 307 def mmap(self): | 323 def mmap(self): |
| 308 return self._mmap | 324 return self._mmap |
| 309 | 325 |
| 310 @property | 326 @property |
| 311 def stacktrace_pattern(self): | 327 def stackfunction_pattern(self): |
| 312 return self._stacktrace_pattern | 328 return self._stackfunction_pattern |
| 329 | |
| 330 @property | |
| 331 def stacksourcefile_pattern(self): | |
| 332 return self._stacksourcefile_pattern | |
| 313 | 333 |
| 314 @property | 334 @property |
| 315 def typeinfo_pattern(self): | 335 def typeinfo_pattern(self): |
| 316 return self._typeinfo_pattern | 336 return self._typeinfo_pattern |
| 317 | 337 |
| 318 | 338 |
| 319 class Policy(object): | 339 class Policy(object): |
| 320 """Represents a policy, a content of a policy file.""" | 340 """Represents a policy, a content of a policy file.""" |
| 321 | 341 |
| 322 def __init__(self, rules, version, components): | 342 def __init__(self, rules, version, components): |
| (...skipping 20 matching lines...) Expand all Loading... | |
| 343 bucket: A Bucket object to be searched for. | 363 bucket: A Bucket object to be searched for. |
| 344 | 364 |
| 345 Returns: | 365 Returns: |
| 346 A string representing a component name. | 366 A string representing a component name. |
| 347 """ | 367 """ |
| 348 if not bucket: | 368 if not bucket: |
| 349 return 'no-bucket' | 369 return 'no-bucket' |
| 350 if bucket.component_cache: | 370 if bucket.component_cache: |
| 351 return bucket.component_cache | 371 return bucket.component_cache |
| 352 | 372 |
| 353 stacktrace = bucket.symbolized_joined_stacktrace | 373 stackfunction = bucket.symbolized_joined_stackfunction |
| 374 stacksourcefile = bucket.symbolized_joined_stacksourcefile | |
| 354 typeinfo = bucket.symbolized_typeinfo | 375 typeinfo = bucket.symbolized_typeinfo |
| 355 if typeinfo.startswith('0x'): | 376 if typeinfo.startswith('0x'): |
| 356 typeinfo = bucket.typeinfo_name | 377 typeinfo = bucket.typeinfo_name |
| 357 | 378 |
| 358 for rule in self._rules: | 379 for rule in self._rules: |
| 359 if (bucket.mmap == rule.mmap and | 380 if (bucket.mmap == rule.mmap and |
| 360 rule.stacktrace_pattern.match(stacktrace) and | 381 (not rule.stackfunction_pattern or |
| 382 rule.stackfunction_pattern.match(stackfunction)) and | |
| 383 (not rule.stacksourcefile_pattern or | |
| 384 rule.stacksourcefile_pattern.match(stacksourcefile)) and | |
| 361 (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))): | 385 (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))): |
| 362 bucket.component_cache = rule.name | 386 bucket.component_cache = rule.name |
| 363 return rule.name | 387 return rule.name |
| 364 | 388 |
| 365 assert False | 389 assert False |
| 366 | 390 |
| 367 @staticmethod | 391 @staticmethod |
| 368 def load(filename, filetype): | 392 def load(filename, filetype): |
| 369 """Loads a policy file of |filename| in a |format|. | 393 """Loads a policy file of |filename| in a |format|. |
| 370 | 394 |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 407 Args: | 431 Args: |
| 408 policy_f: A File/IO object to read. | 432 policy_f: A File/IO object to read. |
| 409 | 433 |
| 410 Returns: | 434 Returns: |
| 411 A loaded policy object. | 435 A loaded policy object. |
| 412 """ | 436 """ |
| 413 policy = json.load(policy_f) | 437 policy = json.load(policy_f) |
| 414 | 438 |
| 415 rules = [] | 439 rules = [] |
| 416 for rule in policy['rules']: | 440 for rule in policy['rules']: |
| 441 stackfunction = None | |
|
M-A Ruel
2013/04/09 03:25:17
stackfunction = rule.get('stackfunction') or rule.
Dai Mikurube (NOT FULLTIME)
2013/04/09 04:31:52
Done.
| |
| 442 if 'stackfunction' in rule: | |
| 443 stackfunction = rule['stackfunction'] | |
| 444 elif 'stacktrace' in rule: | |
| 445 stackfunction = rule['stacktrace'] | |
| 446 | |
| 447 stacksourcefile = None | |
|
M-A Ruel
2013/04/09 03:25:17
same
Dai Mikurube (NOT FULLTIME)
2013/04/09 04:31:52
Done.
| |
| 448 if 'stacksourcefile' in rule: | |
| 449 stacksourcefile = rule['stacksourcefile'] | |
| 450 | |
| 417 rules.append(Rule( | 451 rules.append(Rule( |
| 418 rule['name'], | 452 rule['name'], |
| 419 rule['allocator'] == 'mmap', | 453 rule['allocator'] == 'mmap', |
| 420 rule['stacktrace'], | 454 stackfunction, |
| 455 stacksourcefile, | |
| 421 rule['typeinfo'] if 'typeinfo' in rule else None)) | 456 rule['typeinfo'] if 'typeinfo' in rule else None)) |
| 422 return Policy(rules, policy['version'], policy['components']) | 457 return Policy(rules, policy['version'], policy['components']) |
| 423 | 458 |
| 424 | 459 |
| 425 class PolicySet(object): | 460 class PolicySet(object): |
| 426 """Represents a set of policies.""" | 461 """Represents a set of policies.""" |
| 427 | 462 |
| 428 def __init__(self, policy_directory): | 463 def __init__(self, policy_directory): |
| 429 self._policy_directory = policy_directory | 464 self._policy_directory = policy_directory |
| 430 | 465 |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 486 | 521 |
| 487 class Bucket(object): | 522 class Bucket(object): |
| 488 """Represents a bucket, which is a unit of memory block classification.""" | 523 """Represents a bucket, which is a unit of memory block classification.""" |
| 489 | 524 |
| 490 def __init__(self, stacktrace, mmap, typeinfo, typeinfo_name): | 525 def __init__(self, stacktrace, mmap, typeinfo, typeinfo_name): |
| 491 self._stacktrace = stacktrace | 526 self._stacktrace = stacktrace |
| 492 self._mmap = mmap | 527 self._mmap = mmap |
| 493 self._typeinfo = typeinfo | 528 self._typeinfo = typeinfo |
| 494 self._typeinfo_name = typeinfo_name | 529 self._typeinfo_name = typeinfo_name |
| 495 | 530 |
| 496 self._symbolized_stacktrace = stacktrace | 531 self._symbolized_stackfunction = stacktrace |
| 497 self._symbolized_joined_stacktrace = '' | 532 self._symbolized_joined_stackfunction = '' |
| 533 self._symbolized_stacksourcefile = stacktrace | |
| 534 self._symbolized_joined_stacksourcefile = '' | |
| 498 self._symbolized_typeinfo = typeinfo_name | 535 self._symbolized_typeinfo = typeinfo_name |
| 499 | 536 |
| 500 self.component_cache = '' | 537 self.component_cache = '' |
| 501 | 538 |
| 502 def symbolize(self, symbol_mapping_cache): | 539 def symbolize(self, symbol_mapping_cache): |
| 503 """Makes a symbolized stacktrace and typeinfo with |symbol_mapping_cache|. | 540 """Makes a symbolized stacktrace and typeinfo with |symbol_mapping_cache|. |
| 504 | 541 |
| 505 Args: | 542 Args: |
| 506 symbol_mapping_cache: A SymbolMappingCache object. | 543 symbol_mapping_cache: A SymbolMappingCache object. |
| 507 """ | 544 """ |
| 508 # TODO(dmikurube): Fill explicitly with numbers if symbol not found. | 545 # TODO(dmikurube): Fill explicitly with numbers if symbol not found. |
| 509 self._symbolized_stacktrace = [ | 546 self._symbolized_stackfunction = [ |
| 510 symbol_mapping_cache.lookup(FUNCTION_ADDRESS, address) | 547 symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address) |
| 511 for address in self._stacktrace] | 548 for address in self._stacktrace] |
| 512 self._symbolized_joined_stacktrace = ' '.join(self._symbolized_stacktrace) | 549 self._symbolized_joined_stackfunction = ' '.join( |
| 550 self._symbolized_stackfunction) | |
| 551 self._symbolized_stacksourcefile = [ | |
| 552 symbol_mapping_cache.lookup(SOURCEFILE_SYMBOLS, address) | |
| 553 for address in self._stacktrace] | |
| 554 self._symbolized_joined_stacksourcefile = ' '.join( | |
| 555 self._symbolized_stacksourcefile) | |
| 513 if not self._typeinfo: | 556 if not self._typeinfo: |
| 514 self._symbolized_typeinfo = 'no typeinfo' | 557 self._symbolized_typeinfo = 'no typeinfo' |
| 515 else: | 558 else: |
| 516 self._symbolized_typeinfo = symbol_mapping_cache.lookup( | 559 self._symbolized_typeinfo = symbol_mapping_cache.lookup( |
| 517 TYPEINFO_ADDRESS, self._typeinfo) | 560 TYPEINFO_SYMBOLS, self._typeinfo) |
| 518 if not self._symbolized_typeinfo: | 561 if not self._symbolized_typeinfo: |
| 519 self._symbolized_typeinfo = 'no typeinfo' | 562 self._symbolized_typeinfo = 'no typeinfo' |
| 520 | 563 |
| 521 def clear_component_cache(self): | 564 def clear_component_cache(self): |
| 522 self.component_cache = '' | 565 self.component_cache = '' |
| 523 | 566 |
| 524 @property | 567 @property |
| 525 def stacktrace(self): | 568 def stacktrace(self): |
| 526 return self._stacktrace | 569 return self._stacktrace |
| 527 | 570 |
| 528 @property | 571 @property |
| 529 def mmap(self): | 572 def mmap(self): |
| 530 return self._mmap | 573 return self._mmap |
| 531 | 574 |
| 532 @property | 575 @property |
| 533 def typeinfo(self): | 576 def typeinfo(self): |
| 534 return self._typeinfo | 577 return self._typeinfo |
| 535 | 578 |
| 536 @property | 579 @property |
| 537 def typeinfo_name(self): | 580 def typeinfo_name(self): |
| 538 return self._typeinfo_name | 581 return self._typeinfo_name |
| 539 | 582 |
| 540 @property | 583 @property |
| 541 def symbolized_stacktrace(self): | 584 def symbolized_stackfunction(self): |
| 542 return self._symbolized_stacktrace | 585 return self._symbolized_stackfunction |
| 543 | 586 |
| 544 @property | 587 @property |
| 545 def symbolized_joined_stacktrace(self): | 588 def symbolized_joined_stackfunction(self): |
| 546 return self._symbolized_joined_stacktrace | 589 return self._symbolized_joined_stackfunction |
| 590 | |
| 591 @property | |
| 592 def symbolized_stacksourcefile(self): | |
| 593 return self._symbolized_stacksourcefile | |
| 594 | |
| 595 @property | |
| 596 def symbolized_joined_stacksourcefile(self): | |
| 597 return self._symbolized_joined_stacksourcefile | |
| 547 | 598 |
| 548 @property | 599 @property |
| 549 def symbolized_typeinfo(self): | 600 def symbolized_typeinfo(self): |
| 550 return self._symbolized_typeinfo | 601 return self._symbolized_typeinfo |
| 551 | 602 |
| 552 | 603 |
| 553 class BucketSet(object): | 604 class BucketSet(object): |
| 554 """Represents a set of bucket.""" | 605 """Represents a set of bucket.""" |
| 555 def __init__(self): | 606 def __init__(self): |
| 556 self._buckets = {} | 607 self._buckets = {} |
| 557 self._addresses = { | 608 self._code_addresses = set() |
| 558 FUNCTION_ADDRESS: set(), | 609 self._typeinfo_addresses = set() |
| 559 TYPEINFO_ADDRESS: set(), | |
| 560 } | |
| 561 | 610 |
| 562 def load(self, prefix): | 611 def load(self, prefix): |
| 563 """Loads all related bucket files. | 612 """Loads all related bucket files. |
| 564 | 613 |
| 565 Args: | 614 Args: |
| 566 prefix: A prefix string for bucket file names. | 615 prefix: A prefix string for bucket file names. |
| 567 """ | 616 """ |
| 568 LOGGER.info('Loading bucket files.') | 617 LOGGER.info('Loading bucket files.') |
| 569 | 618 |
| 570 n = 0 | 619 n = 0 |
| (...skipping 13 matching lines...) Expand all Loading... | |
| 584 for line in bucket_f: | 633 for line in bucket_f: |
| 585 words = line.split() | 634 words = line.split() |
| 586 typeinfo = None | 635 typeinfo = None |
| 587 typeinfo_name = '' | 636 typeinfo_name = '' |
| 588 stacktrace_begin = 2 | 637 stacktrace_begin = 2 |
| 589 for index, word in enumerate(words): | 638 for index, word in enumerate(words): |
| 590 if index < 2: | 639 if index < 2: |
| 591 continue | 640 continue |
| 592 if word[0] == 't': | 641 if word[0] == 't': |
| 593 typeinfo = int(word[1:], 16) | 642 typeinfo = int(word[1:], 16) |
| 594 self._addresses[TYPEINFO_ADDRESS].add(typeinfo) | 643 self._typeinfo_addresses.add(typeinfo) |
| 595 elif word[0] == 'n': | 644 elif word[0] == 'n': |
| 596 typeinfo_name = word[1:] | 645 typeinfo_name = word[1:] |
| 597 else: | 646 else: |
| 598 stacktrace_begin = index | 647 stacktrace_begin = index |
| 599 break | 648 break |
| 600 stacktrace = [int(address, 16) for address in words[stacktrace_begin:]] | 649 stacktrace = [int(address, 16) for address in words[stacktrace_begin:]] |
| 601 for frame in stacktrace: | 650 for frame in stacktrace: |
| 602 self._addresses[FUNCTION_ADDRESS].add(frame) | 651 self._code_addresses.add(frame) |
| 603 self._buckets[int(words[0])] = Bucket( | 652 self._buckets[int(words[0])] = Bucket( |
| 604 stacktrace, words[1] == 'mmap', typeinfo, typeinfo_name) | 653 stacktrace, words[1] == 'mmap', typeinfo, typeinfo_name) |
| 605 | 654 |
| 606 def __iter__(self): | 655 def __iter__(self): |
| 607 for bucket_id, bucket_content in self._buckets.iteritems(): | 656 for bucket_id, bucket_content in self._buckets.iteritems(): |
| 608 yield bucket_id, bucket_content | 657 yield bucket_id, bucket_content |
| 609 | 658 |
| 610 def __getitem__(self, bucket_id): | 659 def __getitem__(self, bucket_id): |
| 611 return self._buckets[bucket_id] | 660 return self._buckets[bucket_id] |
| 612 | 661 |
| 613 def get(self, bucket_id): | 662 def get(self, bucket_id): |
| 614 return self._buckets.get(bucket_id) | 663 return self._buckets.get(bucket_id) |
| 615 | 664 |
| 616 def symbolize(self, symbol_mapping_cache): | 665 def symbolize(self, symbol_mapping_cache): |
| 617 for bucket_content in self._buckets.itervalues(): | 666 for bucket_content in self._buckets.itervalues(): |
| 618 bucket_content.symbolize(symbol_mapping_cache) | 667 bucket_content.symbolize(symbol_mapping_cache) |
| 619 | 668 |
| 620 def clear_component_cache(self): | 669 def clear_component_cache(self): |
| 621 for bucket_content in self._buckets.itervalues(): | 670 for bucket_content in self._buckets.itervalues(): |
| 622 bucket_content.clear_component_cache() | 671 bucket_content.clear_component_cache() |
| 623 | 672 |
| 624 def iter_addresses(self, address_type): | 673 def iter_addresses(self, symbol_type): |
| 625 for function in self._addresses[address_type]: | 674 if symbol_type in [FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS]: |
| 626 yield function | 675 for function in self._code_addresses: |
| 676 yield function | |
| 677 else: | |
| 678 for function in self._typeinfo_addresses: | |
| 679 yield function | |
| 627 | 680 |
| 628 | 681 |
| 629 class Dump(object): | 682 class Dump(object): |
| 630 """Represents a heap profile dump.""" | 683 """Represents a heap profile dump.""" |
| 631 | 684 |
| 632 def __init__(self, path, time): | 685 def __init__(self, path, time): |
| 633 self._path = path | 686 self._path = path |
| 634 self._time = time | 687 self._time = time |
| 635 self._stacktrace_lines = [] | 688 self._stacktrace_lines = [] |
| 636 self._global_stats = {} # used only in apply_policy | 689 self._global_stats = {} # used only in apply_policy |
| (...skipping 196 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 833 prefix = Command._find_prefix(dump_path) | 886 prefix = Command._find_prefix(dump_path) |
| 834 symbol_data_sources = SymbolDataSources(prefix) | 887 symbol_data_sources = SymbolDataSources(prefix) |
| 835 symbol_data_sources.prepare() | 888 symbol_data_sources.prepare() |
| 836 bucket_set = BucketSet() | 889 bucket_set = BucketSet() |
| 837 bucket_set.load(prefix) | 890 bucket_set.load(prefix) |
| 838 if multiple: | 891 if multiple: |
| 839 dump_list = DumpList.load(Command._find_all_dumps(dump_path)) | 892 dump_list = DumpList.load(Command._find_all_dumps(dump_path)) |
| 840 else: | 893 else: |
| 841 dump = Dump.load(dump_path) | 894 dump = Dump.load(dump_path) |
| 842 symbol_mapping_cache = SymbolMappingCache() | 895 symbol_mapping_cache = SymbolMappingCache() |
| 843 with open(prefix + '.funcsym', 'a+') as cache_f: | 896 with open(prefix + '.cache.function', 'a+') as cache_f: |
| 844 symbol_mapping_cache.update( | 897 symbol_mapping_cache.update( |
| 845 FUNCTION_ADDRESS, bucket_set, | 898 FUNCTION_SYMBOLS, bucket_set, |
| 846 SymbolFinder(FUNCTION_ADDRESS, symbol_data_sources), cache_f) | 899 SymbolFinder(FUNCTION_SYMBOLS, symbol_data_sources), cache_f) |
| 847 with open(prefix + '.typesym', 'a+') as cache_f: | 900 with open(prefix + '.cache.typeinfo', 'a+') as cache_f: |
| 848 symbol_mapping_cache.update( | 901 symbol_mapping_cache.update( |
| 849 TYPEINFO_ADDRESS, bucket_set, | 902 TYPEINFO_SYMBOLS, bucket_set, |
| 850 SymbolFinder(TYPEINFO_ADDRESS, symbol_data_sources), cache_f) | 903 SymbolFinder(TYPEINFO_SYMBOLS, symbol_data_sources), cache_f) |
| 904 with open(prefix + '.cache.sourcefile', 'a+') as cache_f: | |
| 905 symbol_mapping_cache.update( | |
| 906 SOURCEFILE_SYMBOLS, bucket_set, | |
| 907 SymbolFinder(SOURCEFILE_SYMBOLS, symbol_data_sources), cache_f) | |
| 851 bucket_set.symbolize(symbol_mapping_cache) | 908 bucket_set.symbolize(symbol_mapping_cache) |
| 852 if multiple: | 909 if multiple: |
| 853 return (bucket_set, dump_list) | 910 return (bucket_set, dump_list) |
| 854 else: | 911 else: |
| 855 return (bucket_set, dump) | 912 return (bucket_set, dump) |
| 856 | 913 |
| 857 @staticmethod | 914 @staticmethod |
| 858 def _find_prefix(path): | 915 def _find_prefix(path): |
| 859 return re.sub('\.[0-9][0-9][0-9][0-9]\.heap', '', path) | 916 return re.sub('\.[0-9][0-9][0-9][0-9]\.heap', '', path) |
| 860 | 917 |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 929 bucket_set: A BucketSet object. | 986 bucket_set: A BucketSet object. |
| 930 out: A file object to output. | 987 out: A file object to output. |
| 931 """ | 988 """ |
| 932 for line in dump.iter_stacktrace: | 989 for line in dump.iter_stacktrace: |
| 933 words = line.split() | 990 words = line.split() |
| 934 bucket = bucket_set.get(int(words[BUCKET_ID])) | 991 bucket = bucket_set.get(int(words[BUCKET_ID])) |
| 935 if not bucket: | 992 if not bucket: |
| 936 continue | 993 continue |
| 937 for i in range(0, BUCKET_ID - 1): | 994 for i in range(0, BUCKET_ID - 1): |
| 938 out.write(words[i] + ' ') | 995 out.write(words[i] + ' ') |
| 939 for frame in bucket.symbolized_stacktrace: | 996 for frame in bucket.symbolized_stackfunction: |
| 940 out.write(frame + ' ') | 997 out.write(frame + ' ') |
| 941 out.write('\n') | 998 out.write('\n') |
| 942 | 999 |
| 943 | 1000 |
| 944 class PolicyCommands(Command): | 1001 class PolicyCommands(Command): |
| 945 def __init__(self, command): | 1002 def __init__(self, command): |
| 946 super(PolicyCommands, self).__init__( | 1003 super(PolicyCommands, self).__init__( |
| 947 'Usage: %%prog %s [-p POLICY] <first-dump>' % command) | 1004 'Usage: %%prog %s [-p POLICY] <first-dump>' % command) |
| 948 self._parser.add_option('-p', '--policy', type='string', dest='policy', | 1005 self._parser.add_option('-p', '--policy', type='string', dest='policy', |
| 949 help='profile with POLICY', metavar='POLICY') | 1006 help='profile with POLICY', metavar='POLICY') |
| (...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1114 json_base['policies'][label] = { | 1171 json_base['policies'][label] = { |
| 1115 'legends': policy_set[label].components, | 1172 'legends': policy_set[label].components, |
| 1116 'snapshots': [], | 1173 'snapshots': [], |
| 1117 } | 1174 } |
| 1118 | 1175 |
| 1119 LOGGER.info('Applying a policy %s to...' % label) | 1176 LOGGER.info('Applying a policy %s to...' % label) |
| 1120 for dump in dumps: | 1177 for dump in dumps: |
| 1121 component_sizes = PolicyCommands._apply_policy( | 1178 component_sizes = PolicyCommands._apply_policy( |
| 1122 dump, policy_set[label], bucket_set, dumps[0].time) | 1179 dump, policy_set[label], bucket_set, dumps[0].time) |
| 1123 component_sizes['dump_path'] = dump.path | 1180 component_sizes['dump_path'] = dump.path |
| 1124 component_sizes['dump_time'] = datetime.fromtimestamp( | 1181 component_sizes['dump_time'] = datetime.datetime.fromtimestamp( |
| 1125 dump.time).strftime('%Y-%m-%d %H:%M:%S') | 1182 dump.time).strftime('%Y-%m-%d %H:%M:%S') |
| 1126 json_base['policies'][label]['snapshots'].append(component_sizes) | 1183 json_base['policies'][label]['snapshots'].append(component_sizes) |
| 1127 | 1184 |
| 1128 bucket_set.clear_component_cache() | 1185 bucket_set.clear_component_cache() |
| 1129 | 1186 |
| 1130 json.dump(json_base, out, indent=2, sort_keys=True) | 1187 json.dump(json_base, out, indent=2, sort_keys=True) |
| 1131 | 1188 |
| 1132 return 0 | 1189 return 0 |
| 1133 | 1190 |
| 1134 | 1191 |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1190 out: An IO object to output. | 1247 out: An IO object to output. |
| 1191 """ | 1248 """ |
| 1192 sizes = {} | 1249 sizes = {} |
| 1193 | 1250 |
| 1194 ExpandCommand._accumulate( | 1251 ExpandCommand._accumulate( |
| 1195 dump, policy, bucket_set, component_name, depth, sizes) | 1252 dump, policy, bucket_set, component_name, depth, sizes) |
| 1196 | 1253 |
| 1197 sorted_sizes_list = sorted( | 1254 sorted_sizes_list = sorted( |
| 1198 sizes.iteritems(), key=(lambda x: x[1]), reverse=True) | 1255 sizes.iteritems(), key=(lambda x: x[1]), reverse=True) |
| 1199 total = 0 | 1256 total = 0 |
| 1257 # TODO(dmikurube): Better formatting. | |
| 1200 for size_pair in sorted_sizes_list: | 1258 for size_pair in sorted_sizes_list: |
| 1201 out.write('%10d %s\n' % (size_pair[1], size_pair[0])) | 1259 out.write('%10d %s\n' % (size_pair[1], size_pair[0])) |
| 1202 total += size_pair[1] | 1260 total += size_pair[1] |
| 1203 LOGGER.info('total: %d\n' % total) | 1261 LOGGER.info('total: %d\n' % total) |
| 1204 | 1262 |
| 1205 @staticmethod | 1263 @staticmethod |
| 1206 def _accumulate(dump, policy, bucket_set, component_name, depth, sizes): | 1264 def _accumulate(dump, policy, bucket_set, component_name, depth, sizes): |
| 1207 for line in dump.iter_stacktrace: | 1265 for line in dump.iter_stacktrace: |
| 1208 words = line.split() | 1266 words = line.split() |
| 1209 bucket = bucket_set.get(int(words[BUCKET_ID])) | 1267 bucket = bucket_set.get(int(words[BUCKET_ID])) |
| 1210 component_match = policy.find(bucket) | 1268 component_match = policy.find(bucket) |
| 1211 if component_match == component_name: | 1269 if component_match == component_name: |
| 1212 stacktrace_sequence = '' | 1270 stacktrace_sequence = '' |
| 1213 if bucket.typeinfo: | 1271 if bucket.typeinfo: |
| 1214 stacktrace_sequence += '(type=%s)' % bucket.symbolized_typeinfo | 1272 stacktrace_sequence += '(type=%s)' % bucket.symbolized_typeinfo |
| 1215 stacktrace_sequence += ' (type.name=%s) ' % bucket.typeinfo_name | 1273 stacktrace_sequence += ' (type.name=%s) ' % bucket.typeinfo_name |
| 1216 for stack in bucket.symbolized_stacktrace[ | 1274 for function, sourcefile in zip( |
| 1217 0 : min(len(bucket.symbolized_stacktrace), 1 + depth)]: | 1275 bucket.symbolized_stackfunction[ |
| 1218 stacktrace_sequence += stack + ' ' | 1276 0 : min(len(bucket.symbolized_stackfunction), 1 + depth)], |
| 1277 bucket.symbolized_stacksourcefile[ | |
| 1278 0 : min(len(bucket.symbolized_stacksourcefile), 1 + depth)]): | |
| 1279 stacktrace_sequence += '%s(@%s) ' % (function, sourcefile) | |
| 1219 if not stacktrace_sequence in sizes: | 1280 if not stacktrace_sequence in sizes: |
| 1220 sizes[stacktrace_sequence] = 0 | 1281 sizes[stacktrace_sequence] = 0 |
| 1221 sizes[stacktrace_sequence] += int(words[COMMITTED]) | 1282 sizes[stacktrace_sequence] += int(words[COMMITTED]) |
| 1222 | 1283 |
| 1223 | 1284 |
| 1224 class PProfCommand(Command): | 1285 class PProfCommand(Command): |
| 1225 def __init__(self): | 1286 def __init__(self): |
| 1226 super(PProfCommand, self).__init__( | 1287 super(PProfCommand, self).__init__( |
| 1227 'Usage: %prog pprof [-c COMPONENT] <dump> <policy>') | 1288 'Usage: %prog pprof [-c COMPONENT] <dump> <policy>') |
| 1228 self._parser.add_option('-c', '--component', type='string', | 1289 self._parser.add_option('-c', '--component', type='string', |
| (...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1439 errorcode = COMMANDS[action]().do(sys.argv) | 1500 errorcode = COMMANDS[action]().do(sys.argv) |
| 1440 except ParsingException, e: | 1501 except ParsingException, e: |
| 1441 errorcode = 1 | 1502 errorcode = 1 |
| 1442 sys.stderr.write('Exit by parsing error: %s\n' % e) | 1503 sys.stderr.write('Exit by parsing error: %s\n' % e) |
| 1443 | 1504 |
| 1444 return errorcode | 1505 return errorcode |
| 1445 | 1506 |
| 1446 | 1507 |
| 1447 if __name__ == '__main__': | 1508 if __name__ == '__main__': |
| 1448 sys.exit(main()) | 1509 sys.exit(main()) |
| OLD | NEW |