| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """The deep heap profiler script for Chrome.""" | 6 """The deep heap profiler script for Chrome.""" |
| 7 | 7 |
| 8 from datetime import datetime | 8 from datetime import datetime |
| 9 import json | 9 import json |
| 10 import os | 10 import os |
| 11 import re | 11 import re |
| 12 import shutil |
| 12 import subprocess | 13 import subprocess |
| 13 import sys | 14 import sys |
| 14 import tempfile | 15 import tempfile |
| 15 | 16 |
| 17 FIND_RUNTIME_SYMBOLS_PATH = os.path.join( |
| 18 os.path.dirname(os.path.abspath(__file__)), |
| 19 os.pardir, |
| 20 'find_runtime_symbols') |
| 21 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH) |
| 22 |
| 23 from prepare_symbol_info import prepare_symbol_info |
| 24 from find_runtime_symbols import find_runtime_symbols_list |
| 25 |
| 16 BUCKET_ID = 5 | 26 BUCKET_ID = 5 |
| 17 VIRTUAL = 0 | 27 VIRTUAL = 0 |
| 18 COMMITTED = 1 | 28 COMMITTED = 1 |
| 19 ALLOC_COUNT = 2 | 29 ALLOC_COUNT = 2 |
| 20 FREE_COUNT = 3 | 30 FREE_COUNT = 3 |
| 21 NULL_REGEX = re.compile('') | 31 NULL_REGEX = re.compile('') |
| 22 | 32 |
| 23 # If an executable pprof script is in the directory of deep_memory_profiler, | |
| 24 # use it. Otherwise, use tcmalloc's pprof. | |
| 25 # | |
| 26 # A pprof script in deep_memory_profiler/ is prioritized to allow packaging | |
| 27 # deep_memory_profiler files with a pprof script. The packaged | |
| 28 # deep_memory_profiler is downloaded with pprof by using download.sh. | |
| 29 PPROF_PATH = os.path.join(os.path.dirname(__file__), 'pprof') | |
| 30 if not (os.path.isfile(PPROF_PATH) and os.access(PPROF_PATH, os.X_OK)): | |
| 31 PPROF_PATH = os.path.join(os.path.dirname(__file__), | |
| 32 os.pardir, | |
| 33 os.pardir, | |
| 34 'third_party', | |
| 35 'tcmalloc', | |
| 36 'chromium', | |
| 37 'src', | |
| 38 'pprof') | |
| 39 | |
| 40 # Heap Profile Dump versions | 33 # Heap Profile Dump versions |
| 41 | 34 |
| 42 # DUMP_DEEP_1 is OBSOLETE. | 35 # DUMP_DEEP_1 is OBSOLETE. |
| 43 # DUMP_DEEP_1 DOES NOT distinct mmap regions and malloc chunks. | 36 # DUMP_DEEP_1 DOES NOT distinct mmap regions and malloc chunks. |
| 44 # Their stacktraces DO contain mmap* or tc-* at their tops. | 37 # Their stacktraces DO contain mmap* or tc-* at their tops. |
| 45 # They should be processed by POLICY_DEEP_1. | 38 # They should be processed by POLICY_DEEP_1. |
| 46 DUMP_DEEP_1 = 'DUMP_DEEP_1' | 39 DUMP_DEEP_1 = 'DUMP_DEEP_1' |
| 47 | 40 |
| 48 # DUMP_DEEP_2 is OBSOLETE. | 41 # DUMP_DEEP_2 is OBSOLETE. |
| 49 # DUMP_DEEP_2 DOES distinct mmap regions and malloc chunks. | 42 # DUMP_DEEP_2 DOES distinct mmap regions and malloc chunks. |
| (...skipping 513 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 563 | 556 |
| 564 sorted_sizes_list = sorted( | 557 sorted_sizes_list = sorted( |
| 565 sizes.iteritems(), key=(lambda x: x[1]), reverse=True) | 558 sizes.iteritems(), key=(lambda x: x[1]), reverse=True) |
| 566 total = 0 | 559 total = 0 |
| 567 for size_pair in sorted_sizes_list: | 560 for size_pair in sorted_sizes_list: |
| 568 sys.stdout.write('%10d %s\n' % (size_pair[1], size_pair[0])) | 561 sys.stdout.write('%10d %s\n' % (size_pair[1], size_pair[0])) |
| 569 total += size_pair[1] | 562 total += size_pair[1] |
| 570 sys.stderr.write('total: %d\n' % (total)) | 563 sys.stderr.write('total: %d\n' % (total)) |
| 571 | 564 |
| 572 | 565 |
| 573 def update_symbols(symbol_path, mapping_lines, chrome_path): | 566 def update_symbols(symbol_path, mapping_lines, maps_path): |
| 574 """Updates address/symbol mapping on memory and in a .symbol cache file. | 567 """Updates address/symbol mapping on memory and in a .symbol cache file. |
| 575 | 568 |
| 576 It reads cached address/symbol mapping from a .symbol file if it exists. | 569 It reads cached address/symbol mapping from a .symbol file if it exists. |
| 577 Then, it resolves unresolved addresses from a Chrome binary with pprof. | 570 Then, it resolves unresolved addresses from a Chrome binary with pprof. |
| 578 Both mappings on memory and in a .symbol cache file are updated. | 571 Both mappings on memory and in a .symbol cache file are updated. |
| 579 | 572 |
| 580 Symbol files are formatted as follows: | 573 Symbol files are formatted as follows: |
| 581 <Address> <Symbol> | 574 <Address> <Symbol> |
| 582 <Address> <Symbol> | 575 <Address> <Symbol> |
| 583 <Address> <Symbol> | 576 <Address> <Symbol> |
| 584 ... | 577 ... |
| 585 | 578 |
| 586 Args: | 579 Args: |
| 587 symbol_path: A string representing a path for a .symbol file. | 580 symbol_path: A string representing a path for a .symbol file. |
| 588 mapping_lines: A list of strings containing /proc/.../maps. | 581 mapping_lines: A list of strings containing /proc/.../maps. |
| 589 chrome_path: A string representing a path for a Chrome binary. | 582 maps_path: A string of the path of /proc/.../maps. |
| 590 """ | 583 """ |
| 591 with open(symbol_path, mode='a+') as symbol_f: | 584 with open(symbol_path, mode='a+') as symbol_f: |
| 592 symbol_lines = symbol_f.readlines() | 585 symbol_lines = symbol_f.readlines() |
| 593 if symbol_lines: | 586 if symbol_lines: |
| 594 for line in symbol_lines: | 587 for line in symbol_lines: |
| 595 items = line.split(None, 1) | 588 items = line.split(None, 1) |
| 596 address_symbol_dict[items[0]] = items[1].rstrip() | 589 address_symbol_dict[items[0]] = items[1].rstrip() |
| 597 | 590 |
| 598 unresolved_addresses = sorted( | 591 unresolved_addresses = sorted( |
| 599 a for a in appeared_addresses if a not in address_symbol_dict) | 592 a for a in appeared_addresses if a not in address_symbol_dict) |
| 600 | 593 |
| 601 if unresolved_addresses: | 594 if unresolved_addresses: |
| 602 with tempfile.NamedTemporaryFile( | 595 prepared_data_dir = tempfile.mkdtemp() |
| 603 suffix='maps', prefix="dmprof", mode='w+') as pprof_in: | 596 try: |
| 604 with tempfile.NamedTemporaryFile( | 597 prepare_symbol_info(maps_path, prepared_data_dir) |
| 605 suffix='symbols', prefix="dmprof", mode='w+') as pprof_out: | |
| 606 for line in mapping_lines: | |
| 607 pprof_in.write(line) | |
| 608 | 598 |
| 609 for address in unresolved_addresses: | 599 symbols = find_runtime_symbols_list( |
| 610 pprof_in.write(address + '\n') | 600 prepared_data_dir, unresolved_addresses) |
| 611 | 601 |
| 612 pprof_in.seek(0) | 602 for address, symbol in zip(unresolved_addresses, symbols): |
| 613 | 603 stripped_symbol = symbol.strip() |
| 614 p = subprocess.Popen( | 604 address_symbol_dict[address] = stripped_symbol |
| 615 '%s --symbols %s' % (PPROF_PATH, chrome_path), | 605 symbol_f.write('%s %s\n' % (address, stripped_symbol)) |
| 616 shell=True, stdin=pprof_in, stdout=pprof_out) | 606 finally: |
| 617 p.wait() | 607 shutil.rmtree(prepared_data_dir) |
| 618 | |
| 619 pprof_out.seek(0) | |
| 620 symbols = pprof_out.readlines() | |
| 621 symbol_f.seek(0, 2) | |
| 622 for address, symbol in zip(unresolved_addresses, symbols): | |
| 623 stripped_symbol = symbol.strip() | |
| 624 address_symbol_dict[address] = stripped_symbol | |
| 625 symbol_f.write('%s %s\n' % (address, symbol.strip())) | |
| 626 | 608 |
| 627 | 609 |
| 628 def parse_policy(policy_path): | 610 def parse_policy(policy_path): |
| 629 """Parses policy file. | 611 """Parses policy file. |
| 630 | 612 |
| 631 A policy file contains component's names and their | 613 A policy file contains component's names and their |
| 632 stacktrace pattern written in regular expression. | 614 stacktrace pattern written in regular expression. |
| 633 Those patterns are matched against each symbols of | 615 Those patterns are matched against each symbols of |
| 634 each stacktraces in the order written in the policy file | 616 each stacktraces in the order written in the policy file |
| 635 | 617 |
| (...skipping 126 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 762 new_log.parse_log(buckets) | 744 new_log.parse_log(buckets) |
| 763 except EmptyDumpException: | 745 except EmptyDumpException: |
| 764 sys.stderr.write(' WARNING: ignored an empty dump: %s\n' % path) | 746 sys.stderr.write(' WARNING: ignored an empty dump: %s\n' % path) |
| 765 except ParsingException, e: | 747 except ParsingException, e: |
| 766 sys.stderr.write(' Error in parsing heap profile dump: %s\n' % e) | 748 sys.stderr.write(' Error in parsing heap profile dump: %s\n' % e) |
| 767 sys.exit(1) | 749 sys.exit(1) |
| 768 else: | 750 else: |
| 769 logs.append(new_log) | 751 logs.append(new_log) |
| 770 | 752 |
| 771 sys.stderr.write('getting symbols\n') | 753 sys.stderr.write('getting symbols\n') |
| 772 update_symbols(symbol_path, maps_lines, chrome_path) | 754 update_symbols(symbol_path, maps_lines, maps_path) |
| 773 | 755 |
| 774 # TODO(dmikurube): Many modes now. Split them into separete functions. | 756 # TODO(dmikurube): Many modes now. Split them into separete functions. |
| 775 if action == '--stacktrace': | 757 if action == '--stacktrace': |
| 776 logs[0].dump_stacktrace(buckets) | 758 logs[0].dump_stacktrace(buckets) |
| 777 | 759 |
| 778 elif action == '--csv': | 760 elif action == '--csv': |
| 779 sys.stdout.write(','.join(components)) | 761 sys.stdout.write(','.join(components)) |
| 780 sys.stdout.write('\n') | 762 sys.stdout.write('\n') |
| 781 | 763 |
| 782 for log in logs: | 764 for log in logs: |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 821 | 803 |
| 822 elif action == '--pprof': | 804 elif action == '--pprof': |
| 823 if len(sys.argv) > 5: | 805 if len(sys.argv) > 5: |
| 824 logs[0].dump_for_pprof(policy_list, buckets, maps_lines, sys.argv[5]) | 806 logs[0].dump_for_pprof(policy_list, buckets, maps_lines, sys.argv[5]) |
| 825 else: | 807 else: |
| 826 logs[0].dump_for_pprof(policy_list, buckets, maps_lines, None) | 808 logs[0].dump_for_pprof(policy_list, buckets, maps_lines, None) |
| 827 | 809 |
| 828 | 810 |
| 829 if __name__ == '__main__': | 811 if __name__ == '__main__': |
| 830 sys.exit(main()) | 812 sys.exit(main()) |
| OLD | NEW |