Adding extended report to memdump with resident pages bit reporting.

tools/android/memdump/memsymbols.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file
+
+import base64
+import os
+import sys
+import re
+
+from optparse import OptionParser
+
+""" Extracts the list of resident symbols of a library loaded in a process.

[bulach, 2013/07/22 12:59:39]

nit: remove space before Extracts

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Done.

+
+This scripts combines the extended output of memdump for a given process
+(obtained through memdump -x PID) and the symbol table of a .so loaded in that
+process (obtained through nm -C lib-with-symbols.so), filtering out only those
+symbols that, at the time of the snapshot, were resident in memory (that are,
+the symbols which start address belongs to a mapped page of the .so which was
+resident at the time of the snapshot).
+The aim is to perform a "code coverage"-like profiling of a binary, intersecting
+run-time information (list of resident pages) and debug symbols.
+"""
+
+PAGE_SIZE = 4096

[bulach, 2013/07/22 12:59:39]

nit: _ prefix.
also, below should be _TestBit and _HexAddr

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Done.

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Done.

+
+def test_bit(word, bit):
+  assert(bit >= 0 and bit < 8)
+  return not not ((word >> bit) & 1)
+
+def hex_addr(addr):
+  return hex(addr)[2:].zfill(8)
+
+def main(argv):
+  MAP_RX = re.compile(
+      r'^([0-9a-f]+)-([0-9a-f]+) ([\w-]+) ([0-9a-f]+) .* "(.*)" \[(.*)\]$')
+  NM_RX = re.compile(r'^([0-9a-f]+)\s+\w+\s+.*$')
+
+  parser = OptionParser()
+  parser.add_option("-r", "--reverse",
+                    action="store_true", dest="reverse", default=False,
+                    help="Print out non present symbols")
+  parser.add_option("-v", "--verbose",
+                    action="store_true", dest="verbose", default=False,
+                    help="Print out verbose debug information.")
+
+  (options, args) = parser.parse_args()
+
+  if len(args) != 3:
+    print 'Usage: %s [-v] memdump.file nm.file library.so' % (
+        os.path.basename(argv[0]))
+    return 1
+

[bulach, 2013/07/22 12:59:39]

it'd be nicer to split 54-134 into its own function.

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Done.

+  memdump_file = args[0]
+  nm_file = args[1]
+  lib_name = args[2]
+  resident_pages = set()
+
+  # First process the memdump output and extract the mappings which match the
+  # given library name.
+  memdump_fh = open(memdump_file, 'r')

[Philippe, 2013/07/22 09:20:55]

Nit: will this work with stdin?

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Added special '-' handling to read memdump from stdin.

[pliard, 2013/07/23 09:08:22]

Nice, thanks.

+  for line in memdump_fh:
+    line = line.rstrip('\r\n')
+    if line.startswith('[ PID'):
+      continue
+
+    r = MAP_RX.match(line)
+    if not r:
+      sys.stderr.write('Skipping %s from %s\n' % (line, nm_file))
+      continue
+
+    map_start = int(r.group(1), 16)
+    map_end = int(r.group(2), 16)
+    prot = r.group(3)
+    offset = int(r.group(4), 16)
+    assert(offset % PAGE_SIZE == 0)
+    lib = r.group(5)
+    enc_bitmap = r.group(6)
+
+    #if prot == 'r-xp':

[Philippe, 2013/07/22 09:20:55]

Nit: should we keep this? :)

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Done.

+    #  continue
+    if not lib.endswith(lib_name):
+      continue
+
+    bitmap = base64.b64decode(enc_bitmap)
+    map_pages_count = (map_end - map_start + 1) / PAGE_SIZE
+    bitmap_pages_count = len(bitmap) * 8
+
+    if options.verbose:
+      print 'Found %s: mapped %d pages in mode %s @ offset %s.' % (
+            lib, map_pages_count, prot, hex_addr(offset))
+      print ' Map range in the process VA: [%s - %s]. Len: %s' % (
+          hex_addr(map_start),
+          hex_addr(map_end),
+          hex_addr(map_pages_count * PAGE_SIZE))
+      print ' Corresponding addresses in the binary: [%s - %s]. Len: %s' % (
+          hex_addr(offset),
+          hex_addr(offset + map_end - map_start),
+          hex_addr(map_pages_count * PAGE_SIZE))
+      print ' Bitmap: %d pages' % bitmap_pages_count
+      print ''
+
+    assert(bitmap_pages_count >= map_pages_count)
+    for i in xrange(map_pages_count):
+      bitmap_idx = i / 8
+      bitmap_off = i % 8
+      if (bitmap_idx < len(bitmap) and
+          test_bit(ord(bitmap[bitmap_idx]), bitmap_off)):
+        resident_pages.add(offset/PAGE_SIZE + i)

[Philippe, 2013/07/22 09:20:55]

Nit: spaces around binary operators (also on line 127).

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

Done.

+
+  # Now process the nm symbol table, filtering out the resident symbols.
+  nm_fh = open(nm_file, 'r')
+  last_sym_matched = False
+  for line in nm_fh:
+    line = line.rstrip('\r\n')
+    if line.startswith(' '):
+      if last_sym_matched:
+        print line
+      continue
+
+    r = NM_RX.match(line)
+    if not r:
+      sys.stderr.write('Skipping %s from %s\n' % (line, nm_file))
+      continue
+
+    sym_addr = int(r.group(1),16)
+    sym_page = sym_addr / PAGE_SIZE
+    last_sym_matched = (sym_page in resident_pages)
+    if options.reverse:
+      last_sym_matched = not last_sym_matched
+    if last_sym_matched:
+      print line
+
+if __name__=='__main__':
+    main(sys.argv)

[bulach, 2013/07/22 12:59:39]

nit: return main(sys.argv)

[Primiano Tucci (use gerrit), 2013/07/22 23:22:35]

You mean sys.exit, right? 

[bulach, 2013/07/23 08:10:57]

yep, thanks!