Extend grokdump.py with simple BreakPad symbol files support.

tools/grokdump.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
 #     * Neither the name of Google Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived
 #       from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import bisect
 import cmd
 import ctypes
 import mmap
 import optparse
 import os
 import disasm

[Michael Starzinger, 2012/09/19 09:54:32]

I know you didn't touch, but can we alpha-sort the imports?

[Vyacheslav Egorov (Google), 2012/09/26 12:48:57]

Done.

 import sys
 import types
 import codecs
 import re
 import struct
 
 
 USAGE="""usage: %prog [OPTIONS] [DUMP-FILE]
 
 Minidump analyzer.
@@ skipping 127 matching lines @@
   ("checksum", ctypes.c_uint32),
   ("time_date_stampt", ctypes.c_uint32),
   ("flags", ctypes.c_uint64)
 ])
 
 MINIDUMP_LOCATION_DESCRIPTOR = Descriptor([
   ("data_size", ctypes.c_uint32),
   ("rva", ctypes.c_uint32)
 ])
 
+MINIDUMP_STRING = Descriptor([
+  ("length", ctypes.c_uint32),
+  ("buffer", lambda t: ctypes.c_uint8 * (t.length + 2))
+])
+
 MINIDUMP_DIRECTORY = Descriptor([
   ("stream_type", ctypes.c_uint32),
   ("location", MINIDUMP_LOCATION_DESCRIPTOR.ctype)
 ])
 
 MD_EXCEPTION_MAXIMUM_PARAMETERS = 15
 
 MINIDUMP_EXCEPTION = Descriptor([
   ("code", ctypes.c_uint32),
   ("flags", ctypes.c_uint32),
@@ skipping 200 matching lines @@
   ("ted", ctypes.c_uint64),
   ("stack", MINIDUMP_MEMORY_DESCRIPTOR.ctype),
   ("context", MINIDUMP_LOCATION_DESCRIPTOR.ctype)
 ])
 
 MINIDUMP_THREAD_LIST = Descriptor([
   ("thread_count", ctypes.c_uint32),
   ("threads", lambda t: MINIDUMP_THREAD.ctype * t.thread_count)
 ])
 
+MINIDUMP_RAW_MODULE = Descriptor([
+  ("base_of_image", ctypes.c_uint64),
+  ("size_of_image", ctypes.c_uint32),
+  ("checksum", ctypes.c_uint32),
+  ("time_date_stamp", ctypes.c_uint32),
+  ("module_name_rva", ctypes.c_uint32),
+  ("version_info", ctypes.c_uint32 * 13),
+  ("cv_record", MINIDUMP_LOCATION_DESCRIPTOR.ctype),
+  ("misc_record", MINIDUMP_LOCATION_DESCRIPTOR.ctype),
+  ("reserved0", ctypes.c_uint32 * 2),
+  ("reserved1", ctypes.c_uint32 * 2)
+])
+
+MINIDUMP_MODULE_LIST = Descriptor([
+  ("number_of_modules", ctypes.c_uint32),
+  ("modules", lambda t: MINIDUMP_RAW_MODULE.ctype * t.number_of_modules)
+])
+
 MINIDUMP_RAW_SYSTEM_INFO = Descriptor([
   ("processor_architecture", ctypes.c_uint16)
 ])
 
 MD_CPU_ARCHITECTURE_X86 = 0
 MD_CPU_ARCHITECTURE_AMD64 = 9
 
+class FuncSymbol:
+  def __init__(self, start, size, name):
+    self.start = start
+    self.end = self.start + size
+    self.name = name
+
+  def __cmp__(self, other):
+    if isinstance(other, FuncSymbol):
+      return self.start - other.start
+    return self.start - other
+
+  def Covers(self, addr):
+    return (self.start <= addr) and (addr < self.end)
+
 class MinidumpReader(object):
   """Minidump (.dmp) reader."""
 
   _HEADER_MAGIC = 0x504d444d
 
   def __init__(self, options, minidump_name):
     self.minidump_name = minidump_name
     self.minidump_file = open(minidump_name, "r")
     self.minidump = mmap.mmap(self.minidump_file.fileno(), 0, mmap.MAP_PRIVATE)
     self.header = MINIDUMP_HEADER.Read(self.minidump, 0)
     if self.header.signature != MinidumpReader._HEADER_MAGIC:
       print >>sys.stderr, "Warning: Unsupported minidump header magic!"
     DebugPrint(self.header)
     directories = []
     offset = self.header.stream_directories_rva
     for _ in xrange(self.header.stream_count):
       directories.append(MINIDUMP_DIRECTORY.Read(self.minidump, offset))
       offset += MINIDUMP_DIRECTORY.size
     self.arch = None
     self.exception = None
     self.exception_context = None
     self.memory_list = None
     self.memory_list64 = None
+    self.module_list = None
     self.thread_map = {}
 
+    self.symdir = options.symdir
+    self.modules_with_symbols = []
+    self.symbols = []
+
     # Find MDRawSystemInfo stream and determine arch.
     for d in directories:
       if d.stream_type == MD_SYSTEM_INFO_STREAM:
         system_info = MINIDUMP_RAW_SYSTEM_INFO.Read(
             self.minidump, d.location.rva)
         self.arch = system_info.processor_architecture
         assert self.arch in [MD_CPU_ARCHITECTURE_AMD64, MD_CPU_ARCHITECTURE_X86]
     assert not self.arch is None
 
     for d in directories:
       DebugPrint(d)
       if d.stream_type == MD_EXCEPTION_STREAM:
         self.exception = MINIDUMP_EXCEPTION_STREAM.Read(
           self.minidump, d.location.rva)
         DebugPrint(self.exception)
         if self.arch == MD_CPU_ARCHITECTURE_X86:
           self.exception_context = MINIDUMP_CONTEXT_X86.Read(
               self.minidump, self.exception.thread_context.rva)
         elif self.arch == MD_CPU_ARCHITECTURE_AMD64:
           self.exception_context = MINIDUMP_CONTEXT_AMD64.Read(
               self.minidump, self.exception.thread_context.rva)
         DebugPrint(self.exception_context)
       elif d.stream_type == MD_THREAD_LIST_STREAM:
         thread_list = MINIDUMP_THREAD_LIST.Read(self.minidump, d.location.rva)
         assert ctypes.sizeof(thread_list) == d.location.data_size
         DebugPrint(thread_list)
         for thread in thread_list.threads:
           DebugPrint(thread)
           self.thread_map[thread.id] = thread
+      elif d.stream_type == MD_MODULE_LIST_STREAM:
+        assert self.module_list is None
+        self.module_list = MINIDUMP_MODULE_LIST.Read(
+          self.minidump, d.location.rva)
+        assert ctypes.sizeof(self.module_list) == d.location.data_size
       elif d.stream_type == MD_MEMORY_LIST_STREAM:
         print >>sys.stderr, "Warning: This is not a full minidump!"
         assert self.memory_list is None
         self.memory_list = MINIDUMP_MEMORY_LIST.Read(
           self.minidump, d.location.rva)
         assert ctypes.sizeof(self.memory_list) == d.location.data_size
         DebugPrint(self.memory_list)
       elif d.stream_type == MD_MEMORY_64_LIST_STREAM:
         assert self.memory_list64 is None
         self.memory_list64 = MINIDUMP_MEMORY_LIST64.Read(
@@ skipping 163 matching lines @@
 
   def PointerSize(self):
     if self.arch == MD_CPU_ARCHITECTURE_AMD64:
       return 8
     elif self.arch == MD_CPU_ARCHITECTURE_X86:
       return 4
 
   def Register(self, name):
     return self.exception_context.__getattribute__(name)
 
+  def ReadMinidumpString(self, rva):
+    string = bytearray(MINIDUMP_STRING.Read(self.minidump, rva).buffer)
+    string = string.decode("utf16")
+    return string[0:len(string) - 1]
+
+  # Load FUNC records from a BreakPad symbol file
+  #
+  #    http://code.google.com/p/google-breakpad/wiki/SymbolFiles
+  #
+  def _LoadSymbolsFrom(self, symfile, baseaddr):
+    print "Loading symbols from %s" % (symfile)
+    funcs = []
+    with open(symfile) as f:
+      for line in f:
+        result = re.match(r"^FUNC ([a-f0-9]+) ([a-f0-9]+) ([a-f0-9]+) (.*)$", line)

[Michael Starzinger, 2012/09/19 09:54:32]

More than 80 characters.

[Vyacheslav Egorov (Google), 2012/09/26 12:48:57]

Done.

+        if result is not None:
+          start = int(result.group(1), 16)
+          size = int(result.group(2), 16)
+          name = result.group(4).rstrip()
+          bisect.insort_left(self.symbols, FuncSymbol(baseaddr + start, size, name))

[Michael Starzinger, 2012/09/19 09:54:32]

More than 80 characters.

[Vyacheslav Egorov (Google), 2012/09/26 12:48:57]

Done.

+    print " ... done"
+
+  def TryLoadSymbolsFor(self, modulename, module):
+    try:
+      symfile = os.path.join(self.symdir,
+                             modulename.replace('.', '_') + ".pdb.sym")
+      self._LoadSymbolsFrom(symfile, module.base_of_image)
+      self.modules_with_symbols.append(module)
+    except Exception as e:
+      print "  ... failure (%s)" % (e)
+
+  # Returns true if address is covered by some module that has loaded symbols.
+  def _IsInModuleWithSymbols(self, addr):
+    for module in self.modules_with_symbols:
+      start = module.base_of_image
+      end = start + module.size_of_image
+      if (start <= addr) and (addr < end):
+        return True
+    return False
+
+  # Find symbol covering the given address and return its name in format
+  #     <symbol name>+<offset from the start>
+  def FindSymbol(self, addr):
+    if not self._IsInModuleWithSymbols(addr):
+      return None
+
+    i = bisect.bisect_left(self.symbols, addr)
+    symbol = None
+    if (0 < i) and self.symbols[i - 1].Covers(addr):
+      symbol = self.symbols[i - 1]
+    elif (i < len(self.symbols)) and self.symbols[i].Covers(addr):
+      symbol = self.symbols[i]
+    else:
+      return None
+    diff = addr - symbol.start
+    return "%s+0x%x" % (symbol.name, diff)
+
+
 
 # List of V8 instance types. Obtained by adding the code below to any .cc file.
 #
 # #define DUMP_TYPE(T) printf("  %d: \"%s\",\n", T, #T);
 # struct P {
 #   P() {
 #     printf("INSTANCE_TYPES = {\n");
 #     INSTANCE_TYPE_LIST(DUMP_TYPE)
 #     printf("}\n");
 #   }
@@ skipping 975 matching lines @@
 EIP_PROXIMITY = 64
 
 CONTEXT_FOR_ARCH = {
     MD_CPU_ARCHITECTURE_AMD64:
       ['rax', 'rbx', 'rcx', 'rdx', 'rdi', 'rsi', 'rbp', 'rsp', 'rip',
        'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15'],
     MD_CPU_ARCHITECTURE_X86:
       ['eax', 'ebx', 'ecx', 'edx', 'edi', 'esi', 'ebp', 'esp', 'eip']
 }
 
+KNOWN_MODULES = {'chrome.exe', 'chrome.dll'}
+
+def GetModuleName(reader, module):
+  name = reader.ReadMinidumpString(module.module_name_rva)
+  return str(os.path.basename(str(name).replace("\\", "/")))
 
 def AnalyzeMinidump(options, minidump_name):
   reader = MinidumpReader(options, minidump_name)
   heap = None
   DebugPrint("========================================")
   if reader.exception is None:
     print "Minidump has no exception info"
   else:
     print "Exception info:"
     exception_thread = reader.thread_map[reader.exception.thread_id]
     print "  thread id: %d" % exception_thread.id
     print "  code: %08X" % reader.exception.exception.code
     print "  context:"
     for r in CONTEXT_FOR_ARCH[reader.arch]:
       print "    %s: %s" % (r, reader.FormatIntPtr(reader.Register(r)))
     # TODO(vitalyr): decode eflags.
     print "    eflags: %s" % bin(reader.exception_context.eflags)[2:]
     print

[Michael Starzinger, 2012/09/19 09:54:32]

Can we move this empty print to after the modules are printed?

[Vyacheslav Egorov (Google), 2012/09/26 12:48:57]

I will add another one after modules. I like some empty lines, they make reading
easier.

+    print "  Modules:"

[Michael Starzinger, 2012/09/19 09:54:32]

Lowercase "modules:" would be consistent with the rest of the exception info.

[Vyacheslav Egorov (Google), 2012/09/26 12:48:58]

Done.

+    for module in reader.module_list.modules:
+      name = GetModuleName(reader, module)
+      if name in KNOWN_MODULES:
+        print "    %s at %08X" % (name, module.base_of_image)
+        reader.TryLoadSymbolsFor(name, module)
 
     stack_top = reader.ExceptionSP()
     stack_bottom = exception_thread.stack.start + \
         exception_thread.stack.memory.data_size
     stack_map = {reader.ExceptionIP(): -1}
     for slot in xrange(stack_top, stack_bottom, reader.PointerSize()):
       maybe_address = reader.ReadUIntPtr(slot)
       if not maybe_address in stack_map:
         stack_map[maybe_address] = slot
     heap = V8Heap(reader, stack_map)
 
     print "Disassembly around exception.eip:"
+    print reader.FindSymbol(reader.ExceptionIP())

[Michael Starzinger, 2012/09/19 09:54:32]

This will print "None" when no symbol is found, can we skip printing in that
case? I think the "None" in the output is confusing.

[Vyacheslav Egorov (Google), 2012/09/26 12:48:58]

Done.

     disasm_start = reader.ExceptionIP() - EIP_PROXIMITY
     disasm_bytes = 2 * EIP_PROXIMITY
     if (options.full):
       full_range = reader.FindRegion(reader.ExceptionIP())
       if full_range is not None:
         disasm_start = full_range[0]
         disasm_bytes = full_range[1]
 
     lines = reader.GetDisasmLines(disasm_start, disasm_bytes)
 
     for line in lines:
       print FormatDisasmLine(disasm_start, heap, line)
     print
 
   if heap is None:
     heap = V8Heap(reader, None)
 
   if options.full:
     FullDump(reader, heap)
 
   if options.shell:
     InspectionShell(reader, heap).cmdloop("type help to get help")
   else:
     if reader.exception is not None:
       print "Annotated stack (from exception.esp to bottom):"
       for slot in xrange(stack_top, stack_bottom, reader.PointerSize()):
         maybe_address = reader.ReadUIntPtr(slot)
         heap_object = heap.FindObject(maybe_address)
-        print "%s: %s" % (reader.FormatIntPtr(slot),
-                          reader.FormatIntPtr(maybe_address))
+        maybe_symbol = reader.FindSymbol(maybe_address)
+        if maybe_symbol is None:
+          maybe_symbol = ""
+
+        print "%s: %s %s" % (reader.FormatIntPtr(slot),
+                             reader.FormatIntPtr(maybe_address),
+                             maybe_symbol)

[Michael Starzinger, 2012/09/19 09:54:32]

Just use "maybe_symbol or ''" here and drop the if statement above.

[Vyacheslav Egorov (Google), 2012/09/26 12:48:58]

Done.

         if heap_object:
           heap_object.Print(Printer())
           print
 
   reader.Dispose()
 
 
 if __name__ == "__main__":
   parser = optparse.OptionParser(USAGE)
   parser.add_option("-s", "--shell", dest="shell", action="store_true",
                     help="start an interactive inspector shell")
   parser.add_option("-f", "--full", dest="full", action="store_true",
                     help="dump all information contained in the minidump")
+  parser.add_option("--symdir", dest="symdir", default=".",
+                    help="directory containing *.pdb.sym file with symbols")
   options, args = parser.parse_args()
   if len(args) != 1:
     parser.print_help()
     sys.exit(1)
   AnalyzeMinidump(options, args[0])