Add an end-to-end test for symbolize_trace on macOS.

tracing/tracing/extras/symbolizer/symbolize_trace.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 # pylint: disable=too-many-lines
 
 """
 This script processes trace files and symbolizes stack frames generated by
 Chrome's native heap profiler. This script assumes that the Chrome binary
 referenced in the trace contains symbols, and is the same binary used to emit
@@ skipping 219 matching lines @@
     os.path.join(os.path.dirname(__file__), '..', '..', '..'))
 
 _SYMBOLS_PATH = os.path.abspath(os.path.join(
     _TRACING_DIR,
     'third_party',
     'symbols'))
 sys.path.append(_SYMBOLS_PATH)
 # pylint: disable=import-error
 import symbols.elf_symbolizer as elf_symbolizer
 
-from . import symbolize_trace_atos_regex
-from . import symbolize_trace_macho_reader
+from tracing.extras.symbolizer import symbolize_trace_atos_regex
+from tracing.extras.symbolizer import symbolize_trace_macho_reader
 
 _PY_UTILS_PATH = os.path.abspath(os.path.join(
     _TRACING_DIR,
     '..',
     'common',
     'py_utils'))
 sys.path.append(_PY_UTILS_PATH)
 # pylint: disable=import-error
 import py_utils.cloud_storage as cloud_storage
 
+_UNNAMED_FILE = 'unnamed'
+
+
 class NodeWrapper(object):
   """Wraps an event data node(s).
 
   A node is a reference into a trace event JSON. Wrappers parse nodes to
   provide convenient APIs and update nodes when asked to propagate changes
   back (see ApplyModifications() below).
 
   Here is an example of legacy metadata event that contains stack frame tree:
 
   {
@@ skipping 48 matching lines @@
       },
       ...
     ]
   }
   """
 
   class Region(object):
     def __init__(self, start_address, size, file_path):
       self._start_address = start_address
       self._size = size
-      self._file_path = file_path
+      self._file_path = file_path if file_path else _UNNAMED_FILE
 
     @property
     def start_address(self):
       return self._start_address
 
     @property
     def end_address(self):
       return self._start_address + self._size
 
     @property
@@ skipping 648 matching lines @@
   """Holds file path, addresses to symbolize and stack frames to update.
 
   This class is a link between ELFSymbolizer and a trace file: it specifies
   what to symbolize (addresses) and what to update with the symbolization
   result (frames).
   """
   def __init__(self, file_path):
     self.path = file_path
     self.symbolizable_path = file_path # path to use for symbolization
     self.frames_by_address = collections.defaultdict(list)
+    self.skip_symbolization = False
 
 
 def ResolveSymbolizableFiles(processes):
   """Resolves and groups PCs into list of SymbolizableFiles.
 
   As part of the grouping process, this function resolves PC from each stack
   frame to the corresponding mmap region. Stack frames that failed to resolve
   are symbolized with '<unresolved>'.
   """
   symfile_by_path = {}
   for process in processes:
     if not process.memory_map:
       continue
     for frame in process.stack_frame_map.frame_by_id.itervalues():
       if frame.pc is None:
         continue
       region = process.memory_map.FindRegion(frame.pc)
       if region is None:
         frame.name = '<unresolved>'
         continue
 
       symfile = symfile_by_path.get(region.file_path)
       if symfile is None:
-        symfile = SymbolizableFile(region.file_path)
+        file_path = region.file_path
+        symfile = SymbolizableFile(file_path)
         symfile_by_path[symfile.path] = symfile
 
       relative_pc = frame.pc - region.start_address
       symfile.frames_by_address[relative_pc].append(frame)
   return symfile_by_path.values()
 
 
 def FindInSystemPath(binary_name):
   paths = os.environ['PATH'].split(os.pathsep)
   for path in paths:
@@ skipping 11 matching lines @@
     self.is_win = sys.platform == 'win32'
     if self.is_mac:
       self.binary = 'atos'
       self._matcher = symbolize_trace_atos_regex.AtosRegexMatcher()
     elif self.is_win:
       self.binary = 'addr2line-pdb.exe'
     else:
       self.binary = 'addr2line'
     self.symbolizer_path = FindInSystemPath(self.binary)
 
-  def _SymbolizeLinuxAndAndroid(self, symfile, unsymbolized_name):
+  def _SymbolizeLinuxAndAndroid(self, symfile):
     def _SymbolizerCallback(sym_info, frames):
       # Unwind inline chain to the top.
       while sym_info.inlined_by:
         sym_info = sym_info.inlined_by
 
-      symbolized_name = sym_info.name if sym_info.name else unsymbolized_name
+      symbolized_name = (sym_info.name if sym_info.name else
+                         '<{}>'.format(symfile.path))
       for frame in frames:
         frame.name = symbolized_name
 
     symbolizer = elf_symbolizer.ELFSymbolizer(symfile.symbolizable_path,
                                               self.symbolizer_path,
                                               _SymbolizerCallback,
                                               inlines=True)
 
     for address, frames in symfile.frames_by_address.iteritems():
       # SymbolizeAsync() asserts that the type of address is int. We operate
@@ skipping 53 matching lines @@
     stdout_data = stdout_data.split('\n')
 
     # This is known to be in the same order as stderr_data.
     for i, addr in enumerate(addrs):
       for frame in  symfile.frames_by_address[int(addr, 16)]:
         # Output of addr2line with --functions is always 2 outputs per
         # symbol, function name followed by source line number. Only grab
         # the function name as line info is not always available.
         frame.name = stdout_data[i * 2]
 
-  def Symbolize(self, symfile, unsymbolized_name):
+  def SymbolizeSymfile(self, symfile):
+    if symfile.skip_symbolization:
+      for address, frames in symfile.frames_by_address.iteritems():
+        unsymbolized_name = ('<' + os.path.basename(symfile.symbolizable_path)
+                             + '>')
+        # Only append the address if there's a library.
+        if symfile.symbolizable_path != _UNNAMED_FILE:
+          unsymbolized_name += ' + ' + str(hex(address))
+
+        for frame in frames:
+          frame.name = unsymbolized_name
+      return
+
     if self.is_mac:
       self._SymbolizeMac(symfile)
     elif self.is_win:
       self._SymbolizeWin(symfile)
     else:
-      self._SymbolizeLinuxAndAndroid(symfile, unsymbolized_name)
+      self._SymbolizeLinuxAndAndroid(symfile)
 
   def IsSymbolizableFile(self, file_path):
     if self.is_win:
       extension = os.path.splitext(file_path)[1].lower()
       return extension in ['.dll', '.exe']
     else:
       result = subprocess.check_output(['file', '-0', file_path])
       type_string = result[result.find('\0') + 1:]
       return bool(re.match(r'.*(ELF|Mach-O) (32|64)-bit\b.*',
                            type_string, re.DOTALL))
 
 
 def SymbolizeFiles(symfiles, symbolizer):
   """Symbolizes each file in the given list of SymbolizableFiles
      and updates stack frames with symbolization results."""
 
   if not symfiles:
     print 'Nothing to symbolize.'
     return
 
   print 'Symbolizing...'
 
   def _SubPrintf(message, *args):
     print ('  ' + message).format(*args)
 
   for symfile in symfiles:
-    unsymbolized_name = '<{}>'.format(
-        symfile.path if symfile.path else 'unnamed')
-
     problem = None
     if not os.path.isabs(symfile.symbolizable_path):
       problem = 'not a file'
     elif not os.path.isfile(symfile.symbolizable_path):
       problem = "file doesn't exist"
     elif not symbolizer.IsSymbolizableFile(symfile.symbolizable_path):
       problem = 'file is not symbolizable'
     if problem:
-      _SubPrintf("Won't symbolize {} PCs for '{}': {}.",
-                 len(symfile.frames_by_address),
+      _SubPrintf("Problem with '{}': {}.",
                  symfile.symbolizable_path,
                  problem)
-      for frames in symfile.frames_by_address.itervalues():
-        for frame in frames:
-          frame.name = unsymbolized_name
-      continue
+      symfile.skip_symbolization = True
 
     _SubPrintf('Symbolizing {} PCs from {}...',
                len(symfile.frames_by_address),
                symfile.symbolizable_path)
 
-    symbolizer.Symbolize(symfile, unsymbolized_name)
+    symbolizer.SymbolizeSymfile(symfile)
 
 
 # Matches Android library paths, supports both K (/data/app-lib/<>/lib.so)
 # as well as L+ (/data/app/<>/lib/<>/lib.so). Library name is available
 # via 'name' group.
 ANDROID_PATH_MATCHER = re.compile(
     r'^/data/(?:'
     r'app/[^/]+/lib/[^/]+/|'
     r'app-lib/[^/]+/|'
     r'data/[^/]+/incremental-install-files/lib/'
@@ skipping 14 matching lines @@
       name = match.group('name')
       symfile.symbolizable_path = os.path.join(
           output_path, ANDROID_UNSTRIPPED_SUBPATH, name)
     else:
       # Clobber file path to trigger "not a file" problem in SymbolizeFiles().
       # Without this, files won't be symbolized with "file not found" problem,
       # which is not accurate.
       symfile.symbolizable_path = 'android://{}'.format(symfile.path)
 
 
-def RemapMacFiles(symfiles, symbol_base_directory, version):
+def RemapMacFiles(symfiles, symbol_base_directory, version,
+                  only_symbolize_chrome_symbols):
   suffix = ("Google Chrome Framework.dSYM/Contents/Resources/DWARF/"
             "Google Chrome Framework")
   symbol_sub_dir = os.path.join(symbol_base_directory, version)
   symbolizable_path = os.path.join(symbol_sub_dir, suffix)
 
   for symfile in symfiles:
     if symfile.path.endswith("Google Chrome Framework"):
       symfile.symbolizable_path = symbolizable_path
+    elif only_symbolize_chrome_symbols:
+      symfile.skip_symbolization = True
 
-def RemapWinFiles(symfiles, symbol_base_directory, version, is64bit):
+def RemapWinFiles(symfiles, symbol_base_directory, version, is64bit,
+                  only_symbolize_chrome_symbols):
   folder = "win64" if is64bit else "win"
   symbol_sub_dir = os.path.join(symbol_base_directory,
                                 "chrome-" + folder + "-" + version)
   for symfile in symfiles:
     image = os.path.join(symbol_sub_dir, os.path.basename(symfile.path))
     symbols = image + ".pdb"
     if os.path.isfile(image) and os.path.isfile(symbols):
       symfile.symbolizable_path = image
+    elif only_symbolize_chrome_symbols:
+      symfile.skip_symbolization = True
 
-def Symbolize(options, trace, symbolizer):
+def SymbolizeTrace(options, trace, symbolizer):
   symfiles = ResolveSymbolizableFiles(trace.processes)
 
   # Android trace files don't have any indication they are from Android.
   # So we're checking for Android-specific paths.
   if HaveFilesFromAndroid(symfiles):
     if not options.output_directory:
       sys.exit('The trace file appears to be from Android. Please '
                'specify output directory to properly symbolize it.')
     RemapAndroidFiles(symfiles, os.path.abspath(options.output_directory))
 
 
   if not trace.is_chromium:
+    # A non-chromium trace probably is not coming from the current machine.
+    # Don't attempt to symbolize system symbols, as that will produce the wrong
+    # results.
+    options.only_symbolize_chrome_symbols = True
     if symbolizer.is_mac:
-      RemapMacFiles(symfiles, options.symbol_base_directory, trace.version)
+      RemapMacFiles(symfiles, options.symbol_base_directory, trace.version,
+                    options.only_symbolize_chrome_symbols)
     if symbolizer.is_win:
       RemapWinFiles(symfiles, options.symbol_base_directory, trace.version,
-                    trace.is_64bit)
+                    trace.is_64bit, options.only_symbolize_chrome_symbols)
 
   SymbolizeFiles(symfiles, symbolizer)
 
 
 def OpenTraceFile(file_path, mode):
   if file_path.endswith('.gz'):
     return gzip.open(file_path, mode + 'b')
   else:
     return open(file_path, mode + 't')
 
 
-def FetchAndExtractSymbolsMac(symbol_base_directory, version):
+def FetchAndExtractSymbolsMac(symbol_base_directory, version,
+                              cloud_storage_bucket):
   def GetLocalPath(base_dir, version):
     return os.path.join(base_dir, version + ".tar.bz2")
   def GetSymbolsPath(version):
     return "desktop-*/" + version + "/mac64/Google Chrome.dSYM.tar.bz2"
   def ExtractSymbolTarFile(symbol_sub_dir, symbol_tar_file):
     os.makedirs(symbol_sub_dir)
     with tarfile.open(os.path.expanduser(symbol_tar_file), "r:bz2") as tar:
       tar.extractall(symbol_sub_dir)
 
   symbol_sub_dir = os.path.join(symbol_base_directory, version)
   if os.path.isdir(symbol_sub_dir):
     return True
 
   bzip_path = GetLocalPath(symbol_base_directory, version)
   if not os.path.isfile(bzip_path):
 
-    cloud_storage_bucket = "chrome-unsigned"
     if not cloud_storage.Exists(cloud_storage_bucket, GetSymbolsPath(version)):
       print "Can't find symbols on GCS."
       return False
     print "Downloading symbols files from GCS, please wait."
     cloud_storage.Get(cloud_storage_bucket, GetSymbolsPath(version), bzip_path)
 
   ExtractSymbolTarFile(symbol_sub_dir, bzip_path)
   return True
 
 
-def FetchAndExtractSymbolsWin(symbol_base_directory, version, is64bit):
+def FetchAndExtractSymbolsWin(symbol_base_directory, version, is64bit,
+                              cloud_storage_bucket):
   def DownloadAndExtractZipFile(zip_path, source, destination):
     if not os.path.isfile(zip_path):
-      cloud_storage_bucket = "chrome-unsigned"
       if not cloud_storage.Exists(cloud_storage_bucket, source):
         print "Can't find symbols on GCS."
         return False
       print "Downloading symbols files from GCS, please wait."
       cloud_storage.Get(cloud_storage_bucket, source, zip_path)
       if not os.path.isfile(zip_path):
         print "Can't download symbols on GCS."
         return False
     with zipfile.ZipFile(zip_path, "r") as zip_file:
       for member in zip_file.namelist():
@@ skipping 39 matching lines @@
       help='Trace file to symbolize (.json or .json.gz)')
 
   parser.add_argument(
       '--no-backup', dest='backup', default='true', action='store_false',
       help="Don't create {} files".format(BACKUP_FILE_TAG))
 
   parser.add_argument(
       '--output-directory',
       help='The path to the build output directory, such as out/Debug.')
 
+  parser.add_argument(
+      '--only-symbolize-chrome-symbols',
+      action='store_true',
+      help='Prevents symbolization of non-Chrome [system] symbols.')
+
+  parser.add_argument(
+      '--cloud-storage-bucket', default='chrome-unsigned',
+      help="Bucket that holds symbols for official Chrome builds. "
+           "Used by tests, which don't have access to the default bucket.")
+
   home_dir = os.path.expanduser('~')
   default_dir = os.path.join(home_dir, "symbols")
   parser.add_argument(
       '--symbol-base-directory',
       default=default_dir,
       help='Directory where symbols are downloaded and cached.')
 
   symbolizer = Symbolizer()
   if symbolizer.symbolizer_path is None:
     sys.exit("Can't symbolize - no %s in PATH." % symbolizer.binary)
@@ skipping 12 matching lines @@
     return False
 
   # If the trace is from Chromium, assume that symbols are already present.
   # Otherwise the trace is from Google Chrome. Assume that this is not a local
   # build of Google Chrome with symbols, and that we need to fetch symbols
   # from gcs.
   if not trace.is_chromium:
     has_symbols = False
     if symbolizer.is_mac:
       has_symbols = FetchAndExtractSymbolsMac(options.symbol_base_directory,
-                                              trace.version)
+                                              trace.version,
+                                              options.cloud_storage_bucket)
     if symbolizer.is_win:
       has_symbols = FetchAndExtractSymbolsWin(options.symbol_base_directory,
-                                              trace.version, trace.is_64bit)
+                                              trace.version, trace.is_64bit,
+                                              options.cloud_storage_bucket)
     if not has_symbols:
       print 'Cannot fetch symbols from GCS'
       return False
 
-  Symbolize(options, trace, symbolizer)
+  SymbolizeTrace(options, trace, symbolizer)
 
   if trace.modified:
     trace.ApplyModifications()
 
     if options.backup:
       backup_file_path = trace_file_path + BACKUP_FILE_TAG
       print 'Backing up trace file to {}'.format(backup_file_path)
       os.rename(trace_file_path, backup_file_path)
 
     print 'Updating the trace file...'
     with OpenTraceFile(trace_file_path, 'w') as trace_file:
       json.dump(trace.node, trace_file)
   else:
     print 'No modifications were made - not updating the trace file.'
+  return True
 
 
 if __name__ == '__main__':
   main(sys.argv[1:])