Check that all symbolized methods in the output orderfile are in their own linker section in the or…

tools/cygprofile/symbolize.py

 #!/usr/bin/python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Symbolize log file produced by cypgofile instrumentation.
 
 Given a log file and the binary being profiled (e.g. executable, shared
 library), the script can produce three different outputs: 1) symbols for the
 addresses, 2) function and line numbers for the addresses, or 3) an order file.
@@ skipping 45 matching lines @@
   for call_line in call_lines:
     (sec_timestamp, usec_timestamp) = map(int, call_line[0:2])
     callee_id = call_line[2]
     addr = int(call_line[3], 16)
     if vm_start < addr:
       addr -= vm_start
       call_info.append((sec_timestamp, usec_timestamp, callee_id, addr))
 
   return call_info
 
+def GetStdOutputLines(cmd):
+  p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+  output = p.communicate()[0]
+  return output.split('\n')
 
 def ParseLibSymbols(lib_file):
   """Get output from running nm and greping for text symbols.
 
   Args:
     lib_file: the library or executable that contains the profiled code
 
   Returns:
     list of sorted unique addresses and corresponding size of function symbols
     in lib_file and map of addresses to all symbols at a particular address
   """
   cmd = ['nm', '-S', '-n', lib_file]
-  nm_p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-  output = nm_p.communicate()[0]
-  nm_lines = output.split('\n')
+  nm_lines = GetStdOutputLines(cmd)
 
   nm_symbols = []
   for nm_line in nm_lines:
     if any(str in nm_line for str in (' t ', ' W ', ' T ')):
       nm_symbols.append(nm_line)
 
   nm_index = 0
   unique_addrs = []
   address_map = {}
   while nm_index < len(nm_symbols):
@@ skipping 72 matching lines @@
 
 def FindFunctions(addr, unique_addrs, address_map):
   """Find function symbol names at address addr."""
   return address_map[BinarySearchAddresses(addr, 0, len(unique_addrs) - 1,
                                            unique_addrs)]
 
 
 def AddrToLine(addr, lib_file):
   """Use addr2line to determine line info of a particular address."""
   cmd = ['addr2line', '-f', '-e', lib_file, hex(addr)]
-  p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-  output = (p.communicate()[0]).split('\n')
-  line = output[0]
-  index = 1
-  while index < len(output):
-    line = line + ':' + output[index]
-    index += 1
-  return line
+  output = GetStdOutputLines(cmd)
+  assert(len(output) == 2)
+  return ':'.join(output)
 
+def GetObjectFileNames(obj_dir):
+  """ Gets the list of object files in the output directory. """
+  obj_files = []
+  for (dirpath, dirnames, filenames) in os.walk(obj_dir):
+    for file_name in filenames:
+      if file_name.endswith('.o'):
+        obj_files.append(os.path.join(dirpath, file_name))
+  return obj_files
+
+class WarningCollector(object):
+  def __init__(self, max_warnings):
+    self._warnings = 0
+    self._max_warnings = max_warnings
+
+  def Write(self, message):
+    if self._warnings < self._max_warnings:
+      sys.stderr.write(message)
+    self._warnings += 1
+
+  def WriteEnd(self, message):
+    if self._warnings > self._max_warnings:
+      sys.stderr.write(str(self._warnings - self._max_warnings) +
+                       ' more warnings for: ' + message)

[pasko, 2014/12/11 10:25:30]

does not have a '\n'

[azarchs, 2014/12/11 17:01:15]

Done.

+
+def SymbolToSection(obj_dir):
+  """ Gets a mapping from symbol to linker section name by scanning all
+      of the object files. """
+  object_files = GetObjectFileNames(obj_dir)
+  symbol_to_section_map = {}
+  symbol_warnings = WarningCollector(300)
+  for obj_file in object_files:
+    cmd = ['objdump', '-w', '-t', obj_file]
+    symbol_lines = GetStdOutputLines(cmd)
+    for symbol_line in symbol_lines:
+      items = symbol_line.split()
+      # All of the symbol lines we care about are in the form
+      # 0000000000  g    F   .text.foo     000000000 [.hidden] foo
+      # where g (global) might also be l (local) or w (weak).
+      if len(items) > 4 and items[2] == 'F':
+        # This symbol is a function
+        symbol = items[len(items) - 1]
+        if symbol.startswith('.LTHUNK'):
+          continue
+        section = items[3]
+        if ((symbol in symbol_to_section_map) and
+            (symbol_to_section_map[symbol] != section)):
+          symbol_warnings.Write('WARNING: Symbol ' + symbol +
+                                ' in conflicting sections ' + section +
+                                ' and ' + symbol_to_section_map[symbol] + '\n')

[pasko, 2014/12/11 10:25:30]

please move the '\n' to WarningCollector.Write, less repetitions

[azarchs, 2014/12/11 17:01:14]

Done.

+        elif not section.startswith('.text.'):
+          symbol_warnings.Write('WARNING: Symbol ' + symbol +
+                                ' in incorrect section ' + section + '\n')
+        else:
+          symbol_to_section_map[symbol] = section
+  symbol_warnings.WriteEnd('bad sections')
+  return symbol_to_section_map
 
 def main():
   """Write output for profiled run to standard out.
 
   The format of the output depends on the output type specified as the third
   command line argument.  The default output type is to symbolize the addresses
   of the functions called.
   """
   parser = optparse.OptionParser('usage: %prog [options] log_file lib_file')
   parser.add_option('-t', '--outputType', dest='output_type',
                     default='symbolize', type='string',
                     help='lineize or symbolize or orderfile')
 
   # Option for output type.  The log file and lib file arguments are required
   # by the script and therefore are not options.
   (options, args) = parser.parse_args()
   if len(args) != 2:
     parser.error('expected 2 args: log_file lib_file')
 
   (log_file, lib_file) = args
   output_type = options.output_type
 
+  obj_dir = os.path.abspath(os.path.join(os.path.dirname(lib_file), '../obj'))
+
   lib_name = lib_file.split('/')[-1].strip()
   log_file_lines = map(string.rstrip, open(log_file).readlines())
   call_info = ParseLogLines(log_file_lines)
   (unique_addrs, address_map) = ParseLibSymbols(lib_file)
 
   # Check for duplicate addresses in the log file, and print a warning if
   # duplicates are found. The instrumentation that produces the log file
   # should only print the first time a function is entered.
   addr_list = []
   for call in call_info:
     addr = call[3]
     if addr not in addr_list:
       addr_list.append(addr)
     else:
       print('WARNING: Address ' + hex(addr) + ' (line= ' +
             AddrToLine(addr, lib_file) + ') already profiled.')
 
+  symbol_to_section_map = SymbolToSection(obj_dir)
+
+  unknown_symbol_warnings = WarningCollector(300)
+  symbol_not_found_warnings = WarningCollector(300)
   for call in call_info:
+    addr = call[3]
     if output_type == 'lineize':
-      symbol = AddrToLine(call[3], lib_file)
+      symbol = AddrToLine(addr, lib_file)
       print(str(call[0]) + ' ' + str(call[1]) + '\t' + str(call[2]) + '\t'
             + symbol)
     elif output_type == 'orderfile':
       try:
-        symbols = FindFunctions(call[3], unique_addrs, address_map)
+        symbols = FindFunctions(addr, unique_addrs, address_map)
         for symbol in symbols:
-          print '.text.' + symbol
+          if symbol in symbol_to_section_map:
+            print symbol_to_section_map[symbol]
+          else:
+            unknown_symbol_warnings.write(

[pasko, 2014/12/11 10:25:30]

s/write/Write/ here and below

[azarchs, 2014/12/11 17:01:14]

Done.

+                'WARNING: No known section for symbol ' + symbol + '\n')
         print ''
       except SymbolNotFoundException as e:
-        sys.stderr.write('WARNING: Did not find function in binary. addr: '
-                      + hex(addr) + '\n')
+        symbol_not_found_warnings.write(
+            'WARNING: Did not find function in binary. addr: '
+            + hex(addr) + '\n')
     else:
       try:
-        symbols = FindFunctions(call[3], unique_addrs, address_map)
+        symbols = FindFunctions(addr, unique_addrs, address_map)
         print(str(call[0]) + ' ' + str(call[1]) + '\t' + str(call[2]) + '\t'
               + symbols[0])
         first_symbol = True
         for symbol in symbols:
           if not first_symbol:
             print '\t\t\t\t\t' + symbol
           else:
             first_symbol = False
       except SymbolNotFoundException as e:
-        sys.stderr.write('WARNING: Did not find function in binary. addr: '
-                      + hex(addr) + '\n')
+        symbol_not_found_warnings.write(
+            'WARNING: Did not find function in binary. addr: '
+            + hex(addr) + '\n')
+  unknown_symbol_warnings.WriteEnd('no known section for symbol')
+  symbol_not_found_warnings.WriteEnd('did not find function')
 
 if __name__ == '__main__':
   main()