[Memory Sheriff] --top-symbols for waterfall match

tools/valgrind/test_suppressions.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import argparse

[Lei Zhang, 2013/05/07 20:39:56]

Is everyone running an OS on their desktop with Python 2.7 these days? I
certainly hope so.

[groby-ooo-7-16, 2013/05/07 20:57:59]

Seems to be the case.
On 2013/05/07 20:39:56, Lei Zhang wrote:

 from collections import defaultdict
 import os
 import re
+import subprocess
 import sys
 
 import suppressions
 
 
 def ReadReportsFromFile(filename):
   """ Returns a list of (report_hash, report) and the URL of the report on the
   waterfall.
   """
   input_file = file(filename, 'r')
@@ skipping 19 matching lines @@
       else:
         cur_supp += [" "*3 + line]
     elif line == "{":
       in_suppression = True
       cur_supp = ["{"]
     elif line.find("Suppression (error hash=#") == 0:
       last_hash = line[25:41]
   # The line at the end of the file is assumed to store the URL of the report.
   return reports,line
 
+def Demangle(names):
+  """ Demangle a list of C++ symbols, return a list of human-readable symbols.
+  """
+  args = ['c++filt','-n']
+  args.extend(names)
+  pipe = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  stdout, _ = pipe.communicate()
+  demangled = stdout.split("\n")
+
+  # Each line ends with a newline, so the final entry of the split output
+  # will always be ''.
+  assert len(demangled) == len(names)+1

[Lei Zhang, 2013/05/07 20:39:56]

style nit: spaces around +

[groby-ooo-7-16, 2013/05/07 20:57:59]

Done.

+  return demangled[:-1]
+
+def GetSymbolsFromReport(report):
+  """Extract all symbols from a suppression report."""
+  symbols = []
+  for line in report.splitlines():
+    index = line.find('fun:')
+    if index != -1:
+      symbols.append(line[index+4:])

[Lei Zhang, 2013/05/07 20:39:56]

index + len('fun:'), or have some constant that indicate where 4 came from.

[groby-ooo-7-16, 2013/05/07 20:57:59]

Done.

+  return symbols
+
+def PrintTopSymbols(symbol_reports, top_count):
+  """Print the |top_count| symbols with the most occurrences."""
+  boring_symbols=['malloc', '_Znw*', 'TestBody']

[Lei Zhang, 2013/05/07 20:39:56]

Use BoringCallers() in common.py?

[groby-ooo-7-16, 2013/05/07 20:57:59]

BoringCallers ignores different boringness. Per offline discussion, leaving this
instead of amending BoringCallers
On 2013/05/07 20:39:56, Lei Zhang wrote:

+  sorted_reports = sorted(filter(lambda x:x[0] not in boring_symbols,
+                                 symbol_reports.iteritems()),
+                          key=lambda x:len(x[1]), reverse=True)
+  symbols = symbol_reports.keys()
+  demangled = Demangle(symbols)
+  assert len(demangled) == len(symbols)
+  symboltable = dict(zip(symbols,demangled))

[Lei Zhang, 2013/05/07 20:39:56]

style nit: space after comma.

[groby-ooo-7-16, 2013/05/07 20:57:59]

Done.

+
+  print "\n"
+  print "Top %d symbols" % top_count
+  for (symbol, suppressions) in sorted_reports[:top_count]:
+    print "%4d occurrences : %s" % (len(suppressions), symboltable[symbol])
 
 def main(argv):
   supp = suppressions.GetSuppressions()
 
   # all_reports is a map {report: list of urls containing this report}
   all_reports = defaultdict(list)
   report_hashes = {}
+  symbol_reports = defaultdict(list)
 
-  for f in argv:
+  # Create argument parser.
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--top-symbols', type=int, default=0,
+    help='Print a list of the top <n> symbols')
+  parser.add_argument('reports', metavar='report file', nargs='+',
+    help='List of report files')
+  args = parser.parse_args(argv)
+
+  for f in args.reports:
     f_reports, url = ReadReportsFromFile(f)
     for (hash, report) in f_reports:
       all_reports[report] += [url]
       report_hashes[report] = hash
 
   reports_count = 0
   for r in all_reports:
     cur_supp = supp['common_suppressions']
     if all([re.search("%20Mac%20|mac_valgrind", url)
             for url in all_reports[r]]):
@@ skipping 21 matching lines @@
       reports_count += 1
       print "==================================="
       print "This report observed at"
       for url in all_reports[r]:
         print "  %s" % url
       print "didn't match any suppressions:"
       print "Suppression (error hash=#%s#):" % (report_hashes[r])
       print r
       print "==================================="
 
+      if args.top_symbols > 0:
+        symbols = GetSymbolsFromReport(r)
+        for symbol in symbols:
+          symbol_reports[symbol].append(report_hashes[r])
+
   if reports_count > 0:
     print ("%d unique reports don't match any of the suppressions" %
            reports_count)
+    if args.top_symbols > 0:
+      PrintTopSymbols(symbol_reports, args.top_symbols)
+
   else:
     print "Congratulations! All reports are suppressed!"
     # TODO(timurrrr): also make sure none of the old suppressions
     # were narrowed too much.
 
 
 if __name__ == "__main__":
   main(sys.argv[1:])