Improved GC statistics (cumulative and per collection).

tools/gc-nvp-trace-processor.py

+#!/usr/bin/env python
+#
+# Copyright 2010 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# This is an utility for plotting charts based on GC traces produced by V8 when
+# run with flags --trace-gc --trace-gc-nvp. Relies on gnuplot for actual
+# plotting.
+#
+# Usage: gc-nvp-trace-processor.py <GC-trace-filename>
+#
+
+
+from __future__ import with_statement
+import sys, types, re, subprocess
+
+def flatten(l):
+  flat = []
+  for i in l: flat.extend(i)
+  return flat
+
+def split_nvp(s):
+  t = {}
+  for m in re.finditer(r"(\w+)=(-?\d+)", s):
+    t[m.group(1)] = int(m.group(2))
+  return t
+
+def parse_gc_trace(input):
+  trace = []
+  with open(input) as f:
+    for line in f:
+      info = split_nvp(line)
+      if info and 'pause' in info and info['pause'] > 0:
+        info['i'] = len(trace)
+        trace.append(info)
+  return trace
+
+def extract_field_names(script):
+  fields = { 'data': true, 'in': true }
+
+  for m in re.finditer(r"$(\w+)", script):
+    field_name = m.group(1)
+    if field_name not in fields:
+      fields[field] = field_count
+      field_count = field_count + 1
+
+  return fields
+
+def gnuplot(script):
+  gnuplot = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE)
+  gnuplot.stdin.write(script)
+  gnuplot.stdin.close()
+  gnuplot.wait()
+
+x1y1 = 'x1y1'
+x1y2 = 'x1y2'
+x2y1 = 'x2y1'
+x2y2 = 'x2y2'
+
+class Item(object):
+  def __init__(self, title, field, axis = x1y1, **keywords):
+    self.title = title
+    self.axis = axis
+    self.props = keywords
+    if type(field) is types.ListType:
+      self.field = field
+    else:
+      self.field = [field]
+
+  def fieldrefs(self):
+    return self.field
+
+  def to_gnuplot(self, context):
+    args = ['"%s"' % context.datafile,
+            'using %s' % context.format_fieldref(self.field),
+            'title "%s"' % self.title,
+            'axis %s' % self.axis]
+    if 'style' in self.props:
+      args.append('with %s' % self.props['style'])
+    if 'lc' in self.props:
+      args.append('lc rgb "%s"' % self.props['lc'])
+    if 'fs' in self.props:
+      args.append('fs %s' % self.props['fs'])
+    return ' '.join(args)
+
+class Plot(object):
+  def __init__(self, *items):
+    self.items = items
+
+  def fieldrefs(self):
+    return flatten([item.fieldrefs() for item in self.items])
+
+  def to_gnuplot(self, ctx):
+    return 'plot ' + ', '.join([item.to_gnuplot(ctx) for item in self.items])
+
+class Set(object):
+  def __init__(self, value):
+    self.value = value
+
+  def to_gnuplot(self, ctx):
+    return 'set ' + self.value
+
+  def fieldrefs(self):
+    return []
+
+class Context(object):
+  def __init__(self, datafile, field_to_index):
+    self.datafile = datafile
+    self.field_to_index = field_to_index
+
+  def format_fieldref(self, fieldref):
+    return ':'.join([str(self.field_to_index[field]) for field in fieldref])
+
+def collect_fields(plot):
+  field_to_index = {}
+  fields = []
+
+  def add_field(field):
+    if field not in field_to_index:
+      fields.append(field)
+      field_to_index[field] = len(fields)
+
+  for field in flatten([item.fieldrefs() for item in plot]):
+    add_field(field)
+
+  return (fields, field_to_index)
+
+def is_y2_used(plot):
+  for subplot in plot:
+    if isinstance(subplot, Plot):
+      for item in subplot.items:
+        if item.axis == x1y2 or item.axis == x2y2:
+          return True
+  return False
+
+def get_field(trace_line, field):
+  t = type(field)
+  if t is types.StringType:
+    return trace_line[field]
+  elif t is types.FunctionType:
+    return field(trace_line)
+
+def generate_datafile(datafile_name, trace, fields):
+  with open(datafile_name, 'w') as datafile:
+    for line in trace:
+      data_line = [str(get_field(line, field)) for field in fields]
+      datafile.write('\t'.join(data_line))
+      datafile.write('\n')
+
+def generate_script_and_datafile(plot, trace, datafile, output):
+  (fields, field_to_index) = collect_fields(plot)
+  generate_datafile(datafile, trace, fields)
+  script = [
+      'set terminal png',
+      'set output "%s"' % output,
+      'set autoscale',
+      'set ytics nomirror',
+      'set xtics nomirror',
+      'set key below'
+  ]
+
+  if is_y2_used(plot):
+    script.append('set autoscale y2')
+    script.append('set y2tics')
+
+  context = Context(datafile, field_to_index)
+
+  for item in plot:
+    script.append(item.to_gnuplot(context))
+
+  return '\n'.join(script)
+
+def plot_all(plots, trace, prefix):
+  charts = []
+
+  for plot in plots:
+    outfilename = "%s_%d.png" % (prefix, len(charts))
+    charts.append(outfilename)
+    script = generate_script_and_datafile(plot, trace, '~datafile', outfilename)
+    print 'Plotting %s...' % outfilename
+    gnuplot(script)
+
+  return charts
+
+def reclaimed_bytes(row):
+  return row['total_size_before'] - row['total_size_after']
+
+plots = [
+  [
+    Set('style fill solid 0.5 noborder'),
+    Set('style histogram rowstacked'),
+    Set('style data histograms'),
+    Plot(Item('Marking', 'mark', lc = 'purple'),
+         Item('Sweep', 'sweep', lc = 'blue'),
+         Item('Compaction', 'compact', lc = 'red'),
+         Item('Other',
+              lambda r: r['pause'] - r['mark'] - r['sweep'] - r['compact'],
+              lc = 'grey'))
+  ],
+  [
+    Set('style histogram rowstacked'),
+    Set('style data histograms'),
+    Plot(Item('Heap Size (before GC)', 'total_size_before', x1y2,
+              fs = 'solid 0.4 noborder',
+              lc = 'green'),
+         Item('Total holes (after GC)', 'holes_size_before', x1y2,
+              fs = 'solid 0.4 noborder',
+              lc = 'red'),
+         Item('GC Time', ['i', 'pause'], style = 'lines', lc = 'red'))
+  ],
+  [
+    Set('style histogram rowstacked'),
+    Set('style data histograms'),
+    Plot(Item('Heap Size (after GC)', 'total_size_after', x1y2,
+              fs = 'solid 0.4 noborder',
+              lc = 'green'),
+         Item('Total holes (after GC)', 'holes_size_after', x1y2,
+              fs = 'solid 0.4 noborder',
+              lc = 'red'),
+         Item('GC Time', ['i', 'pause'],
+              style = 'lines',
+              lc = 'red'))
+  ],
+  [
+    Set('style fill solid 0.5 noborder'),
+    Set('style data histograms'),
+    Plot(Item('Allocated', 'allocated'),
+         Item('Reclaimed', reclaimed_bytes),
+         Item('Promoted', 'promoted', style = 'lines', lc = 'black'))
+  ],
+]
+
+def process_trace(filename):
+  trace = parse_gc_trace(filename)
+  total_gc = reduce(lambda t,r: t + r['pause'], trace, 0)
+  max_gc = reduce(lambda t,r: max(t, r['pause']), trace, 0)
+  avg_gc = total_gc / len(trace)
+
+  charts = plot_all(plots, trace, filename)
+
+  with open(filename + '.html', 'w') as out:
+    out.write('<html><body>')
+    out.write('Total in GC: <b>%d</b><br/>' % total_gc)
+    out.write('Max in GC: <b>%d</b><br/>' % max_gc)
+    out.write('Avg in GC: <b>%d</b><br/>' % avg_gc)
+    for chart in charts:
+      out.write('<img src="%s">' % chart)
+      out.write('</body></html>')
+
+  print "%s generated." % (filename + '.html')
+
+if len(sys.argv) != 2:
+  print "Usage: %s <GC-trace-filename>" % sys.argv[0]
+  sys.exit(1)
+
+process_trace(sys.argv[1])