Add tracing/bin/strip_memory_infra_trace

tracing/tracing_build/strip_memory_infra_trace.py

Index: tracing/tracing_build/strip_memory_infra_trace.py
diff --git a/tracing/tracing_build/strip_memory_infra_trace.py b/tracing/tracing_build/strip_memory_infra_trace.py
new file mode 100755
index 0000000000000000000000000000000000000000..9c006c8500df583eef39f72cefdebbc8a8d410a7
--- /dev/null
+++ b/tracing/tracing_build/strip_memory_infra_trace.py
@@ -0,0 +1,97 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Filters a big trace keeping only the last memory-infra dumps."""
+
+import collections
+import gzip
+import json
+
+
+def FormatBytes(value):
+  units = ['B', 'kB', 'MB', 'GB']
+  while abs(value) >= 1000 and len(units) > 1:
+    value /= 1000
+    units = units[1:]

[perezju, 2016/07/27 10:28:14]

nit: units.pop(0)

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

Done.

+  return '%3.1f %s' % (value, units[0])
+

[perezju, 2016/07/27 10:28:13]

nit: extra blank line needed

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

Done.

+def Main(argv):
+  if len(argv) < 2:
+    print 'Usage: %s trace.json[.gz]' % argv[0]
+    return 1
+
+  in_path = argv[1]
+  if in_path.lower().endswith('.gz'):
+    fin = gzip.open(in_path, 'rb')
+  else:
+    fin = open(in_path, 'r')
+
+  out_path = in_path.split('.json')[0] + '-filtered.json'

[perezju, 2016/07/27 10:28:14]

I would move this line to either before opening the file, or right before
needing it when writing the out file below.

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

Done.

+
+  print 'Loading trace (can take 1 min on a z620 for a 1GB trace)...'
+  trace = json.load(fin)
+  print 'Done. Read ' + FormatBytes(fin.tell())
+  fin.close()

[perezju, 2016/07/27 10:28:14]

Here you could still do

  with fin:
    print 'Loading ...'
    trace = json.load(fin)
    print 'Done ...'

to ensure that the file is always properly closed.

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

ah, TIL

+
+  print 'Filtering events'
+  phase_count = collections.defaultdict(int)
+  out_events = []
+  global_dumps = collections.OrderedDict()
+  if isinstance(trace, dict):
+    in_events = trace.get('traceEvents', [])
+  elif isinstance(trace, list) and isinstance(trace[0], dict):
+    in_events = trace
+
+  for evt in in_events:
+    phase = evt.get('ph', '?')
+    phase_count[phase] += 1
+
+    # Drop all diagnostic events for memory-infra debugging.
+    if phase not in ('v', 'V') and evt.get('cat', '').endswith('memory-infra'):
+      continue
+
+    # pass-through all the other non-memory-infra events
+    if phase != 'v':
+      out_events.append(evt)
+      continue
+
+    # Recreate the global dump groups
+    event_id = evt['id']
+    global_dumps.setdefault(event_id, [])
+    global_dumps[event_id].append(evt)
+
+
+  print 'Detected %d memory-infra global dumps' % len(global_dumps)
+  if global_dumps:
+    max_procs = max(len(x) for x in global_dumps.itervalues())
+    print 'Max number of processes seen: %d' % max_procs
+
+  ndumps = 2

[perezju, 2016/07/27 10:28:14]

should this be an arg one can pass from the command line?

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

yeah I was thinking to that, but honestly want to keep it simple until somebody
has the need of this.
The best would be inferring the best size which keeps the trace < 256M, but that
is too much effort :)

[perezju, 2016/07/27 11:02:23]

stgm :)

+  print 'Preserving the last %d memory-infra dumps' % ndumps
+  detailed_dumps = []
+  non_detailed_dumps = []
+  for global_dump in global_dumps.itervalues():
+    lod = global_dump[0].get('args', {}).get('dumps', {}).get('level_of_detail')

[perezju, 2016/07/27 10:28:13]

here this would be clearer/easier:

  try:
    lod = global_dump[0]['args']['dumps']['level_of_detail']
  except KeyError:
    lod = None

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

Ah brilliant.

+    if lod == 'detailed':
+      detailed_dumps.append(global_dump)
+    else:
+      non_detailed_dumps.append(global_dump)
+
+  dumps_to_preserve = detailed_dumps[-ndumps:]

[perezju, 2016/07/27 10:28:14]

nit: you could

  ndumps -= len(dumps_to_preserve)

after this line and then the if condition and the array slicing below become
both cleaner.

[Primiano Tucci (use gerrit), 2016/07/27 10:38:03]

Done.

+  if len(dumps_to_preserve) < ndumps:
+    dumps_to_preserve += non_detailed_dumps[-(ndumps - len(dumps_to_preserve)):]
+
+  for global_dump in dumps_to_preserve:
+    out_events += global_dump
+
+  print '\nEvents histogram for the original trace (count by phase)'
+  print '--------------------------------------------------------'
+  for phase, count in sorted(phase_count.items(), key=lambda x: x[1]):
+    print '%s %d' % (phase, count)
+
+  print '\nWriting filtered trace to ' + out_path,
+  with open(out_path, 'w') as fout:
+    json.dump({'traceEvents': out_events}, fout)
+    num_bytes_written = fout.tell()
+  print ' (%s written)' % FormatBytes(num_bytes_written)