[Interpreter] Add visualization tool for Ignition dispatch counters.

tools/ignition/plot_bytecode_dispatches.py

+#! /usr/bin/python
+#
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+
+import argparse
+import heapq
+import json
+import matplotlib.colors as colors
+import matplotlib.pyplot as plt
+import numpy as np

[rmcilroy, 2016/04/11 16:35:16]

nit - don't use "as" in imports here, I think it just makes the code more
difficult to understand.

[Stefano Sanfilippo, 2016/04/12 11:18:55]

Done.

+import struct
+
+
+__COUNTER_BITS = struct.calcsize("P") * 8  # Size in bits of a pointer
+__COUNTER_MAX = 2**__COUNTER_BITS - 1
+

[rmcilroy, 2016/04/11 16:35:16]

Can you add a couple of unittests (not for the matplot stuff obviously, but for
parsing the data and getting top dispatches).

[Stefano Sanfilippo, 2016/04/12 11:18:55]

The two functions that we should test are:

* warn_if_counter_may_have_saturated
* print_top_counters

However, both of them print something to screen, without returning anything. In
order to make the two unit testable, we could return a list to print externally
or, similarly, turn the function into a generator. We could also pass a file
stream to print to as a function parameter, then use a StringIO object to
retrieve the output as string while testing.

Optionally, we might want to test the logic in the prologue of
plot_dispatches_table, when we turn the dictionary of dictionaries into a
matrix. In that case, the cleanest solution is splitting that part from the
actual plotting commands.

+
+def load_counters_data(stream):
+  raw_data = json.load(stream)
+  labels = sorted(raw_data.keys())
+  counters_matrix = np.empty([len(labels), len(labels)], dtype=int)
+  for from_index, from_name in enumerate(labels):
+    current_row = raw_data[from_name];
+    for to_index, to_name in enumerate(labels):
+      counters_matrix[from_index, to_index] = current_row.get(to_name, 0)
+  return labels, counters_matrix
+
+
+def build_name_for_coordinates(labels, coordinates):
+  return " -> ".join(labels[c] for c in coordinates)
+
+
+def check_saturated_counters(labels, counters_matrix):

[rmcilroy, 2016/04/11 16:35:16]

warn_if_counters_are_saturated

[Stefano Sanfilippo, 2016/04/12 11:18:55]

Done.

+  # np.where() returns two arrays of x and y coordinates,
+  # zip them together to get (x,y) tuples.
+  saturated_cells = zip(*np.where(counters_matrix == __COUNTER_MAX))
+  for coordinates in saturated_cells:
+    name = build_name_for_coordinates(labels, coordinates)
+    print "WARNING: counter may have saturated: {}".format(name)
+
+
+def cells_coordinates_content_generator(counters_matrix):
+  """Yield tuples (cell_content, (x, y)) for each cell in counters_matrix."""
+  cells_iterator = np.nditer(counters_matrix, flags=['multi_index'])
+  while not cells_iterator.finished:
+    yield (cells_iterator[0], cells_iterator.multi_index)
+    cells_iterator.iternext()
+
+
+def print_top_counters(labels, counters_matrix, limit):
+  top_counters = heapq.nlargest(

[rmcilroy, 2016/04/11 16:35:16]

As discussed offline, I think this tool would be much simpler if you kept the
results in the JSON (nested dictionary) form and only moved to a matrix when you
are printing the plot. Could you have another stab at this please.

[Stefano Sanfilippo, 2016/04/12 11:18:55]

Done.

+    limit,
+    cells_coordinates_content_generator(counters_matrix),
+    key=lambda x: x[0]
+  )
+  print "Top {} dispatch counters:".format(limit)
+  for counter_value, counter_coordinates in top_counters:
+    counter_name = build_name_for_coordinates(labels, counter_coordinates)
+    print "  {:>12}  {}".format(counter_value, counter_name)
+
+
+def prepare_data_for_plot(labels, counters_matrix):
+  xlabels = labels
+  ylabels = list(reversed(xlabels))
+  counters_matrix = np.flipud(counters_matrix)
+  return xlabels, ylabels, counters_matrix
+
+
+def plot_handler_to_handler_counters(counters_matrix, xlabels, ylabels, figure,
+                                     axis):
+  image = axis.pcolor(
+    counters_matrix,
+    cmap='jet',
+    norm=colors.LogNorm(),
+    edgecolor='grey',
+    linestyle='dotted',
+    linewidth=0.5
+  )
+
+  axis.xaxis.set(
+    ticks=np.arange(0.5, len(xlabels)),
+    label="From bytecode handler"
+  )
+  axis.xaxis.tick_top()
+  axis.set_xlim(0, len(xlabels))
+  axis.set_xticklabels(xlabels, rotation='vertical')
+
+  axis.yaxis.set(
+    ticks=np.arange(0.5, len(ylabels)),
+    label="To bytecode handler",
+    ticklabels=ylabels
+  )
+  axis.set_ylim(0, len(ylabels))
+
+  figure.colorbar(
+    image,
+    ax=axis,
+    fraction=0.01,
+    pad=0.01
+  )
+
+
+def parse_command_line():
+  command_line_parser = argparse.ArgumentParser(
+    description="Plot Ignition counters file.",

[rmcilroy, 2016/04/11 16:35:16]

Could you add more description of what this tool is and how to use it (like the
one in linux_perf_report.py).

[Stefano Sanfilippo, 2016/04/12 11:18:55]

Done.

+  )
+  command_line_parser.add_argument(
+    "--plot_size", "-s",
+    metavar="N",
+    default=30,
+    help="shorter side, in inches, of the output plot",
+  )
+  command_line_parser.add_argument(
+    "--interactive", "-i",
+    action="store_true",
+    help="open an interactive viewer, rather than writing to file.",
+  )
+  command_line_parser.add_argument(
+    "--print_top", "-t",

[rmcilroy, 2016/04/11 16:35:16]

I think you need to rename the file since this isn't just about plotting a file.
How about bytecode_dispatch_report.py

[Stefano Sanfilippo, 2016/04/12 11:18:55]

Done.

+    metavar="N",
+    type=int,
+    default=0,
+    help="print the top N counters"
+  )

[rmcilroy, 2016/04/11 16:35:16]

You should probably have another option for doing the plot, and make the default
operation of this script to print the top 10 dispatches without doing a plot
(since that's probably more immediately useful to implementers).

[Stefano Sanfilippo, 2016/04/12 11:18:55]

Done.

+  command_line_parser.add_argument(
+    "--output_filename", "-o",
+    metavar="<output filename>",
+    default="v8.ignition_dispatches_table.svg",
+    help=("file to save the plot file to. File type is deduced from the "
+          "extension. PDF, SVG, PNG supported")
+  )
+  command_line_parser.add_argument(
+    "input_filename",
+    metavar="<input filename>",
+    default="v8.ignition_dispatches_table.json",
+    nargs='?',
+    help="Ignition counters JSON file",
+  )
+
+  return command_line_parser.parse_args()
+
+
+def main():
+  program_options = parse_command_line()
+
+  with open(program_options.input_filename) as stream:
+    labels, counters_matrix = load_counters_data(stream)
+
+  check_saturated_counters(labels, counters_matrix)
+  if program_options.print_top:
+    print_top_counters(labels, counters_matrix, program_options.print_top)
+
+  xlabels, ylabels, counters_matrix = prepare_data_for_plot(labels,
+                                                            counters_matrix)
+
+  figure, axis = plt.subplots()
+  plot_handler_to_handler_counters(counters_matrix, xlabels, ylabels, figure,
+                                   axis)
+
+  if program_options.interactive:
+    plt.show()
+  else:
+    figure.set_size_inches(program_options.plot_size, program_options.plot_size)
+    plt.savefig(program_options.output_filename)
+
+
+if __name__ == "__main__":
+  main()