Dump JSON data for stacked graphs in endure_result_parser.py.

chrome/test/functional/perf/endure_result_parser.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to parse perf data from Chrome Endure test executions, to be graphed.
 
 This script connects via HTTP to a buildbot master in order to scrape and parse
 perf data from Chrome Endure tests that have been run.  The perf data is then
 stored in local text files to be graphed by the Chrome Endure graphing code.
@@ skipping 94 matching lines @@
     elif int(line_dict['rev']) < int(revision):
       break
   if not overwritten:
     existing_lines.insert(0, simplejson.dumps(new_line))
 
   with open(data_file, 'w') as f:
     f.write('\n'.join(existing_lines))
   os.chmod(data_file, 0755)
 
 
-def OutputPerfData(revision, graph_name, values, units, units_x, dest_dir):
+def OutputPerfData(revision, graph_name, values, units, units_x, dest_dir,
+                   is_stacked=False, stack_order=[]):
   """Outputs perf data to a local text file to be graphed.
 
   Args:
     revision: The string revision number associated with the perf data.
     graph_name: The string name of the graph on which to plot the data.
     values: A dict which maps a description to a value.  A value is either a
         single data value to be graphed, or a list of 2-tuples
         representing (x, y) points to be graphed for long-running tests.
     units: The string description for the y-axis units on the graph.
     units_x: The string description for the x-axis units on the graph.  Should
         be set to None if the results are not for long-running graphs.
     dest_dir: The name of the destination directory to which to write.
+    is_stacked: True to draw a "stacked" graph.  First-come values are
+        stacked at bottom by default.
+    stack_order: A list that contains key strings in the order to stack values
+        in the graph.
   """
   # Update graphs.dat, which contains metadata associated with each graph.
   existing_graphs = []
   graphs_file = os.path.join(dest_dir, 'graphs.dat')
   if os.path.exists(graphs_file):
     with open(graphs_file, 'r') as f:
       existing_graphs = simplejson.loads(f.read())
   is_new_graph = True
   for graph in existing_graphs:
     if graph['name'] == graph_name:
@@ skipping 28 matching lines @@
       points = []
       for point in value:
         points.append([str(point[0]), str(point[1])])
       new_traces[description] = points
     else:
       new_traces[description] = [str(value), str(0.0)]
   new_line = {
     'traces': new_traces,
     'rev': revision
   }
+  if is_stacked:
+    new_line['stack'] = True
+    new_line['stack_order'] = stack_order
 
   WriteToDataFile(new_line, existing_lines, revision, data_file)
 
 
 def OutputEventData(revision, event_dict, dest_dir):
   """Outputs event data to a local text file to be graphed.
 
   Args:
     revision: The string revision number associated with the event data.
     event_dict: A dict which maps a description to an array of tuples
@@ skipping 107 matching lines @@
       # Issue warning but continue to the next stdio link.
       logging.warning('Error reading test stdio URL "%s": %s', stdio_url,
                       str(e))
     finally:
       if fp:
         fp.close()
 
     perf_data_raw = []
 
     def AppendRawPerfData(graph_name, description, value, units, units_x,
-                          webapp_name, test_name):
+                          webapp_name, test_name, is_stacked=False):
       perf_data_raw.append({
         'graph_name': graph_name,
         'description': description,
         'value': value,
         'units': units,
         'units_x': units_x,
         'webapp_name': webapp_name,
         'test_name': test_name,
+        'stack': is_stacked,
       })
 
     # First scan for short-running perf test results.
     for match in re.findall(
         r'RESULT ([^:]+): ([^=]+)= ([-\d\.]+) (\S+)', url_contents):
       AppendRawPerfData(match[0], match[1], eval(match[2]), match[3], None,
                         stdio_url_data['webapp_name'],
                         stdio_url_data['webapp_name'])
 
     # Next scan for long-running perf test results.
     for match in re.findall(
         r'RESULT ([^:]+): ([^=]+)= (\[[^\]]+\]) (\S+) (\S+)', url_contents):
+      # TODO(dmikurube): Change the condition to use stacked graph when we
+      # determine how to specify it.
       AppendRawPerfData(match[0], match[1], eval(match[2]), match[3], match[4],
                         stdio_url_data['webapp_name'],
-                        stdio_url_data['test_name'])
+                        stdio_url_data['test_name'],
+                        match[0].endswith('-DMP'))
 
     # Next scan for events in the test results.
     for match in re.findall(
         r'RESULT _EVENT_: ([^=]+)= (\[[^\]]+\])', url_contents):
       AppendRawPerfData('_EVENT_', match[0], eval(match[1]), None, None,
                         stdio_url_data['webapp_name'],
                         stdio_url_data['test_name'])
 
     # For each graph_name/description pair that refers to a long-running test
     # result or an event, concatenate all the results together (assume results
     # in the input file are in the correct order).  For short-running test
     # results, keep just one if more than one is specified.
     perf_data = {}  # Maps a graph-line key to a perf data dictionary.
     for data in perf_data_raw:
       key_graph = data['graph_name']
       key_description = data['description']
       if not key_graph in perf_data:
         perf_data[key_graph] = {
           'graph_name': data['graph_name'],
           'value': {},
           'units': data['units'],
           'units_x': data['units_x'],
           'webapp_name': data['webapp_name'],
           'test_name': data['test_name'],
         }
+      perf_data[key_graph]['stack'] = data['stack']
+      if 'stack_order' not in perf_data[key_graph]:
+        perf_data[key_graph]['stack_order'] = []
+      if (data['stack'] and
+          data['description'] not in perf_data[key_graph]['stack_order']):
+        perf_data[key_graph]['stack_order'].append(data['description'])
+
       if data['graph_name'] != '_EVENT_' and not data['units_x']:
         # Short-running test result.
         perf_data[key_graph]['value'][key_description] = data['value']
       else:
         # Long-running test result or event.
         if key_description in perf_data[key_graph]['value']:
           perf_data[key_graph]['value'][key_description] += data['value']
         else:
           perf_data[key_graph]['value'][key_description] = data['value']
 
     # Finally, for each graph-line in |perf_data|, update the associated local
     # graph data files if necessary.
     for perf_data_key in perf_data:
       perf_data_dict = perf_data[perf_data_key]
 
       dest_dir = os.path.join(LOCAL_GRAPH_DIR, perf_data_dict['webapp_name'])
       if not os.path.exists(dest_dir):
         os.mkdir(dest_dir)  # Webapp name directory.
         os.chmod(dest_dir, 0755)
       dest_dir = os.path.join(dest_dir, perf_data_dict['test_name'])
 
       SetupBaseGraphDirIfNeeded(perf_data_dict['webapp_name'],
                                 perf_data_dict['test_name'], dest_dir)
       if perf_data_dict['graph_name'] == '_EVENT_':
         OutputEventData(revision, perf_data_dict['value'], dest_dir)
       else:
         OutputPerfData(revision, perf_data_dict['graph_name'],
                        perf_data_dict['value'],
                        perf_data_dict['units'], perf_data_dict['units_x'],
-                       dest_dir)
+                       dest_dir,
+                       perf_data_dict['stack'], perf_data_dict['stack_order'])
 
   return True
 
 
 def UpdatePerfDataFiles():
   """Updates the Chrome Endure graph data files with the latest test results.
 
   For each known Chrome Endure slave, we scan its latest test results looking
   for any new test data.  Any new data that is found is then appended to the
   data files used to display the Chrome Endure graphs.
@@ skipping 203 matching lines @@
   if not success:
     logging.error('Failed to update perf data files.')
     sys.exit(0)
 
   GenerateIndexPage()
   logging.debug('All done!')
 
 
 if __name__ == '__main__':
   main()