Allow multiple performance values in endure_result_parser.py.

chrome/test/functional/perf/endure_result_parser.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to parse perf data from Chrome Endure test executions, to be graphed.
 
 This script connects via HTTP to a buildbot master in order to scrape and parse
 perf data from Chrome Endure tests that have been run.  The perf data is then
 stored in local text files to be graphed by the Chrome Endure graphing code.
@@ skipping 100 matching lines @@
     elif int(line_dict['rev']) < int(revision):
       break
   if not overwritten:
     existing_lines.insert(0, simplejson.dumps(new_line))
 
   with open(data_file, 'w') as f:
     f.write('\n'.join(existing_lines))
   os.chmod(data_file, 0755)
 
 
-def OutputPerfData(revision, graph_name, description, value, units, units_x,
-                   dest_dir):
+def OutputPerfData(revision, graph_name, values, units, units_x, dest_dir):
   """Outputs perf data to a local text file to be graphed.
 
   Args:
     revision: The string revision number associated with the perf data.
     graph_name: The string name of the graph on which to plot the data.
-    description: A string description of the perf value to be graphed.
-    value: Either a single data value to be graphed, or a list of 2-tuples
+    values: A dict which maps a description to a value.  A value is either a
+        single data value to be graphed, or a list of 2-tuples
         representing (x, y) points to be graphed for long-running tests.
     units: The string description for the y-axis units on the graph.
     units_x: The string description for the x-axis units on the graph.  Should
         be set to None if the results are not for long-running graphs.
     dest_dir: The name of the destination directory to which to write.
   """
   # Update graphs.dat, which contains metadata associated with each graph.
   existing_graphs = []
   graphs_file = os.path.join(dest_dir, 'graphs.dat')
   if os.path.exists(graphs_file):
@@ skipping 19 matching lines @@
     os.chmod(graphs_file, 0755)
 
   # Update summary data file, containing the actual data to be graphed.
   data_file_name = graph_name + '-summary.dat'
   existing_lines = []
   data_file = os.path.join(dest_dir, data_file_name)
   if os.path.exists(data_file):
     with open(data_file, 'r') as f:
       existing_lines = f.readlines()
   existing_lines = map(lambda x: x.strip(), existing_lines)
-  if units_x:
-    points = []
-    for point in value:
-      points.append([str(point[0]), str(point[1])])
-    new_traces = {
-      description: points
-    }
-  else:
-    new_traces = {
-      description: [str(value), str(0.0)]
-    }
+  new_traces = {}
+  for description in values:
+    value = values[description]
+    if units_x:
+      points = []
+      for point in value:
+        points.append([str(point[0]), str(point[1])])
+      new_traces[description] = points
+    else:
+      new_traces[description] = [str(value), str(0.0)]
   new_line = {
     'traces': new_traces,
     'rev': revision
   }
 
   WriteToDataFile(new_line, existing_lines, revision, data_file)
 
 
-def OutputEventData(revision, description, event_list, dest_dir):
+def OutputEventData(revision, event_dict, dest_dir):
   """Outputs event data to a local text file to be graphed.
 
   Args:
     revision: The string revision number associated with the event data.
-    description: A string description of the event values to be graphed.
-    event_list: An array of tuples representing event data to be graphed.
+    event_dict: A dict which maps a description to an array of tuples
+        representing event data to be graphed.
     dest_dir: The name of the destination directory to which to write.
   """
   data_file_name = '_EVENT_-summary.dat'
   existing_lines = []
   data_file = os.path.join(dest_dir, data_file_name)
   if os.path.exists(data_file):
     with open(data_file, 'r') as f:
       existing_lines = f.readlines()
   existing_lines = map(lambda x: x.strip(), existing_lines)
 
-  value_list = []
-  for event_time, event_data in event_list:
-    value_list.append([str(event_time), event_data])
-  new_events = {
-    description: value_list
-  }
+  new_events = {}
+  for description in event_dict:
+    event_list = event_dict[description]
+    value_list = []
+    for event_time, event_data in event_list:
+      value_list.append([str(event_time), event_data])
+    new_events[description] = value_list
 
   new_line = {
     'rev': revision,
     'events': new_events
   }
 
   WriteToDataFile(new_line, existing_lines, revision, data_file)
 
 
 def UpdatePerfDataForSlaveAndBuild(slave_info, build_num):
@@ skipping 117 matching lines @@
       AppendRawPerfData('_EVENT_', match[0], eval(match[1]), None, None,
                         stdio_url_data['webapp_name'],
                         stdio_url_data['test_name'])
 
     # For each graph_name/description pair that refers to a long-running test
     # result or an event, concatenate all the results together (assume results
     # in the input file are in the correct order).  For short-running test
     # results, keep just one if more than one is specified.
     perf_data = {}  # Maps a graph-line key to a perf data dictionary.
     for data in perf_data_raw:
-      key = data['graph_name'] + '|' + data['description']
+      key1 = data['graph_name']

[dennis_jeffrey, 2012/09/26 00:34:29]

use a more descriptive variable name, such as "key_graph", instead of "key1"

[Dai Mikurube (NOT FULLTIME), 2012/09/26 02:44:51]

Done.

+      key2 = data['description']

[dennis_jeffrey, 2012/09/26 00:34:29]

use a more descriptive variable name, such as "key_description" or "key_desc",
instead of "key2"

[Dai Mikurube (NOT FULLTIME), 2012/09/26 02:44:51]

Done.

+      if not key1 in perf_data:
+        perf_data[key1] = {
+          'graph_name': data['graph_name'],
+          'value': {},
+          'units': data['units'],
+          'units_x': data['units_x'],
+          'webapp_name': data['webapp_name'],
+          'test_name': data['test_name'],
+        }
       if data['graph_name'] != '_EVENT_' and not data['units_x']:
         # Short-running test result.
-        perf_data[key] = data
+        perf_data[key1]['value'][key2] = data['value']
       else:
         # Long-running test result or event.
-        if key in perf_data:
-          perf_data[key]['value'] += data['value']
+        if key2 in perf_data[key1]['value']:
+          perf_data[key1]['value'][key2] += data['value']
         else:
-          perf_data[key] = data
+          perf_data[key1]['value'][key2] = data['value']
 
     # Finally, for each graph-line in |perf_data|, update the associated local
     # graph data files if necessary.
     for perf_data_key in perf_data:
       perf_data_dict = perf_data[perf_data_key]
 
       dest_dir = os.path.join(LOCAL_GRAPH_DIR, perf_data_dict['webapp_name'])
       if not os.path.exists(dest_dir):
         os.mkdir(dest_dir)  # Webapp name directory.
         os.chmod(dest_dir, 0755)
       dest_dir = os.path.join(dest_dir, perf_data_dict['test_name'])
 
       SetupBaseGraphDirIfNeeded(perf_data_dict['webapp_name'],
                                 perf_data_dict['test_name'], dest_dir)
       if perf_data_dict['graph_name'] == '_EVENT_':
-        OutputEventData(revision, perf_data_dict['description'],
-                        perf_data_dict['value'], dest_dir)
+        OutputEventData(revision, perf_data_dict['value'], dest_dir)
       else:
         OutputPerfData(revision, perf_data_dict['graph_name'],
-                       perf_data_dict['description'], perf_data_dict['value'],
+                       perf_data_dict['value'],
                        perf_data_dict['units'], perf_data_dict['units_x'],
                        dest_dir)
 
   return True
 
 
 def UpdatePerfDataFiles():
   """Updates the Chrome Endure graph data files with the latest test results.
 
   For each known Chrome Endure slave, we scan its latest test results looking
@@ skipping 205 matching lines @@
   if not success:
     logging.error('Failed to update perf data files.')
     sys.exit(0)
 
   GenerateIndexPage()
   logging.debug('All done!')
 
 
 if __name__ == '__main__':
   main()