Switches to a Skia-specific appengine entry point that uses condensed data and taskqueue writes.

bench/bench_graph_svg.py

 '''
 Created on May 16, 2011
 
 @author: bungeman
 '''
 import bench_util
 import getopt
 import httplib
 import itertools
 import json
 import os
 import re
 import sys
 import urllib
 import urllib2
 import xml.sax.saxutils
 
 # We throw out any measurement outside this range, and log a warning.
 MIN_REASONABLE_TIME = 0
 MAX_REASONABLE_TIME = 99999
 
 # Constants for prefixes in output title used in buildbot.
 TITLE_PREAMBLE = 'Bench_Performance_for_Skia_'
 TITLE_PREAMBLE_LENGTH = len(TITLE_PREAMBLE)
 
-# Number of data points to send to appengine at once.
-DATA_POINT_BATCHSIZE = 66
-
-def grouper(n, iterable):
-    """Groups list into list of lists for a given size. See itertools doc:
-      http://docs.python.org/2/library/itertools.html#module-itertools
-    """
-    args = [iter(iterable)] * n
-    return [[n for n in t if n] for t in itertools.izip_longest(*args)]
-
-
 def usage():
     """Prints simple usage information."""
 
     print '-a <url> the url to use for adding bench values to app engine app.'
     print '   Example: "https://skiadash.appspot.com/add_point".'
     print '   If not set, will skip this step.'
     print '-b <bench> the bench to show.'
     print '-c <config> the config to show (GPU, 8888, 565, etc).'
     print '-d <dir> a directory containing bench_r<revision>_<scalar> files.'
     print '-e <file> file containing expected bench values/ranges.'
@@ skipping 359 matching lines @@
             raise Exception('Bench values out of range:\n' +
                             '\n'.join(exceptions))
 
     def write_to_appengine(line_data_dict, url, newest_revision, bot):
         """Writes latest bench values to appengine datastore.
           line_data_dict: dictionary from create_lines.
           url: the appengine url used to send bench values to write
           newest_revision: the latest revision that this script reads
           bot: the bot platform the bench is run on
         """
-        data = []
+        config_data_dic = {}
         for label in line_data_dict.iterkeys():
             if not label.bench.endswith('.skp') or label.time_type:
                 # filter out non-picture and non-walltime benches
                 continue
             config = label.config
             rev, val = line_data_dict[label][-1]
             # This assumes that newest_revision is >= the revision of the last
             # data point we have for each line.
             if rev != newest_revision:
                 continue
-            data.append({'master': 'Skia', 'bot': bot,
-                         'test': config + '/' + label.bench.replace('.skp', ''),
-                         'revision': rev, 'value': val, 'error': 0})
-        for curr_data in grouper(DATA_POINT_BATCHSIZE, data):
-            req = urllib2.Request(appengine_url,
-                urllib.urlencode({'data': json.dumps(curr_data)}))
-            try:
-              urllib2.urlopen(req)
-            except urllib2.HTTPError, e:
-                sys.stderr.write("HTTPError for JSON data %s: %s\n" % (
-                    data, e))
-            except urllib2.URLError, e:
-                sys.stderr.write("URLError for JSON data %s: %s\n" % (
-                    data, e))
-            except httplib.HTTPException, e:
-                sys.stderr.write("HTTPException for JSON data %s: %s\n" % (
-                    data, e))
+            if config not in config_data_dic:
+                config_data_dic[config] = []
+            config_data_dic[config].append(label.bench.replace('.skp', '') +
+                ':%.2f' % val)
+        for config in config_data_dic:
+            if config_data_dic[config]:
+                data = {'master': 'Skia', 'bot': bot, 'test': config,
+                        'revision': newest_revision,
+                        'benches': ','.join(config_data_dic[config])}
+                req = urllib2.Request(appengine_url,
+                    urllib.urlencode({'data': json.dumps(data)}))
+                try:
+                    urllib2.urlopen(req)
+                except urllib2.HTTPError, e:
+                    sys.stderr.write("HTTPError for JSON data %s: %s\n" % (
+                        data, e))
+                except urllib2.URLError, e:
+                    sys.stderr.write("URLError for JSON data %s: %s\n" % (
+                        data, e))
+                except httplib.HTTPException, e:
+                    sys.stderr.write("HTTPException for JSON data %s: %s\n" % (
+                        data, e))
 
     try:
         for option, value in opts:
             if option == "-a":
                 appengine_url = value
             elif option == "-b":
                 bench_of_interest = value
             elif option == "-c":
                 config_of_interest = value
             elif option == "-d":
@@ skipping 586 matching lines @@
     print '<a id="rev_link" xlink:href="" target="_top">'
     print '<text id="revision" x="0" y=%s style="' % qa(font_size*2)
     print 'font-size: %s; ' % qe(font_size)
     print 'stroke: #0000dd; text-decoration: underline; '
     print '"> </text></a>'
 
     print '</svg>'
 
 if __name__ == "__main__":
     main()