Switch buildershandler over to using the json mirror.

Tools/TestResultServer/handlers/buildershandler.py

 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 20 matching lines @@
 import logging
 import re
 import sys
 import traceback
 import urllib2
 import webapp2
 
 from google.appengine.api import memcache
 
 MASTERS = [
-    {'name': 'ChromiumWin', 'url': 'http://build.chromium.org/p/chromium.win', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumMac', 'url': 'http://build.chromium.org/p/chromium.mac', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumLinux', 'url': 'http://build.chromium.org/p/chromium.linux', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumChromiumOS', 'url': 'http://build.chromium.org/p/chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
-    {'name': 'ChromiumGPU', 'url': 'http://build.chromium.org/p/chromium.gpu', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumGPUFYI', 'url': 'http://build.chromium.org/p/chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
-    {'name': 'ChromiumWebkit', 'url': 'http://build.chromium.org/p/chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
-    {'name': 'ChromiumFYI', 'url': 'http://build.chromium.org/p/chromium.fyi', 'groups': ['@ToT Chromium FYI']},
-    {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']},
+    {'name': 'ChromiumWin', 'url_name': 'chromium.win', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumMac', 'url_name': 'chromium.mac', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumLinux', 'url_name': 'chromium.linux', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumChromiumOS', 'url_name': 'chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
+    {'name': 'ChromiumGPU', 'url_name': 'chromium.gpu', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumGPUFYI', 'url_name': 'chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
+    {'name': 'ChromiumWebkit', 'url_name': 'chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
+    {'name': 'ChromiumFYI', 'url_name': 'chromium.fyi', 'groups': ['@ToT Chromium FYI']},
+    {'name': 'V8', 'url_name': 'client.v8', 'groups': ['@ToT V8']},
 ]
 
 # Buildbot steps that have test in the name, but don't run tests.
 NON_TEST_STEP_NAMES = [
     'archive',
     'Run tests',
     'find isolated tests',
     'read test spec',
     'Download latest chromedriver',
     'compile tests',
@@ skipping 10 matching lines @@
     'python_tests(chrome',
     'run_all_tests.py',
     'test_report',
     'test CronetSample',
     'test_mini_installer',
     'telemetry_unittests',
     'webkit_python_tests',
     'webkit_unit_tests',
 ]
 
-class FetchBuildersException(Exception): pass
+BUILDS_URL = 'http://chrome-build-extract.appspot.com/get_builds?builder=%s&master=%s&num_builds=1'
+MASTER_URL = 'http://chrome-build-extract.appspot.com/get_master/%s'
 
 
-def master_json_url(master_url):
-    return master_url + '/json/builders'
-
-
-def builder_json_url(master_url, builder):
-    return master_json_url(master_url) + '/' + urllib2.quote(builder)
-
-
-def cached_build_json_url(master_url, builder, build_number):
-    return builder_json_url(master_url, builder) + '/builds/' + str(build_number)
+class FetchBuildersException(Exception):
+    pass
 
 
 def fetch_json(url):
     logging.debug('Fetching %s' % url)
     fetched_json = {}
     try:
         resp = urllib2.urlopen(url)
     except:
         exc_info = sys.exc_info()
         logging.warning('Error while fetching %s: %s', url, exc_info[1])
         return fetched_json
 
     try:
         fetched_json = json.load(resp)
     except:
         exc_info = sys.exc_info()
         logging.warning('Unable to parse JSON response from %s: %s', url, exc_info[1])
 
     return fetched_json
 
 
-def get_latest_build(build_data):
-    cached_builds = []
-    if 'cachedBuilds' in build_data:
-        cached_builds = build_data['cachedBuilds']
-
-    current_builds = build_data['currentBuilds']
-
-    latest_cached_builds = set(cached_builds) - set(current_builds)
-    if len(latest_cached_builds) != 0:
-        latest_cached_builds = sorted(list(latest_cached_builds))
-        latest_build = latest_cached_builds[-1]
-    elif len(current_builds) != 0:
-        latest_build = current_builds[0]
-    else:
-        basedir = build_data['basedir'] if 'basedir' in build_data else 'current builder'
-        logging.info('No cached or current builds for %s', basedir)
-        return None
-
-    return latest_build
-
-
 def dump_json(data):
     return json.dumps(data, separators=(',', ':'), sort_keys=True)
 
 
-def fetch_buildbot_data(masters, force_update=False):
-    if force_update:
-        logging.info('Starting a forced buildbot update. Failure to fetch a master\'s data will not abort the fetch.')
-
+def fetch_buildbot_data(masters):
     start_time = datetime.datetime.now()
     master_data = masters[:]
     for master in master_data:
-        master_url = master['url']
+        master_url = MASTER_URL % master['url_name']
+        builders = fetch_json(master_url)
+        if not builders:
+            msg = 'Aborting fetch. Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
+            logging.warning(msg)
+            raise FetchBuildersException(msg)
+
         tests_object = master.setdefault('tests', {})
-        master['tests'] = tests_object
 
-        builders = fetch_json(master_json_url(master_url))
-        if not builders:
-            msg = 'Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
-            logging.warning(msg)
-            if force_update:
-                continue
-            else:
-                logging.warning('Aborting fetch.')
-                raise FetchBuildersException(msg)
-
-        for builder in builders:
-            build_data = fetch_json(builder_json_url(master_url, builder))
-
-            latest_build = get_latest_build(build_data)
-            if not latest_build:
-                logging.info('Skipping builder %s because it lacked cached or current builds.', builder)
+        for builder in builders['builders'].keys():
+            build = fetch_json(BUILDS_URL % (urllib2.quote(builder), master['url_name']))
+            if not build:
+                logging.info('Skipping builder %s on master %s due to empty data.', builder, master['url_name'])
                 continue
 
-            build = fetch_json(cached_build_json_url(master_url, builder, latest_build))
-            if not build:
-                logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder)
-            for step in build['steps']:
+            if not build['builds']:
+                logging.info('Skipping builder %s on master %s due to empty builds list.', builder, master['url_name'])
+                continue
+
+            for step in build['builds'][0]['steps']:
                 step_name = step['name']
 
                 if not 'test' in step_name:
                     continue
 
                 if any(name in step_name for name in NON_TEST_STEP_NAMES):
                     continue
 
                 if re.search('_only|_ignore|_perf$', step_name):
                     continue
@@ skipping 12 matching lines @@
     delta = datetime.datetime.now() - start_time
 
     logging.info('Fetched buildbot data in %s seconds.', delta.seconds)
 
     return dump_json(output_data)
 
 
 class UpdateBuilders(webapp2.RequestHandler):
     """Fetch and update the cached buildbot data."""
     def get(self):
-        force_update = True if self.request.get('force') else False
         try:
-            buildbot_data = fetch_buildbot_data(MASTERS, force_update)
+            buildbot_data = fetch_buildbot_data(MASTERS)
             memcache.set('buildbot_data', buildbot_data)
             self.response.set_status(200)
             self.response.out.write("ok")
         except FetchBuildersException, ex:
             logging.error('Not updating builders because fetch failed: %s', str(ex))
             self.response.set_status(500)
             self.response.out.write(ex.message)
 
 
-
 class GetBuilders(webapp2.RequestHandler):
     """Return a list of masters mapped to their respective builders, possibly using cached data."""
     def get(self):
-        callback = self.request.get('callback')
-
         buildbot_data = memcache.get('buildbot_data')
 
         if not buildbot_data:
             logging.warning('No buildbot data in memcache. If this message repeats, something is probably wrong with memcache.')
+            try:
+                buildbot_data = fetch_buildbot_data(MASTERS)
+                memcache.set('buildbot_data', buildbot_data)
+            except FetchBuildersException, ex:
+                logging.error('Builders fetch failed: %s', str(ex))
+                self.response.set_status(500)
+                self.response.out.write(ex.message)
+                return
 
-            # Since we have no cached buildbot data, we would rather have missing masters than no data at all.
-            buildbot_data = fetch_buildbot_data(MASTERS, True)
-            try:
-                memcache.set('buildbot_data', buildbot_data)
-            except ValueError, err:
-                logging.error(str(err))
-
+        callback = self.request.get('callback')
         if callback:
             buildbot_data = callback + '(' + buildbot_data + ');'
 
         self.response.out.write(buildbot_data)