Switch buildershandler over to using the json mirror.

Tools/TestResultServer/handlers/buildershandler.py

Index: Tools/TestResultServer/handlers/buildershandler.py
diff --git a/Tools/TestResultServer/handlers/buildershandler.py b/Tools/TestResultServer/handlers/buildershandler.py
index 84dfed7e44765ad041cb81a954bfbacd4656eb90..dc1e73740ab6b8f94343915d620c0299b699822c 100644
--- a/Tools/TestResultServer/handlers/buildershandler.py
+++ b/Tools/TestResultServer/handlers/buildershandler.py
@@ -38,15 +38,15 @@ import webapp2
 from google.appengine.api import memcache
 
 MASTERS = [
-    {'name': 'ChromiumWin', 'url': 'http://build.chromium.org/p/chromium.win', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumMac', 'url': 'http://build.chromium.org/p/chromium.mac', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumLinux', 'url': 'http://build.chromium.org/p/chromium.linux', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumChromiumOS', 'url': 'http://build.chromium.org/p/chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
-    {'name': 'ChromiumGPU', 'url': 'http://build.chromium.org/p/chromium.gpu', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumGPUFYI', 'url': 'http://build.chromium.org/p/chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
-    {'name': 'ChromiumWebkit', 'url': 'http://build.chromium.org/p/chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
-    {'name': 'ChromiumFYI', 'url': 'http://build.chromium.org/p/chromium.fyi', 'groups': ['@ToT Chromium FYI']},
-    {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']},
+    {'name': 'ChromiumWin', 'url_name': 'chromium.win', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumMac', 'url_name': 'chromium.mac', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumLinux', 'url_name': 'chromium.linux', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumChromiumOS', 'url_name': 'chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
+    {'name': 'ChromiumGPU', 'url_name': 'chromium.gpu', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumGPUFYI', 'url_name': 'chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
+    {'name': 'ChromiumWebkit', 'url_name': 'chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
+    {'name': 'ChromiumFYI', 'url_name': 'chromium.fyi', 'groups': ['@ToT Chromium FYI']},
+    {'name': 'V8', 'url_name': 'client.v8', 'groups': ['@ToT V8']},
 ]
 
 # Buildbot steps that have test in the name, but don't run tests.
@@ -77,19 +77,12 @@ TEST_STEPS_THAT_DO_NOT_UPLOAD_YET = [
     'webkit_unit_tests',
 ]
 
-class FetchBuildersException(Exception): pass
+BUILDS_URL = 'http://chrome-build-extract.appspot.com/get_builds?builder=%s&master=%s&num_builds=1'
+MASTER_URL = 'http://chrome-build-extract.appspot.com/get_master/%s'
 
 
-def master_json_url(master_url):
-    return master_url + '/json/builders'
-
-
-def builder_json_url(master_url, builder):
-    return master_json_url(master_url) + '/' + urllib2.quote(builder)
-
-
-def cached_build_json_url(master_url, builder, build_number):
-    return builder_json_url(master_url, builder) + '/builds/' + str(build_number)
+class FetchBuildersException(Exception):
+    pass
 
 
 def fetch_json(url):
@@ -111,64 +104,34 @@ def fetch_json(url):
     return fetched_json
 
 
-def get_latest_build(build_data):
-    cached_builds = []
-    if 'cachedBuilds' in build_data:
-        cached_builds = build_data['cachedBuilds']
-
-    current_builds = build_data['currentBuilds']
-
-    latest_cached_builds = set(cached_builds) - set(current_builds)
-    if len(latest_cached_builds) != 0:
-        latest_cached_builds = sorted(list(latest_cached_builds))
-        latest_build = latest_cached_builds[-1]
-    elif len(current_builds) != 0:
-        latest_build = current_builds[0]
-    else:
-        basedir = build_data['basedir'] if 'basedir' in build_data else 'current builder'
-        logging.info('No cached or current builds for %s', basedir)
-        return None
-
-    return latest_build
-
-
 def dump_json(data):
     return json.dumps(data, separators=(',', ':'), sort_keys=True)
 
 
-def fetch_buildbot_data(masters, force_update=False):
-    if force_update:
-        logging.info('Starting a forced buildbot update. Failure to fetch a master\'s data will not abort the fetch.')
-
+def fetch_buildbot_data(masters):
     start_time = datetime.datetime.now()
     master_data = masters[:]
     for master in master_data:
-        master_url = master['url']
-        tests_object = master.setdefault('tests', {})
-        master['tests'] = tests_object
-
-        builders = fetch_json(master_json_url(master_url))
+        master_url = MASTER_URL % master['url_name']
+        builders = fetch_json(master_url)
         if not builders:
-            msg = 'Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
+            msg = 'Aborting fetch. Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
             logging.warning(msg)
-            if force_update:
-                continue
-            else:
-                logging.warning('Aborting fetch.')
-                raise FetchBuildersException(msg)
+            raise FetchBuildersException(msg)
+
+        tests_object = master.setdefault('tests', {})
 
-        for builder in builders:
-            build_data = fetch_json(builder_json_url(master_url, builder))
+        for builder in builders['builders'].keys():
+            build = fetch_json(BUILDS_URL % (urllib2.quote(builder), master['url_name']))
+            if not build:
+                logging.info('Skipping builder %s on master %s due to empty data.', builder, master['url_name'])
+                continue
 
-            latest_build = get_latest_build(build_data)
-            if not latest_build:
-                logging.info('Skipping builder %s because it lacked cached or current builds.', builder)
+            if not build['builds']:
+                logging.info('Skipping builder %s on master %s due to empty builds list.', builder, master['url_name'])
                 continue
 
-            build = fetch_json(cached_build_json_url(master_url, builder, latest_build))
-            if not build:
-                logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder)
-            for step in build['steps']:
+            for step in build['builds'][0]['steps']:
                 step_name = step['name']
 
                 if not 'test' in step_name:
@@ -201,9 +164,8 @@ def fetch_buildbot_data(masters, force_update=False):
 class UpdateBuilders(webapp2.RequestHandler):
     """Fetch and update the cached buildbot data."""
     def get(self):
-        force_update = True if self.request.get('force') else False
         try:
-            buildbot_data = fetch_buildbot_data(MASTERS, force_update)
+            buildbot_data = fetch_buildbot_data(MASTERS)
             memcache.set('buildbot_data', buildbot_data)
             self.response.set_status(200)
             self.response.out.write("ok")
@@ -213,24 +175,23 @@ class UpdateBuilders(webapp2.RequestHandler):
             self.response.out.write(ex.message)
 
 
-
 class GetBuilders(webapp2.RequestHandler):
     """Return a list of masters mapped to their respective builders, possibly using cached data."""
     def get(self):
-        callback = self.request.get('callback')
-
         buildbot_data = memcache.get('buildbot_data')
 
         if not buildbot_data:
             logging.warning('No buildbot data in memcache. If this message repeats, something is probably wrong with memcache.')
-
-            # Since we have no cached buildbot data, we would rather have missing masters than no data at all.
-            buildbot_data = fetch_buildbot_data(MASTERS, True)
             try:
+                buildbot_data = fetch_buildbot_data(MASTERS)
                 memcache.set('buildbot_data', buildbot_data)
-            except ValueError, err:
-                logging.error(str(err))
+            except FetchBuildersException, ex:
+                logging.error('Builders fetch failed: %s', str(ex))
+                self.response.set_status(500)
+                self.response.out.write(ex.message)
+                return
 
+        callback = self.request.get('callback')
         if callback:
             buildbot_data = callback + '(' + buildbot_data + ');'