Initial check-in of tryconsole.

tryconsole/tryconsole.py

+#!/usr/bin/python
+# Copyright (c) 2009-2010 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""AJAX-y console for try jobs."""
+
+import datetime
+import logging
+import os
+import pickle
+import re
+import time
+from google.appengine.api import memcache
+from google.appengine.api import urlfetch
+from google.appengine.api.labs import taskqueue
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import template
+from google.appengine.ext.webapp.util import run_wsgi_app
+
+
+BUILDBOT_URLS = [
+    'http://build.chromium.org/buildbot/try-server',
+]
+HISTORY_SIZE = 50
+
+
+class BuildResult(db.Model):
+  """Entity for storing the current status of a single trybot build."""
+  # TODO(bradnelson): make these required?
+  master_url = db.StringProperty()
+  builder_name = db.StringProperty()
+  build_number = db.IntegerProperty()
+
+  slave_name = db.StringProperty()
+
+  raw_reason = db.StringProperty()
+  username = db.StringProperty()
+  patch_name = db.StringProperty()
+
+  revision = db.IntegerProperty()
+  got_revision = db.IntegerProperty()
+
+  account_name = db.StringProperty()
+
+  changed_at = db.DateTimeProperty()
+  start = db.DateTimeProperty()
+  end = db.DateTimeProperty()
+
+  steps_and_logfiles = db.BlobProperty()
+
+  eta_seconds = db.IntegerProperty()
+  page_built = db.DateTimeProperty()
+  last_checked = db.DateTimeProperty()
+
+
+class PerformanceSamples(db.Model):
+  """A Sample of builds running, used to generate graphs."""
+  interval = db.IntegerProperty()
+  running_builds = db.BlobProperty()
+  measured_at = db.DateTimeProperty()
+
+
+def GetBuilders(master_url):
+  """Given the URL of a trybot master, get a list of builders for that master.
+
+  Arguments:
+    master_url: the URL of the master.
+  Returns:
+    A list of strings containing the builder names.
+  """
+  # TODO(bradnelson): use the json interface in place of html scraping.
+  # Returned cached copy if we have one.
+  blist = memcache.get(master_url + '/builders')
+  if blist:
+    return blist
+  # Grab the webpage.
+  url = master_url + '/builders'
+  try:
+    result = urlfetch.fetch(url)
+  except urlfetch.DownloadError:
+    return []
+  if result.status_code != 200:
+    return []
+  # Pick out the proper list.
+  text = re.search('<ol>(.*)</ol>', result.content, re.DOTALL)
+  if not text:
+    return []
+  # Pick out the builder name.
+  blist = re.findall('<li><a href[^>]+>([^<]+)</a></li>', text.group(1))
+  # Add to cache.
+  memcache.add('/builders', blist, 1000)
+  return blist
+
+
+def GetBuilderInfo(master_url, builder_name):
+  """Given a trybot master and builder name, get information about a builder.
+
+  Arguments:
+    master_url: the URL of the master.
+    builder_name: the name of a builder on this master.
+  Returns:
+    A dict filled with a range of values that describe the state of this
+    builder. It contains things like which builds are currently running.
+  """
+  # TODO(bradnelson): use the json interface in place of html scraping.
+  # Returned cached copy if we have one.
+  key = '%s/builders/%s' % (master_url, builder_name)
+  info = memcache.get(key)
+  if info:
+    return info
+  # Prep it.
+  info = {
+      'master_url': master_url,
+      'name': builder_name,
+      'currently_building': [],
+      'recent_builds': [],
+  }
+  # Grab the webpage.
+  url = master_url + '/builders/' + builder_name
+  try:
+    result = urlfetch.fetch(url)
+  except urlfetch.DownloadError:
+    return info
+  if result.status_code != 200:
+    return info
+  # Pick out the currently building list.
+  text = re.search('<h2>Currently Building:</h2>[ \n\r]+<ul>(.*?)</ul>',
+                   result.content, re.DOTALL)
+  if text:
+    # Pick out info about each one currently building.
+    builders = re.findall(r'<li>([^:]+): <a[^>]+>#([0-9]+)</a> '
+                          r'ETA ([0-9]+)s \(([^)]+)\) \[([^]]+)\]</li>',
+                          text.group(1))
+    for b in builders:
+      e = {
+          'slave_name': str(b[0]),
+          'build_number': int(b[1]),
+          'eta_seconds': int(b[2]),
+          'eta_time': str(b[3]),
+          'stage': str(b[4]),
+      }
+      info['currently_building'].append(e)
+  # Pick out the recently built list.
+  text = re.search('<h2>Recent Builds:</h2>[ \n\r]+<ul>(.*?)</ul>',
+                   result.content, re.DOTALL)
+  if text:
+    # Pick out info about each.
+    builders = re.findall(r'>#([0-9]+)</a>:', text.group(1))
+    for b in builders:
+      e = {
+          'build_number': int(b),
+      }
+      info['recent_builds'].append(e)
+  # Find highest builder number.
+  build_nums = [0]
+  build_nums += [i['build_number'] for i in info['currently_building']]
+  build_nums += [i['build_number'] for i in info['recent_builds']]
+  info['latest'] = max(build_nums)
+  # Add to cache.
+  memcache.add(key, info, 1)
+  return info
+
+
+def CheckForBuild(master_url, builder_name, build_number):
+  """Check if a particular build exists.
+
+  Arguments:
+    master_url: the URL of the master.
+    builder_name: the name of a builder on this master.
+    build_number: integer builder number.
+  Returns:
+    True or False depending on if this build exists in the data-store.
+  """
+  q = db.GqlQuery('SELECT __key__ FROM BuildResult '
+                  'WHERE master_url=:1 '
+                  'AND builder_name=:2 '
+                  'AND build_number=:3',
+                  master_url, builder_name, build_number)
+  b = q.fetch(1)
+  if not b or len(b) != 1:
+    return False
+  else:
+    return True
+
+
+def UpdateBuildResult(br):
+  """Update data-store information on a particular build.
+
+  Checks the master for fresh information on a given build.
+  Arguments:
+    br: A build result to update.
+  """
+  # TODO(bradnelson): use the json interface in place of html scraping.
+  # Advance time.
+  br.last_checked = datetime.datetime.now()
+  # Grab the webpage.
+  url = '%s/builders/%s/builds/%d' % (
+      br.master_url, br.builder_name, br.build_number)
+  try:
+    result = urlfetch.fetch(url)
+  except urlfetch.DownloadError:
+    br.put()
+    logging.debug('Download Error Updating %s | %s | %d',
+                  br.master_url, br.builder_name, br.build_number)
+    return
+  # Handle 404 specially.
+  if result.status_code == 404:
+    # Put in an empty entry for expired ones.
+    # But only do it if we have nothing else.
+    if not CheckForBuild(br.master_url, br.builder_name, br.build_number):
+      br.end = datetime.datetime.fromordinal(1)
+      br.put()
+  if result.status_code != 200:
+    br.put()
+    logging.debug('Result Code %d Updating %s | %s | %d',
+                  result.status_code,
+                  br.master_url, br.builder_name, br.build_number)
+    return
+
+  # Pick out content.
+  content = result.content
+
+  # Pick out steps and logfiles.
+  text = re.search('<h2>Build In Progress</h2><div>'
+                   r'ETA ([0-9]+)s \(([^)]+)\)</div>', content)
+  if text:
+    br.eta_seconds = int(text.group(1))
+  else:
+    br.eta_seconds = 0
+  # Pick out steps and logfiles.
+  text = re.search('<h2>Steps and Logfiles:</h2>[ \n\r]+'
+                   '<ol>(.+?)</ol>[ \n\r]*<h2>',
+                   content, re.DOTALL)
+  if text:
+    br.steps_and_logfiles = text.group(1)
+  # Pick out reason.
+  text = re.search('<h2>Reason:</h2>[\n\r]+([^\n\r].+?)[\n\r]+<h2>',
+                   content, re.DOTALL)
+  if text:
+    br.raw_reason = text.group(1)
+    m = re.match("'([^:]+): ([^']+)' try job", br.raw_reason)
+    if m:
+      br.username = m.group(1)
+      br.patch_name = m.group(2)
+  # Pick out source stamp.
+  text = re.search('<h2>SourceStamp:</h2>[\n\r ]+<ul>(.+?)</ul>',
+                   content, re.DOTALL)
+  if text:
+    m = re.search('<li>Revision: ([0-9]+)</li>', text.group(1))
+    if m:
+      br.revision = int(m.group(1))
+    m = re.search('<li>Got Revision: ([0-9]+)</li>', text.group(1))
+    if m:
+      br.got_revision = int(m.group(1))
+  # Pick out all changes.
+  text = re.search('<h2>All Changes</h2>[\n\r ]+<ol>(.+?)</ol>',
+                   content, re.DOTALL)
+  if text:
+    # Pick out who changed it.
+    m = re.search('<p>Changed by: <b>(.+?)</b><br />', text.group(1))
+    if m:
+      br.account_name = m.group(1)
+    # Pick out when the change occured.
+    m = re.search('Changed at: <b>(.+?)</b><br />', text.group(1))
+    if m:
+      tm = time.strptime(m.group(1), '%a %d %b %Y %H:%M:%S')
+      br.changed_at = datetime.datetime(*(tm)[0:6])
+  # Pick out timing.
+  text = re.search('<h2>Timing</h2>[\n\r ]+<table>(.+?)</table>',
+                   content, re.DOTALL)
+  if text:
+    m = re.search('<tr><td>Start</td><td>(.+?)</td></tr>', text.group(1))
+    if m:
+      tm = time.strptime(m.group(1), '%a %b %d %H:%M:%S %Y')
+      br.start = datetime.datetime(*(tm)[0:6])
+    m = re.search('<tr><td>End</td><td>(.+?)</td></tr>', text.group(1))
+    if m:
+      tm = time.strptime(m.group(1), '%a %b %d %H:%M:%S %Y')
+      br.end = datetime.datetime(*(tm)[0:6])
+  # Pick page built.
+  text = re.search('Page built: (.+?)[\n\r ]+</div>',
+                   content, re.DOTALL)
+  if text:
+    tm = time.strptime(text.group(1), '%a %d %b %Y %H:%M:%S')
+    br.page_built = datetime.datetime(*(tm)[0:6])
+  # Store it.
+  br.put()
+  # Log it.
+  logging.debug('Updating %s | %s | %d',
+                br.master_url, br.builder_name, br.build_number)
+
+
+def IntroduceBuildResult(master_url, builder_name, build_number):
+  """Introduces information into the data-store on a particular build.
+
+  Checks the master for fresh information on a given build.
+  Arguments:
+    master_url: the URL of the master.
+    builder_name: the name of a builder on this master.
+    build_number: integer builder number.
+  """
+  # Get the current entry if any.
+  q = BuildResult.gql('WHERE master_url=:1 and '
+                      'builder_name=:2 and '
+                      'build_number=:3',
+                      master_url, builder_name, build_number)
+  br = q.fetch(1)
+  if br:
+    br = br[0]
+  else:
+    br = BuildResult()
+  # Set what we know.
+  br.master_url = master_url
+  br.builder_name = builder_name
+  br.build_number = build_number
+  # Update it.
+  UpdateBuildResult(br)
+
+
+class UpdateBuilders(webapp.RequestHandler):
+  """Handles web requests to update all builders on a given trybot master."""
+
+  def post(self):
+    # Update info on the current set of builders.
+    master_url = self.request.get('master_url')
+    builders = GetBuilders(master_url)
+    for builder_name in builders:
+      taskqueue.Task(
+          url='/update_builder',
+          params={'master_url': master_url,
+                  'builder_name': builder_name}).add(
+                      queue_name='update-builder')
+
+
+def GetLatestBuilderBuild(master_url, builder_name):
+  """Given a builder, find the latest build number for it.
+
+  Arguments:
+    master_url: the URL of the master.
+    builder_name: the name of a builder on this master.
+  Returns:
+    The integer value of the latest build number for this builder.
+  """
+  q = BuildResult.gql('WHERE master_url=:1 '
+                      'AND builder_name=:2 '
+                      'ORDER BY build_number DESC',
+                      master_url, builder_name)
+  latest = q.fetch(1)
+  if not latest or len(latest) != 1:
+    return 0
+  else:
+    return latest[0].build_number
+
+
+class UpdateBuilder(webapp.RequestHandler):
+  """Handle a web request to poll the status of a given builder."""
+
+  def post(self):
+    """Handle post requests."""
+    # Update info on one builder.
+    master_url = self.request.get('master_url')
+    builder_name = self.request.get('builder_name')
+    # Get latest build known to builder.
+    info = GetBuilderInfo(master_url, builder_name)
+    latest_known = info.get('latest', 1)
+    # Get latest build in the datastore.
+    latest_stored = GetLatestBuilderBuild(master_url, builder_name)
+    # We're done if they match,.
+    if latest_stored >= latest_known: return
+    # Add a task to introduce up to that build.
+    taskqueue.Task(
+        url='/introduce_build',
+        params={'master_url': master_url,
+                'builder_name': builder_name,
+                'begin': str(latest_stored + 1),
+                'end': str(latest_known)}).add(
+                    queue_name='introduce-build')
+
+
+class IntroduceBuild(webapp.RequestHandler):
+  """Handles a web request to introduce a new build to the datastore."""
+
+  def get(self):
+    """Handle get requests as post requests."""
+    self.post()
+
+  def post(self):
+    """Handle post requests."""
+    # Update info on one builder.
+    master_url = self.request.get('master_url')
+    builder_name = self.request.get('builder_name')
+    begin = int(self.request.get('begin'))
+    end = int(self.request.get('end'))
+    # Done if begin > end.
+    if begin > end: return
+    # Update one past current one.
+    IntroduceBuildResult(master_url, builder_name, begin)
+    # Log it.
+    logging.debug('Introducing %s | %s | %d',
+                  master_url, builder_name, begin)
+    # Advance to the next one.
+    begin += 1
+    # Done if begin > end.
+    if begin > end: return
+    # Trigger self again.
+    taskqueue.Task(
+        url='/introduce_build',
+        params={'master_url': master_url,
+                'builder_name': builder_name,
+                'begin': str(begin),
+                'end': str(end)}).add(
+                    queue_name='introduce-build')
+
+
+class UpdateBuilds(webapp.RequestHandler):
+  """Handle a web request to update the most stale build still in progress."""
+
+  def post(self):
+    """Handle post requests."""
+    # Update info on one build.
+    q = BuildResult.gql('WHERE end=null ORDER BY last_checked')
+    stale = q.fetch(1)
+    if not stale:
+      return
+    for br in stale:
+      UpdateBuildResult(br)
+
+
+def GetBuildListFromDB(count):
+  """Fetch a list of the keys for the most recent builds.
+
+  Arguments:
+    count: a limit on the number of builds to fetch.
+  Returns:
+    A list of database keys for the latest builds.
+  """
+  q = db.GqlQuery('SELECT __key__ FROM BuildResult ORDER BY start DESC')
+  builds = q.fetch(count)
+  # List out the keys for these builds.
+  info = ''
+  for b in builds:
+    info += '%s\n' % b
+  return info
+
+
+def GetBuildList(count):
+  """Fetch a list of the keys for the most recent builds, cached.
+
+  Cache recent builds in the memcache, to reduce load on the data-store.
+  Arguments:
+    count: a limit on the number of builds to fetch.
+  Returns:
+    A list of database keys for the latest builds.
+  """
+  key = 'listbuilds//%d' % count
+  info = memcache.get(key)
+  if not info:
+    info = GetBuildListFromDB(count)
+    memcache.set(key, info, 20)
+  return info
+
+
+class ListBuilds(webapp.RequestHandler):
+  """Handle a web request for the latest builds."""
+
+  def get(self):
+    """Handle get requests."""
+    self.response.headers['Content-Type'] = 'text/plain'
+    # Get a different number of builds based on if this is the initial set.
+    if self.request.get('initial', 0):
+      info = GetBuildList(300)
+    else:
+      info = GetBuildList(50)
+    # Write it out.
+    self.response.out.write(info)
+
+
+class BuilderDump(webapp.RequestHandler):
+  """Handle a web request for information on a particular builder."""
+
+  def get(self):
+    """Handle get requests."""
+    self.response.headers['Content-Type'] = 'text/plain'
+    q = BuildResult.gql('WHERE builder_name=:1 '
+                        'and build_number >= :2 '
+                        'ORDER BY build_number',
+                        self.request.get('builder'),
+                        int(self.request.get('start')))
+    builds = q.fetch(30)
+    for i in builds:
+      self.response.out.write('%d\n' % i.build_number)
+
+
+class BuildInfo(webapp.RequestHandler):
+  """Handle a web request for information on a particular build."""
+
+  def get(self):
+    """Handle get requests."""
+    self.response.headers['Content-Type'] = 'text/plain'
+    build_id = str(self.request.get('id'))
+    # Decide the key to use for the memory cache.
+    key = 'buildinfo3//' + build_id
+    # Try to get a cached copy.
+    cinfo = memcache.get(key)
+    if cinfo:
+      cache_on_client = cinfo[0]
+      info = cinfo[1]
+    else:
+      # Load from datastore.
+      br = BuildResult.get(build_id)
+      if not br: return
+      # Write out info on this build.
+      info = ''
+      info += ('master_url %s\n' % br.master_url)
+      info += ('builder_name %s\n' % br.builder_name)
+      info += ('build_number %s\n' % br.build_number)
+      info += ('slave_name %s\n' % br.slave_name)
+      info += ('raw_reason %s\n' % br.raw_reason)
+      info += ('username %s\n' % br.username)
+      info += ('patch_name %s\n' % br.patch_name)
+      info += ('got_revision %s\n' % br.got_revision)
+      info += ('account_name %s\n' % br.account_name)
+      info += ('changed_at %s\n' % br.changed_at)
+      info += ('start %s\n' % br.start)
+      info += ('end %s\n' % br.end)
+      info += ('eta_seconds %s\n' % br.eta_seconds)
+      info += ('page_built %s\n' % br.page_built)
+      info += ('last_checked %s\n' % br.last_checked)
+      logs = re.sub('[\r\n\t]', ' ', br.steps_and_logfiles)
+      info += ('steps_and_logfiles %s\n' % logs)
+      # Only allow caching on client if its done building.
+      cache_on_client = br.end is not None
+      if cache_on_client:
+        timeout = 0
+      else:
+        timeout = 15
+      memcache.set(key, (cache_on_client, info), time=timeout)
+    # Allow the client to cache it for a day, assumming its done.
+    if cache_on_client:
+      self.response.headers['Cache-Control'] = 'public, max-age=86400'
+    # Write out the info.
+    self.response.out.write(info)
+
+
+class StalePage(webapp.RequestHandler):
+  """Return a long cached value for requests to historical stale pages."""
+
+  def get(self):
+    """Handle get requests."""
+    self.response.headers['Content-Type'] = 'text/plain'
+    self.response.headers['Cache-Control'] = 'public, max-age=86400'
+
+
+def ChromeFrameMe(handler):
+  """Handle web requests to instruct users to install ChromeFrame.
+
+  Arguments:
+    handler: the current webapp.RequestHandler.
+  Returns:
+    True or False indicating success.
+  """
+  agent = handler.request.headers.get('USER_AGENT', '')
+  if agent.find('MSIE') >= 0 and agent.find('chromeframe') < 0:
+    path = os.path.join(os.path.dirname(__file__),
+                        'templates/chrome_frame.html')
+    handler.response.out.write(template.render(path, {}))
+    return True
+  return False
+
+
+class MainPage(webapp.RequestHandler):
+  """Handle web requests for the main tryconsole page."""
+
+  def get(self):
+    """Handle get requests."""
+    if ChromeFrameMe(self): return
+    path = os.path.join(os.path.dirname(__file__), 'templates/status.html')
+    self.response.out.write(template.render(path, {}))
+
+
+class PerformanceRaw(webapp.RequestHandler):
+  """Handle web requests for graphed performance data."""
+
+  def get(self):
+    """Handle get requests."""
+    now = datetime.datetime.now()
+    self.response.headers['Content-Type'] = 'text/plain'
+    interval = int(self.request.get('interval'))
+    start = int(self.request.get('start'))
+    stop = int(self.request.get('stop'))
+    q = PerformanceSamples.gql('WHERE interval=:1 and '
+                               'measured_at>=:2 and '
+                               'measured_at<=:3 '
+                               'ORDER BY measured_at',
+                               interval,
+                               now - datetime.timedelta(seconds=start),
+                               now - datetime.timedelta(seconds=stop))
+    samples = q.fetch(1000)
+    for s in samples:
+      measured_at = int(time.mktime(s.measured_at.timetuple()))
+      srb = pickle.loads(s.running_builds)
+      for b in srb:
+        self.response.out.write(
+            '%s|%s|%d|%d|%d\n' % (b[0], b[1], measured_at,
+                                  srb[b]['count'], srb[b]['samples']))
+
+
+class TallyMerge(webapp.RequestHandler):
+  """Handle web requests to generate overview performance data."""
+
+  def get(self):
+    """Handle get requests."""
+    now = datetime.datetime.now()
+    src = int(self.request.get('src'))
+    dst = int(self.request.get('dst'))
+    q = PerformanceSamples.gql('WHERE interval=:1 and measured_at>=:2',
+                               src, now - datetime.timedelta(seconds=dst))
+    samples = q.fetch(300)
+    # Tally each.
+    running_builds = {}
+    for s in samples:
+      srb = pickle.loads(s.running_builds)
+      for ss in srb:
+        old = running_builds.get(ss, {'count': 0, 'samples': 0})
+        running_builds[ss] = {
+            'count': old['count'] + srb[ss]['count'],
+            'samples': old['samples'] + srb[ss]['samples'],
+        }
+    # Store the total.
+    ps = PerformanceSamples()
+    ps.running_builds = pickle.dumps(running_builds)
+    ps.interval = dst
+    ps.measured_at = now
+    ps.put()
+
+
+class TallyRunning(webapp.RequestHandler):
+  """Handle web requests to record a sample of builds in progress."""
+
+  def get(self):
+    """Handle get requests."""
+    now = datetime.datetime.now()
+    q = BuildResult.gql('WHERE end=null ORDER BY start DESC')
+    builds = q.fetch(300)
+    # Find set of unique builders.
+    builders = set()
+    for b in builds:
+      builders.add((b.master_url, b.builder_name))
+    # Add in tally for each builder.
+    running_builds = {}
+    for bldr in builders:
+      count = 0
+      for b in builds:
+        if b.master_url == bldr[0] and b.builder_name == bldr[1]:
+          count += 1
+      # Add an entry for this builder.
+      running_builds[bldr] = {
+          'count': count,
+          'samples': 1,
+      }
+    # Write to database.
+    ps = PerformanceSamples()
+    ps.running_builds = pickle.dumps(running_builds)
+    ps.interval = 60
+    ps.measured_at = now
+    ps.put()
+
+
+class StartWork(webapp.RequestHandler):
+  """Handle web cron requests to initiate an update work cycle."""
+
+  def get(self):
+    """Handle get requests."""
+    # Run tasks for master.
+    for master_url in BUILDBOT_URLS:
+      taskqueue.Task(
+          url='/update_builders',
+          params={'master_url': master_url}).add(
+              queue_name='update-builders')
+    # Initiate updates on existing builds.
+    for t in range(0, 10):
+      taskqueue.Task(
+          url='/update_builds',
+          countdown=6*t).add(queue_name='update-builds')
+
+
+application = webapp.WSGIApplication([
+    ('/', MainPage),
+    ('/start_work', StartWork),
+    ('/update_builders', UpdateBuilders),
+    ('/update_builder', UpdateBuilder),
+    ('/update_builds', UpdateBuilds),
+    ('/introduce_build', IntroduceBuild),
+    ('/tally_running', TallyRunning),
+    ('/tally_merge', TallyMerge),
+    ('/performance_raw', PerformanceRaw),
+    ('/list_builds', ListBuilds),
+    ('/build_info2', BuildInfo),
+    ('/build_info', StalePage),
+    ('/builder_dump', BuilderDump),
+], debug=True)
+
+
+def main():
+  run_wsgi_app(application)
+
+
+if __name__ == '__main__':
+  main()