| Index: buildlogparse.py
|
| diff --git a/buildlogparse.py b/buildlogparse.py
|
| new file mode 100644
|
| index 0000000000000000000000000000000000000000..f99a34fbb476d641773170e722347ce3d839aebc
|
| --- /dev/null
|
| +++ b/buildlogparse.py
|
| @@ -0,0 +1,465 @@
|
| +# buildlogparse.py: Proxy and rendering layer for build.chromium.org.
|
| +# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
| +# Use of this source code is governed by a BSD-style license that can be
|
| +# found in the LICENSE file.
|
| +
|
| +import jinja2
|
| +import json
|
| +import logging
|
| +import os
|
| +import re
|
| +import time
|
| +import urlparse
|
| +import webapp2
|
| +import zlib
|
| +
|
| +from google.appengine.api import urlfetch
|
| +from google.appengine.ext import db
|
| +
|
| +import utils
|
| +
|
| +
|
| +VERSION_ID = os.environ['CURRENT_VERSION_ID']
|
| +
|
| +jinja_environment = jinja2.Environment(
|
| + loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__),
|
| + 'templates')),
|
| + autoescape=True,
|
| + extensions=['jinja2.ext.autoescape'])
|
| +jinja_environment.filters['delta_time'] = utils.delta_time
|
| +jinja_environment.filters['nl2br'] = utils.nl2br
|
| +jinja_environment.filters['time_since'] = utils.time_since
|
| +jinja_environment.filters['rot13_email'] = utils.rot13_email
|
| +jinja_environment.filters['cl_comment'] = utils.cl_comment
|
| +
|
| +if os.environ.get('HTTP_HOST'):
|
| + APP_URL = os.environ['HTTP_HOST']
|
| +else:
|
| + APP_URL = os.environ['SERVER_NAME']
|
| +
|
| +# Note: All of these replacements occur AFTER jinja autoescape.
|
| +# This way we can add <html> tags in the replacements, but do note that spaces
|
| +# are .
|
| +REPLACEMENTS = [
|
| + # Find ../../scripts/.../*.py scripts and add links to them.
|
| + (r'\.\./\.\./\.\./scripts/(.*)\.py',
|
| + r'<a href="https://code.google.com/p/chromium/codesearch#chromium/tools/'
|
| + r'build/scripts/\1.py">../../scripts/\1.py</a>'),
|
| +
|
| + # Find ../../chrome/.../*.cc files and add links to them.
|
| + (r'\.\./\.\./chrome/(.*)\.cc:(\d+)',
|
| + r'<a href="https://code.google.com/p/chromium/codesearch#chromium/src/'
|
| + r'chrome/\1.cc&l=\2">../../chrome/\1.cc:\2</a>'),
|
| +
|
| + # Searches for codereview issue numbers, and add codereview links.
|
| + (r'apply_issue(.*)-i (\d{8})(.*)-s (.*)',
|
| + r'apply_issue\1-i <a href="\4/\2">\2</a>\3-s \4'),
|
| +
|
| + # Add green labels to PASSED or OK items.
|
| + (r'\[(( PASSED )|'
|
| + r'( OK ))\]',
|
| + r'<span class="label label-success">[\1]</span>'),
|
| +
|
| + # Add red labels to FAILED items.
|
| + (r'\[( FAILED )\]',
|
| + r'<span class="label label-important">[\1]</span>'),
|
| +
|
| + # Add black labels ot RUN items.
|
| + (r'\[( RUN )\]',
|
| + r'<span class="label label-inverse">[\1]</span>'),
|
| +
|
| + # Add badges to running tests.
|
| + (r'\[(( )*\d+/\d+)\](( )+)(\d+\.\d+s) '
|
| + r'([\w/]+\.[\w/]+) \(([\d.s]+)\)',
|
| + r'<span class="badge badge-success">\1</span>\3<span class="badge">'
|
| + r'\5</span> \6 <span class="badge">\7</span>'),
|
| +
|
| + # Add gray labels to [==========] blocks.
|
| + (r'\[([-=]{10})\]',
|
| + r'<span class="label">[\1]</span>'),
|
| +
|
| + # Find .cc and .h files and add codesite links to them.
|
| + (r'\.\./\.\./([\w/-]+)\.(cc|h): ',
|
| + r'<a href="https://code.google.com/p/chromium/codesearch#chromium/src/'
|
| + r'\1.\2">../../\1.\2</a>: '),
|
| +
|
| + # Find source files with line numbers and add links to them.
|
| + (r'\.\./\.\./([\w/-]+)\.(cc|h):(\d+): ',
|
| + r'<a href="https://code.google.com/p/chromium/codesearch#chromium/src/'
|
| + r'\1.\2&l=\3">../../\1.\2:\3</a>: '),
|
| +
|
| + # Add badges to compiling items.
|
| + (r'\[(\d+/\d+)\] (CXX|AR|STAMP|CC|ACTION|RULE|COPY)',
|
| + r'<span class="badge badge-info">\1</span> '
|
| + r'<span class="badge">\2</span>'),
|
| +
|
| + # Bold the LHS of A=B text.
|
| + (r'^(( )*)(\w+)=([\w:/-_.]+)',
|
| + r'\1<strong>\3</strong>=\4'),
|
| +]
|
| +
|
| +
|
| +########
|
| +# Models
|
| +########
|
| +class BuildLogModel(db.Model):
|
| + # Used for caching finished build logs.
|
| + url = db.StringProperty()
|
| + data = db.BlobProperty()
|
| +
|
| +class BuildbotCacheModel(db.Model):
|
| + # Used for caching finished build data.
|
| + url = db.StringProperty()
|
| + data = db.BlobProperty()
|
| +
|
| +class BuildLogResultModel(db.Model):
|
| + # Used for caching finished and parsed build logs.
|
| + url = db.StringProperty()
|
| + version = db.StringProperty()
|
| + data = db.BlobProperty()
|
| +
|
| +
|
| +def emit(source, out):
|
| + # TODO(hinoka): This currently employs a "lookback" strategy
|
| + # (Find [PASS/FAIL], then goes back and marks all of the lines.)
|
| + # This should be switched to a "scan twice" strategy. 1st pass creates a
|
| + # Test Name -> PASS/FAIL/INCOMPLETE dictionary, and 2nd pass marks the lines.
|
| + attr = []
|
| + if source == 'header':
|
| + attr.append('text-info')
|
| + lines = []
|
| + current_test = None
|
| + current_test_line = 0
|
| + for line in out.split('\n'):
|
| + if line:
|
| + test_match = re.search(r'\[ RUN \]\s*([^() ]*)\s*', line)
|
| + line_attr = attr[:]
|
| + if test_match:
|
| + # This line is a "We're running a test" line.
|
| + current_test = test_match.group(1).strip()
|
| + current_test_line = len(lines)
|
| + elif '[ OK ]' in line or '[ PASSED ]' in line:
|
| + line_attr.append('text-success')
|
| + test_match = re.search(r'\[ OK \]\s*([^(), ]*)\s*', line)
|
| + if test_match:
|
| + finished_test = test_match.group(1).strip()
|
| + for line_item in lines[current_test_line:]:
|
| + if finished_test == current_test:
|
| + line_item[2].append('text-success')
|
| + else:
|
| + line_item[2].append('text-error')
|
| + current_test = None
|
| + elif '[ FAILED ]' in line:
|
| + line_attr.append('text-error')
|
| + test_match = re.search(r'\[ FAILED \]\s*([^(), ]*)\s*', line)
|
| + if test_match:
|
| + finished_test = test_match.group(1).strip()
|
| + for line_item in lines[current_test_line:]:
|
| + if finished_test == current_test:
|
| + line_item[2].append('text-error')
|
| + current_test = None
|
| + elif re.search(r'\[.{10}\]', line):
|
| + current_test = None
|
| + elif re.search(r'\[\s*\d+/\d+\]\s*\d+\.\d+s\s+[\w/]+\.'
|
| + r'[\w/]+\s+\([\d.s]+\)', line):
|
| + # runtest.py output: [20/200] 23.3s [TestSuite.TestName] (5.3s)
|
| + current_test = None
|
| + line_attr.append('text-success')
|
| + elif 'aborting test' in line:
|
| + current_test = None
|
| + elif current_test:
|
| + line_attr.append('text-warning')
|
| +
|
| + line = line.replace(' ', ' ')
|
| + for rep_from, rep_to in REPLACEMENTS:
|
| + line = re.sub(rep_from, rep_to, line)
|
| + lines.append((line, line_attr))
|
| + return (source, lines)
|
| +
|
| +
|
| +class BuildbotPassthrough(webapp2.RequestHandler):
|
| + def get(self, path):
|
| + # TODO(hinoka): Page caching.
|
| + url = 'http://build.chromium.org/p/%s' % path
|
| + s = urlfetch.fetch(url.replace(' ', '%20'),
|
| + method=urlfetch.GET, deadline=60).content
|
| + s = s.replace('default.css', '../../static/default-old.css')
|
| + self.response.out.write(s)
|
| +
|
| +
|
| +class BuildStep(webapp2.RequestHandler):
|
| + @staticmethod
|
| + def get_build_step(url):
|
| + build_step = BuildbotCacheModel.all().filter('url =', url).get()
|
| + if build_step:
|
| + return json.loads(build_step.data)
|
| + else:
|
| + s = urlfetch.fetch(url.replace(' ', '%20'),
|
| + method=urlfetch.GET, deadline=60).content
|
| + logging.info(s)
|
| + build_step_data = json.loads(s)
|
| + # Cache if completed.
|
| + if not build_step_data['currentStep']:
|
| + build_step = BuildbotCacheModel(url=url, data=s)
|
| + build_step.put()
|
| + return build_step_data
|
| +
|
| + @utils.render_iff_new_flag_set('step.html', jinja_environment)
|
| + def get(self, master, builder, step, new=None):
|
| + """Parses a build step page."""
|
| + # Fetch the page.
|
| + if new:
|
| + json_url = ('http://build.chromium.org/p/%s/'
|
| + 'json/builders/%s/builds/%s' % (master, builder, step))
|
| + result = BuildStep.get_build_step(json_url)
|
| +
|
| + # Add on some extraneous info.
|
| + build_properties = dict((name, value) for name, value, _
|
| + in result['properties'])
|
| + failed_steps = ['<strong>%s</strong>' % s['name'] for s in result['steps']
|
| + if s['results'][0] == 2]
|
| + if len(failed_steps) == 1:
|
| + result['failed_steps'] = failed_steps[0]
|
| + elif len(failed_steps) == 2:
|
| + logging.info(failed_steps)
|
| + result['failed_steps'] = '%s and %s' % tuple(failed_steps)
|
| + elif failed_steps:
|
| + # Oxford comma.
|
| + result['failed_steps'] = '%s, and %s' % (
|
| + ', '.join(failed_steps[:-1], failed_steps[-1]))
|
| + else:
|
| + result['failed_steps'] = None
|
| +
|
| + if 'rietveld' in build_properties:
|
| + result['rietveld'] = build_properties['rietveld']
|
| + result['breadcrumbs'] = [
|
| + ('Master %s' % master, '/buildbot/%s' % master),
|
| + ('Builder %s' % builder, '/buildbot/%s/builders/%s' %
|
| + (master, builder)),
|
| + ('Slave %s' % result['slave'],
|
| + '/buildbot/%s/buildslaves/%s' % (master, result['slave'])),
|
| + ('Build Number %s' % step,
|
| + '/buildbot/%s/builders/%s/builds/%s' %
|
| + (master, builder, step)),
|
| + ]
|
| + result['url'] = self.request.url.split('?')[0]
|
| + return result
|
| + else:
|
| + url = ('http://build.chromium.org/p/%s/'
|
| + 'builders/%s/builds/%s' % (master, builder, step))
|
| + s = urlfetch.fetch(url.replace(' ', '%20'),
|
| + method=urlfetch.GET, deadline=60).content
|
| + s = s.replace('../../../default.css', '/static/default-old.css')
|
| + s = s.replace('<a href="../../../about">About</a>',
|
| + '<a href="../../../about">About</a>'
|
| + ' - <a href="%s?new=true">New Layout</a>' %
|
| + self.request.url.split('?')[0])
|
| + return s
|
| +
|
| +
|
| +class BuildSlave(webapp2.RequestHandler):
|
| + """Parses a build slave page."""
|
| + @utils.render_iff_new_flag_set('slave.html', jinja_environment)
|
| + def get(self, master, slave, new=None):
|
| + # Fetch the page.
|
| + if new:
|
| + json_url = ('http://build.chromium.org/p/%s/'
|
| + 'json/slaves/%s' % (master, slave))
|
| + logging.info(json_url)
|
| + s = urlfetch.fetch(json_url.replace(' ', '%20'),
|
| + method=urlfetch.GET, deadline=60).content
|
| +
|
| + result = json.loads(s)
|
| + result['breadcrumbs'] = [
|
| + ('Master %s' % master,
|
| + '/buildbot/%s?new=true' % master),
|
| + ('All Slaves',
|
| + '/buildbot/%s/buildslaves?new=true' % master),
|
| + ('Slave %s' % slave,
|
| + '/buildbot/%s/buildslaves/%s?new=true' % (master, slave)),
|
| + ]
|
| + result['url'] = self.request.url.split('?')[0]
|
| + result['master'] = master
|
| + result['slave'] = slave
|
| + return result
|
| + else:
|
| + url = ('http://build.chromium.org/p/%s/buildslaves/%s' %
|
| + (master, slave))
|
| + s = urlfetch.fetch(url.replace(' ', '%20'),
|
| + method=urlfetch.GET, deadline=60).content
|
| + s = s.replace('../default.css', '/static/default-old.css')
|
| + s = s.replace('<a href="../about">About</a>',
|
| + '<a href="../about">About</a>'
|
| + ' - <a href="%s?new=true">New Layout</a>' %
|
| + self.request.url.split('?')[0])
|
| + return s
|
| +
|
| +
|
| +class MainPage(webapp2.RequestHandler):
|
| + """Parses a buildlog page."""
|
| + @utils.render('buildbot.html', jinja_environment)
|
| + @utils.expect_request_param('url')
|
| + def get(self, url):
|
| + if not url:
|
| + return {}
|
| +
|
| + # Redirect the page if we detect a different type of URL.
|
| + _, _, path, _, _, _ = urlparse.urlparse(url)
|
| + logging.info(path)
|
| + step_m = re.match(r'^/((p/)?)(.*)/builders/(.*)/builds/(\d+)$', path)
|
| + if step_m:
|
| + self.redirect('/buildbot/%s/builders/%s/builds/%s' % step_m.groups()[2:])
|
| + return {}
|
| +
|
| + log_m = re.match(
|
| + r'^/((p/)?)(.*)/builders/(.*)/builds/(\d+)/steps/(.*)/logs/(.*)', path)
|
| + if log_m:
|
| + self.redirect('/buildbot/%s/builders/%s/builds/%s/steps/%s'
|
| + '/logs/%s?new=true' % log_m.groups()[2:])
|
| + return {}
|
| +
|
| + self.error(404)
|
| + return {'error': 'Url not found: %s' % url}
|
| +
|
| +class BuildLog(webapp2.RequestHandler):
|
| + @staticmethod
|
| + def fetch_buildlog(url):
|
| + """Fetch buildlog from either the datastore cache or the remote url.
|
| + Caches the log once fetched."""
|
| + buildlog = BuildLogModel.all().filter('url =', url).get()
|
| + if buildlog:
|
| + return zlib.decompress(buildlog.data)
|
| + else:
|
| + log_fetch_start = time.time()
|
| + s = urlfetch.fetch(url.replace(' ', '%20'),
|
| + method=urlfetch.GET, deadline=60).content
|
| + logging.info('Log fetching time: %2f' % (time.time() - log_fetch_start))
|
| + # Cache this build log.
|
| + # TODO(hinoka): This should be in Google Storage.
|
| + compressed_data = zlib.compress(s)
|
| + if len(compressed_data) < 10**6:
|
| + buildlog = BuildLogModel(url=url, data=compressed_data)
|
| + buildlog.put()
|
| + return s
|
| +
|
| + @utils.render_iff_new_flag_set('logs.html', jinja_environment)
|
| + def get(self, master, builder, build, step, logname, new):
|
| + # Lets fetch the build data first to determine if this is a running step.
|
| + json_url = ('http://build.chromium.org/p/%s/'
|
| + 'json/builders/%s/builds/%s' % (master, builder, build))
|
| + build_data = BuildStep.get_build_step(json_url)
|
| + steps = dict([(_step['name'], _step) for _step in build_data['steps']])
|
| + # Construct the url to the log file.
|
| + url = ('http://build.chromium.org/'
|
| + 'p/%s/builders/%s/builds/%s/steps/%s/logs/%s' %
|
| + (master, builder, build, step, logname))
|
| + current_step = steps[step]
|
| + if not current_step['isFinished']:
|
| + # We're not finished with this step, redirect over to the real buildbot.
|
| + self.redirect(url)
|
| + return {} # Empty dict to keep the decorator happy.
|
| +
|
| + if new:
|
| + logging.info('New layout')
|
| + # New layout: We want to fetch the processed json blob.
|
| + # Check for cached results or fetch the page if none exists.
|
| + cached_result = BuildLogResultModel.all().filter(
|
| + 'url =', url).filter('version =', VERSION_ID).get()
|
| + if cached_result:
|
| + logging.info('Returning cached data')
|
| + return json.loads(zlib.decompress(cached_result.data))
|
| + else:
|
| + # Fetch the log from the buildbot master.
|
| + s = BuildLog.fetch_buildlog(url)
|
| +
|
| + # Parse the log output to add colors.
|
| + parse_time_start = time.time()
|
| + all_output = re.findall(r'<span class="(header|stdout)">(.*?)</span>',
|
| + s, re.S)
|
| + result_output = []
|
| + current_source = None
|
| + current_string = ''
|
| + for source, output in all_output:
|
| + if source == current_source:
|
| + current_string += output
|
| + continue
|
| + else:
|
| + # We hit a new source, we want to emit whatever we had left and
|
| + # start anew.
|
| + if current_string:
|
| + result_output.append(emit(current_source, current_string))
|
| + current_string = output
|
| + current_source = source
|
| + if current_string:
|
| + result_output.append(emit(current_source, current_string))
|
| + logging.info('Parse time: %2f' % (time.time() - parse_time_start))
|
| +
|
| + # Add build PASS/FAIL banner.
|
| + ret_code_m = re.search('program finished with exit code (-?\d+)', s)
|
| + if ret_code_m:
|
| + ret_code = int(ret_code_m.group(1))
|
| + if ret_code == 0:
|
| + status = 'OK'
|
| + else:
|
| + status = 'ERROR'
|
| + else:
|
| + ret_code = None
|
| + status = None
|
| +
|
| + final_result = {
|
| + 'output': result_output,
|
| + 'org_url': url,
|
| + 'url': self.request.url.split('?')[0],
|
| + 'name': step,
|
| + 'breadcrumbs': [
|
| + ('Master %s' % master,
|
| + '/buildbot/%s/waterfall' % master),
|
| + ('Builder %s' % builder,
|
| + '/buildbot/%s/builders/%s' %
|
| + (master, builder)),
|
| + ('Slave %s' % build_data['slave'],
|
| + '/buildbot/%s/buildslaves/%s' %
|
| + (master, build_data['slave'])),
|
| + ('Build Number %s ' % build,
|
| + '/buildbot/%s/builders/%s/builds/%s' %
|
| + (master, builder, build)),
|
| + ('Step %s' % step, '/buildbot/%s/builders/%s/builds/%s'
|
| + '/steps/%s/logs/%s' %
|
| + (master, builder, build, step, logname))
|
| + ],
|
| + 'status': status,
|
| + 'ret_code': ret_code,
|
| + 'debug': self.request.get('debug'),
|
| + 'size': len(s),
|
| + 'slave': build_data['slave']
|
| + }
|
| + # Cache parsed logs.
|
| + # TODO(hinoka): This should be in Google storage, where the grass is
|
| + # green and size limits don't exist.
|
| + compressed_result = zlib.compress(json.dumps(final_result))
|
| + if len(compressed_result) < 10**6:
|
| + cached_result = BuildLogResultModel(
|
| + url=url, version=VERSION_ID, data=compressed_result)
|
| + cached_result.put()
|
| +
|
| + return final_result
|
| + else:
|
| + # Fetch the log from the buildbot master.
|
| + logging.info('Old layout')
|
| + s = BuildLog.fetch_buildlog(url)
|
| + s = s.replace('default.css', '../../static/default-old.css')
|
| + s = s.replace('<a href="stdio/text">(view as text)</a>',
|
| + '<a href="stdio/text">(view as text)</a><br/><br/>'
|
| + '<a href="%s?new=true">(New layout)</a>' %
|
| + self.request.url.split('?')[0])
|
| + return s
|
| +
|
| +
|
| +app = webapp2.WSGIApplication([
|
| + ('/buildbot/', MainPage),
|
| + ('/buildbot/(.*)/builders/(.*)/builds/(\d+)/steps/(.*)/logs/(.*)/?',
|
| + BuildLog),
|
| + ('/buildbot/(.*)/builders/(.*)/builds/(\d+)/?', BuildStep),
|
| + ('/buildbot/(.*)/buildslaves/(.*)/?', BuildSlave),
|
| + ('/buildbot/(.*)', BuildbotPassthrough),
|
| + ], debug=True)
|
|
|
Chromium Code Reviews
Issue 13892003:
Added buildbot appengine frontend for chromium-build app (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/chromium-build