OLD | NEW |
---|---|
(Empty) | |
1 | |
2 | |
3 import webapp2 | |
4 from google.appengine.ext import db | |
5 from datetime import timedelta | |
6 import cStringIO | |
7 import time | |
8 import jinja2 | |
9 import datetime | |
10 import re | |
11 import logging | |
12 import urllib | |
13 from google.appengine.api import urlfetch | |
14 import base64 | |
15 import urlparse | |
16 import os | |
17 import json | |
18 import Queue | |
19 import os | |
20 import zlib | |
21 from google.appengine.api import users | |
22 from google.appengine.api import memcache | |
23 from google.appengine.ext import deferred | |
24 from google.appengine.api import files | |
25 from google.appengine.api import mail | |
agable
2013/04/15 19:33:30
Please cleanup imports to be only what you actuall
Ryan Tseng
2013/04/17 22:53:48
Done.
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
26 | |
27 VERSION_ID = os.environ['CURRENT_VERSION_ID'] | |
28 | |
29 jinja_environment = jinja2.Environment( | |
30 loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__), | |
31 'templates')), | |
32 autoescape=True, | |
33 extensions=['jinja2.ext.autoescape']) | |
34 | |
35 if os.environ.get('HTTP_HOST'): | |
36 APP_URL = os.environ['HTTP_HOST'] | |
37 else: | |
38 APP_URL = os.environ['SERVER_NAME'] | |
39 | |
40 REPLACEMENTS = [ | |
41 # Find ../../scripts/.../*.py scripts and add links to them. | |
42 (r'\.\./\.\./\.\./scripts/(.*)\.py', | |
43 r'<a href="https://code.google.com/p/chromium/codesearch#chromium/tools/' | |
44 r'build/scripts/\1.py">../../scripts/\1.py</a>'), | |
45 | |
46 # Find ../../chrome/.../*.cc files and add links to them. | |
47 (r'\.\./\.\./chrome/(.*)\.cc:(\d+)', | |
48 r'<a href="https://code.google.com/p/chromium/codesearch#chromium/src/' | |
49 r'chrome/\1.cc&l=\2">../../chrome/\1.cc:\2</a>'), | |
50 | |
51 # Searches for codereview issue numbers, and add codereview links. | |
52 (r'apply_issue(.*)-i (\d{8})(.*)-s (.*)', | |
53 r'apply_issue\1-i <a href="\4/\2">\2</a>\3-s \4'), | |
54 | |
55 # Add green labels to PASSED items. | |
56 (r'\[( PASSED )\]', | |
57 r'<span class="label label-success">[\1]</span>'), | |
58 | |
59 # Add red labels to FAILED items. | |
60 (r'\[( FAILED )\]', | |
61 r'<span class="label label-important">[\1]</span>'), | |
62 | |
63 # Add black labels ot RUN items. | |
64 (r'\[( RUN )\]', | |
65 r'<span class="label label-inverse">[\1]</span>'), | |
66 | |
67 # Add badges to running tests. | |
68 (r'\[(( )*\d+/\d+)\](( )+)(\d+\.\d+s) ' | |
69 r'([\w/]+\.[\w/]+) \(([\d.s]+)\)', | |
70 r'<span class="badge badge-success">\1</span>\3<span class="badge">' | |
71 r'\5</span> \6 <span class="badge">\7</span>'), | |
72 | |
73 # Add gray labels to [==========] blocks. | |
74 (r'\[([-=]{10})\]', | |
75 r'<span class="label">[\1]</span>'), | |
76 | |
77 # Find .cc and .h files and add codesite links to them. | |
78 (r'\.\./\.\./([\w/-]+)\.(cc|h): ', | |
79 r'<a href="https://code.google.com/p/chromium/codesearch#chromium/src/' | |
80 r'\1.\2">../../\1.\2</a>: '), | |
81 | |
82 # Find source files with line numbers and add links to them. | |
83 (r'\.\./\.\./([\w/-]+)\.(cc|h):(\d+): ', | |
84 r'<a href="https://code.google.com/p/chromium/codesearch#chromium/src/' | |
85 r'\1.\2&l=\3">../../\1.\2:\3</a>: '), | |
86 | |
87 # Add badges to compiling items. | |
88 (r'\[(\d+/\d+)\] (CXX|AR|STAMP|CC|ACTION|RULE|COPY)', | |
89 r'<span class="badge badge-info">\1</span> ' | |
90 r'<span class="badge">\2</span>'), | |
91 | |
92 # Bold the LHS of A=B text. | |
93 (r'^(( )*)(\w+)=([\w:/-_.]+)', | |
94 r'\1<strong>\3</strong>=\4'), | |
95 ] | |
96 | |
97 ############### | |
98 # Jinja filters | |
99 ############### | |
100 | |
101 def delta_time(delta): | |
102 hours = int(delta/60/60) | |
103 minutes = int((delta - hours * 3600)/60) | |
104 seconds = int(delta - (hours * 3600) - (minutes * 60)) | |
105 result = '' | |
106 if hours > 1: | |
107 result += '%d hrs ' % hours | |
agable
2013/04/15 19:33:30
nit: add commas after hr/hrs, min/mins, and a peri
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
108 elif hours: | |
109 result += '%d hr ' % hours | |
110 if minutes > 1: | |
111 result += '%d mins ' % minutes | |
112 elif minutes: | |
113 result += '%d min ' % minutes | |
114 if not hours: | |
115 if seconds > 1: | |
116 result += '%d secs' % seconds | |
117 else: | |
118 result += '%d sec' % seconds | |
119 return result | |
120 jinja_environment.filters['delta_time'] = delta_time | |
121 | |
122 def time_since(timestamp): | |
123 delta = time.time() - timestamp | |
124 return delta_time(delta) | |
125 jinja_environment.filters['time_since'] = time_since | |
126 | |
127 def nl2br(value): | |
128 return value.replace('\n','<br>\n') | |
129 jinja_environment.filters['nl2br'] = nl2br | |
130 | |
131 def cl_comment(value): | |
132 """Add links to https:// addresses, BUG=####, and trim excessive newlines.""" | |
133 value = re.sub(r'(https?://.*)', r'<a href="\1">\1</a>', value) | |
134 value = re.sub( | |
135 r'BUG=(\d+)', r'BUG=<a href="http://crbug.com/\1">\1</a>', value) | |
136 # value = re.sub(r'\n\n', r'\n', value) | |
137 value = re.sub(r'\n', r'<br>', value) | |
138 return value | |
139 jinja_environment.filters['cl_comment'] = cl_comment | |
140 | |
141 ######## | |
142 # Models | |
143 ######## | |
144 | |
145 class BuildLogModel(db.Model): | |
146 # Used for caching finished build logs. | |
147 url = db.StringProperty() | |
148 data = db.BlobProperty() | |
149 | |
150 class BuildLogResultModel(db.Model): | |
151 # Used for caching finished and parsed build logs. | |
152 url = db.StringProperty() | |
153 version = db.StringProperty() | |
154 data = db.BlobProperty() | |
155 | |
156 | |
157 ############ | |
158 # Decorators | |
159 ############ | |
160 def render(template_filename): | |
agable
2013/04/15 19:33:30
Docstring, similar to the one for render_json belo
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
161 def _render(fn): | |
162 def wrapper(self, *args, **kwargs): | |
163 results = fn(self, *args, **kwargs) | |
164 template = jinja_environment.get_template(template_filename) | |
165 self.response.out.write(template.render(results)) | |
166 return wrapper | |
167 return _render | |
168 | |
169 def render_json(fn): | |
170 # The function is expected to return a dict, and we want to render json. | |
agable
2013/04/15 19:33:30
Make this a real docstring.
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
171 def wrapper(self, *args, **kwargs): | |
172 results = fn(self, *args, **kwargs) | |
173 self.response.out.write(json.dumps(results)) | |
174 return wrapper | |
175 | |
176 def return_json_if_flag_is_set_else_render(template_filename): | |
agable
2013/04/15 19:33:30
maybe_return_json?
Ryan Tseng
2013/04/17 22:53:48
Works
| |
177 """If the variable 'json' exists in the request, return a json object. | |
178 Otherwise render the page using the template""" | |
179 def _render(fn): | |
180 def wrapper(self, *args, **kwargs): | |
181 results = fn(self, *args, **kwargs) | |
182 if self.request.get('json'): | |
183 self.response.out.write(json.dumps(results)) | |
184 else: | |
185 template = jinja_environment.get_template(template_filename) | |
186 self.response.out.write(template.render(results)) | |
187 return wrapper | |
188 return _render | |
189 | |
190 def login_required(fn): | |
191 """Redirect user to a login page.""" | |
192 def wrapper(self, *args, **kwargs): | |
193 user = users.get_current_user() | |
194 if not user: | |
195 self.redirect(users.create_login_url(self.request.uri)) | |
196 return | |
197 else: | |
198 return fn(self, *args, **kwargs) | |
199 return wrapper | |
200 | |
201 def google_login_required(fn): | |
202 """Return 403 unless the user is logged in from a @google.com domain""" | |
203 def wrapper(self, *args, **kwargs): | |
204 user = users.get_current_user() | |
205 if not user: | |
206 self.redirect(users.create_login_url(self.request.uri)) | |
207 return | |
208 email_match = re.match('^(.*)@(.*)$', user.email()) | |
209 if email_match: | |
210 _, domain = email_match.groups() | |
211 if domain == 'google.com': | |
212 return fn(self, *args, **kwargs) | |
213 self.error(403) # Unrecognized email or unauthroized domain. | |
214 self.response.out.write('unauthroized email %s' % user.user_id()) | |
215 return wrapper | |
216 | |
217 def admin_required(fn): | |
218 """Return 403 unless an admin is logged in""" | |
agable
2013/04/15 19:33:30
Give all of these docstrings periods -- they're se
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
219 def wrapper(self, *args, **kwargs): | |
220 user = users.get_current_user() | |
221 if not user: | |
222 self.redirect(users.create_login_url(self.request.uri)) | |
223 return | |
224 elif not users.is_current_user_admin(): | |
225 self.error(403) | |
226 return | |
227 else: | |
228 return fn(self, *args, **kwargs) | |
229 return wrapper | |
230 | |
231 def expect_request(*request_args): | |
agable
2013/04/15 19:33:30
expect_request_param? expect_request sounds like i
Ryan Tseng
2013/04/17 22:53:48
Works for me. Done
| |
232 """Strips out the expected args from a request and feeds it into the function | |
233 as the arguments. Optionally, typecast the argument from a string into a | |
234 different class. Examples include: | |
235 name (Get the request object called "name") | |
236 time as timestamp (Get "time", pass it in as "timestamp") | |
237 """ | |
238 def _decorator(fn): | |
239 def wrapper(self, *args, **kwargs): | |
240 request_kwargs = {} | |
241 for arg in request_args: | |
242 arg_match = re.match(r'^(\((\w+)\))?\s*(\w+)( as (\w+))?$', arg) | |
243 if arg_match: | |
244 _, target_type_name, name, _, target_name = arg_match.groups() | |
245 if not target_name: | |
246 target_name = name | |
247 request_item = self.request.get(name) | |
248 request_kwargs[target_name] = request_item | |
249 else: | |
250 raise Exception('Incorrect format %s' % arg) | |
251 kwargs.update(request_kwargs) | |
252 return fn(self, *args, **kwargs) | |
253 return wrapper | |
254 return _decorator | |
agable
2013/04/15 19:33:30
All these wrappers are really nice and general. On
Ryan Tseng
2013/04/17 22:53:48
Or I can do that now :)
| |
255 | |
256 def emit(source, out): | |
257 # TODO(hinoka): This currently employs a "lookback" strategy | |
258 # (Find [PASS/FAIL], then goes back and marks all of the lines.) | |
259 # This should be switched to a "scan twice" strategy. 1st pass creates a | |
260 # Test Name -> PASS/FAIL/INCOMPLETE dictionary, and 2nd pass marks the lines. | |
261 title = source | |
agable
2013/04/15 19:33:30
Remove this, title is never used.
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
262 attr = [] | |
263 if source == 'header': | |
264 attr.append('text-info') | |
265 lines = [] | |
266 current_test = None | |
267 current_test_line = 0 | |
268 for line in out.split('\n'): | |
269 if line: | |
270 test_match = re.search(r'\[ RUN \]\s*([^() ]*)\s*', line) | |
agable
2013/04/15 19:33:30
Here you're searching for [ RUN ], while earli
Ryan Tseng
2013/04/17 22:53:48
This set of regex is a bit special in that its not
| |
271 line_attr = attr[:] | |
272 if test_match: | |
273 # This line is a "We're running a test" line. | |
274 current_test = test_match.group(1).strip() | |
275 current_test_line = len(lines) | |
276 elif '[ OK ]' in line or '[ PASSED ]' in line: | |
277 line_attr.append('text-success') | |
278 test_match = re.search(r'\[ OK \]\s*([^(), ]*)\s*', line) | |
279 if test_match: | |
280 finished_test = test_match.group(1).strip() | |
281 for line_item in lines[current_test_line:]: | |
282 if finished_test == current_test: | |
283 line_item[2].append('text-success') | |
284 else: | |
285 line_item[2].append('text-error') | |
286 current_test = None | |
287 elif '[ FAILED ]' in line: | |
288 line_attr.append('text-error') | |
289 test_match = re.search(r'\[ FAILED \]\s*([^(), ]*)\s*', line) | |
290 if test_match: | |
291 finished_test = test_match.group(1).strip() | |
292 for line_item in lines[current_test_line:]: | |
293 if finished_test == current_test: | |
294 line_item[2].append('text-error') | |
295 current_test = None | |
296 elif re.search(r'\[.{10}\]', line): | |
297 current_test = None | |
298 elif re.search(r'\[\s*\d+/\d+\]\s*\d+\.\d+s\s+[\w/]+\.' | |
agable
2013/04/15 19:33:30
Document your regexes :)
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
299 r'[\w/]+\s+\([\d.s]+\)', line): | |
300 current_test = None | |
301 line_attr.append('text-success') | |
302 elif 'aborting test' in line: | |
303 current_test = None | |
304 elif current_test: | |
305 line_attr.append('text-warning') | |
306 | |
307 if len(line) > 160: | |
agable
2013/04/15 19:33:30
Why 160?
Ryan Tseng
2013/04/17 22:53:48
That was arbitrary. I think I'll remove this and
| |
308 line_abbr = line[:160] | |
309 line_abbr = line_abbr.replace(' ', ' ') | |
310 line = line.replace(' ', ' ') | |
311 if 'apply_issue' in line: | |
312 logging.warning(line) | |
313 for rep_from, rep_to in REPLACEMENTS: | |
314 line_abbr = re.sub(rep_from, rep_to, line_abbr) | |
315 line = re.sub(rep_from, rep_to, line) | |
316 lines.append((line_abbr, line, line_attr)) | |
317 else: | |
318 line = line.replace(' ', ' ') | |
319 for rep_from, rep_to in REPLACEMENTS: | |
320 line = re.sub(rep_from, rep_to, line) | |
321 lines.append((None, line, line_attr)) | |
agable
2013/04/15 19:33:30
Can pull this duplicated code (line.replace; for f
Ryan Tseng
2013/04/17 22:53:48
Removed line_abbr anyways.
| |
322 return (title, lines) | |
agable
2013/04/15 19:33:30
Remove 'return title', it is identical to the inpu
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
323 | |
324 | |
325 class BuildStep(webapp2.RequestHandler): | |
326 """Prases a build step page.""" | |
agable
2013/04/15 19:33:30
Parses.
He how prases the build step page.
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
327 @render('step.html') | |
328 @expect_request('url') | |
329 def get(self, url): | |
330 if not url: | |
331 self.redirect('/buildbot/') | |
agable
2013/04/15 19:33:30
See comment below about having url be a required u
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
332 | |
333 # Fetch the page. | |
334 sch, netloc, path, _, _, _ = urlparse.urlparse(url) | |
335 url_m = re.match(r'^/((p/)?)(.*)/builders/(.*)/builds/(\d+)$', path) | |
agable
2013/04/15 19:33:30
Offline comment about this (p/)? to follow.
Ryan Tseng
2013/04/17 22:53:48
?
| |
336 if not url_m: | |
337 self.redirect('/buildbot/') | |
338 prefix, _, master, builder, step = url_m.groups() | |
339 json_url = '%s://%s/%s%s/json/builders/%s/builds/%s' % ( | |
340 sch, netloc, prefix, master, builder, step) | |
341 s = urlfetch.fetch(json_url.replace(' ', '%20'), | |
342 method=urlfetch.GET, deadline=60).content | |
343 logging.info(s) | |
344 | |
345 result = json.loads(s) | |
346 | |
347 # Add on some extraneous info. | |
348 build_properties = dict((name, value) for name, value, _ | |
349 in result['properties']) | |
350 | |
351 if 'rietveld' in build_properties: | |
352 result['rietveld'] = build_properties['rietveld'] | |
353 result['breadcrumbs'] = [ | |
354 ('Master %s' % master, '#'), | |
355 ('Builder %s' % builder, '#'), | |
356 ('Build Number %s' % step, '#'), | |
357 ('Slave %s' % result['slave'], '#') | |
358 ] | |
359 return result | |
360 | |
361 | |
362 class MainPage(webapp2.RequestHandler): | |
363 """Parses a buildlog page.""" | |
364 @render('main.html') | |
365 @expect_request('url') | |
agable
2013/04/15 19:33:30
Having a *required* url parameter is kinda weird.
Ryan Tseng
2013/04/17 22:53:48
Done. MainPage now just parses the url and redire
| |
366 def get(self, url): | |
agable
2013/04/15 19:33:30
I'd reorder the steps this method performs for bet
Ryan Tseng
2013/04/17 22:53:48
Refactored to just do #1. The rest has also been
| |
367 if not url: | |
368 return {} | |
369 | |
370 # Redirect the page if we detect a different type of URL. | |
371 sch, netloc, path, _, _, _ = urlparse.urlparse(url) | |
372 logging.info(path) | |
373 if re.match(r'^/((p/)?)(.*)/builders/(.*)/builds/(\d+)$', path): | |
374 self.redirect('/buildbot/step?url=%s' % url) | |
375 return {} | |
376 | |
377 buildlog_query = BuildLogModel.all().filter('url =', url) | |
378 buildlog = buildlog_query.get() | |
agable
2013/04/15 19:33:30
377 and 378 can be one line.
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
379 log_fetch_start = time.time() | |
380 if buildlog: | |
381 s = zlib.decompress(buildlog.data) | |
382 else: | |
383 s = urlfetch.fetch(url, method=urlfetch.GET, deadline=60).content | |
384 log_fetch_time = time.time() - log_fetch_start | |
385 all_output = re.findall(r'<span class="(header|stdout)">(.*?)</span>', | |
agable
2013/04/15 19:33:30
Don't bother performing this regex unless the cach
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
386 s, re.S) | |
387 | |
388 cached_result = BuildLogResultModel.all().filter( | |
389 'url =', url).filter('version =', VERSION_ID).get() | |
390 parse_time_start = time.time() | |
391 if cached_result: | |
392 result_output = json.loads(zlib.decompress(cached_result.data)) | |
393 else: | |
394 result_output = [] | |
395 current_source = None | |
396 current_string = '' | |
397 for source, output in all_output: | |
398 if source == current_source: | |
399 current_string += output | |
400 continue | |
401 else: | |
402 # We hit a new source, we want to emit whatever we had left and | |
403 # start anew. | |
404 if current_string: | |
405 result_output.append(emit(current_source, current_string)) | |
406 current_string = output | |
407 current_source = source | |
408 if current_string: | |
409 result_output.append(emit(current_source, current_string)) | |
410 compressed_result = zlib.compress(json.dumps(result_output)) | |
411 if len(compressed_result) < 1000 * 1000: | |
agable
2013/04/15 19:33:30
Use 10**6
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
412 cached_result = BuildLogResultModel( | |
413 url=url, version=VERSION_ID, data=compressed_result) | |
414 cached_result.put() | |
415 | |
416 url_re = r'/[p]/([\w.]+)/builders/(\w+)/builds/(\w+)/steps/(\w+)/logs/.*' | |
417 master_name, builder_name, build_number, step = re.search( | |
418 url_re, url).groups() | |
419 | |
420 ret_code_m = re.search('program finished with exit code (-?\d+)', s) | |
421 if ret_code_m: | |
422 ret_code = int(ret_code_m.group(1)) | |
423 if ret_code == 0: | |
424 status = 'OK' | |
425 else: | |
426 status = 'ERROR' | |
427 else: | |
428 status = 'RUNNING' | |
429 ret_code = None | |
430 | |
431 if ret_code is not None and not buildlog: | |
432 # Cache this build log if not already. | |
433 compressed_data = zlib.compress(s) | |
434 if len(compressed_data) < 1000 * 1000: | |
435 buildlog = BuildLogModel(url=url, data=compressed_data) | |
436 buildlog.put() | |
437 parse_time = time.time() - parse_time_start | |
438 | |
439 return { | |
440 'output': result_output, | |
441 'url': url, | |
442 'name': step, | |
443 'breadcrumbs': [ | |
444 ('Master %s' % master_name, | |
445 'http://build.chromium.org/p/%s/waterfall' % master_name), | |
446 ('Builder %s' % builder_name, | |
447 'http://build.chromium.org/p/%s/builders/%s' % | |
448 (master_name, builder_name)), | |
449 ('Build Number %s ' % build_number, | |
450 'http://build.chromium.org/p/%s/builders/%s/builds/%s' % | |
451 (master_name, builder_name, build_number)), | |
452 ('Step %s' % step, url) | |
453 ], | |
454 'status': status, | |
455 'ret_code': ret_code, | |
456 'log_fetch_time': log_fetch_time, | |
457 'parse_time': parse_time, | |
458 'compressed_size': len(buildlog.data) if buildlog else -1, | |
459 'compressed_report': len(cached_result.data) if cached_result else -1, | |
460 'url': url, | |
461 'debug': self.request.get('debug'), | |
462 'size': len(s) | |
463 } | |
agable
2013/04/15 19:33:30
Could cache the compressed version of this whole j
Ryan Tseng
2013/04/17 22:53:48
done :) (That's what line 388/412 is)
Well, it cac
| |
464 | |
465 | |
466 def webapp_add_wsgi_middleware(app): | |
467 from google.appengine.ext.appstats import recording | |
468 app = recording.appstats_wsgi_middleware(app) | |
469 return app | |
470 | |
471 | |
472 app = webapp2.WSGIApplication([ | |
473 ('/buildbot/', MainPage), | |
474 ('/buildbot/step/?', BuildStep), | |
agable
2013/04/15 19:33:30
See comments on MainPage and BuildStep get methods
Ryan Tseng
2013/04/17 22:53:48
Done.
| |
475 ], debug=True) | |
476 app = webapp_add_wsgi_middleware(app) | |
OLD | NEW |