Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(135)

Side by Side Diff: tools/buildbot/scripts/master/log_parser/process_log.py

Issue 28034: Move the trace sorting from the JS up to the Python, matching what's done for... (Closed) Base URL: svn://chrome-svn/chrome/trunk/src/
Patch Set: Created 11 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/buildbot/perf/dashboard/ui/generic_plotter.html ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Defines various log processors used by buildbot steps. 6 """Defines various log processors used by buildbot steps.
7 7
8 Current approach is to set an instance of log processor in 8 Current approach is to set an instance of log processor in
9 the ProcessLogTestStep implementation and it will call process() 9 the ProcessLogTestStep implementation and it will call process()
10 method upon completion with full data from process stdio. 10 method upon completion with full data from process stdio.
(...skipping 276 matching lines...) Expand 10 before | Expand all | Expand 10 after
287 self.traces = {} 287 self.traces = {}
288 288
289 def IsImportant(self): 289 def IsImportant(self):
290 """A graph is 'important' if any of its traces is.""" 290 """A graph is 'important' if any of its traces is."""
291 for trace in self.traces.itervalues(): 291 for trace in self.traces.itervalues():
292 if trace.important: 292 if trace.important:
293 return True 293 return True
294 return False 294 return False
295 295
296 296
297 class JSONGraphEncoder(simplejson.JSONEncoder):
298 def default(self, obj):
299 if isinstance(obj, Graph):
300 return {'units': obj.units, 'traces': obj.traces}
301 if isinstance(obj, Trace):
302 # NB: We do not encode the 'important' value: the plot doesn't need it.
303 return [obj.value, obj.stddev]
304 return simplejson.JSONEncoder.default(self, obj)
305
306
307 class GraphingLogProcessor(PerformanceLogProcessor): 297 class GraphingLogProcessor(PerformanceLogProcessor):
308 """Parent class for any log processor expecting standard data to be graphed. 298 """Parent class for any log processor expecting standard data to be graphed.
309 299
310 The log will be parsed looking for any lines of the form 300 The log will be parsed looking for any lines of the form
311 <*>RESULT <graph_name>: <trace_name>= <value> <units> 301 <*>RESULT <graph_name>: <trace_name>= <value> <units>
312 or 302 or
313 <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] <units> 303 <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] <units>
314 or 304 or
315 <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units> 305 <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
316 For example, 306 For example,
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after
404 This method may be overridden by subclasses wanting a different standard 394 This method may be overridden by subclasses wanting a different standard
405 deviation calcuation (or some other sort of error value entirely). 395 deviation calcuation (or some other sort of error value entirely).
406 396
407 Args: 397 Args:
408 value_list: the list of values to use in the calculation 398 value_list: the list of values to use in the calculation
409 trace_name: the trace that produced the data (not used in the base 399 trace_name: the trace that produced the data (not used in the base
410 implementation, but subclasses may use it) 400 implementation, but subclasses may use it)
411 """ 401 """
412 return chromium_utils.FilteredMeanAndStandardDeviation(value_list) 402 return chromium_utils.FilteredMeanAndStandardDeviation(value_list)
413 403
404 def __BuildSummaryJSON(self, graph):
405 """Sorts the traces and returns a summary JSON encoding of the graph.
406
407 Although JS objects are not ordered, according to the spec, in practice
408 everyone iterates in order, since not doing so is a compatibility problem.
409 So we'll count on it here and produce an ordered list of traces.
410
411 But since Python dicts are *not* ordered, we'll need to construct the JSON
412 manually so we don't lose the trace order.
413 """
414 traces = []
415 remaining_traces = graph.traces.copy()
416
417 def AddTrace(trace_name):
418 if trace_name in remaining_traces:
419 traces.append(trace_name)
420 del remaining_traces[trace_name]
421
422 # First pull out any important traces in alphabetical order, and their
423 # _ref traces even if not important.
424 keys = [x for x in graph.traces.keys() if graph.traces[x].important]
425 keys.sort()
426 for name in keys:
427 AddTrace(name)
428 AddTrace(name + "_ref")
429
430 # Now append any other traces that have corresponding _ref traces, in
431 # alphabetical order.
432 keys = [x for x in graph.traces.keys() if x + "_ref" in remaining_traces]
433 keys.sort()
434 for name in keys:
435 AddTrace(name)
436 AddTrace(name + "_ref")
437
438 # Finally, append any remaining traces, in alphabetical order.
439 keys = remaining_traces.keys()
440 keys.sort()
441 traces.extend(keys)
442
443 # Now build the JSON.
444 trace_json = ', '.join(['"%s": [%s, %s]' %
445 (x, graph.traces[x].value, graph.traces[x].stddev) for x in traces])
446 return '{"traces": {%s}, "rev": "%s"}' % (trace_json, self._revision)
447
414 def __CreateSummaryOutput(self): 448 def __CreateSummaryOutput(self):
415 """Write the summary data file and collect the waterfall display text. 449 """Write the summary data file and collect the waterfall display text.
416 450
417 The summary file contains JSON-encoded data. 451 The summary file contains JSON-encoded data.
418 452
419 The waterfall contains lines for each important trace, in the form 453 The waterfall contains lines for each important trace, in the form
420 tracename: value< (refvalue)> 454 tracename: value< (refvalue)>
421 """ 455 """
422 for graph_name, graph in self._graphs.iteritems(): 456 for graph_name, graph in self._graphs.iteritems():
423 # Write a line in the applicable summary file for each graph. 457 # Write a line in the applicable summary file for each graph.
424 if self._ShouldWriteResults(): 458 if self._ShouldWriteResults():
425 filename = os.path.join(self._output_dir, 459 filename = os.path.join(self._output_dir,
426 "%s-summary.dat" % graph_name) 460 "%s-summary.dat" % graph_name)
427 json = simplejson.dumps({'rev': self._revision, 461 Prepend(filename, self.__BuildSummaryJSON(graph) + "\n")
428 'traces': graph.traces}, cls=JSONGraphEncoder)
429 Prepend(filename, json + "\n")
430 462
431 # Add a line to the waterfall for each important trace. 463 # Add a line to the waterfall for each important trace.
432 for trace_name, trace in graph.traces.iteritems(): 464 for trace_name, trace in graph.traces.iteritems():
433 if trace_name.endswith("_ref"): 465 if trace_name.endswith("_ref"):
434 continue 466 continue
435 if trace.important: 467 if trace.important:
436 display = "%s: %s" % (trace_name, FormatHumanReadable(trace.value)) 468 display = "%s: %s" % (trace_name, FormatHumanReadable(trace.value))
437 if graph.traces.get(trace_name + '_ref'): 469 if graph.traces.get(trace_name + '_ref'):
438 display += " (%s)" % FormatHumanReadable( 470 display += " (%s)" % FormatHumanReadable(
439 graph.traces[trace_name + '_ref'].value) 471 graph.traces[trace_name + '_ref'].value)
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after
550 FormatFloat(mean), 582 FormatFloat(mean),
551 FormatFloat(stddev), 583 FormatFloat(stddev),
552 self._JoinWithSpacesAndNewLine(times))) 584 self._JoinWithSpacesAndNewLine(times)))
553 585
554 filename = os.path.join(self._output_dir, 586 filename = os.path.join(self._output_dir,
555 '%s_%s.dat' % (self._revision, trace_name)) 587 '%s_%s.dat' % (self._revision, trace_name))
556 file = open(filename, 'w') 588 file = open(filename, 'w')
557 file.write(''.join(file_data)) 589 file.write(''.join(file_data))
558 file.close() 590 file.close()
559 os.chmod(filename, READABLE_FILE_PERMISSIONS) 591 os.chmod(filename, READABLE_FILE_PERMISSIONS)
OLDNEW
« no previous file with comments | « tools/buildbot/perf/dashboard/ui/generic_plotter.html ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698