OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """A tool used to run a Chrome test executable and process the output. | 6 """A tool used to run a Chrome test executable and process the output. |
7 | 7 |
8 This script is used by the buildbot slaves. It must be run from the outer | 8 This script is used by the buildbot slaves. It must be run from the outer |
9 build directory, e.g. chrome-release/build/. | 9 build directory, e.g. chrome-release/build/. |
10 | 10 |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
63 | 63 |
64 # Directory to write JSON for test results into. | 64 # Directory to write JSON for test results into. |
65 DEST_DIR = 'gtest_results' | 65 DEST_DIR = 'gtest_results' |
66 | 66 |
67 # Names of httpd configuration file under different platforms. | 67 # Names of httpd configuration file under different platforms. |
68 HTTPD_CONF = { | 68 HTTPD_CONF = { |
69 'linux': 'httpd2_linux.conf', | 69 'linux': 'httpd2_linux.conf', |
70 'mac': 'httpd2_mac.conf', | 70 'mac': 'httpd2_mac.conf', |
71 'win': 'httpd.conf' | 71 'win': 'httpd.conf' |
72 } | 72 } |
73 # Regex matching git comment lines containing svn revision info. | |
74 GIT_SVN_ID_RE = re.compile(r'^git-svn-id: .*@([0-9]+) .*$') | |
75 # Regex for the master branch commit position. | |
76 GIT_CR_POS_RE = re.compile(r'^Cr-Commit-Position: refs/heads/master@{#(\d+)}$') | |
77 | 73 |
78 # The directory that this script is in. | 74 # The directory that this script is in. |
79 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | 75 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) |
80 | 76 |
81 LOG_PROCESSOR_CLASSES = { | 77 LOG_PROCESSOR_CLASSES = { |
82 'gtest': gtest_utils.GTestLogParser, | 78 'gtest': gtest_utils.GTestLogParser, |
83 'graphing': performance_log_processor.GraphingLogProcessor, | 79 'graphing': performance_log_processor.GraphingLogProcessor, |
84 } | 80 } |
85 | 81 |
86 | 82 |
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
201 def _GetMaster(): | 197 def _GetMaster(): |
202 """Return the master name for the current host.""" | 198 """Return the master name for the current host.""" |
203 return chromium_utils.GetActiveMaster() | 199 return chromium_utils.GetActiveMaster() |
204 | 200 |
205 | 201 |
206 def _GetMasterString(master): | 202 def _GetMasterString(master): |
207 """Returns a message describing what the master is.""" | 203 """Returns a message describing what the master is.""" |
208 return '[Running for master: "%s"]' % master | 204 return '[Running for master: "%s"]' % master |
209 | 205 |
210 | 206 |
211 def _GetGitCommitPositionFromLog(log): | |
212 """Returns either the commit position or svn rev from a git log.""" | |
213 # Parse from the bottom up, in case the commit message embeds the message | |
214 # from a different commit (e.g., for a revert). | |
215 for r in [GIT_CR_POS_RE, GIT_SVN_ID_RE]: | |
216 for line in reversed(log.splitlines()): | |
217 m = r.match(line.strip()) | |
218 if m: | |
219 return m.group(1) | |
220 return None | |
221 | |
222 | |
223 def _GetGitCommitPosition(dir_path): | |
224 """Extracts the commit position or svn revision number of the HEAD commit.""" | |
225 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
226 p = subprocess.Popen( | |
227 [git_exe, 'log', '-n', '1', '--pretty=format:%B', 'HEAD'], | |
228 cwd=dir_path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
229 (log, _) = p.communicate() | |
230 if p.returncode != 0: | |
231 return None | |
232 return _GetGitCommitPositionFromLog(log) | |
233 | |
234 | |
235 def _IsGitDirectory(dir_path): | 207 def _IsGitDirectory(dir_path): |
236 """Checks whether the given directory is in a git repository. | 208 """Checks whether the given directory is in a git repository. |
237 | 209 |
238 Args: | 210 Args: |
239 dir_path: The directory path to be tested. | 211 dir_path: The directory path to be tested. |
240 | 212 |
241 Returns: | 213 Returns: |
242 True if given directory is in a git repository, False otherwise. | 214 True if given directory is in a git repository, False otherwise. |
243 """ | 215 """ |
244 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | 216 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' |
245 with open(os.devnull, 'w') as devnull: | 217 with open(os.devnull, 'w') as devnull: |
246 p = subprocess.Popen([git_exe, 'rev-parse', '--git-dir'], | 218 p = subprocess.Popen([git_exe, 'rev-parse', '--git-dir'], |
247 cwd=dir_path, stdout=devnull, stderr=devnull) | 219 cwd=dir_path, stdout=devnull, stderr=devnull) |
248 return p.wait() == 0 | 220 return p.wait() == 0 |
249 | 221 |
250 | 222 |
251 def _GetRevision(in_directory): | |
252 """Returns the SVN revision, git commit position, or git hash. | |
253 | |
254 Args: | |
255 in_directory: A directory in the repository to be checked. | |
256 | |
257 Returns: | |
258 An SVN revision as a string if the given directory is in a SVN repository, | |
259 or a git commit position number, or if that's not available, a git hash. | |
260 If all of that fails, an empty string is returned. | |
261 """ | |
262 import xml.dom.minidom | |
263 if not os.path.exists(os.path.join(in_directory, '.svn')): | |
264 if _IsGitDirectory(in_directory): | |
265 svn_rev = _GetGitCommitPosition(in_directory) | |
266 if svn_rev: | |
267 return svn_rev | |
268 return _GetGitRevision(in_directory) | |
269 else: | |
270 return '' | |
271 | |
272 # Note: Not thread safe: http://bugs.python.org/issue2320 | 223 # Note: Not thread safe: http://bugs.python.org/issue2320 |
273 output = subprocess.Popen(['svn', 'info', '--xml'], | 224 output = subprocess.Popen(['svn', 'info', '--xml'], |
274 cwd=in_directory, | 225 cwd=in_directory, |
275 shell=(sys.platform == 'win32'), | 226 shell=(sys.platform == 'win32'), |
276 stdout=subprocess.PIPE).communicate()[0] | 227 stdout=subprocess.PIPE).communicate()[0] |
277 try: | 228 try: |
278 dom = xml.dom.minidom.parseString(output) | 229 dom = xml.dom.minidom.parseString(output) |
279 return dom.getElementsByTagName('entry')[0].getAttribute('revision') | 230 return dom.getElementsByTagName('entry')[0].getAttribute('revision') |
280 except xml.parsers.expat.ExpatError: | 231 except xml.parsers.expat.ExpatError: |
281 return '' | 232 return '' |
282 return '' | 233 return '' |
283 | 234 |
284 | 235 |
285 def _GetGitRevision(in_directory): | |
286 """Returns the git hash tag for the given directory. | |
287 | |
288 Args: | |
289 in_directory: The directory where git is to be run. | |
290 | |
291 Returns: | |
292 The git SHA1 hash string. | |
293 """ | |
294 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
295 p = subprocess.Popen( | |
296 [git_exe, 'rev-parse', 'HEAD'], | |
297 cwd=in_directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
298 (stdout, _) = p.communicate() | |
299 return stdout.strip() | |
300 | |
301 | |
302 def _GenerateJSONForTestResults(options, log_processor): | 236 def _GenerateJSONForTestResults(options, log_processor): |
303 """Generates or updates a JSON file from the gtest results XML and upload the | 237 """Generates or updates a JSON file from the gtest results XML and upload the |
304 file to the archive server. | 238 file to the archive server. |
305 | 239 |
306 The archived JSON file will be placed at: | 240 The archived JSON file will be placed at: |
307 www-dir/DEST_DIR/buildname/testname/results.json | 241 www-dir/DEST_DIR/buildname/testname/results.json |
308 on the archive server. NOTE: This will be deprecated. | 242 on the archive server. NOTE: This will be deprecated. |
309 | 243 |
310 Args: | 244 Args: |
311 options: command-line options that are supposed to have build_dir, | 245 options: command-line options that are supposed to have build_dir, |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
353 | 287 |
354 print _GetMasterString(generate_json_options.master_name) | 288 print _GetMasterString(generate_json_options.master_name) |
355 | 289 |
356 generator = None | 290 generator = None |
357 | 291 |
358 try: | 292 try: |
359 if options.revision: | 293 if options.revision: |
360 generate_json_options.chrome_revision = options.revision | 294 generate_json_options.chrome_revision = options.revision |
361 else: | 295 else: |
362 chrome_dir = chromium_utils.FindUpwardParent(build_dir, 'third_party') | 296 chrome_dir = chromium_utils.FindUpwardParent(build_dir, 'third_party') |
363 generate_json_options.chrome_revision = _GetRevision(chrome_dir) | 297 generate_json_options.chrome_revision = \ |
| 298 slave_utils.GetRevision(chrome_dir) |
364 | 299 |
365 if options.webkit_revision: | 300 if options.webkit_revision: |
366 generate_json_options.webkit_revision = options.webkit_revision | 301 generate_json_options.webkit_revision = options.webkit_revision |
367 else: | 302 else: |
368 webkit_dir = chromium_utils.FindUpward( | 303 webkit_dir = chromium_utils.FindUpward( |
369 build_dir, 'third_party', 'WebKit', 'Source') | 304 build_dir, 'third_party', 'WebKit', 'Source') |
370 generate_json_options.webkit_revision = _GetRevision(webkit_dir) | 305 generate_json_options.webkit_revision = \ |
| 306 slave_utils.GetRevision(webkit_dir) |
371 | 307 |
372 # Generate results JSON file and upload it to the appspot server. | 308 # Generate results JSON file and upload it to the appspot server. |
373 generator = gtest_slave_utils.GenerateJSONResults( | 309 generator = gtest_slave_utils.GenerateJSONResults( |
374 results_map, generate_json_options) | 310 results_map, generate_json_options) |
375 | 311 |
376 except Exception as e: | 312 except Exception as e: |
377 print 'Unexpected error while generating JSON: %s' % e | 313 print 'Unexpected error while generating JSON: %s' % e |
378 sys.excepthook(*sys.exc_info()) | 314 sys.excepthook(*sys.exc_info()) |
379 return False | 315 return False |
380 | 316 |
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
470 else: | 406 else: |
471 return LOG_PROCESSOR_CLASSES[options.annotate] | 407 return LOG_PROCESSOR_CLASSES[options.annotate] |
472 else: | 408 else: |
473 raise KeyError('"%s" is not a valid GTest parser!' % options.annotate) | 409 raise KeyError('"%s" is not a valid GTest parser!' % options.annotate) |
474 elif options.generate_json_file: | 410 elif options.generate_json_file: |
475 return LOG_PROCESSOR_CLASSES['gtest'] | 411 return LOG_PROCESSOR_CLASSES['gtest'] |
476 | 412 |
477 return None | 413 return None |
478 | 414 |
479 | 415 |
480 def _GetCommitPos(build_properties): | |
481 """Extracts the commit position from the build properties, if its there.""" | |
482 if 'got_revision_cp' not in build_properties: | |
483 return None | |
484 commit_pos = build_properties['got_revision_cp'] | |
485 return int(re.search(r'{#(\d+)}', commit_pos).group(1)) | |
486 | |
487 | |
488 def _GetMainRevision(options): | 416 def _GetMainRevision(options): |
489 """Return revision to use as the numerical x-value in the perf dashboard. | 417 slave_utils.GetMainRevision( |
490 | 418 options.build_properties, options.build_dir, options.revision) |
491 This will be used as the value of "rev" in the data passed to | |
492 results_dashboard.SendResults. | |
493 | |
494 In order or priority, this function could return: | |
495 1. The value of the --revision flag (IF it can be parsed as an int). | |
496 2. The value of "got_revision_cp" in build properties. | |
497 3. An SVN number, git commit position, or git commit hash. | |
498 """ | |
499 if options.revision and options.revision.isdigit(): | |
500 return options.revision | |
501 commit_pos_num = _GetCommitPos(options.build_properties) | |
502 if commit_pos_num is not None: | |
503 return commit_pos_num | |
504 # TODO(sullivan,qyearsley): Don't fall back to _GetRevision if it returns | |
505 # a git commit, since this should be a numerical revision. Instead, abort | |
506 # and fail. | |
507 return _GetRevision(os.path.dirname(os.path.abspath(options.build_dir))) | |
508 | 419 |
509 | 420 |
510 def _GetBlinkRevision(options): | 421 def _GetBlinkRevision(options): |
511 if options.webkit_revision: | 422 slave_utils.GetBlinkRevision(options.build_dir, options.webkit_revision) |
512 webkit_revision = options.webkit_revision | |
513 else: | |
514 try: | |
515 webkit_dir = chromium_utils.FindUpward( | |
516 os.path.abspath(options.build_dir), 'third_party', 'WebKit', 'Source') | |
517 webkit_revision = _GetRevision(webkit_dir) | |
518 except Exception: | |
519 webkit_revision = None | |
520 return webkit_revision | |
521 | 423 |
522 | 424 |
523 def _GetTelemetryRevisions(options): | 425 def _GetTelemetryRevisions(options): |
524 """Fills in the same revisions fields that process_log_utils does.""" | 426 return slave_utils.GetTelemetryRevisions( |
525 | 427 options.build_properties, |
526 versions = {} | 428 _GetMainRevision(options), |
527 versions['rev'] = _GetMainRevision(options) | 429 _GetBlinkRevision(options), |
528 versions['webkit_rev'] = _GetBlinkRevision(options) | 430 options.point_id) |
529 versions['webrtc_rev'] = options.build_properties.get('got_webrtc_revision') | |
530 versions['v8_rev'] = options.build_properties.get('got_v8_revision') | |
531 versions['ver'] = options.build_properties.get('version') | |
532 versions['git_revision'] = options.build_properties.get('git_revision') | |
533 versions['point_id'] = options.point_id | |
534 # There are a lot of "bad" revisions to check for, so clean them all up here. | |
535 for key in versions.keys(): | |
536 if not versions[key] or versions[key] == 'undefined': | |
537 del versions[key] | |
538 return versions | |
539 | 431 |
540 | 432 |
541 def _CreateLogProcessor(log_processor_class, options, telemetry_info): | 433 def _CreateLogProcessor(log_processor_class, options, telemetry_info): |
542 """Creates a log processor instance. | 434 """Creates a log processor instance. |
543 | 435 |
544 Args: | 436 Args: |
545 log_processor_class: A subclass of PerformanceLogProcessor or similar class. | 437 log_processor_class: A subclass of PerformanceLogProcessor or similar class. |
546 options: Command-line options (from OptionParser). | 438 options: Command-line options (from OptionParser). |
547 telemetry_info: dict of info for run_benchmark runs. | 439 telemetry_info: dict of info for run_benchmark runs. |
548 | 440 |
(...skipping 1358 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1907 finally: | 1799 finally: |
1908 if did_launch_dbus: | 1800 if did_launch_dbus: |
1909 # It looks like the command line argument --exit-with-session | 1801 # It looks like the command line argument --exit-with-session |
1910 # isn't working to clean up the spawned dbus-daemon. Kill it | 1802 # isn't working to clean up the spawned dbus-daemon. Kill it |
1911 # manually. | 1803 # manually. |
1912 _ShutdownDBus() | 1804 _ShutdownDBus() |
1913 | 1805 |
1914 | 1806 |
1915 if '__main__' == __name__: | 1807 if '__main__' == __name__: |
1916 sys.exit(main()) | 1808 sys.exit(main()) |
OLD | NEW |