| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2012 the V8 project authors. All rights reserved. | |
| 4 # Redistribution and use in source and binary forms, with or without | |
| 5 # modification, are permitted provided that the following conditions are | |
| 6 # met: | |
| 7 # | |
| 8 # * Redistributions of source code must retain the above copyright | |
| 9 # notice, this list of conditions and the following disclaimer. | |
| 10 # * Redistributions in binary form must reproduce the above | |
| 11 # copyright notice, this list of conditions and the following | |
| 12 # disclaimer in the documentation and/or other materials provided | |
| 13 # with the distribution. | |
| 14 # * Neither the name of Google Inc. nor the names of its | |
| 15 # contributors may be used to endorse or promote products derived | |
| 16 # from this software without specific prior written permission. | |
| 17 # | |
| 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 29 | |
| 30 | |
| 31 import imp | |
| 32 import optparse | |
| 33 import os | |
| 34 from os.path import join, dirname, abspath, basename, isdir, exists | |
| 35 import platform | |
| 36 import re | |
| 37 import signal | |
| 38 import subprocess | |
| 39 import sys | |
| 40 import tempfile | |
| 41 import time | |
| 42 import threading | |
| 43 import utils | |
| 44 from Queue import Queue, Empty | |
| 45 | |
| 46 | |
| 47 VERBOSE = False | |
| 48 | |
| 49 | |
| 50 # --------------------------------------------- | |
| 51 # --- P r o g r e s s I n d i c a t o r s --- | |
| 52 # --------------------------------------------- | |
| 53 | |
| 54 | |
| 55 class ProgressIndicator(object): | |
| 56 | |
| 57 def __init__(self, cases): | |
| 58 self.cases = cases | |
| 59 self.queue = Queue(len(cases)) | |
| 60 for case in cases: | |
| 61 self.queue.put_nowait(case) | |
| 62 self.succeeded = 0 | |
| 63 self.remaining = len(cases) | |
| 64 self.total = len(cases) | |
| 65 self.failed = [ ] | |
| 66 self.crashed = 0 | |
| 67 self.terminate = False | |
| 68 self.lock = threading.Lock() | |
| 69 | |
| 70 def PrintFailureHeader(self, test): | |
| 71 if test.IsNegative(): | |
| 72 negative_marker = '[negative] ' | |
| 73 else: | |
| 74 negative_marker = '' | |
| 75 print "=== %(label)s %(negative)s===" % { | |
| 76 'label': test.GetLabel(), | |
| 77 'negative': negative_marker | |
| 78 } | |
| 79 print "Path: %s" % "/".join(test.path) | |
| 80 | |
| 81 def Run(self, tasks): | |
| 82 self.Starting() | |
| 83 threads = [] | |
| 84 # Spawn N-1 threads and then use this thread as the last one. | |
| 85 # That way -j1 avoids threading altogether which is a nice fallback | |
| 86 # in case of threading problems. | |
| 87 for i in xrange(tasks - 1): | |
| 88 thread = threading.Thread(target=self.RunSingle, args=[]) | |
| 89 threads.append(thread) | |
| 90 thread.start() | |
| 91 try: | |
| 92 self.RunSingle() | |
| 93 # Wait for the remaining threads | |
| 94 for thread in threads: | |
| 95 # Use a timeout so that signals (ctrl-c) will be processed. | |
| 96 thread.join(timeout=10000000) | |
| 97 except Exception, e: | |
| 98 # If there's an exception we schedule an interruption for any | |
| 99 # remaining threads. | |
| 100 self.terminate = True | |
| 101 # ...and then reraise the exception to bail out | |
| 102 raise | |
| 103 self.Done() | |
| 104 return not self.failed | |
| 105 | |
| 106 def RunSingle(self): | |
| 107 while not self.terminate: | |
| 108 try: | |
| 109 test = self.queue.get_nowait() | |
| 110 except Empty: | |
| 111 return | |
| 112 case = test.case | |
| 113 self.lock.acquire() | |
| 114 self.AboutToRun(case) | |
| 115 self.lock.release() | |
| 116 try: | |
| 117 start = time.time() | |
| 118 output = case.Run() | |
| 119 case.duration = (time.time() - start) | |
| 120 except BreakNowException: | |
| 121 self.terminate = True | |
| 122 except IOError, e: | |
| 123 assert self.terminate | |
| 124 return | |
| 125 if self.terminate: | |
| 126 return | |
| 127 self.lock.acquire() | |
| 128 if output.UnexpectedOutput(): | |
| 129 self.failed.append(output) | |
| 130 if output.HasCrashed(): | |
| 131 self.crashed += 1 | |
| 132 else: | |
| 133 self.succeeded += 1 | |
| 134 self.remaining -= 1 | |
| 135 self.HasRun(output) | |
| 136 self.lock.release() | |
| 137 | |
| 138 | |
| 139 def EscapeCommand(command): | |
| 140 parts = [] | |
| 141 for part in command: | |
| 142 if ' ' in part: | |
| 143 # Escape spaces and double quotes. We may need to escape more characters | |
| 144 # for this to work properly. | |
| 145 parts.append('"%s"' % part.replace('"', '\\"')) | |
| 146 else: | |
| 147 parts.append(part) | |
| 148 return " ".join(parts) | |
| 149 | |
| 150 | |
| 151 class SimpleProgressIndicator(ProgressIndicator): | |
| 152 | |
| 153 def Starting(self): | |
| 154 print 'Running %i tests' % len(self.cases) | |
| 155 | |
| 156 def Done(self): | |
| 157 print | |
| 158 for failed in self.failed: | |
| 159 self.PrintFailureHeader(failed.test) | |
| 160 if failed.output.stderr: | |
| 161 print "--- stderr ---" | |
| 162 print failed.output.stderr.strip() | |
| 163 if failed.output.stdout: | |
| 164 print "--- stdout ---" | |
| 165 print failed.output.stdout.strip() | |
| 166 print "Command: %s" % EscapeCommand(failed.command) | |
| 167 if failed.HasCrashed(): | |
| 168 print "--- CRASHED ---" | |
| 169 if failed.HasTimedOut(): | |
| 170 print "--- TIMEOUT ---" | |
| 171 if len(self.failed) == 0: | |
| 172 print "===" | |
| 173 print "=== All tests succeeded" | |
| 174 print "===" | |
| 175 else: | |
| 176 print | |
| 177 print "===" | |
| 178 print "=== %i tests failed" % len(self.failed) | |
| 179 if self.crashed > 0: | |
| 180 print "=== %i tests CRASHED" % self.crashed | |
| 181 print "===" | |
| 182 | |
| 183 | |
| 184 class VerboseProgressIndicator(SimpleProgressIndicator): | |
| 185 | |
| 186 def AboutToRun(self, case): | |
| 187 print 'Starting %s...' % case.GetLabel() | |
| 188 sys.stdout.flush() | |
| 189 | |
| 190 def HasRun(self, output): | |
| 191 if output.UnexpectedOutput(): | |
| 192 if output.HasCrashed(): | |
| 193 outcome = 'CRASH' | |
| 194 else: | |
| 195 outcome = 'FAIL' | |
| 196 else: | |
| 197 outcome = 'pass' | |
| 198 print 'Done running %s: %s' % (output.test.GetLabel(), outcome) | |
| 199 | |
| 200 | |
| 201 class DotsProgressIndicator(SimpleProgressIndicator): | |
| 202 | |
| 203 def AboutToRun(self, case): | |
| 204 pass | |
| 205 | |
| 206 def HasRun(self, output): | |
| 207 total = self.succeeded + len(self.failed) | |
| 208 if (total > 1) and (total % 50 == 1): | |
| 209 sys.stdout.write('\n') | |
| 210 if output.UnexpectedOutput(): | |
| 211 if output.HasCrashed(): | |
| 212 sys.stdout.write('C') | |
| 213 sys.stdout.flush() | |
| 214 elif output.HasTimedOut(): | |
| 215 sys.stdout.write('T') | |
| 216 sys.stdout.flush() | |
| 217 else: | |
| 218 sys.stdout.write('F') | |
| 219 sys.stdout.flush() | |
| 220 else: | |
| 221 sys.stdout.write('.') | |
| 222 sys.stdout.flush() | |
| 223 | |
| 224 | |
| 225 class CompactProgressIndicator(ProgressIndicator): | |
| 226 | |
| 227 def __init__(self, cases, templates): | |
| 228 super(CompactProgressIndicator, self).__init__(cases) | |
| 229 self.templates = templates | |
| 230 self.last_status_length = 0 | |
| 231 self.start_time = time.time() | |
| 232 | |
| 233 def Starting(self): | |
| 234 pass | |
| 235 | |
| 236 def Done(self): | |
| 237 self.PrintProgress('Done') | |
| 238 | |
| 239 def AboutToRun(self, case): | |
| 240 self.PrintProgress(case.GetLabel()) | |
| 241 | |
| 242 def HasRun(self, output): | |
| 243 if output.UnexpectedOutput(): | |
| 244 self.ClearLine(self.last_status_length) | |
| 245 self.PrintFailureHeader(output.test) | |
| 246 stdout = output.output.stdout.strip() | |
| 247 if len(stdout): | |
| 248 print self.templates['stdout'] % stdout | |
| 249 stderr = output.output.stderr.strip() | |
| 250 if len(stderr): | |
| 251 print self.templates['stderr'] % stderr | |
| 252 print "Command: %s" % EscapeCommand(output.command) | |
| 253 if output.HasCrashed(): | |
| 254 print "--- CRASHED ---" | |
| 255 if output.HasTimedOut(): | |
| 256 print "--- TIMEOUT ---" | |
| 257 | |
| 258 def Truncate(self, str, length): | |
| 259 if length and (len(str) > (length - 3)): | |
| 260 return str[:(length-3)] + "..." | |
| 261 else: | |
| 262 return str | |
| 263 | |
| 264 def PrintProgress(self, name): | |
| 265 self.ClearLine(self.last_status_length) | |
| 266 elapsed = time.time() - self.start_time | |
| 267 status = self.templates['status_line'] % { | |
| 268 'passed': self.succeeded, | |
| 269 'remaining': (((self.total - self.remaining) * 100) // self.total), | |
| 270 'failed': len(self.failed), | |
| 271 'test': name, | |
| 272 'mins': int(elapsed) / 60, | |
| 273 'secs': int(elapsed) % 60 | |
| 274 } | |
| 275 status = self.Truncate(status, 78) | |
| 276 self.last_status_length = len(status) | |
| 277 print status, | |
| 278 sys.stdout.flush() | |
| 279 | |
| 280 | |
| 281 class ColorProgressIndicator(CompactProgressIndicator): | |
| 282 | |
| 283 def __init__(self, cases): | |
| 284 templates = { | |
| 285 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\0
33[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s", | |
| 286 'stdout': "\033[1m%s\033[0m", | |
| 287 'stderr': "\033[31m%s\033[0m", | |
| 288 } | |
| 289 super(ColorProgressIndicator, self).__init__(cases, templates) | |
| 290 | |
| 291 def ClearLine(self, last_line_length): | |
| 292 print "\033[1K\r", | |
| 293 | |
| 294 | |
| 295 class MonochromeProgressIndicator(CompactProgressIndicator): | |
| 296 | |
| 297 def __init__(self, cases): | |
| 298 templates = { | |
| 299 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(
failed) 4d]: %(test)s", | |
| 300 'stdout': '%s', | |
| 301 'stderr': '%s', | |
| 302 } | |
| 303 super(MonochromeProgressIndicator, self).__init__(cases, templates) | |
| 304 | |
| 305 def ClearLine(self, last_line_length): | |
| 306 print ("\r" + (" " * last_line_length) + "\r"), | |
| 307 | |
| 308 | |
| 309 PROGRESS_INDICATORS = { | |
| 310 'verbose': VerboseProgressIndicator, | |
| 311 'dots': DotsProgressIndicator, | |
| 312 'color': ColorProgressIndicator, | |
| 313 'mono': MonochromeProgressIndicator | |
| 314 } | |
| 315 | |
| 316 | |
| 317 # ------------------------- | |
| 318 # --- F r a m e w o r k --- | |
| 319 # ------------------------- | |
| 320 | |
| 321 class BreakNowException(Exception): | |
| 322 def __init__(self, value): | |
| 323 self.value = value | |
| 324 def __str__(self): | |
| 325 return repr(self.value) | |
| 326 | |
| 327 | |
| 328 class CommandOutput(object): | |
| 329 | |
| 330 def __init__(self, exit_code, timed_out, stdout, stderr): | |
| 331 self.exit_code = exit_code | |
| 332 self.timed_out = timed_out | |
| 333 self.stdout = stdout | |
| 334 self.stderr = stderr | |
| 335 self.failed = None | |
| 336 | |
| 337 | |
| 338 class TestCase(object): | |
| 339 | |
| 340 def __init__(self, context, path, mode): | |
| 341 self.path = path | |
| 342 self.context = context | |
| 343 self.duration = None | |
| 344 self.mode = mode | |
| 345 | |
| 346 def IsNegative(self): | |
| 347 return False | |
| 348 | |
| 349 def TestsIsolates(self): | |
| 350 return False | |
| 351 | |
| 352 def CompareTime(self, other): | |
| 353 return cmp(other.duration, self.duration) | |
| 354 | |
| 355 def DidFail(self, output): | |
| 356 if output.failed is None: | |
| 357 output.failed = self.IsFailureOutput(output) | |
| 358 return output.failed | |
| 359 | |
| 360 def IsFailureOutput(self, output): | |
| 361 return output.exit_code != 0 | |
| 362 | |
| 363 def GetSource(self): | |
| 364 return "(no source available)" | |
| 365 | |
| 366 def RunCommand(self, command): | |
| 367 full_command = self.context.processor(command) | |
| 368 output = Execute(full_command, | |
| 369 self.context, | |
| 370 self.context.GetTimeout(self, self.mode)) | |
| 371 self.Cleanup() | |
| 372 return TestOutput(self, | |
| 373 full_command, | |
| 374 output, | |
| 375 self.context.store_unexpected_output) | |
| 376 | |
| 377 def BeforeRun(self): | |
| 378 pass | |
| 379 | |
| 380 def AfterRun(self, result): | |
| 381 pass | |
| 382 | |
| 383 def GetCustomFlags(self, mode): | |
| 384 return None | |
| 385 | |
| 386 def Run(self): | |
| 387 self.BeforeRun() | |
| 388 result = None | |
| 389 try: | |
| 390 result = self.RunCommand(self.GetCommand()) | |
| 391 except: | |
| 392 self.terminate = True | |
| 393 raise BreakNowException("User pressed CTRL+C or IO went wrong") | |
| 394 finally: | |
| 395 self.AfterRun(result) | |
| 396 return result | |
| 397 | |
| 398 def Cleanup(self): | |
| 399 return | |
| 400 | |
| 401 | |
| 402 class TestOutput(object): | |
| 403 | |
| 404 def __init__(self, test, command, output, store_unexpected_output): | |
| 405 self.test = test | |
| 406 self.command = command | |
| 407 self.output = output | |
| 408 self.store_unexpected_output = store_unexpected_output | |
| 409 | |
| 410 def UnexpectedOutput(self): | |
| 411 if self.HasCrashed(): | |
| 412 outcome = CRASH | |
| 413 elif self.HasTimedOut(): | |
| 414 outcome = TIMEOUT | |
| 415 elif self.HasFailed(): | |
| 416 outcome = FAIL | |
| 417 else: | |
| 418 outcome = PASS | |
| 419 return not outcome in self.test.outcomes | |
| 420 | |
| 421 def HasPreciousOutput(self): | |
| 422 return self.UnexpectedOutput() and self.store_unexpected_output | |
| 423 | |
| 424 def HasCrashed(self): | |
| 425 if utils.IsWindows(): | |
| 426 return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.outpu
t.exit_code) | |
| 427 else: | |
| 428 # Timed out tests will have exit_code -signal.SIGTERM. | |
| 429 if self.output.timed_out: | |
| 430 return False | |
| 431 return self.output.exit_code < 0 and \ | |
| 432 self.output.exit_code != -signal.SIGABRT | |
| 433 | |
| 434 def HasTimedOut(self): | |
| 435 return self.output.timed_out | |
| 436 | |
| 437 def HasFailed(self): | |
| 438 execution_failed = self.test.DidFail(self.output) | |
| 439 if self.test.IsNegative(): | |
| 440 return not execution_failed | |
| 441 else: | |
| 442 return execution_failed | |
| 443 | |
| 444 | |
| 445 def KillProcessWithID(pid): | |
| 446 if utils.IsWindows(): | |
| 447 os.popen('taskkill /T /F /PID %d' % pid) | |
| 448 else: | |
| 449 os.kill(pid, signal.SIGTERM) | |
| 450 | |
| 451 | |
| 452 MAX_SLEEP_TIME = 0.1 | |
| 453 INITIAL_SLEEP_TIME = 0.0001 | |
| 454 SLEEP_TIME_FACTOR = 1.25 | |
| 455 | |
| 456 SEM_INVALID_VALUE = -1 | |
| 457 SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h | |
| 458 | |
| 459 def Win32SetErrorMode(mode): | |
| 460 prev_error_mode = SEM_INVALID_VALUE | |
| 461 try: | |
| 462 import ctypes | |
| 463 prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode) | |
| 464 except ImportError: | |
| 465 pass | |
| 466 return prev_error_mode | |
| 467 | |
| 468 def RunProcess(context, timeout, args, **rest): | |
| 469 if context.verbose: print "#", " ".join(args) | |
| 470 popen_args = args | |
| 471 prev_error_mode = SEM_INVALID_VALUE | |
| 472 if utils.IsWindows(): | |
| 473 popen_args = subprocess.list2cmdline(args) | |
| 474 if context.suppress_dialogs: | |
| 475 # Try to change the error mode to avoid dialogs on fatal errors. Don't | |
| 476 # touch any existing error mode flags by merging the existing error mode. | |
| 477 # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. | |
| 478 error_mode = SEM_NOGPFAULTERRORBOX | |
| 479 prev_error_mode = Win32SetErrorMode(error_mode) | |
| 480 Win32SetErrorMode(error_mode | prev_error_mode) | |
| 481 process = subprocess.Popen( | |
| 482 shell = utils.IsWindows(), | |
| 483 args = popen_args, | |
| 484 **rest | |
| 485 ) | |
| 486 if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_I
NVALID_VALUE: | |
| 487 Win32SetErrorMode(prev_error_mode) | |
| 488 # Compute the end time - if the process crosses this limit we | |
| 489 # consider it timed out. | |
| 490 if timeout is None: end_time = None | |
| 491 else: end_time = time.time() + timeout | |
| 492 timed_out = False | |
| 493 # Repeatedly check the exit code from the process in a | |
| 494 # loop and keep track of whether or not it times out. | |
| 495 exit_code = None | |
| 496 sleep_time = INITIAL_SLEEP_TIME | |
| 497 while exit_code is None: | |
| 498 if (not end_time is None) and (time.time() >= end_time): | |
| 499 # Kill the process and wait for it to exit. | |
| 500 KillProcessWithID(process.pid) | |
| 501 exit_code = process.wait() | |
| 502 timed_out = True | |
| 503 else: | |
| 504 exit_code = process.poll() | |
| 505 time.sleep(sleep_time) | |
| 506 sleep_time = sleep_time * SLEEP_TIME_FACTOR | |
| 507 if sleep_time > MAX_SLEEP_TIME: | |
| 508 sleep_time = MAX_SLEEP_TIME | |
| 509 return (process, exit_code, timed_out) | |
| 510 | |
| 511 | |
| 512 def PrintError(str): | |
| 513 sys.stderr.write(str) | |
| 514 sys.stderr.write('\n') | |
| 515 | |
| 516 | |
| 517 def CheckedUnlink(name): | |
| 518 # On Windows, when run with -jN in parallel processes, | |
| 519 # OS often fails to unlink the temp file. Not sure why. | |
| 520 # Need to retry. | |
| 521 # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch | |
| 522 retry_count = 0 | |
| 523 while retry_count < 30: | |
| 524 try: | |
| 525 os.unlink(name) | |
| 526 return | |
| 527 except OSError, e: | |
| 528 retry_count += 1 | |
| 529 time.sleep(retry_count * 0.1) | |
| 530 PrintError("os.unlink() " + str(e)) | |
| 531 | |
| 532 def Execute(args, context, timeout=None): | |
| 533 (fd_out, outname) = tempfile.mkstemp() | |
| 534 (fd_err, errname) = tempfile.mkstemp() | |
| 535 (process, exit_code, timed_out) = RunProcess( | |
| 536 context, | |
| 537 timeout, | |
| 538 args = args, | |
| 539 stdout = fd_out, | |
| 540 stderr = fd_err, | |
| 541 ) | |
| 542 os.close(fd_out) | |
| 543 os.close(fd_err) | |
| 544 output = file(outname).read() | |
| 545 errors = file(errname).read() | |
| 546 CheckedUnlink(outname) | |
| 547 CheckedUnlink(errname) | |
| 548 return CommandOutput(exit_code, timed_out, output, errors) | |
| 549 | |
| 550 | |
| 551 def ExecuteNoCapture(args, context, timeout=None): | |
| 552 (process, exit_code, timed_out) = RunProcess( | |
| 553 context, | |
| 554 timeout, | |
| 555 args = args, | |
| 556 ) | |
| 557 return CommandOutput(exit_code, False, "", "") | |
| 558 | |
| 559 | |
| 560 def CarCdr(path): | |
| 561 if len(path) == 0: | |
| 562 return (None, [ ]) | |
| 563 else: | |
| 564 return (path[0], path[1:]) | |
| 565 | |
| 566 | |
| 567 # Use this to run several variants of the tests, e.g.: | |
| 568 # VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']] | |
| 569 VARIANT_FLAGS = [[], | |
| 570 ['--stress-opt', '--always-opt'], | |
| 571 ['--nocrankshaft']] | |
| 572 | |
| 573 | |
| 574 class TestConfiguration(object): | |
| 575 | |
| 576 def __init__(self, context, root): | |
| 577 self.context = context | |
| 578 self.root = root | |
| 579 | |
| 580 def Contains(self, path, file): | |
| 581 if len(path) > len(file): | |
| 582 return False | |
| 583 for i in xrange(len(path)): | |
| 584 if not path[i].match(file[i]): | |
| 585 return False | |
| 586 return True | |
| 587 | |
| 588 def GetTestStatus(self, sections, defs): | |
| 589 pass | |
| 590 | |
| 591 def VariantFlags(self): | |
| 592 return VARIANT_FLAGS | |
| 593 | |
| 594 | |
| 595 | |
| 596 | |
| 597 class TestSuite(object): | |
| 598 | |
| 599 def __init__(self, name): | |
| 600 self.name = name | |
| 601 | |
| 602 def GetName(self): | |
| 603 return self.name | |
| 604 | |
| 605 | |
| 606 class TestRepository(TestSuite): | |
| 607 | |
| 608 def __init__(self, path): | |
| 609 normalized_path = abspath(path) | |
| 610 super(TestRepository, self).__init__(basename(normalized_path)) | |
| 611 self.path = normalized_path | |
| 612 self.is_loaded = False | |
| 613 self.config = None | |
| 614 | |
| 615 def GetConfiguration(self, context): | |
| 616 if self.is_loaded: | |
| 617 return self.config | |
| 618 self.is_loaded = True | |
| 619 file = None | |
| 620 try: | |
| 621 (file, pathname, description) = imp.find_module('testcfg', [ self.path ]) | |
| 622 module = imp.load_module('testcfg', file, pathname, description) | |
| 623 self.config = module.GetConfiguration(context, self.path) | |
| 624 finally: | |
| 625 if file: | |
| 626 file.close() | |
| 627 return self.config | |
| 628 | |
| 629 def GetBuildRequirements(self, path, context): | |
| 630 return self.GetConfiguration(context).GetBuildRequirements() | |
| 631 | |
| 632 def DownloadData(self, context): | |
| 633 config = self.GetConfiguration(context) | |
| 634 if 'DownloadData' in dir(config): | |
| 635 config.DownloadData() | |
| 636 | |
| 637 def AddTestsToList(self, result, current_path, path, context, mode): | |
| 638 config = self.GetConfiguration(context) | |
| 639 for v in config.VariantFlags(): | |
| 640 tests = config.ListTests(current_path, path, mode, v) | |
| 641 for t in tests: t.variant_flags = v | |
| 642 result += tests | |
| 643 | |
| 644 def GetTestStatus(self, context, sections, defs): | |
| 645 self.GetConfiguration(context).GetTestStatus(sections, defs) | |
| 646 | |
| 647 | |
| 648 class LiteralTestSuite(TestSuite): | |
| 649 | |
| 650 def __init__(self, tests): | |
| 651 super(LiteralTestSuite, self).__init__('root') | |
| 652 self.tests = tests | |
| 653 | |
| 654 def GetBuildRequirements(self, path, context): | |
| 655 (name, rest) = CarCdr(path) | |
| 656 result = [ ] | |
| 657 for test in self.tests: | |
| 658 if not name or name.match(test.GetName()): | |
| 659 result += test.GetBuildRequirements(rest, context) | |
| 660 return result | |
| 661 | |
| 662 def DownloadData(self, path, context): | |
| 663 (name, rest) = CarCdr(path) | |
| 664 for test in self.tests: | |
| 665 if not name or name.match(test.GetName()): | |
| 666 test.DownloadData(context) | |
| 667 | |
| 668 def ListTests(self, current_path, path, context, mode, variant_flags): | |
| 669 (name, rest) = CarCdr(path) | |
| 670 result = [ ] | |
| 671 for test in self.tests: | |
| 672 test_name = test.GetName() | |
| 673 if not name or name.match(test_name): | |
| 674 full_path = current_path + [test_name] | |
| 675 test.AddTestsToList(result, full_path, path, context, mode) | |
| 676 return result | |
| 677 | |
| 678 def GetTestStatus(self, context, sections, defs): | |
| 679 for test in self.tests: | |
| 680 test.GetTestStatus(context, sections, defs) | |
| 681 | |
| 682 | |
| 683 SUFFIX = { | |
| 684 'debug' : '_g', | |
| 685 'release' : '' } | |
| 686 FLAGS = { | |
| 687 'debug' : ['--nobreak-on-abort', '--nodead-code-elimination', | |
| 688 '--nofold-constants', '--enable-slow-asserts', | |
| 689 '--debug-code', '--verify-heap'], | |
| 690 'release' : ['--nobreak-on-abort', '--nodead-code-elimination', | |
| 691 '--nofold-constants']} | |
| 692 TIMEOUT_SCALEFACTOR = { | |
| 693 'debug' : 4, | |
| 694 'release' : 1 } | |
| 695 | |
| 696 | |
| 697 class Context(object): | |
| 698 | |
| 699 def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, sup
press_dialogs, store_unexpected_output): | |
| 700 self.workspace = workspace | |
| 701 self.buildspace = buildspace | |
| 702 self.verbose = verbose | |
| 703 self.vm_root = vm | |
| 704 self.timeout = timeout | |
| 705 self.processor = processor | |
| 706 self.suppress_dialogs = suppress_dialogs | |
| 707 self.store_unexpected_output = store_unexpected_output | |
| 708 | |
| 709 def GetVm(self, mode): | |
| 710 name = self.vm_root + SUFFIX[mode] | |
| 711 if utils.IsWindows() and not name.endswith('.exe'): | |
| 712 name = name + '.exe' | |
| 713 return name | |
| 714 | |
| 715 def GetVmCommand(self, testcase, mode): | |
| 716 return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode) | |
| 717 | |
| 718 def GetVmFlags(self, testcase, mode): | |
| 719 flags = testcase.GetCustomFlags(mode) | |
| 720 if flags is None: | |
| 721 flags = FLAGS[mode] | |
| 722 return testcase.variant_flags + flags | |
| 723 | |
| 724 def GetTimeout(self, testcase, mode): | |
| 725 result = self.timeout * TIMEOUT_SCALEFACTOR[mode] | |
| 726 if '--stress-opt' in self.GetVmFlags(testcase, mode): | |
| 727 return result * 4 | |
| 728 else: | |
| 729 return result | |
| 730 | |
| 731 def RunTestCases(cases_to_run, progress, tasks): | |
| 732 progress = PROGRESS_INDICATORS[progress](cases_to_run) | |
| 733 result = 0 | |
| 734 try: | |
| 735 result = progress.Run(tasks) | |
| 736 except Exception, e: | |
| 737 print "\n", e | |
| 738 return result | |
| 739 | |
| 740 | |
| 741 def BuildRequirements(context, requirements, mode, scons_flags): | |
| 742 command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)] | |
| 743 + requirements | |
| 744 + scons_flags) | |
| 745 output = ExecuteNoCapture(command_line, context) | |
| 746 return output.exit_code == 0 | |
| 747 | |
| 748 | |
| 749 # ------------------------------------------- | |
| 750 # --- T e s t C o n f i g u r a t i o n --- | |
| 751 # ------------------------------------------- | |
| 752 | |
| 753 | |
| 754 SKIP = 'skip' | |
| 755 FAIL = 'fail' | |
| 756 PASS = 'pass' | |
| 757 OKAY = 'okay' | |
| 758 TIMEOUT = 'timeout' | |
| 759 CRASH = 'crash' | |
| 760 SLOW = 'slow' | |
| 761 | |
| 762 | |
| 763 class Expression(object): | |
| 764 pass | |
| 765 | |
| 766 | |
| 767 class Constant(Expression): | |
| 768 | |
| 769 def __init__(self, value): | |
| 770 self.value = value | |
| 771 | |
| 772 def Evaluate(self, env, defs): | |
| 773 return self.value | |
| 774 | |
| 775 | |
| 776 class Variable(Expression): | |
| 777 | |
| 778 def __init__(self, name): | |
| 779 self.name = name | |
| 780 | |
| 781 def GetOutcomes(self, env, defs): | |
| 782 if self.name in env: return ListSet([env[self.name]]) | |
| 783 else: return Nothing() | |
| 784 | |
| 785 def Evaluate(self, env, defs): | |
| 786 return env[self.name] | |
| 787 | |
| 788 | |
| 789 class Outcome(Expression): | |
| 790 | |
| 791 def __init__(self, name): | |
| 792 self.name = name | |
| 793 | |
| 794 def GetOutcomes(self, env, defs): | |
| 795 if self.name in defs: | |
| 796 return defs[self.name].GetOutcomes(env, defs) | |
| 797 else: | |
| 798 return ListSet([self.name]) | |
| 799 | |
| 800 | |
| 801 class Set(object): | |
| 802 pass | |
| 803 | |
| 804 | |
| 805 class ListSet(Set): | |
| 806 | |
| 807 def __init__(self, elms): | |
| 808 self.elms = elms | |
| 809 | |
| 810 def __str__(self): | |
| 811 return "ListSet%s" % str(self.elms) | |
| 812 | |
| 813 def Intersect(self, that): | |
| 814 if not isinstance(that, ListSet): | |
| 815 return that.Intersect(self) | |
| 816 return ListSet([ x for x in self.elms if x in that.elms ]) | |
| 817 | |
| 818 def Union(self, that): | |
| 819 if not isinstance(that, ListSet): | |
| 820 return that.Union(self) | |
| 821 return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ]) | |
| 822 | |
| 823 def IsEmpty(self): | |
| 824 return len(self.elms) == 0 | |
| 825 | |
| 826 | |
| 827 class Everything(Set): | |
| 828 | |
| 829 def Intersect(self, that): | |
| 830 return that | |
| 831 | |
| 832 def Union(self, that): | |
| 833 return self | |
| 834 | |
| 835 def IsEmpty(self): | |
| 836 return False | |
| 837 | |
| 838 | |
| 839 class Nothing(Set): | |
| 840 | |
| 841 def Intersect(self, that): | |
| 842 return self | |
| 843 | |
| 844 def Union(self, that): | |
| 845 return that | |
| 846 | |
| 847 def IsEmpty(self): | |
| 848 return True | |
| 849 | |
| 850 | |
| 851 class Operation(Expression): | |
| 852 | |
| 853 def __init__(self, left, op, right): | |
| 854 self.left = left | |
| 855 self.op = op | |
| 856 self.right = right | |
| 857 | |
| 858 def Evaluate(self, env, defs): | |
| 859 if self.op == '||' or self.op == ',': | |
| 860 return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs) | |
| 861 elif self.op == 'if': | |
| 862 return False | |
| 863 elif self.op == '==': | |
| 864 inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(
env, defs)) | |
| 865 return not inter.IsEmpty() | |
| 866 elif self.op == '!=': | |
| 867 inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(
env, defs)) | |
| 868 return inter.IsEmpty() | |
| 869 else: | |
| 870 assert self.op == '&&' | |
| 871 return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs) | |
| 872 | |
| 873 def GetOutcomes(self, env, defs): | |
| 874 if self.op == '||' or self.op == ',': | |
| 875 return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env,
defs)) | |
| 876 elif self.op == 'if': | |
| 877 if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs) | |
| 878 else: return Nothing() | |
| 879 else: | |
| 880 assert self.op == '&&' | |
| 881 return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(e
nv, defs)) | |
| 882 | |
| 883 | |
| 884 def IsAlpha(str): | |
| 885 for char in str: | |
| 886 if not (char.isalpha() or char.isdigit() or char == '_'): | |
| 887 return False | |
| 888 return True | |
| 889 | |
| 890 | |
| 891 class Tokenizer(object): | |
| 892 """A simple string tokenizer that chops expressions into variables, | |
| 893 parens and operators""" | |
| 894 | |
| 895 def __init__(self, expr): | |
| 896 self.index = 0 | |
| 897 self.expr = expr | |
| 898 self.length = len(expr) | |
| 899 self.tokens = None | |
| 900 | |
| 901 def Current(self, length = 1): | |
| 902 if not self.HasMore(length): return "" | |
| 903 return self.expr[self.index:self.index+length] | |
| 904 | |
| 905 def HasMore(self, length = 1): | |
| 906 return self.index < self.length + (length - 1) | |
| 907 | |
| 908 def Advance(self, count = 1): | |
| 909 self.index = self.index + count | |
| 910 | |
| 911 def AddToken(self, token): | |
| 912 self.tokens.append(token) | |
| 913 | |
| 914 def SkipSpaces(self): | |
| 915 while self.HasMore() and self.Current().isspace(): | |
| 916 self.Advance() | |
| 917 | |
| 918 def Tokenize(self): | |
| 919 self.tokens = [ ] | |
| 920 while self.HasMore(): | |
| 921 self.SkipSpaces() | |
| 922 if not self.HasMore(): | |
| 923 return None | |
| 924 if self.Current() == '(': | |
| 925 self.AddToken('(') | |
| 926 self.Advance() | |
| 927 elif self.Current() == ')': | |
| 928 self.AddToken(')') | |
| 929 self.Advance() | |
| 930 elif self.Current() == '$': | |
| 931 self.AddToken('$') | |
| 932 self.Advance() | |
| 933 elif self.Current() == ',': | |
| 934 self.AddToken(',') | |
| 935 self.Advance() | |
| 936 elif IsAlpha(self.Current()): | |
| 937 buf = "" | |
| 938 while self.HasMore() and IsAlpha(self.Current()): | |
| 939 buf += self.Current() | |
| 940 self.Advance() | |
| 941 self.AddToken(buf) | |
| 942 elif self.Current(2) == '&&': | |
| 943 self.AddToken('&&') | |
| 944 self.Advance(2) | |
| 945 elif self.Current(2) == '||': | |
| 946 self.AddToken('||') | |
| 947 self.Advance(2) | |
| 948 elif self.Current(2) == '==': | |
| 949 self.AddToken('==') | |
| 950 self.Advance(2) | |
| 951 elif self.Current(2) == '!=': | |
| 952 self.AddToken('!=') | |
| 953 self.Advance(2) | |
| 954 else: | |
| 955 return None | |
| 956 return self.tokens | |
| 957 | |
| 958 | |
| 959 class Scanner(object): | |
| 960 """A simple scanner that can serve out tokens from a given list""" | |
| 961 | |
| 962 def __init__(self, tokens): | |
| 963 self.tokens = tokens | |
| 964 self.length = len(tokens) | |
| 965 self.index = 0 | |
| 966 | |
| 967 def HasMore(self): | |
| 968 return self.index < self.length | |
| 969 | |
| 970 def Current(self): | |
| 971 return self.tokens[self.index] | |
| 972 | |
| 973 def Advance(self): | |
| 974 self.index = self.index + 1 | |
| 975 | |
| 976 | |
| 977 def ParseAtomicExpression(scan): | |
| 978 if scan.Current() == "true": | |
| 979 scan.Advance() | |
| 980 return Constant(True) | |
| 981 elif scan.Current() == "false": | |
| 982 scan.Advance() | |
| 983 return Constant(False) | |
| 984 elif IsAlpha(scan.Current()): | |
| 985 name = scan.Current() | |
| 986 scan.Advance() | |
| 987 return Outcome(name.lower()) | |
| 988 elif scan.Current() == '$': | |
| 989 scan.Advance() | |
| 990 if not IsAlpha(scan.Current()): | |
| 991 return None | |
| 992 name = scan.Current() | |
| 993 scan.Advance() | |
| 994 return Variable(name.lower()) | |
| 995 elif scan.Current() == '(': | |
| 996 scan.Advance() | |
| 997 result = ParseLogicalExpression(scan) | |
| 998 if (not result) or (scan.Current() != ')'): | |
| 999 return None | |
| 1000 scan.Advance() | |
| 1001 return result | |
| 1002 else: | |
| 1003 return None | |
| 1004 | |
| 1005 | |
| 1006 BINARIES = ['==', '!='] | |
| 1007 def ParseOperatorExpression(scan): | |
| 1008 left = ParseAtomicExpression(scan) | |
| 1009 if not left: return None | |
| 1010 while scan.HasMore() and (scan.Current() in BINARIES): | |
| 1011 op = scan.Current() | |
| 1012 scan.Advance() | |
| 1013 right = ParseOperatorExpression(scan) | |
| 1014 if not right: | |
| 1015 return None | |
| 1016 left = Operation(left, op, right) | |
| 1017 return left | |
| 1018 | |
| 1019 | |
| 1020 def ParseConditionalExpression(scan): | |
| 1021 left = ParseOperatorExpression(scan) | |
| 1022 if not left: return None | |
| 1023 while scan.HasMore() and (scan.Current() == 'if'): | |
| 1024 scan.Advance() | |
| 1025 right = ParseOperatorExpression(scan) | |
| 1026 if not right: | |
| 1027 return None | |
| 1028 left = Operation(left, 'if', right) | |
| 1029 return left | |
| 1030 | |
| 1031 | |
| 1032 LOGICALS = ["&&", "||", ","] | |
| 1033 def ParseLogicalExpression(scan): | |
| 1034 left = ParseConditionalExpression(scan) | |
| 1035 if not left: return None | |
| 1036 while scan.HasMore() and (scan.Current() in LOGICALS): | |
| 1037 op = scan.Current() | |
| 1038 scan.Advance() | |
| 1039 right = ParseConditionalExpression(scan) | |
| 1040 if not right: | |
| 1041 return None | |
| 1042 left = Operation(left, op, right) | |
| 1043 return left | |
| 1044 | |
| 1045 | |
| 1046 def ParseCondition(expr): | |
| 1047 """Parses a logical expression into an Expression object""" | |
| 1048 tokens = Tokenizer(expr).Tokenize() | |
| 1049 if not tokens: | |
| 1050 print "Malformed expression: '%s'" % expr | |
| 1051 return None | |
| 1052 scan = Scanner(tokens) | |
| 1053 ast = ParseLogicalExpression(scan) | |
| 1054 if not ast: | |
| 1055 print "Malformed expression: '%s'" % expr | |
| 1056 return None | |
| 1057 if scan.HasMore(): | |
| 1058 print "Malformed expression: '%s'" % expr | |
| 1059 return None | |
| 1060 return ast | |
| 1061 | |
| 1062 | |
| 1063 class ClassifiedTest(object): | |
| 1064 | |
| 1065 def __init__(self, case, outcomes): | |
| 1066 self.case = case | |
| 1067 self.outcomes = outcomes | |
| 1068 | |
| 1069 def TestsIsolates(self): | |
| 1070 return self.case.TestsIsolates() | |
| 1071 | |
| 1072 | |
| 1073 class Configuration(object): | |
| 1074 """The parsed contents of a configuration file""" | |
| 1075 | |
| 1076 def __init__(self, sections, defs): | |
| 1077 self.sections = sections | |
| 1078 self.defs = defs | |
| 1079 | |
| 1080 def ClassifyTests(self, cases, env): | |
| 1081 sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)
] | |
| 1082 all_rules = reduce(list.__add__, [s.rules for s in sections], []) | |
| 1083 unused_rules = set(all_rules) | |
| 1084 result = [ ] | |
| 1085 all_outcomes = set([]) | |
| 1086 for case in cases: | |
| 1087 matches = [ r for r in all_rules if r.Contains(case.path) ] | |
| 1088 outcomes = set([]) | |
| 1089 for rule in matches: | |
| 1090 outcomes = outcomes.union(rule.GetOutcomes(env, self.defs)) | |
| 1091 unused_rules.discard(rule) | |
| 1092 if not outcomes: | |
| 1093 outcomes = [PASS] | |
| 1094 case.outcomes = outcomes | |
| 1095 all_outcomes = all_outcomes.union(outcomes) | |
| 1096 result.append(ClassifiedTest(case, outcomes)) | |
| 1097 return (result, list(unused_rules), all_outcomes) | |
| 1098 | |
| 1099 | |
| 1100 class Section(object): | |
| 1101 """A section of the configuration file. Sections are enabled or | |
| 1102 disabled prior to running the tests, based on their conditions""" | |
| 1103 | |
| 1104 def __init__(self, condition): | |
| 1105 self.condition = condition | |
| 1106 self.rules = [ ] | |
| 1107 | |
| 1108 def AddRule(self, rule): | |
| 1109 self.rules.append(rule) | |
| 1110 | |
| 1111 | |
| 1112 class Rule(object): | |
| 1113 """A single rule that specifies the expected outcome for a single | |
| 1114 test.""" | |
| 1115 | |
| 1116 def __init__(self, raw_path, path, value): | |
| 1117 self.raw_path = raw_path | |
| 1118 self.path = path | |
| 1119 self.value = value | |
| 1120 | |
| 1121 def GetOutcomes(self, env, defs): | |
| 1122 set = self.value.GetOutcomes(env, defs) | |
| 1123 assert isinstance(set, ListSet) | |
| 1124 return set.elms | |
| 1125 | |
| 1126 def Contains(self, path): | |
| 1127 if len(self.path) > len(path): | |
| 1128 return False | |
| 1129 for i in xrange(len(self.path)): | |
| 1130 if not self.path[i].match(path[i]): | |
| 1131 return False | |
| 1132 return True | |
| 1133 | |
| 1134 | |
| 1135 HEADER_PATTERN = re.compile(r'\[([^]]+)\]') | |
| 1136 RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)') | |
| 1137 DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$') | |
| 1138 PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$') | |
| 1139 | |
| 1140 | |
| 1141 def ReadConfigurationInto(path, sections, defs): | |
| 1142 current_section = Section(Constant(True)) | |
| 1143 sections.append(current_section) | |
| 1144 prefix = [] | |
| 1145 for line in utils.ReadLinesFrom(path): | |
| 1146 header_match = HEADER_PATTERN.match(line) | |
| 1147 if header_match: | |
| 1148 condition_str = header_match.group(1).strip() | |
| 1149 condition = ParseCondition(condition_str) | |
| 1150 new_section = Section(condition) | |
| 1151 sections.append(new_section) | |
| 1152 current_section = new_section | |
| 1153 continue | |
| 1154 rule_match = RULE_PATTERN.match(line) | |
| 1155 if rule_match: | |
| 1156 path = prefix + SplitPath(rule_match.group(1).strip()) | |
| 1157 value_str = rule_match.group(2).strip() | |
| 1158 value = ParseCondition(value_str) | |
| 1159 if not value: | |
| 1160 return False | |
| 1161 current_section.AddRule(Rule(rule_match.group(1), path, value)) | |
| 1162 continue | |
| 1163 def_match = DEF_PATTERN.match(line) | |
| 1164 if def_match: | |
| 1165 name = def_match.group(1).lower() | |
| 1166 value = ParseCondition(def_match.group(2).strip()) | |
| 1167 if not value: | |
| 1168 return False | |
| 1169 defs[name] = value | |
| 1170 continue | |
| 1171 prefix_match = PREFIX_PATTERN.match(line) | |
| 1172 if prefix_match: | |
| 1173 prefix = SplitPath(prefix_match.group(1).strip()) | |
| 1174 continue | |
| 1175 print "Malformed line: '%s'." % line | |
| 1176 return False | |
| 1177 return True | |
| 1178 | |
| 1179 | |
| 1180 # --------------- | |
| 1181 # --- M a i n --- | |
| 1182 # --------------- | |
| 1183 | |
| 1184 | |
| 1185 ARCH_GUESS = utils.GuessArchitecture() | |
| 1186 TIMEOUT_DEFAULT = 60; | |
| 1187 | |
| 1188 | |
| 1189 def BuildOptions(): | |
| 1190 result = optparse.OptionParser() | |
| 1191 result.add_option("-m", "--mode", help="The test modes in which to run (comma-
separated)", | |
| 1192 default='release') | |
| 1193 result.add_option("-v", "--verbose", help="Verbose output", | |
| 1194 default=False, action="store_true") | |
| 1195 result.add_option("-S", dest="scons_flags", help="Flag to pass through to scon
s", | |
| 1196 default=[], action="append") | |
| 1197 result.add_option("-p", "--progress", | |
| 1198 help="The style of progress indicator (verbose, dots, color, mono)", | |
| 1199 choices=PROGRESS_INDICATORS.keys(), default="mono") | |
| 1200 result.add_option("--no-build", help="Don't build requirements", | |
| 1201 default=False, action="store_true") | |
| 1202 result.add_option("--build-only", help="Only build requirements, don't run the
tests", | |
| 1203 default=False, action="store_true") | |
| 1204 result.add_option("--build-system", help="Build system in use (scons or gyp)", | |
| 1205 default='scons') | |
| 1206 result.add_option("--report", help="Print a summary of the tests to be run", | |
| 1207 default=False, action="store_true") | |
| 1208 result.add_option("--download-data", help="Download missing test suite data", | |
| 1209 default=False, action="store_true") | |
| 1210 result.add_option("-s", "--suite", help="A test suite", | |
| 1211 default=[], action="append") | |
| 1212 result.add_option("-t", "--timeout", help="Timeout in seconds", | |
| 1213 default=-1, type="int") | |
| 1214 result.add_option("--arch", help='The architecture to run tests for', | |
| 1215 default='none') | |
| 1216 result.add_option("--snapshot", help="Run the tests with snapshot turned on", | |
| 1217 default=False, action="store_true") | |
| 1218 result.add_option("--simulator", help="Run tests with architecture simulator", | |
| 1219 default='none') | |
| 1220 result.add_option("--special-command", default=None) | |
| 1221 result.add_option("--valgrind", help="Run tests through valgrind", | |
| 1222 default=False, action="store_true") | |
| 1223 result.add_option("--cat", help="Print the source of the tests", | |
| 1224 default=False, action="store_true") | |
| 1225 result.add_option("--warn-unused", help="Report unused rules", | |
| 1226 default=False, action="store_true") | |
| 1227 result.add_option("-j", help="The number of parallel tasks to run", | |
| 1228 default=1, type="int") | |
| 1229 result.add_option("--time", help="Print timing information after running", | |
| 1230 default=False, action="store_true") | |
| 1231 result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for cra
shing tests", | |
| 1232 dest="suppress_dialogs", default=True, action="store_true") | |
| 1233 result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for c
rashing tests", | |
| 1234 dest="suppress_dialogs", action="store_false") | |
| 1235 result.add_option("--mips-arch-variant", help="mips architecture variant: mips
32r1/mips32r2", default="mips32r2"); | |
| 1236 result.add_option("--shell", help="Path to V8 shell", default="d8") | |
| 1237 result.add_option("--isolates", help="Whether to test isolates", default=False
, action="store_true") | |
| 1238 result.add_option("--store-unexpected-output", | |
| 1239 help="Store the temporary JS files from tests that fails", | |
| 1240 dest="store_unexpected_output", default=True, action="store_true") | |
| 1241 result.add_option("--no-store-unexpected-output", | |
| 1242 help="Deletes the temporary JS files from tests that fails", | |
| 1243 dest="store_unexpected_output", action="store_false") | |
| 1244 result.add_option("--stress-only", | |
| 1245 help="Only run tests with --always-opt --stress-opt", | |
| 1246 default=False, action="store_true") | |
| 1247 result.add_option("--nostress", | |
| 1248 help="Don't run crankshaft --always-opt --stress-op test", | |
| 1249 default=False, action="store_true") | |
| 1250 result.add_option("--shard-count", | |
| 1251 help="Split testsuites into this number of shards", | |
| 1252 default=1, type="int") | |
| 1253 result.add_option("--shard-run", | |
| 1254 help="Run this shard from the split up tests.", | |
| 1255 default=1, type="int") | |
| 1256 result.add_option("--noprof", help="Disable profiling support", | |
| 1257 default=False) | |
| 1258 return result | |
| 1259 | |
| 1260 | |
| 1261 def ProcessOptions(options): | |
| 1262 global VERBOSE | |
| 1263 VERBOSE = options.verbose | |
| 1264 options.mode = options.mode.split(',') | |
| 1265 for mode in options.mode: | |
| 1266 if not mode in ['debug', 'release']: | |
| 1267 print "Unknown mode %s" % mode | |
| 1268 return False | |
| 1269 if options.simulator != 'none': | |
| 1270 # Simulator argument was set. Make sure arch and simulator agree. | |
| 1271 if options.simulator != options.arch: | |
| 1272 if options.arch == 'none': | |
| 1273 options.arch = options.simulator | |
| 1274 else: | |
| 1275 print "Architecture %s does not match sim %s" %(options.arch, options.si
mulator) | |
| 1276 return False | |
| 1277 # Ensure that the simulator argument is handed down to scons. | |
| 1278 options.scons_flags.append("simulator=" + options.simulator) | |
| 1279 else: | |
| 1280 # If options.arch is not set by the command line and no simulator setting | |
| 1281 # was found, set the arch to the guess. | |
| 1282 if options.arch == 'none': | |
| 1283 options.arch = ARCH_GUESS | |
| 1284 options.scons_flags.append("arch=" + options.arch) | |
| 1285 # Simulators are slow, therefore allow a longer default timeout. | |
| 1286 if options.timeout == -1: | |
| 1287 if options.arch in ['android', 'arm', 'mipsel']: | |
| 1288 options.timeout = 2 * TIMEOUT_DEFAULT; | |
| 1289 else: | |
| 1290 options.timeout = TIMEOUT_DEFAULT; | |
| 1291 if options.snapshot: | |
| 1292 options.scons_flags.append("snapshot=on") | |
| 1293 global VARIANT_FLAGS | |
| 1294 if options.mips_arch_variant: | |
| 1295 options.scons_flags.append("mips_arch_variant=" + options.mips_arch_variant) | |
| 1296 | |
| 1297 if options.stress_only: | |
| 1298 VARIANT_FLAGS = [['--stress-opt', '--always-opt']] | |
| 1299 if options.nostress: | |
| 1300 VARIANT_FLAGS = [[],['--nocrankshaft']] | |
| 1301 if options.shell.endswith("d8"): | |
| 1302 if options.special_command: | |
| 1303 options.special_command += " --test" | |
| 1304 else: | |
| 1305 options.special_command = "@ --test" | |
| 1306 if options.noprof: | |
| 1307 options.scons_flags.append("prof=off") | |
| 1308 options.scons_flags.append("profilingsupport=off") | |
| 1309 if options.build_system == 'gyp': | |
| 1310 if options.build_only: | |
| 1311 print "--build-only not supported for gyp, please build manually." | |
| 1312 options.build_only = False | |
| 1313 return True | |
| 1314 | |
| 1315 | |
| 1316 def DoSkip(case): | |
| 1317 return (SKIP in case.outcomes) or (SLOW in case.outcomes) | |
| 1318 | |
| 1319 | |
| 1320 REPORT_TEMPLATE = """\ | |
| 1321 Total: %(total)i tests | |
| 1322 * %(skipped)4d tests will be skipped | |
| 1323 * %(timeout)4d tests are expected to timeout sometimes | |
| 1324 * %(nocrash)4d tests are expected to be flaky but not crash | |
| 1325 * %(pass)4d tests are expected to pass | |
| 1326 * %(fail_ok)4d tests are expected to fail that we won't fix | |
| 1327 * %(fail)4d tests are expected to fail that we should fix\ | |
| 1328 """ | |
| 1329 | |
| 1330 def PrintReport(cases): | |
| 1331 def IsFlaky(o): | |
| 1332 return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o) | |
| 1333 def IsFailOk(o): | |
| 1334 return (len(o) == 2) and (FAIL in o) and (OKAY in o) | |
| 1335 unskipped = [c for c in cases if not DoSkip(c)] | |
| 1336 print REPORT_TEMPLATE % { | |
| 1337 'total': len(cases), | |
| 1338 'skipped': len(cases) - len(unskipped), | |
| 1339 'timeout': len([t for t in unskipped if TIMEOUT in t.outcomes]), | |
| 1340 'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]), | |
| 1341 'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]), | |
| 1342 'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]), | |
| 1343 'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]]) | |
| 1344 } | |
| 1345 | |
| 1346 | |
| 1347 class Pattern(object): | |
| 1348 | |
| 1349 def __init__(self, pattern): | |
| 1350 self.pattern = pattern | |
| 1351 self.compiled = None | |
| 1352 | |
| 1353 def match(self, str): | |
| 1354 if not self.compiled: | |
| 1355 pattern = "^" + self.pattern.replace('*', '.*') + "$" | |
| 1356 self.compiled = re.compile(pattern) | |
| 1357 return self.compiled.match(str) | |
| 1358 | |
| 1359 def __str__(self): | |
| 1360 return self.pattern | |
| 1361 | |
| 1362 | |
| 1363 def SplitPath(s): | |
| 1364 stripped = [ c.strip() for c in s.split('/') ] | |
| 1365 return [ Pattern(s) for s in stripped if len(s) > 0 ] | |
| 1366 | |
| 1367 | |
| 1368 def GetSpecialCommandProcessor(value): | |
| 1369 if (not value) or (value.find('@') == -1): | |
| 1370 def ExpandCommand(args): | |
| 1371 return args | |
| 1372 return ExpandCommand | |
| 1373 else: | |
| 1374 pos = value.find('@') | |
| 1375 import urllib | |
| 1376 import shlex | |
| 1377 prefix = shlex.split(urllib.unquote(value[:pos])) | |
| 1378 suffix = shlex.split(urllib.unquote(value[pos+1:])) | |
| 1379 def ExpandCommand(args): | |
| 1380 return prefix + args + suffix | |
| 1381 return ExpandCommand | |
| 1382 | |
| 1383 | |
| 1384 BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message', 'preparser'] | |
| 1385 | |
| 1386 | |
| 1387 def GetSuites(test_root): | |
| 1388 def IsSuite(path): | |
| 1389 return isdir(path) and exists(join(path, 'testcfg.py')) | |
| 1390 return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ] | |
| 1391 | |
| 1392 | |
| 1393 def FormatTime(d): | |
| 1394 millis = round(d * 1000) % 1000 | |
| 1395 return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) | |
| 1396 | |
| 1397 def ShardTests(tests, options): | |
| 1398 if options.shard_count < 2: | |
| 1399 return tests | |
| 1400 if options.shard_run < 1 or options.shard_run > options.shard_count: | |
| 1401 print "shard-run not a valid number, should be in [1:shard-count]" | |
| 1402 print "defaulting back to running all tests" | |
| 1403 return tests | |
| 1404 count = 0 | |
| 1405 shard = [] | |
| 1406 for test in tests: | |
| 1407 if count % options.shard_count == options.shard_run - 1: | |
| 1408 shard.append(test) | |
| 1409 count += 1 | |
| 1410 return shard | |
| 1411 | |
| 1412 def Main(): | |
| 1413 parser = BuildOptions() | |
| 1414 (options, args) = parser.parse_args() | |
| 1415 if not ProcessOptions(options): | |
| 1416 parser.print_help() | |
| 1417 return 1 | |
| 1418 | |
| 1419 workspace = abspath(join(dirname(sys.argv[0]), '..')) | |
| 1420 suites = GetSuites(join(workspace, 'test')) | |
| 1421 repositories = [TestRepository(join(workspace, 'test', name)) for name in suit
es] | |
| 1422 repositories += [TestRepository(a) for a in options.suite] | |
| 1423 | |
| 1424 root = LiteralTestSuite(repositories) | |
| 1425 if len(args) == 0: | |
| 1426 paths = [SplitPath(t) for t in BUILT_IN_TESTS] | |
| 1427 else: | |
| 1428 paths = [ ] | |
| 1429 for arg in args: | |
| 1430 path = SplitPath(arg) | |
| 1431 paths.append(path) | |
| 1432 | |
| 1433 # Check for --valgrind option. If enabled, we overwrite the special | |
| 1434 # command flag with a command that uses the run-valgrind.py script. | |
| 1435 if options.valgrind: | |
| 1436 run_valgrind = join(workspace, "tools", "run-valgrind.py") | |
| 1437 options.special_command = "python -u " + run_valgrind + " @" | |
| 1438 | |
| 1439 if options.build_system == 'gyp': | |
| 1440 SUFFIX['debug'] = '' | |
| 1441 | |
| 1442 shell = abspath(options.shell) | |
| 1443 buildspace = dirname(shell) | |
| 1444 | |
| 1445 context = Context(workspace, buildspace, VERBOSE, | |
| 1446 shell, | |
| 1447 options.timeout, | |
| 1448 GetSpecialCommandProcessor(options.special_command), | |
| 1449 options.suppress_dialogs, | |
| 1450 options.store_unexpected_output) | |
| 1451 # First build the required targets | |
| 1452 if not options.no_build: | |
| 1453 reqs = [ ] | |
| 1454 for path in paths: | |
| 1455 reqs += root.GetBuildRequirements(path, context) | |
| 1456 reqs = list(set(reqs)) | |
| 1457 if len(reqs) > 0: | |
| 1458 if options.j != 1: | |
| 1459 options.scons_flags += ['-j', str(options.j)] | |
| 1460 if not BuildRequirements(context, reqs, options.mode, options.scons_flags)
: | |
| 1461 return 1 | |
| 1462 | |
| 1463 # Just return if we are only building the targets for running the tests. | |
| 1464 if options.build_only: | |
| 1465 return 0 | |
| 1466 | |
| 1467 # Get status for tests | |
| 1468 sections = [ ] | |
| 1469 defs = { } | |
| 1470 root.GetTestStatus(context, sections, defs) | |
| 1471 config = Configuration(sections, defs) | |
| 1472 | |
| 1473 # Download missing test suite data if requested. | |
| 1474 if options.download_data: | |
| 1475 for path in paths: | |
| 1476 root.DownloadData(path, context) | |
| 1477 | |
| 1478 # List the tests | |
| 1479 all_cases = [ ] | |
| 1480 all_unused = [ ] | |
| 1481 unclassified_tests = [ ] | |
| 1482 globally_unused_rules = None | |
| 1483 for path in paths: | |
| 1484 for mode in options.mode: | |
| 1485 env = { | |
| 1486 'mode': mode, | |
| 1487 'system': utils.GuessOS(), | |
| 1488 'arch': options.arch, | |
| 1489 'simulator': options.simulator, | |
| 1490 'isolates': options.isolates | |
| 1491 } | |
| 1492 test_list = root.ListTests([], path, context, mode, []) | |
| 1493 unclassified_tests += test_list | |
| 1494 (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env) | |
| 1495 if globally_unused_rules is None: | |
| 1496 globally_unused_rules = set(unused_rules) | |
| 1497 else: | |
| 1498 globally_unused_rules = globally_unused_rules.intersection(unused_rules) | |
| 1499 all_cases += ShardTests(cases, options) | |
| 1500 all_unused.append(unused_rules) | |
| 1501 | |
| 1502 if options.cat: | |
| 1503 visited = set() | |
| 1504 for test in unclassified_tests: | |
| 1505 key = tuple(test.path) | |
| 1506 if key in visited: | |
| 1507 continue | |
| 1508 visited.add(key) | |
| 1509 print "--- begin source: %s ---" % test.GetLabel() | |
| 1510 source = test.GetSource().strip() | |
| 1511 print source | |
| 1512 print "--- end source: %s ---" % test.GetLabel() | |
| 1513 return 0 | |
| 1514 | |
| 1515 if options.warn_unused: | |
| 1516 for rule in globally_unused_rules: | |
| 1517 print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path]
) | |
| 1518 | |
| 1519 if not options.isolates: | |
| 1520 all_cases = [c for c in all_cases if not c.TestsIsolates()] | |
| 1521 | |
| 1522 if options.report: | |
| 1523 PrintReport(all_cases) | |
| 1524 | |
| 1525 result = None | |
| 1526 cases_to_run = [ c for c in all_cases if not DoSkip(c) ] | |
| 1527 if len(cases_to_run) == 0: | |
| 1528 print "No tests to run." | |
| 1529 return 0 | |
| 1530 else: | |
| 1531 try: | |
| 1532 start = time.time() | |
| 1533 if RunTestCases(cases_to_run, options.progress, options.j): | |
| 1534 result = 0 | |
| 1535 else: | |
| 1536 result = 1 | |
| 1537 duration = time.time() - start | |
| 1538 except KeyboardInterrupt: | |
| 1539 print "Interrupted" | |
| 1540 return 1 | |
| 1541 | |
| 1542 if options.time: | |
| 1543 # Write the times to stderr to make it easy to separate from the | |
| 1544 # test output. | |
| 1545 print | |
| 1546 sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration)) | |
| 1547 timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None
] | |
| 1548 timed_tests.sort(lambda a, b: a.CompareTime(b)) | |
| 1549 index = 1 | |
| 1550 for entry in timed_tests[:20]: | |
| 1551 t = FormatTime(entry.duration) | |
| 1552 sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel())) | |
| 1553 index += 1 | |
| 1554 | |
| 1555 return result | |
| 1556 | |
| 1557 | |
| 1558 if __name__ == '__main__': | |
| 1559 sys.exit(Main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 14348002:
Remove SCons related files (Closed)
Base URL: https://v8.googlecode.com/svn/branches/bleeding_edge