| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Runs each test cases as a single shard, single process execution. | |
| 7 | |
| 8 Similar to sharding_supervisor.py but finer grained. It runs each test case | |
| 9 individually instead of running per shard. Runs multiple instances in parallel. | |
| 10 """ | |
| 11 | |
| 12 import datetime | |
| 13 import fnmatch | |
| 14 import json | |
| 15 import logging | |
| 16 import optparse | |
| 17 import os | |
| 18 import Queue | |
| 19 import random | |
| 20 import re | |
| 21 import subprocess | |
| 22 import sys | |
| 23 import threading | |
| 24 import time | |
| 25 from xml.dom import minidom | |
| 26 import xml.parsers.expat | |
| 27 | |
| 28 import run_isolated | |
| 29 | |
| 30 # Scripts using run_test_cases as a library expect this function. | |
| 31 from run_isolated import fix_python_path | |
| 32 | |
| 33 | |
| 34 # These are known to influence the way the output is generated. | |
| 35 KNOWN_GTEST_ENV_VARS = [ | |
| 36 'GTEST_ALSO_RUN_DISABLED_TESTS', | |
| 37 'GTEST_BREAK_ON_FAILURE', | |
| 38 'GTEST_CATCH_EXCEPTIONS', | |
| 39 'GTEST_COLOR', | |
| 40 'GTEST_FILTER', | |
| 41 'GTEST_OUTPUT', | |
| 42 'GTEST_PRINT_TIME', | |
| 43 'GTEST_RANDOM_SEED', | |
| 44 'GTEST_REPEAT', | |
| 45 'GTEST_SHARD_INDEX', | |
| 46 'GTEST_SHARD_STATUS_FILE', | |
| 47 'GTEST_SHUFFLE', | |
| 48 'GTEST_THROW_ON_FAILURE', | |
| 49 'GTEST_TOTAL_SHARDS', | |
| 50 ] | |
| 51 | |
| 52 # These needs to be poped out before running a test. | |
| 53 GTEST_ENV_VARS_TO_REMOVE = [ | |
| 54 'GTEST_ALSO_RUN_DISABLED_TESTS', | |
| 55 'GTEST_FILTER', | |
| 56 'GTEST_OUTPUT', | |
| 57 'GTEST_RANDOM_SEED', | |
| 58 # TODO(maruel): Handle. | |
| 59 'GTEST_REPEAT', | |
| 60 'GTEST_SHARD_INDEX', | |
| 61 # TODO(maruel): Handle. | |
| 62 'GTEST_SHUFFLE', | |
| 63 'GTEST_TOTAL_SHARDS', | |
| 64 ] | |
| 65 | |
| 66 | |
| 67 RUN_PREFIX = '[ RUN ] ' | |
| 68 OK_PREFIX = '[ OK ] ' | |
| 69 FAILED_PREFIX = '[ FAILED ] ' | |
| 70 | |
| 71 | |
| 72 if subprocess.mswindows: | |
| 73 import msvcrt # pylint: disable=F0401 | |
| 74 from ctypes import wintypes | |
| 75 from ctypes import windll | |
| 76 | |
| 77 def ReadFile(handle, desired_bytes): | |
| 78 """Calls kernel32.ReadFile().""" | |
| 79 c_read = wintypes.DWORD() | |
| 80 buff = wintypes.create_string_buffer(desired_bytes+1) | |
| 81 windll.kernel32.ReadFile( | |
| 82 handle, buff, desired_bytes, wintypes.byref(c_read), None) | |
| 83 # NULL terminate it. | |
| 84 buff[c_read.value] = '\x00' | |
| 85 return wintypes.GetLastError(), buff.value | |
| 86 | |
| 87 def PeekNamedPipe(handle): | |
| 88 """Calls kernel32.PeekNamedPipe(). Simplified version.""" | |
| 89 c_avail = wintypes.DWORD() | |
| 90 c_message = wintypes.DWORD() | |
| 91 success = windll.kernel32.PeekNamedPipe( | |
| 92 handle, None, 0, None, wintypes.byref(c_avail), | |
| 93 wintypes.byref(c_message)) | |
| 94 if not success: | |
| 95 raise OSError(wintypes.GetLastError()) | |
| 96 return c_avail.value | |
| 97 | |
| 98 def recv_multi_impl(conns, maxsize, timeout): | |
| 99 """Reads from the first available pipe. | |
| 100 | |
| 101 If timeout is None, it's blocking. If timeout is 0, it is not blocking. | |
| 102 """ | |
| 103 # TODO(maruel): Use WaitForMultipleObjects(). Python creates anonymous pipes | |
| 104 # for proc.stdout and proc.stderr but they are implemented as named pipes on | |
| 105 # Windows. Since named pipes are not waitable object, they can't be passed | |
| 106 # as-is to WFMO(). So this means N times CreateEvent(), N times ReadFile() | |
| 107 # and finally WFMO(). This requires caching the events handles in the Popen | |
| 108 # object and remembering the pending ReadFile() calls. This will require | |
| 109 # some re-architecture. | |
| 110 maxsize = max(maxsize or 16384, 1) | |
| 111 if timeout: | |
| 112 start = time.time() | |
| 113 handles = [msvcrt.get_osfhandle(conn.fileno()) for conn in conns] | |
| 114 while handles: | |
| 115 for i, handle in enumerate(handles): | |
| 116 try: | |
| 117 avail = min(PeekNamedPipe(handle), maxsize) | |
| 118 if avail: | |
| 119 return i, ReadFile(handle, avail)[1] | |
| 120 if (timeout and (time.time() - start) >= timeout) or timeout == 0: | |
| 121 return None, None | |
| 122 # Polling rocks. | |
| 123 time.sleep(0.001) | |
| 124 except OSError: | |
| 125 handles.pop(i) | |
| 126 break | |
| 127 # Nothing to wait for. | |
| 128 return None, None | |
| 129 | |
| 130 else: | |
| 131 import fcntl # pylint: disable=F0401 | |
| 132 import select | |
| 133 | |
| 134 def recv_multi_impl(conns, maxsize, timeout): | |
| 135 """Reads from the first available pipe. | |
| 136 | |
| 137 If timeout is None, it's blocking. If timeout is 0, it is not blocking. | |
| 138 """ | |
| 139 try: | |
| 140 r, _, _ = select.select(conns, [], [], timeout) | |
| 141 except select.error: | |
| 142 return None, None | |
| 143 if not r: | |
| 144 return None, None | |
| 145 | |
| 146 conn = r[0] | |
| 147 # Temporarily make it non-blocking. | |
| 148 flags = fcntl.fcntl(conn, fcntl.F_GETFL) | |
| 149 if not conn.closed: | |
| 150 # pylint: disable=E1101 | |
| 151 fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK) | |
| 152 try: | |
| 153 data = conn.read(max(maxsize or 16384, 1)) | |
| 154 return conns.index(conn), data | |
| 155 finally: | |
| 156 if not conn.closed: | |
| 157 fcntl.fcntl(conn, fcntl.F_SETFL, flags) | |
| 158 | |
| 159 | |
| 160 class Failure(Exception): | |
| 161 pass | |
| 162 | |
| 163 | |
| 164 class Popen(subprocess.Popen): | |
| 165 """Adds timeout support on stdout and stderr. | |
| 166 | |
| 167 Inspired by | |
| 168 http://code.activestate.com/recipes/440554-module-to-allow-asynchronous-subpro
cess-use-on-win/ | |
| 169 """ | |
| 170 def __init__(self, *args, **kwargs): | |
| 171 self.start = time.time() | |
| 172 self.end = None | |
| 173 super(Popen, self).__init__(*args, **kwargs) | |
| 174 | |
| 175 def duration(self): | |
| 176 """Duration of the child process. | |
| 177 | |
| 178 It is greater or equal to the actual time the child process ran. It can be | |
| 179 significantly higher than the real value if neither .wait() nor .poll() was | |
| 180 used. | |
| 181 """ | |
| 182 return (self.end or time.time()) - self.start | |
| 183 | |
| 184 def wait(self): | |
| 185 ret = super(Popen, self).wait() | |
| 186 if not self.end: | |
| 187 # communicate() uses wait() internally. | |
| 188 self.end = time.time() | |
| 189 return ret | |
| 190 | |
| 191 def poll(self): | |
| 192 ret = super(Popen, self).poll() | |
| 193 if ret is not None and not self.end: | |
| 194 self.end = time.time() | |
| 195 return ret | |
| 196 | |
| 197 def yield_any(self, timeout=None): | |
| 198 """Yields output until the process terminates or is killed by a timeout. | |
| 199 | |
| 200 Yielded values are in the form (pipename, data). | |
| 201 | |
| 202 If timeout is None, it is blocking. If timeout is 0, it doesn't block. This | |
| 203 is not generally useful to use timeout=0. | |
| 204 """ | |
| 205 remaining = 0 | |
| 206 while self.poll() is None: | |
| 207 if timeout: | |
| 208 # While these float() calls seem redundant, they are to force | |
| 209 # ResetableTimeout to "render" itself into a float. At each call, the | |
| 210 # resulting value could be different, depending if a .reset() call | |
| 211 # occurred. | |
| 212 remaining = max(float(timeout) - self.duration(), 0.001) | |
| 213 else: | |
| 214 remaining = timeout | |
| 215 t, data = self.recv_any(timeout=remaining) | |
| 216 if data or timeout == 0: | |
| 217 yield (t, data) | |
| 218 if timeout and self.duration() >= float(timeout): | |
| 219 break | |
| 220 if self.poll() is None and timeout and self.duration() >= float(timeout): | |
| 221 logging.debug('Kill %s %s', self.duration(), float(timeout)) | |
| 222 self.kill() | |
| 223 self.wait() | |
| 224 # Read all remaining output in the pipes. | |
| 225 while True: | |
| 226 t, data = self.recv_any() | |
| 227 if not data: | |
| 228 break | |
| 229 yield (t, data) | |
| 230 | |
| 231 def recv_any(self, maxsize=None, timeout=None): | |
| 232 """Reads from stderr and if empty, from stdout. | |
| 233 | |
| 234 If timeout is None, it is blocking. If timeout is 0, it doesn't block. | |
| 235 """ | |
| 236 pipes = [ | |
| 237 x for x in ((self.stderr, 'stderr'), (self.stdout, 'stdout')) if x[0] | |
| 238 ] | |
| 239 if len(pipes) == 2 and self.stderr.fileno() == self.stdout.fileno(): | |
| 240 pipes.pop(0) | |
| 241 if not pipes: | |
| 242 return None, None | |
| 243 conns, names = zip(*pipes) | |
| 244 index, data = recv_multi_impl(conns, maxsize, timeout) | |
| 245 if index is None: | |
| 246 return index, data | |
| 247 if not data: | |
| 248 self._close(names[index]) | |
| 249 return None, None | |
| 250 if self.universal_newlines: | |
| 251 data = self._translate_newlines(data) | |
| 252 return names[index], data | |
| 253 | |
| 254 def recv_out(self, maxsize=None, timeout=None): | |
| 255 """Reads from stdout asynchronously.""" | |
| 256 return self._recv('stdout', maxsize, timeout) | |
| 257 | |
| 258 def recv_err(self, maxsize=None, timeout=None): | |
| 259 """Reads from stderr asynchronously.""" | |
| 260 return self._recv('stderr', maxsize, timeout) | |
| 261 | |
| 262 def _close(self, which): | |
| 263 getattr(self, which).close() | |
| 264 setattr(self, which, None) | |
| 265 | |
| 266 def _recv(self, which, maxsize, timeout): | |
| 267 conn = getattr(self, which) | |
| 268 if conn is None: | |
| 269 return None | |
| 270 data = recv_multi_impl([conn], maxsize, timeout) | |
| 271 if not data: | |
| 272 return self._close(which) | |
| 273 if self.universal_newlines: | |
| 274 data = self._translate_newlines(data) | |
| 275 return data | |
| 276 | |
| 277 | |
| 278 def call_with_timeout(cmd, timeout, **kwargs): | |
| 279 """Runs an executable with an optional timeout. | |
| 280 | |
| 281 timeout 0 or None disables the timeout. | |
| 282 """ | |
| 283 proc = Popen( | |
| 284 cmd, | |
| 285 stdin=subprocess.PIPE, | |
| 286 stdout=subprocess.PIPE, | |
| 287 **kwargs) | |
| 288 if timeout: | |
| 289 out = '' | |
| 290 err = '' | |
| 291 for t, data in proc.yield_any(timeout): | |
| 292 if t == 'stdout': | |
| 293 out += data | |
| 294 else: | |
| 295 err += data | |
| 296 else: | |
| 297 # This code path is much faster. | |
| 298 out, err = proc.communicate() | |
| 299 return out, err, proc.returncode, proc.duration() | |
| 300 | |
| 301 | |
| 302 class QueueWithProgress(Queue.PriorityQueue): | |
| 303 """Implements progress support in join().""" | |
| 304 def __init__(self, maxsize, *args, **kwargs): | |
| 305 Queue.PriorityQueue.__init__(self, *args, **kwargs) | |
| 306 self.progress = Progress(maxsize) | |
| 307 | |
| 308 def set_progress(self, progress): | |
| 309 """Replace the current progress, mainly used when a progress should be | |
| 310 shared between queues.""" | |
| 311 self.progress = progress | |
| 312 | |
| 313 def task_done(self): | |
| 314 """Contrary to Queue.task_done(), it wakes self.all_tasks_done at each task | |
| 315 done. | |
| 316 """ | |
| 317 self.all_tasks_done.acquire() | |
| 318 try: | |
| 319 unfinished = self.unfinished_tasks - 1 | |
| 320 if unfinished < 0: | |
| 321 raise ValueError('task_done() called too many times') | |
| 322 self.unfinished_tasks = unfinished | |
| 323 # This is less efficient, because we want the Progress to be updated. | |
| 324 self.all_tasks_done.notify_all() | |
| 325 except Exception as e: | |
| 326 logging.exception('task_done threw an exception.\n%s', e) | |
| 327 finally: | |
| 328 self.all_tasks_done.release() | |
| 329 | |
| 330 def wake_up(self): | |
| 331 """Wakes up all_tasks_done. | |
| 332 | |
| 333 Unlike task_done(), do not substract one from self.unfinished_tasks. | |
| 334 """ | |
| 335 # TODO(maruel): This is highly inefficient, since the listener is awaken | |
| 336 # twice; once per output, once per task. There should be no relationship | |
| 337 # between the number of output and the number of input task. | |
| 338 self.all_tasks_done.acquire() | |
| 339 try: | |
| 340 self.all_tasks_done.notify_all() | |
| 341 finally: | |
| 342 self.all_tasks_done.release() | |
| 343 | |
| 344 def join(self): | |
| 345 """Calls print_update() whenever possible.""" | |
| 346 self.progress.print_update() | |
| 347 self.all_tasks_done.acquire() | |
| 348 try: | |
| 349 while self.unfinished_tasks: | |
| 350 self.progress.print_update() | |
| 351 self.all_tasks_done.wait(60.) | |
| 352 self.progress.print_update() | |
| 353 finally: | |
| 354 self.all_tasks_done.release() | |
| 355 | |
| 356 | |
| 357 class ThreadPool(run_isolated.ThreadPool): | |
| 358 QUEUE_CLASS = QueueWithProgress | |
| 359 | |
| 360 def __init__(self, progress, *args, **kwargs): | |
| 361 super(ThreadPool, self).__init__(*args, **kwargs) | |
| 362 self.tasks.set_progress(progress) | |
| 363 | |
| 364 def _output_append(self, out): | |
| 365 """Also wakes up the listener on new completed test_case.""" | |
| 366 super(ThreadPool, self)._output_append(out) | |
| 367 self.tasks.wake_up() | |
| 368 | |
| 369 | |
| 370 class Progress(object): | |
| 371 """Prints progress and accepts updates thread-safely.""" | |
| 372 def __init__(self, size): | |
| 373 # To be used in the primary thread | |
| 374 self.last_printed_line = '' | |
| 375 self.index = 0 | |
| 376 self.start = time.time() | |
| 377 self.size = size | |
| 378 self.use_cr_only = True | |
| 379 self.unfinished_commands = set() | |
| 380 | |
| 381 # To be used in all threads. | |
| 382 self.queued_lines = Queue.Queue() | |
| 383 | |
| 384 def update_item(self, name, index=True, size=False): | |
| 385 """Queue information to print out. | |
| 386 | |
| 387 |index| notes that the index should be incremented. | |
| 388 |size| note that the total size should be incremented. | |
| 389 """ | |
| 390 # This code doesn't need lock because it's only using self.queued_lines. | |
| 391 self.queued_lines.put((name, index, size)) | |
| 392 | |
| 393 def print_update(self): | |
| 394 """Prints the current status.""" | |
| 395 # Flush all the logging output so it doesn't appear within this output. | |
| 396 for handler in logging.root.handlers: | |
| 397 handler.flush() | |
| 398 | |
| 399 while True: | |
| 400 try: | |
| 401 name, index, size = self.queued_lines.get_nowait() | |
| 402 except Queue.Empty: | |
| 403 break | |
| 404 | |
| 405 if size: | |
| 406 self.size += 1 | |
| 407 if index: | |
| 408 self.index += 1 | |
| 409 if not name: | |
| 410 continue | |
| 411 | |
| 412 if index: | |
| 413 alignment = str(len(str(self.size))) | |
| 414 next_line = ('[%' + alignment + 'd/%d] %6.2fs %s') % ( | |
| 415 self.index, | |
| 416 self.size, | |
| 417 time.time() - self.start, | |
| 418 name) | |
| 419 # Fill it with whitespace only if self.use_cr_only is set. | |
| 420 prefix = '' | |
| 421 if self.use_cr_only: | |
| 422 if self.last_printed_line: | |
| 423 prefix = '\r' | |
| 424 if self.use_cr_only: | |
| 425 suffix = ' ' * max(0, len(self.last_printed_line) - len(next_line)) | |
| 426 else: | |
| 427 suffix = '\n' | |
| 428 line = '%s%s%s' % (prefix, next_line, suffix) | |
| 429 self.last_printed_line = next_line | |
| 430 else: | |
| 431 line = '\n%s\n' % name.strip('\n') | |
| 432 self.last_printed_line = '' | |
| 433 | |
| 434 sys.stdout.write(line) | |
| 435 | |
| 436 # Ensure that all the output is flush to prevent it from getting mixed with | |
| 437 # other output streams (like the logging streams). | |
| 438 sys.stdout.flush() | |
| 439 | |
| 440 if self.unfinished_commands: | |
| 441 logging.debug('Waiting for the following commands to finish:\n%s', | |
| 442 '\n'.join(self.unfinished_commands)) | |
| 443 | |
| 444 | |
| 445 def setup_gtest_env(): | |
| 446 """Copy the enviroment variables and setup for running a gtest.""" | |
| 447 env = os.environ.copy() | |
| 448 for name in GTEST_ENV_VARS_TO_REMOVE: | |
| 449 env.pop(name, None) | |
| 450 | |
| 451 # Forcibly enable color by default, if not already disabled. | |
| 452 env.setdefault('GTEST_COLOR', 'on') | |
| 453 | |
| 454 return env | |
| 455 | |
| 456 | |
| 457 def gtest_list_tests(cmd, cwd): | |
| 458 """List all the test cases for a google test. | |
| 459 | |
| 460 See more info at http://code.google.com/p/googletest/. | |
| 461 """ | |
| 462 cmd = cmd[:] | |
| 463 cmd.append('--gtest_list_tests') | |
| 464 env = setup_gtest_env() | |
| 465 timeout = 0. | |
| 466 try: | |
| 467 out, err, returncode, _ = call_with_timeout( | |
| 468 cmd, | |
| 469 timeout, | |
| 470 stderr=subprocess.PIPE, | |
| 471 env=env, | |
| 472 cwd=cwd) | |
| 473 except OSError, e: | |
| 474 raise Failure('Failed to run %s\ncwd=%s\n%s' % (' '.join(cmd), cwd, str(e))) | |
| 475 if returncode: | |
| 476 raise Failure( | |
| 477 'Failed to run %s\nstdout:\n%s\nstderr:\n%s' % | |
| 478 (' '.join(cmd), out, err), returncode) | |
| 479 # pylint: disable=E1103 | |
| 480 if err and not err.startswith('Xlib: extension "RANDR" missing on display '): | |
| 481 logging.error('Unexpected spew in gtest_list_tests:\n%s\n%s', err, cmd) | |
| 482 return out | |
| 483 | |
| 484 | |
| 485 def filter_shards(tests, index, shards): | |
| 486 """Filters the shards. | |
| 487 | |
| 488 Watch out about integer based arithmetics. | |
| 489 """ | |
| 490 # The following code could be made more terse but I liked the extra clarity. | |
| 491 assert 0 <= index < shards | |
| 492 total = len(tests) | |
| 493 quotient, remainder = divmod(total, shards) | |
| 494 # 1 item of each remainder is distributed over the first 0:remainder shards. | |
| 495 # For example, with total == 5, index == 1, shards == 3 | |
| 496 # min_bound == 2, max_bound == 4. | |
| 497 min_bound = quotient * index + min(index, remainder) | |
| 498 max_bound = quotient * (index + 1) + min(index + 1, remainder) | |
| 499 return tests[min_bound:max_bound] | |
| 500 | |
| 501 | |
| 502 def filter_bad_tests(tests, disabled, fails, flaky): | |
| 503 """Filters out DISABLED_, FAILS_ or FLAKY_ test cases.""" | |
| 504 def starts_with(a, b, prefix): | |
| 505 return a.startswith(prefix) or b.startswith(prefix) | |
| 506 | |
| 507 def valid(test): | |
| 508 if not '.' in test: | |
| 509 logging.error('Ignoring unknown test %s', test) | |
| 510 return False | |
| 511 fixture, case = test.split('.', 1) | |
| 512 if not disabled and starts_with(fixture, case, 'DISABLED_'): | |
| 513 return False | |
| 514 if not fails and starts_with(fixture, case, 'FAILS_'): | |
| 515 return False | |
| 516 if not flaky and starts_with(fixture, case, 'FLAKY_'): | |
| 517 return False | |
| 518 return True | |
| 519 | |
| 520 return [test for test in tests if valid(test)] | |
| 521 | |
| 522 | |
| 523 def chromium_valid(test, pre, manual): | |
| 524 """Returns True if the test case is valid to be selected.""" | |
| 525 def starts_with(a, b, prefix): | |
| 526 return a.startswith(prefix) or b.startswith(prefix) | |
| 527 | |
| 528 if not '.' in test: | |
| 529 logging.error('Ignoring unknown test %s', test) | |
| 530 return False | |
| 531 fixture, case = test.split('.', 1) | |
| 532 if not pre and starts_with(fixture, case, 'PRE_'): | |
| 533 return False | |
| 534 if not manual and starts_with(fixture, case, 'MANUAL_'): | |
| 535 return False | |
| 536 if test == 'InProcessBrowserTest.Empty': | |
| 537 return False | |
| 538 return True | |
| 539 | |
| 540 | |
| 541 def chromium_filter_bad_tests(tests, disabled, fails, flaky, pre, manual): | |
| 542 """Filters out PRE_, MANUAL_, and other weird Chromium-specific test cases.""" | |
| 543 tests = filter_bad_tests(tests, disabled, fails, flaky) | |
| 544 return [test for test in tests if chromium_valid(test, pre, manual)] | |
| 545 | |
| 546 | |
| 547 def parse_gtest_cases(out, seed): | |
| 548 """Returns the flattened list of test cases in the executable. | |
| 549 | |
| 550 The returned list is sorted so it is not dependent on the order of the linked | |
| 551 objects. Then |seed| is applied to deterministically shuffle the list if | |
| 552 |seed| is a positive value. The rationale is that the probability of two test | |
| 553 cases stomping on each other when run simultaneously is high for test cases in | |
| 554 the same fixture. By shuffling the tests, the probability of these badly | |
| 555 written tests running simultaneously, let alone being in the same shard, is | |
| 556 lower. | |
| 557 | |
| 558 Expected format is a concatenation of this: | |
| 559 TestFixture1 | |
| 560 TestCase1 | |
| 561 TestCase2 | |
| 562 """ | |
| 563 tests = [] | |
| 564 fixture = None | |
| 565 lines = out.splitlines() | |
| 566 while lines: | |
| 567 line = lines.pop(0) | |
| 568 if not line: | |
| 569 break | |
| 570 if not line.startswith(' '): | |
| 571 fixture = line | |
| 572 else: | |
| 573 case = line[2:] | |
| 574 if case.startswith('YOU HAVE'): | |
| 575 # It's a 'YOU HAVE foo bar' line. We're done. | |
| 576 break | |
| 577 assert ' ' not in case | |
| 578 tests.append(fixture + case) | |
| 579 tests = sorted(tests) | |
| 580 if seed: | |
| 581 # Sadly, python's random module doesn't permit local seeds. | |
| 582 state = random.getstate() | |
| 583 try: | |
| 584 # This is totally deterministic. | |
| 585 random.seed(seed) | |
| 586 random.shuffle(tests) | |
| 587 finally: | |
| 588 random.setstate(state) | |
| 589 return tests | |
| 590 | |
| 591 | |
| 592 def list_test_cases( | |
| 593 cmd, cwd, index, shards, disabled, fails, flaky, pre, manual, seed): | |
| 594 """Returns the list of test cases according to the specified criterias.""" | |
| 595 tests = parse_gtest_cases(gtest_list_tests(cmd, cwd), seed) | |
| 596 | |
| 597 # TODO(maruel): Splitting shards before filtering bad test cases could result | |
| 598 # in inbalanced shards. | |
| 599 if shards: | |
| 600 tests = filter_shards(tests, index, shards) | |
| 601 return chromium_filter_bad_tests(tests, disabled, fails, flaky, pre, manual) | |
| 602 | |
| 603 | |
| 604 class RunSome(object): | |
| 605 """Thread-safe object deciding if testing should continue.""" | |
| 606 def __init__( | |
| 607 self, expected_count, retries, min_failures, max_failure_ratio, | |
| 608 max_failures): | |
| 609 """Determines if it is better to give up testing after an amount of failures | |
| 610 and successes. | |
| 611 | |
| 612 Arguments: | |
| 613 - expected_count is the expected number of elements to run. | |
| 614 - retries is how many time a failing element can be retried. retries should | |
| 615 be set to the maximum number of retries per failure. This permits | |
| 616 dampening the curve to determine threshold where to stop. | |
| 617 - min_failures is the minimal number of failures to tolerate, to put a lower | |
| 618 limit when expected_count is small. This value is multiplied by the number | |
| 619 of retries. | |
| 620 - max_failure_ratio is the ratio of permitted failures, e.g. 0.1 to stop | |
| 621 after 10% of failed test cases. | |
| 622 - max_failures is the absolute maximum number of tolerated failures or None. | |
| 623 | |
| 624 For large values of expected_count, the number of tolerated failures will be | |
| 625 at maximum "(expected_count * retries) * max_failure_ratio". | |
| 626 | |
| 627 For small values of expected_count, the number of tolerated failures will be | |
| 628 at least "min_failures * retries". | |
| 629 """ | |
| 630 assert 0 < expected_count | |
| 631 assert 0 <= retries < 100 | |
| 632 assert 0 <= min_failures | |
| 633 assert 0. < max_failure_ratio < 1. | |
| 634 # Constants. | |
| 635 self._expected_count = expected_count | |
| 636 self._retries = retries | |
| 637 self._min_failures = min_failures | |
| 638 self._max_failure_ratio = max_failure_ratio | |
| 639 | |
| 640 self._min_failures_tolerated = self._min_failures * (self._retries + 1) | |
| 641 # Pre-calculate the maximum number of allowable failures. Note that | |
| 642 # _max_failures can be lower than _min_failures. | |
| 643 self._max_failures_tolerated = round( | |
| 644 (expected_count * (retries + 1)) * max_failure_ratio) | |
| 645 if max_failures is not None: | |
| 646 # Override the ratio if necessary. | |
| 647 self._max_failures_tolerated = min( | |
| 648 self._max_failures_tolerated, max_failures) | |
| 649 self._min_failures_tolerated = min( | |
| 650 self._min_failures_tolerated, max_failures) | |
| 651 | |
| 652 # Variables. | |
| 653 self._lock = threading.Lock() | |
| 654 self._passed = 0 | |
| 655 self._failures = 0 | |
| 656 self.stopped = False | |
| 657 | |
| 658 def should_stop(self): | |
| 659 """Stops once a threshold was reached. This includes retries.""" | |
| 660 with self._lock: | |
| 661 if self.stopped: | |
| 662 return True | |
| 663 # Accept at least the minimum number of failures. | |
| 664 if self._failures <= self._min_failures_tolerated: | |
| 665 return False | |
| 666 if self._failures >= self._max_failures_tolerated: | |
| 667 self.stopped = True | |
| 668 return self.stopped | |
| 669 | |
| 670 def got_result(self, passed): | |
| 671 with self._lock: | |
| 672 if passed: | |
| 673 self._passed += 1 | |
| 674 else: | |
| 675 self._failures += 1 | |
| 676 | |
| 677 def __str__(self): | |
| 678 return '%s(%d, %d, %d, %.3f)' % ( | |
| 679 self.__class__.__name__, | |
| 680 self._expected_count, | |
| 681 self._retries, | |
| 682 self._min_failures, | |
| 683 self._max_failure_ratio) | |
| 684 | |
| 685 | |
| 686 class RunAll(object): | |
| 687 """Never fails.""" | |
| 688 stopped = False | |
| 689 | |
| 690 @staticmethod | |
| 691 def should_stop(): | |
| 692 return False | |
| 693 | |
| 694 @staticmethod | |
| 695 def got_result(_): | |
| 696 pass | |
| 697 | |
| 698 | |
| 699 def process_output(lines, test_cases): | |
| 700 """Yield the data of each test cases. | |
| 701 | |
| 702 Expects the test cases to be run in the order of the list. | |
| 703 | |
| 704 Handles the following google-test behavior: | |
| 705 - Test case crash causing a partial number of test cases to be run. | |
| 706 - Invalid test case name so the test case wasn't run at all. | |
| 707 | |
| 708 This function automatically distribute the startup cost across each test case. | |
| 709 """ | |
| 710 test_cases = test_cases[:] | |
| 711 test_case = None | |
| 712 test_case_data = None | |
| 713 # Accumulates the junk between test cases. | |
| 714 accumulation = '' | |
| 715 eat_last_lines = False | |
| 716 | |
| 717 for line in lines: | |
| 718 if eat_last_lines: | |
| 719 test_case_data['output'] += line | |
| 720 continue | |
| 721 | |
| 722 i = line.find(RUN_PREFIX) | |
| 723 if i > 0 and test_case_data: | |
| 724 # This may occur specifically in browser_tests, because the test case is | |
| 725 # run in a child process. If the child process doesn't terminate its | |
| 726 # output with a LF, it may cause the "[ RUN ]" line to be improperly | |
| 727 # printed out in the middle of a line. | |
| 728 test_case_data['output'] += line[:i] | |
| 729 line = line[i:] | |
| 730 i = 0 | |
| 731 if i >= 0: | |
| 732 if test_case: | |
| 733 # The previous test case had crashed. No idea about its duration | |
| 734 test_case_data['returncode'] = 1 | |
| 735 test_case_data['duration'] = 0 | |
| 736 test_case_data['crashed'] = True | |
| 737 yield test_case_data | |
| 738 | |
| 739 test_case = line[len(RUN_PREFIX):].strip().split(' ', 1)[0] | |
| 740 # Accept the test case even if it was unexpected. | |
| 741 if test_case in test_cases: | |
| 742 test_cases.remove(test_case) | |
| 743 else: | |
| 744 logging.warning('Unexpected test case: %s', test_case) | |
| 745 test_case_data = { | |
| 746 'test_case': test_case, | |
| 747 'returncode': None, | |
| 748 'duration': None, | |
| 749 'output': accumulation + line, | |
| 750 } | |
| 751 accumulation = '' | |
| 752 | |
| 753 elif test_case: | |
| 754 test_case_data['output'] += line | |
| 755 i = line.find(OK_PREFIX) | |
| 756 if i >= 0: | |
| 757 result = 0 | |
| 758 line = line[i + len(OK_PREFIX):] | |
| 759 else: | |
| 760 i = line.find(FAILED_PREFIX) | |
| 761 if i >= 0: | |
| 762 line = line[i + len(FAILED_PREFIX):] | |
| 763 result = 1 | |
| 764 if i >= 0: | |
| 765 # The test completed. It's important to make sure the test case name | |
| 766 # match too, since it could be a fake output. | |
| 767 if line.startswith(test_case): | |
| 768 line = line[len(test_case):] | |
| 769 match = re.search(r' \((\d+) ms\)', line) | |
| 770 if match: | |
| 771 test_case_data['duration'] = float(match.group(1)) / 1000. | |
| 772 else: | |
| 773 # Make sure duration is at least not None since the test case ran. | |
| 774 test_case_data['duration'] = 0 | |
| 775 test_case_data['returncode'] = result | |
| 776 if not test_cases: | |
| 777 # Its the last test case. Eat all the remaining lines. | |
| 778 eat_last_lines = True | |
| 779 continue | |
| 780 yield test_case_data | |
| 781 test_case = None | |
| 782 test_case_data = None | |
| 783 else: | |
| 784 accumulation += line | |
| 785 | |
| 786 # It's guaranteed here that the lines generator is exhausted. | |
| 787 if eat_last_lines: | |
| 788 yield test_case_data | |
| 789 test_case = None | |
| 790 test_case_data = None | |
| 791 | |
| 792 if test_case_data: | |
| 793 # This means the last one likely crashed. | |
| 794 test_case_data['crashed'] = True | |
| 795 test_case_data['duration'] = 0 | |
| 796 test_case_data['returncode'] = 1 | |
| 797 test_case_data['output'] += accumulation | |
| 798 yield test_case_data | |
| 799 | |
| 800 # If test_cases is not empty, these test cases were not run. | |
| 801 for t in test_cases: | |
| 802 yield { | |
| 803 'test_case': t, | |
| 804 'returncode': None, | |
| 805 'duration': None, | |
| 806 'output': None, | |
| 807 } | |
| 808 | |
| 809 | |
| 810 def convert_to_lines(generator): | |
| 811 """Turn input coming from a generator into lines. | |
| 812 | |
| 813 It is Windows-friendly. | |
| 814 """ | |
| 815 accumulator = '' | |
| 816 for data in generator: | |
| 817 items = (accumulator + data).splitlines(True) | |
| 818 for item in items[:-1]: | |
| 819 yield item | |
| 820 if items[-1].endswith(('\r', '\n')): | |
| 821 yield items[-1] | |
| 822 accumulator = '' | |
| 823 else: | |
| 824 accumulator = items[-1] | |
| 825 if accumulator: | |
| 826 yield accumulator | |
| 827 | |
| 828 | |
| 829 def chromium_filter_tests(data): | |
| 830 """Returns a generator that removes funky PRE_ chromium-specific tests.""" | |
| 831 return (d for d in data if chromium_valid(d['test_case'], False, True)) | |
| 832 | |
| 833 | |
| 834 class ResetableTimeout(object): | |
| 835 """A resetable timeout that acts as a float. | |
| 836 | |
| 837 At each reset, the timeout is increased so that it still has the equivalent | |
| 838 of the original timeout value, but according to 'now' at the time of the | |
| 839 reset. | |
| 840 """ | |
| 841 def __init__(self, timeout): | |
| 842 assert timeout >= 0. | |
| 843 self.timeout = float(timeout) | |
| 844 self.last_reset = time.time() | |
| 845 | |
| 846 def reset(self): | |
| 847 """Respendish the timeout.""" | |
| 848 now = time.time() | |
| 849 self.timeout += max(0., now - self.last_reset) | |
| 850 self.last_reset = now | |
| 851 return now | |
| 852 | |
| 853 @staticmethod | |
| 854 def __bool__(): | |
| 855 return True | |
| 856 | |
| 857 def __float__(self): | |
| 858 """To be used as a timeout value for a function call.""" | |
| 859 return self.timeout | |
| 860 | |
| 861 | |
| 862 class Runner(object): | |
| 863 """Immutable settings to run many test cases in a loop.""" | |
| 864 def __init__( | |
| 865 self, cmd, cwd_dir, timeout, progress, retries, decider, verbose, | |
| 866 add_task, add_serial_task): | |
| 867 self.cmd = cmd[:] | |
| 868 self.cwd_dir = cwd_dir | |
| 869 self.timeout = timeout | |
| 870 self.progress = progress | |
| 871 # The number of retries. For example if 2, the test case will be tried 3 | |
| 872 # times in total. | |
| 873 self.retries = retries | |
| 874 self.decider = decider | |
| 875 self.verbose = verbose | |
| 876 self.add_task = add_task | |
| 877 self.add_serial_task = add_serial_task | |
| 878 # It is important to remove the shard environment variables since it could | |
| 879 # conflict with --gtest_filter. | |
| 880 self.env = setup_gtest_env() | |
| 881 | |
| 882 def map(self, priority, test_cases, try_count): | |
| 883 """Traces a single test case and returns its output. | |
| 884 | |
| 885 try_count is 0 based, the original try is 0. | |
| 886 """ | |
| 887 if self.decider.should_stop(): | |
| 888 raise StopIteration() | |
| 889 cmd = self.cmd + ['--gtest_filter=%s' % ':'.join(test_cases)] | |
| 890 if '--gtest_print_time' not in cmd: | |
| 891 cmd.append('--gtest_print_time') | |
| 892 proc = Popen( | |
| 893 cmd, | |
| 894 cwd=self.cwd_dir, | |
| 895 stdout=subprocess.PIPE, | |
| 896 stderr=subprocess.STDOUT, | |
| 897 env=self.env) | |
| 898 | |
| 899 # Use an intelligent timeout that can be reset. The idea is simple, the | |
| 900 # timeout is set to the value of the timeout for a single test case. | |
| 901 # Everytime a test case is parsed, the timeout is reset to its full value. | |
| 902 # proc.yield_any() uses float() to extract the instantaneous value of | |
| 903 # 'timeout'. | |
| 904 timeout = ResetableTimeout(self.timeout) | |
| 905 | |
| 906 # Create a pipeline of generators. | |
| 907 gen_lines = convert_to_lines(data for _, data in proc.yield_any(timeout)) | |
| 908 # It needs to be valid utf-8 otherwise it can't be stored. | |
| 909 # TODO(maruel): Be more intelligent than decoding to ascii. | |
| 910 gen_lines_utf8 = ( | |
| 911 line.decode('ascii', 'ignore').encode('utf-8') for line in gen_lines) | |
| 912 gen_test_cases = process_output(gen_lines_utf8, test_cases) | |
| 913 gen_test_cases_filtered = chromium_filter_tests(gen_test_cases) | |
| 914 | |
| 915 last_timestamp = proc.start | |
| 916 got_failure_at_least_once = False | |
| 917 results = [] | |
| 918 for i in gen_test_cases_filtered: | |
| 919 results.append(i) | |
| 920 now = timeout.reset() | |
| 921 test_case_has_passed = (i['returncode'] == 0) | |
| 922 if i['duration'] is None: | |
| 923 assert not test_case_has_passed | |
| 924 # Do not notify self.decider, because an early crash in a large cluster | |
| 925 # could cause the test to quit early. | |
| 926 else: | |
| 927 i['duration'] = max(i['duration'], now - last_timestamp) | |
| 928 # A new test_case completed. | |
| 929 self.decider.got_result(test_case_has_passed) | |
| 930 | |
| 931 need_to_retry = not test_case_has_passed and try_count < self.retries | |
| 932 got_failure_at_least_once |= not test_case_has_passed | |
| 933 last_timestamp = now | |
| 934 | |
| 935 # Create the line to print out. | |
| 936 if i['duration'] is not None: | |
| 937 duration = '(%.2fs)' % i['duration'] | |
| 938 else: | |
| 939 duration = '<unknown>' | |
| 940 if try_count: | |
| 941 line = '%s %s - retry #%d' % (i['test_case'], duration, try_count) | |
| 942 else: | |
| 943 line = '%s %s' % (i['test_case'], duration) | |
| 944 if self.verbose or not test_case_has_passed or try_count > 0: | |
| 945 # Print output in one of three cases: | |
| 946 # - --verbose was specified. | |
| 947 # - The test failed. | |
| 948 # - The wasn't the first attempt (this is needed so the test parser can | |
| 949 # detect that a test has been successfully retried). | |
| 950 if i['output']: | |
| 951 line += '\n' + i['output'] | |
| 952 self.progress.update_item(line, True, need_to_retry) | |
| 953 | |
| 954 if need_to_retry: | |
| 955 priority = self._retry(priority, i['test_case'], try_count) | |
| 956 | |
| 957 # Delay yielding when only one test case is running, in case of a | |
| 958 # crash-after-succeed. | |
| 959 if len(test_cases) > 1: | |
| 960 yield i | |
| 961 | |
| 962 if proc.returncode and not got_failure_at_least_once: | |
| 963 if len(test_cases) == 1: | |
| 964 # Crash after pass. | |
| 965 results[-1]['returncode'] = proc.returncode | |
| 966 | |
| 967 if try_count < self.retries: | |
| 968 # This is tricky, one of the test case failed but each did print that | |
| 969 # they succeeded! Retry them *all* individually. | |
| 970 if not self.verbose and not try_count: | |
| 971 # Print all the output as one shot when not verbose to be sure the | |
| 972 # potential stack trace is printed. | |
| 973 output = ''.join(i['output'] for i in results) | |
| 974 self.progress.update_item(output, False, False) | |
| 975 for i in results: | |
| 976 priority = self._retry(priority, i['test_case'], try_count) | |
| 977 self.progress.update_item('', False, True) | |
| 978 | |
| 979 # Only yield once the process completed when there is only one test case as | |
| 980 # a safety precaution. | |
| 981 if len(test_cases) == 1: | |
| 982 yield i | |
| 983 | |
| 984 def _retry(self, priority, test_case, try_count): | |
| 985 if try_count + 1 < self.retries: | |
| 986 # The test failed and needs to be retried normally. | |
| 987 # Leave a buffer of ~40 test cases before retrying. | |
| 988 priority += 40 | |
| 989 self.add_task(priority, self.map, priority, [test_case], try_count + 1) | |
| 990 else: | |
| 991 # This test only has one retry left, so the final retry should be | |
| 992 # done serially. | |
| 993 self.add_serial_task( | |
| 994 priority, self.map, priority, [test_case], try_count + 1) | |
| 995 return priority | |
| 996 | |
| 997 | |
| 998 def get_test_cases( | |
| 999 cmd, cwd, whitelist, blacklist, index, shards, seed, disabled, fails, flaky, | |
| 1000 manual): | |
| 1001 """Returns the filtered list of test cases. | |
| 1002 | |
| 1003 This is done synchronously. | |
| 1004 """ | |
| 1005 try: | |
| 1006 # List all the test cases if a whitelist is used. | |
| 1007 tests = list_test_cases( | |
| 1008 cmd, | |
| 1009 cwd, | |
| 1010 index=index, | |
| 1011 shards=shards, | |
| 1012 disabled=disabled, | |
| 1013 fails=fails, | |
| 1014 flaky=flaky, | |
| 1015 pre=False, | |
| 1016 manual=manual, | |
| 1017 seed=seed) | |
| 1018 except Failure, e: | |
| 1019 print('Failed to list test cases') | |
| 1020 print(e.args[0]) | |
| 1021 return None | |
| 1022 | |
| 1023 if shards: | |
| 1024 # This is necessary for Swarm log parsing. | |
| 1025 print('Note: This is test shard %d of %d.' % (index+1, shards)) | |
| 1026 | |
| 1027 # Filters the test cases with the two lists. | |
| 1028 if blacklist: | |
| 1029 tests = [ | |
| 1030 t for t in tests if not any(fnmatch.fnmatch(t, s) for s in blacklist) | |
| 1031 ] | |
| 1032 if whitelist: | |
| 1033 tests = [ | |
| 1034 t for t in tests if any(fnmatch.fnmatch(t, s) for s in whitelist) | |
| 1035 ] | |
| 1036 logging.info('Found %d test cases in %s' % (len(tests), ' '.join(cmd))) | |
| 1037 return tests | |
| 1038 | |
| 1039 | |
| 1040 def dump_results_as_json(result_file, results): | |
| 1041 """Write the results out to a json file.""" | |
| 1042 base_path = os.path.dirname(result_file) | |
| 1043 if base_path and not os.path.isdir(base_path): | |
| 1044 os.makedirs(base_path) | |
| 1045 with open(result_file, 'wb') as f: | |
| 1046 json.dump(results, f, sort_keys=True, indent=2) | |
| 1047 | |
| 1048 | |
| 1049 def dump_results_as_xml(gtest_output, results, now): | |
| 1050 """Write the results out to a xml file in google-test compatible format.""" | |
| 1051 # TODO(maruel): Print all the test cases, including the ones that weren't run | |
| 1052 # and the retries. | |
| 1053 test_suites = {} | |
| 1054 for test_case, result in results['test_cases'].iteritems(): | |
| 1055 suite, case = test_case.split('.', 1) | |
| 1056 test_suites.setdefault(suite, {})[case] = result[0] | |
| 1057 | |
| 1058 with open(gtest_output, 'wb') as f: | |
| 1059 # Sanity warning: hand-rolling XML. What could possibly go wrong? | |
| 1060 f.write('<?xml version="1.0" ?>\n') | |
| 1061 # TODO(maruel): File the fields nobody reads anyway. | |
| 1062 # disabled="%d" errors="%d" failures="%d" | |
| 1063 f.write( | |
| 1064 ('<testsuites name="AllTests" tests="%d" time="%f" timestamp="%s">\n') | |
| 1065 % (results['expected'], results['duration'], now)) | |
| 1066 for suite_name, suite in test_suites.iteritems(): | |
| 1067 # TODO(maruel): disabled="0" errors="0" failures="0" time="0" | |
| 1068 f.write('<testsuite name="%s" tests="%d">\n' % (suite_name, len(suite))) | |
| 1069 for case_name, case in suite.iteritems(): | |
| 1070 if case['returncode'] == 0: | |
| 1071 f.write( | |
| 1072 ' <testcase classname="%s" name="%s" status="run" time="%f"/>\n' % | |
| 1073 (suite_name, case_name, case['duration'])) | |
| 1074 else: | |
| 1075 f.write( | |
| 1076 ' <testcase classname="%s" name="%s" status="run" time="%f">\n' % | |
| 1077 (suite_name, case_name, (case['duration'] or 0))) | |
| 1078 # While at it, hand-roll CDATA escaping too. | |
| 1079 output = ']]><![CDATA['.join((case['output'] or '').split(']]>')) | |
| 1080 # TODO(maruel): message="" type="" | |
| 1081 f.write('<failure><![CDATA[%s]]></failure></testcase>\n' % output) | |
| 1082 f.write('</testsuite>\n') | |
| 1083 f.write('</testsuites>') | |
| 1084 | |
| 1085 | |
| 1086 def append_gtest_output_to_xml(final_xml, filepath): | |
| 1087 """Combines the shard xml file with the final xml file.""" | |
| 1088 try: | |
| 1089 with open(filepath) as shard_xml_file: | |
| 1090 shard_xml = minidom.parse(shard_xml_file) | |
| 1091 except xml.parsers.expat.ExpatError as e: | |
| 1092 logging.error('Failed to parse %s: %s', filepath, e) | |
| 1093 return final_xml | |
| 1094 except IOError as e: | |
| 1095 logging.error('Failed to load %s: %s', filepath, e) | |
| 1096 # If the shard crashed, gtest will not have generated an xml file. | |
| 1097 return final_xml | |
| 1098 | |
| 1099 if not final_xml: | |
| 1100 # Out final xml is empty, let's prepopulate it with the first one we see. | |
| 1101 return shard_xml | |
| 1102 | |
| 1103 final_testsuites_by_name = dict( | |
| 1104 (suite.getAttribute('name'), suite) | |
| 1105 for suite in final_xml.documentElement.getElementsByTagName('testsuite')) | |
| 1106 | |
| 1107 for testcase in shard_xml.documentElement.getElementsByTagName('testcase'): | |
| 1108 # Don't bother updating the final xml if there is no data. | |
| 1109 status = testcase.getAttribute('status') | |
| 1110 if status == 'notrun': | |
| 1111 continue | |
| 1112 | |
| 1113 name = testcase.getAttribute('name') | |
| 1114 # Look in our final xml to see if it's there. | |
| 1115 to_remove = [] | |
| 1116 final_testsuite = final_testsuites_by_name[ | |
| 1117 testcase.getAttribute('classname')] | |
| 1118 for final_testcase in final_testsuite.getElementsByTagName('testcase'): | |
| 1119 # Trim all the notrun testcase instances to add the new instance there. | |
| 1120 # This is to make sure it works properly in case of a testcase being run | |
| 1121 # multiple times. | |
| 1122 if (final_testcase.getAttribute('name') == name and | |
| 1123 final_testcase.getAttribute('status') == 'notrun'): | |
| 1124 to_remove.append(final_testcase) | |
| 1125 | |
| 1126 for item in to_remove: | |
| 1127 final_testsuite.removeChild(item) | |
| 1128 # Reparent the XML node. | |
| 1129 final_testsuite.appendChild(testcase) | |
| 1130 | |
| 1131 return final_xml | |
| 1132 | |
| 1133 | |
| 1134 def running_serial_warning(): | |
| 1135 return ['*****************************************************', | |
| 1136 '*****************************************************', | |
| 1137 '*****************************************************', | |
| 1138 'WARNING: The remaining tests are going to be retried', | |
| 1139 'serially. All tests should be isolated and be able to pass', | |
| 1140 'regardless of what else is running.', | |
| 1141 'If you see a test that can only pass serially, that test is', | |
| 1142 'probably broken and should be fixed.', | |
| 1143 '*****************************************************', | |
| 1144 '*****************************************************', | |
| 1145 '*****************************************************'] | |
| 1146 | |
| 1147 | |
| 1148 def gen_gtest_output_dir(cwd, gtest_output): | |
| 1149 """Converts gtest_output to an actual path that can be used in parallel. | |
| 1150 | |
| 1151 Returns a 'corrected' gtest_output value. | |
| 1152 """ | |
| 1153 if not gtest_output.startswith('xml'): | |
| 1154 raise Failure('Can\'t parse --gtest_output=%s' % gtest_output) | |
| 1155 # Figure out the result filepath in case we can't parse it, it'd be | |
| 1156 # annoying to error out *after* running the tests. | |
| 1157 if gtest_output == 'xml': | |
| 1158 gtest_output = os.path.join(cwd, 'test_detail.xml') | |
| 1159 else: | |
| 1160 match = re.match(r'xml\:(.+)', gtest_output) | |
| 1161 if not match: | |
| 1162 raise Failure('Can\'t parse --gtest_output=%s' % gtest_output) | |
| 1163 # If match.group(1) is an absolute path, os.path.join() will do the right | |
| 1164 # thing. | |
| 1165 if match.group(1).endswith((os.path.sep, '/')): | |
| 1166 gtest_output = os.path.join(cwd, match.group(1), 'test_detail.xml') | |
| 1167 else: | |
| 1168 gtest_output = os.path.join(cwd, match.group(1)) | |
| 1169 | |
| 1170 base_path = os.path.dirname(gtest_output) | |
| 1171 if base_path and not os.path.isdir(base_path): | |
| 1172 os.makedirs(base_path) | |
| 1173 | |
| 1174 # Emulate google-test' automatic increasing index number. | |
| 1175 while True: | |
| 1176 try: | |
| 1177 # Creates a file exclusively. | |
| 1178 os.close(os.open(gtest_output, os.O_CREAT|os.O_EXCL|os.O_RDWR, 0666)) | |
| 1179 # It worked, we are done. | |
| 1180 return gtest_output | |
| 1181 except OSError: | |
| 1182 pass | |
| 1183 logging.debug('%s existed', gtest_output) | |
| 1184 base, ext = os.path.splitext(gtest_output) | |
| 1185 match = re.match(r'^(.+?_)(\d+)$', base) | |
| 1186 if match: | |
| 1187 base = match.group(1) + str(int(match.group(2)) + 1) | |
| 1188 else: | |
| 1189 base = base + '_0' | |
| 1190 gtest_output = base + ext | |
| 1191 | |
| 1192 | |
| 1193 def calc_cluster_default(num_test_cases, jobs): | |
| 1194 """Calculates a desired number for clusters depending on the number of test | |
| 1195 cases and parallel jobs. | |
| 1196 """ | |
| 1197 if not num_test_cases: | |
| 1198 return 0 | |
| 1199 chunks = 6 * jobs | |
| 1200 if chunks >= num_test_cases: | |
| 1201 # Too many chunks, use 1~5 test case per thread. Not enough to start | |
| 1202 # chunking. | |
| 1203 value = num_test_cases / jobs | |
| 1204 else: | |
| 1205 # Use chunks that are spread across threads. | |
| 1206 value = (num_test_cases + chunks - 1) / chunks | |
| 1207 # Limit to 10 test cases per cluster. | |
| 1208 return min(10, max(1, value)) | |
| 1209 | |
| 1210 | |
| 1211 def run_test_cases( | |
| 1212 cmd, cwd, test_cases, jobs, timeout, clusters, retries, run_all, | |
| 1213 max_failures, no_cr, gtest_output, result_file, verbose): | |
| 1214 """Runs test cases in parallel. | |
| 1215 | |
| 1216 Arguments: | |
| 1217 - cmd: command to run. | |
| 1218 - cwd: working directory. | |
| 1219 - test_cases: list of preprocessed test cases to run. | |
| 1220 - jobs: number of parallel execution threads to do. | |
| 1221 - timeout: individual test case timeout. Modulated when used with | |
| 1222 clustering. | |
| 1223 - clusters: number of test cases to lump together in a single execution. 0 | |
| 1224 means the default automatic value which depends on len(test_cases) and | |
| 1225 jobs. Capped to len(test_cases) / jobs. | |
| 1226 - retries: number of times a test case can be retried. | |
| 1227 - run_all: If true, do not early return even if all test cases fail. | |
| 1228 - max_failures is the absolute maximum number of tolerated failures or None. | |
| 1229 - no_cr: makes output friendly to piped logs. | |
| 1230 - gtest_output: saves results as xml. | |
| 1231 - result_file: saves results as json. | |
| 1232 - verbose: print more details. | |
| 1233 | |
| 1234 It may run a subset of the test cases if too many test cases failed, as | |
| 1235 determined with max_failures, retries and run_all. | |
| 1236 """ | |
| 1237 assert 0 <= retries <= 100000 | |
| 1238 if not test_cases: | |
| 1239 return 0 | |
| 1240 if run_all: | |
| 1241 decider = RunAll() | |
| 1242 else: | |
| 1243 # If 10% of test cases fail, just too bad. | |
| 1244 decider = RunSome(len(test_cases), retries, 2, 0.1, max_failures) | |
| 1245 | |
| 1246 if not clusters: | |
| 1247 clusters = calc_cluster_default(len(test_cases), jobs) | |
| 1248 else: | |
| 1249 # Limit the value. | |
| 1250 clusters = min(clusters, len(test_cases) / jobs) | |
| 1251 | |
| 1252 logging.debug('%d test cases with clusters of %d', len(test_cases), clusters) | |
| 1253 | |
| 1254 if gtest_output: | |
| 1255 gtest_output = gen_gtest_output_dir(cwd, gtest_output) | |
| 1256 progress = Progress(len(test_cases)) | |
| 1257 serial_tasks = QueueWithProgress(0) | |
| 1258 serial_tasks.set_progress(progress) | |
| 1259 | |
| 1260 def add_serial_task(priority, func, *args, **kwargs): | |
| 1261 """Adds a serial task, to be executed later.""" | |
| 1262 assert isinstance(priority, int) | |
| 1263 assert callable(func) | |
| 1264 serial_tasks.put((priority, func, args, kwargs)) | |
| 1265 | |
| 1266 with ThreadPool(progress, jobs, jobs, len(test_cases)) as pool: | |
| 1267 runner = Runner( | |
| 1268 cmd, cwd, timeout, progress, retries, decider, verbose, | |
| 1269 pool.add_task, add_serial_task) | |
| 1270 function = runner.map | |
| 1271 progress.use_cr_only = not no_cr | |
| 1272 # Cluster the test cases right away. | |
| 1273 for i in xrange((len(test_cases) + clusters - 1) / clusters): | |
| 1274 cluster = test_cases[i*clusters : (i+1)*clusters] | |
| 1275 pool.add_task(i, function, i, cluster, 0) | |
| 1276 results = pool.join() | |
| 1277 | |
| 1278 # Retry any failed tests serially. | |
| 1279 if not serial_tasks.empty(): | |
| 1280 progress.update_item('\n'.join(running_serial_warning()), index=False, | |
| 1281 size=False) | |
| 1282 progress.print_update() | |
| 1283 | |
| 1284 while not serial_tasks.empty(): | |
| 1285 _priority, func, args, kwargs = serial_tasks.get() | |
| 1286 for out in func(*args, **kwargs): | |
| 1287 results.append(out) | |
| 1288 serial_tasks.task_done() | |
| 1289 progress.print_update() | |
| 1290 | |
| 1291 # Call join since that is a standard call once a queue has been emptied. | |
| 1292 serial_tasks.join() | |
| 1293 | |
| 1294 duration = time.time() - pool.tasks.progress.start | |
| 1295 | |
| 1296 cleaned = {} | |
| 1297 for i in results: | |
| 1298 cleaned.setdefault(i['test_case'], []).append(i) | |
| 1299 results = cleaned | |
| 1300 | |
| 1301 # Total time taken to run each test case. | |
| 1302 test_case_duration = dict( | |
| 1303 (test_case, sum((i.get('duration') or 0) for i in item)) | |
| 1304 for test_case, item in results.iteritems()) | |
| 1305 | |
| 1306 # Classify the results | |
| 1307 success = [] | |
| 1308 flaky = [] | |
| 1309 fail = [] | |
| 1310 nb_runs = 0 | |
| 1311 for test_case in sorted(results): | |
| 1312 items = results[test_case] | |
| 1313 nb_runs += len(items) | |
| 1314 if not any(i['returncode'] == 0 for i in items): | |
| 1315 fail.append(test_case) | |
| 1316 elif len(items) > 1 and any(i['returncode'] == 0 for i in items): | |
| 1317 flaky.append(test_case) | |
| 1318 elif len(items) == 1 and items[0]['returncode'] == 0: | |
| 1319 success.append(test_case) | |
| 1320 else: | |
| 1321 # The test never ran. | |
| 1322 assert False, items | |
| 1323 missing = sorted(set(test_cases) - set(success) - set(flaky) - set(fail)) | |
| 1324 | |
| 1325 saved = { | |
| 1326 'test_cases': results, | |
| 1327 'expected': len(test_cases), | |
| 1328 'success': success, | |
| 1329 'flaky': flaky, | |
| 1330 'fail': fail, | |
| 1331 'missing': missing, | |
| 1332 'duration': duration, | |
| 1333 } | |
| 1334 if result_file: | |
| 1335 dump_results_as_json(result_file, saved) | |
| 1336 if gtest_output: | |
| 1337 dump_results_as_xml(gtest_output, saved, datetime.datetime.now()) | |
| 1338 sys.stdout.write('\n') | |
| 1339 if not results: | |
| 1340 return 1 | |
| 1341 | |
| 1342 if flaky: | |
| 1343 print('Flaky tests:') | |
| 1344 for test_case in sorted(flaky): | |
| 1345 items = results[test_case] | |
| 1346 print(' %s (tried %d times)' % (test_case, len(items))) | |
| 1347 | |
| 1348 if fail: | |
| 1349 print('Failed tests:') | |
| 1350 for test_case in sorted(fail): | |
| 1351 print(' %s' % test_case) | |
| 1352 | |
| 1353 if not decider.should_stop() and missing: | |
| 1354 print('Missing tests:') | |
| 1355 for test_case in sorted(missing): | |
| 1356 print(' %s' % test_case) | |
| 1357 | |
| 1358 print('Summary:') | |
| 1359 if decider.should_stop(): | |
| 1360 print(' ** STOPPED EARLY due to high failure rate **') | |
| 1361 output = [ | |
| 1362 ('Success', success), | |
| 1363 ('Flaky', flaky), | |
| 1364 ('Fail', fail), | |
| 1365 ] | |
| 1366 if missing: | |
| 1367 output.append(('Missing', missing)) | |
| 1368 total_expected = len(test_cases) | |
| 1369 for name, items in output: | |
| 1370 number = len(items) | |
| 1371 print( | |
| 1372 ' %7s: %4d %6.2f%% %7.2fs' % ( | |
| 1373 name, | |
| 1374 number, | |
| 1375 number * 100. / total_expected, | |
| 1376 sum(test_case_duration.get(item, 0) for item in items))) | |
| 1377 print(' %.2fs Done running %d tests with %d executions. %.2f test/s' % ( | |
| 1378 duration, | |
| 1379 len(results), | |
| 1380 nb_runs, | |
| 1381 nb_runs / duration if duration else 0)) | |
| 1382 return int(bool(fail) or decider.stopped or bool(missing)) | |
| 1383 | |
| 1384 | |
| 1385 class OptionParserWithLogging(run_isolated.OptionParserWithLogging): | |
| 1386 def __init__(self, **kwargs): | |
| 1387 run_isolated.OptionParserWithLogging.__init__( | |
| 1388 self, | |
| 1389 log_file=os.environ.get('RUN_TEST_CASES_LOG_FILE', ''), | |
| 1390 **kwargs) | |
| 1391 | |
| 1392 | |
| 1393 class OptionParserWithTestSharding(OptionParserWithLogging): | |
| 1394 """Adds automatic handling of test sharding""" | |
| 1395 def __init__(self, **kwargs): | |
| 1396 OptionParserWithLogging.__init__(self, **kwargs) | |
| 1397 | |
| 1398 def as_digit(variable, default): | |
| 1399 return int(variable) if variable.isdigit() else default | |
| 1400 | |
| 1401 group = optparse.OptionGroup(self, 'Which shard to select') | |
| 1402 group.add_option( | |
| 1403 '-I', '--index', | |
| 1404 type='int', | |
| 1405 default=as_digit(os.environ.get('GTEST_SHARD_INDEX', ''), None), | |
| 1406 help='Shard index to select') | |
| 1407 group.add_option( | |
| 1408 '-S', '--shards', | |
| 1409 type='int', | |
| 1410 default=as_digit(os.environ.get('GTEST_TOTAL_SHARDS', ''), None), | |
| 1411 help='Total number of shards to calculate from the --index to select') | |
| 1412 self.add_option_group(group) | |
| 1413 | |
| 1414 def parse_args(self, *args, **kwargs): | |
| 1415 options, args = OptionParserWithLogging.parse_args(self, *args, **kwargs) | |
| 1416 if bool(options.shards) != bool(options.index is not None): | |
| 1417 self.error('Use both --index X --shards Y or none of them') | |
| 1418 return options, args | |
| 1419 | |
| 1420 | |
| 1421 class OptionParserWithTestShardingAndFiltering(OptionParserWithTestSharding): | |
| 1422 """Adds automatic handling of test sharding and filtering.""" | |
| 1423 def __init__(self, *args, **kwargs): | |
| 1424 OptionParserWithTestSharding.__init__(self, *args, **kwargs) | |
| 1425 | |
| 1426 group = optparse.OptionGroup(self, 'Which test cases to select') | |
| 1427 group.add_option( | |
| 1428 '-w', '--whitelist', | |
| 1429 default=[], | |
| 1430 action='append', | |
| 1431 help='filter to apply to test cases to run, wildcard-style, defaults ' | |
| 1432 'to all test') | |
| 1433 group.add_option( | |
| 1434 '-b', '--blacklist', | |
| 1435 default=[], | |
| 1436 action='append', | |
| 1437 help='filter to apply to test cases to skip, wildcard-style, defaults ' | |
| 1438 'to no test') | |
| 1439 group.add_option( | |
| 1440 '-T', '--test-case-file', | |
| 1441 help='File containing the exact list of test cases to run') | |
| 1442 group.add_option( | |
| 1443 '--gtest_filter', | |
| 1444 default=os.environ.get('GTEST_FILTER', ''), | |
| 1445 help='Select test cases like google-test does, separated with ":"') | |
| 1446 group.add_option( | |
| 1447 '--seed', | |
| 1448 type='int', | |
| 1449 default=os.environ.get('GTEST_RANDOM_SEED', '1'), | |
| 1450 help='Deterministically shuffle the test list if non-0. default: ' | |
| 1451 '%default') | |
| 1452 group.add_option( | |
| 1453 '-d', '--disabled', | |
| 1454 action='store_true', | |
| 1455 default=int(os.environ.get('GTEST_ALSO_RUN_DISABLED_TESTS', '0')), | |
| 1456 help='Include DISABLED_ tests') | |
| 1457 group.add_option( | |
| 1458 '--gtest_also_run_disabled_tests', | |
| 1459 action='store_true', | |
| 1460 dest='disabled', | |
| 1461 help='same as --disabled') | |
| 1462 self.add_option_group(group) | |
| 1463 | |
| 1464 group = optparse.OptionGroup( | |
| 1465 self, 'Which test cases to select; chromium-specific') | |
| 1466 group.add_option( | |
| 1467 '-f', '--fails', | |
| 1468 action='store_true', | |
| 1469 help='Include FAILS_ tests') | |
| 1470 group.add_option( | |
| 1471 '-F', '--flaky', | |
| 1472 action='store_true', | |
| 1473 help='Include FLAKY_ tests') | |
| 1474 group.add_option( | |
| 1475 '-m', '--manual', | |
| 1476 action='store_true', | |
| 1477 help='Include MANUAL_ tests') | |
| 1478 group.add_option( | |
| 1479 '--run-manual', | |
| 1480 action='store_true', | |
| 1481 dest='manual', | |
| 1482 help='same as --manual') | |
| 1483 self.add_option_group(group) | |
| 1484 | |
| 1485 def parse_args(self, *args, **kwargs): | |
| 1486 options, args = OptionParserWithTestSharding.parse_args( | |
| 1487 self, *args, **kwargs) | |
| 1488 | |
| 1489 if options.gtest_filter: | |
| 1490 # Override any other option. | |
| 1491 # Based on UnitTestOptions::FilterMatchesTest() in | |
| 1492 # http://code.google.com/p/googletest/source/browse/#svn%2Ftrunk%2Fsrc | |
| 1493 if '-' in options.gtest_filter: | |
| 1494 options.whitelist, options.blacklist = options.gtest_filter.split('-', | |
| 1495 1) | |
| 1496 else: | |
| 1497 options.whitelist = options.gtest_filter | |
| 1498 options.blacklist = '' | |
| 1499 options.whitelist = [i for i in options.whitelist.split(':') if i] | |
| 1500 options.blacklist = [i for i in options.blacklist.split(':') if i] | |
| 1501 | |
| 1502 return options, args | |
| 1503 | |
| 1504 @staticmethod | |
| 1505 def process_gtest_options(cmd, cwd, options): | |
| 1506 """Grabs the test cases.""" | |
| 1507 if options.test_case_file: | |
| 1508 with open(options.test_case_file, 'r') as f: | |
| 1509 # Do not shuffle or alter the file in any way in that case except to | |
| 1510 # strip whitespaces. | |
| 1511 return [l for l in (l.strip() for l in f) if l] | |
| 1512 else: | |
| 1513 return get_test_cases( | |
| 1514 cmd, | |
| 1515 cwd, | |
| 1516 options.whitelist, | |
| 1517 options.blacklist, | |
| 1518 options.index, | |
| 1519 options.shards, | |
| 1520 options.seed, | |
| 1521 options.disabled, | |
| 1522 options.fails, | |
| 1523 options.flaky, | |
| 1524 options.manual) | |
| 1525 | |
| 1526 | |
| 1527 class OptionParserTestCases(OptionParserWithTestShardingAndFiltering): | |
| 1528 def __init__(self, *args, **kwargs): | |
| 1529 OptionParserWithTestShardingAndFiltering.__init__(self, *args, **kwargs) | |
| 1530 self.add_option( | |
| 1531 '-j', '--jobs', | |
| 1532 type='int', | |
| 1533 default=run_isolated.num_processors(), | |
| 1534 help='Number of parallel jobs; default=%default') | |
| 1535 self.add_option( | |
| 1536 '--use-less-jobs', | |
| 1537 action='store_const', | |
| 1538 const=run_isolated.num_processors() - 1, | |
| 1539 dest='jobs', | |
| 1540 help='Starts less parallel jobs than the default, used to help reduce' | |
| 1541 'contention between threads if all the tests are very CPU heavy.') | |
| 1542 self.add_option( | |
| 1543 '-t', '--timeout', | |
| 1544 type='int', | |
| 1545 default=75, | |
| 1546 help='Timeout for a single test case, in seconds default:%default') | |
| 1547 self.add_option( | |
| 1548 '--clusters', | |
| 1549 type='int', | |
| 1550 help='Number of test cases to cluster together, clamped to ' | |
| 1551 'len(test_cases) / jobs; the default is automatic') | |
| 1552 | |
| 1553 | |
| 1554 def process_args(argv): | |
| 1555 parser = OptionParserTestCases( | |
| 1556 usage='%prog <options> [gtest]', | |
| 1557 verbose=int(os.environ.get('ISOLATE_DEBUG', 0))) | |
| 1558 parser.add_option( | |
| 1559 '--run-all', | |
| 1560 action='store_true', | |
| 1561 help='Do not fail early when a large number of test cases fail') | |
| 1562 parser.add_option( | |
| 1563 '--max-failures', type='int', | |
| 1564 help='Limit the number of failures before aborting') | |
| 1565 parser.add_option( | |
| 1566 '--retries', type='int', default=2, | |
| 1567 help='Number of times each test case should be retried in case of ' | |
| 1568 'failure.') | |
| 1569 parser.add_option( | |
| 1570 '--no-dump', | |
| 1571 action='store_true', | |
| 1572 help='do not generate a .run_test_cases file') | |
| 1573 parser.add_option( | |
| 1574 '--no-cr', | |
| 1575 action='store_true', | |
| 1576 help='Use LF instead of CR for status progress') | |
| 1577 parser.add_option( | |
| 1578 '--result', | |
| 1579 help='Override the default name of the generated .run_test_cases file') | |
| 1580 | |
| 1581 group = optparse.OptionGroup(parser, 'google-test compability flags') | |
| 1582 group.add_option( | |
| 1583 '--gtest_list_tests', | |
| 1584 action='store_true', | |
| 1585 help='List all the test cases unformatted. Keeps compatibility with the ' | |
| 1586 'executable itself.') | |
| 1587 group.add_option( | |
| 1588 '--gtest_output', | |
| 1589 default=os.environ.get('GTEST_OUTPUT', ''), | |
| 1590 help='XML output to generate') | |
| 1591 parser.add_option_group(group) | |
| 1592 | |
| 1593 options, args = parser.parse_args(argv) | |
| 1594 | |
| 1595 if not args: | |
| 1596 parser.error( | |
| 1597 'Please provide the executable line to run, if you need fancy things ' | |
| 1598 'like xvfb, start this script from *inside* xvfb, it\'ll be much faster' | |
| 1599 '.') | |
| 1600 | |
| 1601 if options.run_all and options.max_failures is not None: | |
| 1602 parser.error('Use only one of --run-all or --max-failures') | |
| 1603 return parser, options, fix_python_path(args) | |
| 1604 | |
| 1605 | |
| 1606 def main(argv): | |
| 1607 """CLI frontend to validate arguments.""" | |
| 1608 run_isolated.disable_buffering() | |
| 1609 parser, options, cmd = process_args(argv) | |
| 1610 | |
| 1611 if options.gtest_list_tests: | |
| 1612 # Special case, return the output of the target unmodified. | |
| 1613 return subprocess.call(cmd + ['--gtest_list_tests']) | |
| 1614 | |
| 1615 cwd = os.getcwd() | |
| 1616 test_cases = parser.process_gtest_options(cmd, cwd, options) | |
| 1617 | |
| 1618 if options.no_dump: | |
| 1619 result_file = None | |
| 1620 else: | |
| 1621 result_file = options.result | |
| 1622 if not result_file: | |
| 1623 if cmd[0] == sys.executable: | |
| 1624 result_file = '%s.run_test_cases' % cmd[1] | |
| 1625 else: | |
| 1626 result_file = '%s.run_test_cases' % cmd[0] | |
| 1627 | |
| 1628 if not test_cases: | |
| 1629 # The fact of not running any test is considered a failure. This is to | |
| 1630 # prevent silent failure with an invalid --gtest_filter argument or because | |
| 1631 # of a misconfigured unit test. | |
| 1632 if test_cases is not None: | |
| 1633 print('Found no test to run') | |
| 1634 if result_file: | |
| 1635 dump_results_as_json(result_file, { | |
| 1636 'test_cases': [], | |
| 1637 'expected': 0, | |
| 1638 'success': [], | |
| 1639 'flaky': [], | |
| 1640 'fail': [], | |
| 1641 'missing': [], | |
| 1642 'duration': 0, | |
| 1643 }) | |
| 1644 return 1 | |
| 1645 | |
| 1646 if options.disabled: | |
| 1647 cmd.append('--gtest_also_run_disabled_tests') | |
| 1648 if options.manual: | |
| 1649 cmd.append('--run-manual') | |
| 1650 | |
| 1651 try: | |
| 1652 return run_test_cases( | |
| 1653 cmd, | |
| 1654 cwd, | |
| 1655 test_cases, | |
| 1656 options.jobs, | |
| 1657 options.timeout, | |
| 1658 options.clusters, | |
| 1659 options.retries, | |
| 1660 options.run_all, | |
| 1661 options.max_failures, | |
| 1662 options.no_cr, | |
| 1663 options.gtest_output, | |
| 1664 result_file, | |
| 1665 options.verbose) | |
| 1666 except Failure as e: | |
| 1667 print >> sys.stderr, e.args[0] | |
| 1668 return 1 | |
| 1669 | |
| 1670 | |
| 1671 if __name__ == '__main__': | |
| 1672 sys.exit(main(sys.argv[1:])) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 19917006:
Move all googletest related scripts into googletest/ (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/swarm_client