| OLD | NEW |
| (Empty) |
| 1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 """Module containing methods/classes related to running parallel test jobs.""" | |
| 6 | |
| 7 import sys | |
| 8 import threading | |
| 9 import time | |
| 10 | |
| 11 import cros_build_lib as cros_lib | |
| 12 | |
| 13 class ParallelJob(threading.Thread): | |
| 14 """Small wrapper for threading. Thread that releases a semaphores on exit.""" | |
| 15 | |
| 16 def __init__(self, starting_semaphore, ending_semaphore, target, args): | |
| 17 """Initializes an instance of a job. | |
| 18 | |
| 19 Args: | |
| 20 starting_semaphore: Semaphore used by caller to wait on such that | |
| 21 there isn't more than a certain number of threads running. Should | |
| 22 be initialized to a value for the number of threads wanting to be run | |
| 23 at a time. | |
| 24 ending_semaphore: Semaphore is released every time a job ends. Should be | |
| 25 initialized to 0 before starting first job. Should be acquired once for | |
| 26 each job. Threading.Thread.join() has a bug where if the run function | |
| 27 terminates too quickly join() will hang forever. | |
| 28 target: The func to run. | |
| 29 args: Args to pass to the fun. | |
| 30 """ | |
| 31 threading.Thread.__init__(self, target=target, args=args) | |
| 32 self._target = target | |
| 33 self._args = args | |
| 34 self._starting_semaphore = starting_semaphore | |
| 35 self._ending_semaphore = ending_semaphore | |
| 36 self._output = None | |
| 37 self._completed = False | |
| 38 | |
| 39 def run(self): | |
| 40 """Thread override. Runs the method specified and sets output.""" | |
| 41 try: | |
| 42 self._output = self._target(*self._args) | |
| 43 finally: | |
| 44 # Our own clean up. | |
| 45 self._Cleanup() | |
| 46 self._completed = True | |
| 47 # From threading.py to avoid a refcycle. | |
| 48 del self._target, self._args | |
| 49 | |
| 50 def GetOutput(self): | |
| 51 """Returns the output of the method run.""" | |
| 52 assert self._completed, 'GetOutput called before thread was run.' | |
| 53 return self._output | |
| 54 | |
| 55 def _Cleanup(self): | |
| 56 """Releases semaphores for a waiting caller.""" | |
| 57 self._starting_semaphore.release() | |
| 58 self._ending_semaphore.release() | |
| 59 | |
| 60 def __str__(self): | |
| 61 return '%s(%s)' % (self._target, self._args) | |
| 62 | |
| 63 | |
| 64 def RunParallelJobs(number_of_simultaneous_jobs, jobs, jobs_args, | |
| 65 print_status): | |
| 66 """Runs set number of specified jobs in parallel. | |
| 67 | |
| 68 Args: | |
| 69 number_of_simultaneous_jobs: Max number of threads to be run in parallel. | |
| 70 jobs: Array of methods to run. | |
| 71 jobs_args: Array of args associated with method calls. | |
| 72 print_status: True if you'd like this to print out .'s as it runs jobs. | |
| 73 Returns: | |
| 74 Returns an array of results corresponding to each thread. | |
| 75 """ | |
| 76 def _TwoTupleize(x, y): | |
| 77 return (x, y) | |
| 78 | |
| 79 threads = [] | |
| 80 job_start_semaphore = threading.Semaphore(number_of_simultaneous_jobs) | |
| 81 join_semaphore = threading.Semaphore(0) | |
| 82 assert len(jobs) == len(jobs_args), 'Length of args array is wrong.' | |
| 83 | |
| 84 # Create the parallel jobs. | |
| 85 for job, args in map(_TwoTupleize, jobs, jobs_args): | |
| 86 thread = ParallelJob(job_start_semaphore, join_semaphore, target=job, | |
| 87 args=args) | |
| 88 threads.append(thread) | |
| 89 | |
| 90 # Cache sudo access. | |
| 91 cros_lib.RunCommand(['sudo', 'echo', 'Caching sudo credentials'], | |
| 92 print_cmd=False, redirect_stdout=True, | |
| 93 redirect_stderr=True) | |
| 94 | |
| 95 # We use a semaphore to ensure we don't run more jobs than required. | |
| 96 # After each thread finishes, it releases (increments semaphore). | |
| 97 # Acquire blocks of num jobs reached and continues when a thread finishes. | |
| 98 for next_thread in threads: | |
| 99 job_start_semaphore.acquire(blocking=True) | |
| 100 next_thread.start() | |
| 101 | |
| 102 # Wait on the rest of the threads to finish. | |
| 103 for thread in threads: | |
| 104 while not join_semaphore.acquire(blocking=False): | |
| 105 time.sleep(5) | |
| 106 if print_status: | |
| 107 print >> sys.stderr, '.', | |
| 108 | |
| 109 return [thread.GetOutput() for thread in threads] | |
| OLD | NEW |