OLD | NEW |
1 # Copyright 2009 Google Inc. All Rights Reserved. | 1 # Copyright 2009 Google Inc. All Rights Reserved. |
2 # | 2 # |
3 # Licensed under the Apache License, Version 2.0 (the "License"); | 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
4 # you may not use this file except in compliance with the License. | 4 # you may not use this file except in compliance with the License. |
5 # You may obtain a copy of the License at | 5 # You may obtain a copy of the License at |
6 # | 6 # |
7 # http://www.apache.org/licenses/LICENSE-2.0 | 7 # http://www.apache.org/licenses/LICENSE-2.0 |
8 # | 8 # |
9 # Unless required by applicable law or agreed to in writing, software | 9 # Unless required by applicable law or agreed to in writing, software |
10 # distributed under the License is distributed on an "AS IS" BASIS, | 10 # distributed under the License is distributed on an "AS IS" BASIS, |
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 # See the License for the specific language governing permissions and | 12 # See the License for the specific language governing permissions and |
13 # limitations under the License. | 13 # limitations under the License. |
14 | 14 |
15 """Generic utils.""" | 15 """Generic utils.""" |
16 | 16 |
17 import errno | 17 import errno |
18 import logging | 18 import logging |
19 import os | 19 import os |
20 import Queue | |
21 import re | 20 import re |
22 import stat | 21 import stat |
23 import subprocess | 22 import subprocess |
24 import sys | 23 import sys |
25 import time | 24 import time |
26 import threading | |
27 import traceback | |
28 import xml.dom.minidom | 25 import xml.dom.minidom |
29 import xml.parsers.expat | 26 import xml.parsers.expat |
30 | 27 |
31 | 28 |
32 class CheckCallError(OSError): | 29 class CheckCallError(OSError): |
33 """CheckCall() returned non-0.""" | 30 """CheckCall() returned non-0.""" |
34 def __init__(self, command, cwd, retcode, stdout, stderr=None): | 31 def __init__(self, command, cwd, retcode, stdout, stderr=None): |
35 OSError.__init__(self, command, cwd, retcode, stdout, stderr) | 32 OSError.__init__(self, command, cwd, retcode, stdout, stderr) |
36 self.command = command | 33 self.command = command |
37 self.cwd = cwd | 34 self.cwd = cwd |
(...skipping 311 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
349 config_path = FindFileUpwards(config_file, path) | 346 config_path = FindFileUpwards(config_file, path) |
350 | 347 |
351 if not config_path: | 348 if not config_path: |
352 print "Can't find", config_file | 349 print "Can't find", config_file |
353 return None | 350 return None |
354 | 351 |
355 env = {} | 352 env = {} |
356 execfile(config_path, env) | 353 execfile(config_path, env) |
357 config_dir = os.path.dirname(config_path) | 354 config_dir = os.path.dirname(config_path) |
358 return config_dir, env['entries'] | 355 return config_dir, env['entries'] |
359 | |
360 | |
361 class ThreadPool: | |
362 """A thread pool class that lets one schedule jobs on many worker threads.""" | |
363 | |
364 def __init__(self, threads=1): | |
365 self._threads = threads | |
366 self._queue = Queue.Queue() | |
367 self._jobs_left = 0 | |
368 self._condition = threading.Condition() | |
369 self._workers = [] | |
370 | |
371 class Worker(threading.Thread): | |
372 """Internal worker class that executes jobs from the ThreadPool queue.""" | |
373 | |
374 def __init__(self, pool): | |
375 threading.Thread.__init__(self) | |
376 self._pool = pool | |
377 self._done = False | |
378 self.exceptions = [] | |
379 | |
380 def Done(self): | |
381 """Terminates the worker threads.""" | |
382 self._done = True | |
383 | |
384 def run(self): | |
385 """Executes jobs from the pool's queue.""" | |
386 while not self._done: | |
387 f = self._pool._queue.get() | |
388 try: | |
389 try: | |
390 f(self) | |
391 except Exception, e: | |
392 # Catch all exceptions, otherwise we can't join the thread. Print | |
393 # the backtrace now, but keep the exception so that we can raise it | |
394 # on the main thread. | |
395 type, value, tb = sys.exc_info() | |
396 traceback.print_exception(type, value, tb) | |
397 self.exceptions.append(e) | |
398 finally: | |
399 self._pool._JobDone() | |
400 | |
401 def _AddJobToQueue(self, job): | |
402 self._condition.acquire() | |
403 self._queue.put(job) | |
404 self._jobs_left += 1 | |
405 self._condition.release() | |
406 | |
407 def _JobDone(self): | |
408 self._condition.acquire() | |
409 try: | |
410 assert self._jobs_left | |
411 self._jobs_left -= 1 | |
412 if self._jobs_left == 0: | |
413 self._condition.notify() | |
414 finally: | |
415 self._condition.release() | |
416 | |
417 def _JoinQueue(self): | |
418 self._condition.acquire() | |
419 try: | |
420 while self._jobs_left: | |
421 self._condition.wait() | |
422 finally: | |
423 self._condition.release() | |
424 | |
425 def Start(self): | |
426 """Starts the thread pool. Spawns worker threads.""" | |
427 assert not self._workers | |
428 for i in xrange(0, self._threads): | |
429 worker = self.Worker(self) | |
430 self._workers.append(worker) | |
431 worker.start() | |
432 | |
433 def Stop(self): | |
434 """Stops the thread pool. Joins all worker threads.""" | |
435 assert self._workers | |
436 for i in xrange(0, len(self._workers)): | |
437 wrapped = lambda thread: thread.Done() | |
438 self._AddJobToQueue(wrapped) | |
439 self._JoinQueue() | |
440 for worker in self._workers: | |
441 worker.join() | |
442 try: | |
443 for worker in self._workers: | |
444 for e in worker.exceptions: | |
445 # If we collected exceptions, raise them now. | |
446 raise e | |
447 finally: | |
448 self._workers = [] | |
449 | |
450 def AddJob(self, function): | |
451 """Adds a job to the queue. | |
452 | |
453 A job is a simple closure, that will get executed on one of the worker | |
454 threads.""" | |
455 wrapped = lambda worker: function() | |
456 self._AddJobToQueue(wrapped) | |
457 | |
458 def WaitJobs(self): | |
459 """Waits for all jobs to be completed.""" | |
460 assert self._workers | |
461 self._JoinQueue() | |
OLD | NEW |