| OLD | NEW |
| 1 # Copyright 2009 Google Inc. All Rights Reserved. | 1 # Copyright 2009 Google Inc. All Rights Reserved. |
| 2 # | 2 # |
| 3 # Licensed under the Apache License, Version 2.0 (the "License"); | 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 # you may not use this file except in compliance with the License. | 4 # you may not use this file except in compliance with the License. |
| 5 # You may obtain a copy of the License at | 5 # You may obtain a copy of the License at |
| 6 # | 6 # |
| 7 # http://www.apache.org/licenses/LICENSE-2.0 | 7 # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 # | 8 # |
| 9 # Unless required by applicable law or agreed to in writing, software | 9 # Unless required by applicable law or agreed to in writing, software |
| 10 # distributed under the License is distributed on an "AS IS" BASIS, | 10 # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 # See the License for the specific language governing permissions and | 12 # See the License for the specific language governing permissions and |
| 13 # limitations under the License. | 13 # limitations under the License. |
| 14 | 14 |
| 15 """Generic utils.""" | 15 """Generic utils.""" |
| 16 | 16 |
| 17 import errno | 17 import errno |
| 18 import logging | 18 import logging |
| 19 import os | 19 import os |
| 20 import Queue | |
| 21 import re | 20 import re |
| 22 import stat | 21 import stat |
| 23 import subprocess | 22 import subprocess |
| 24 import sys | 23 import sys |
| 25 import time | 24 import time |
| 26 import threading | |
| 27 import traceback | |
| 28 import xml.dom.minidom | 25 import xml.dom.minidom |
| 29 import xml.parsers.expat | 26 import xml.parsers.expat |
| 30 | 27 |
| 31 | 28 |
| 32 class CheckCallError(OSError): | 29 class CheckCallError(OSError): |
| 33 """CheckCall() returned non-0.""" | 30 """CheckCall() returned non-0.""" |
| 34 def __init__(self, command, cwd, retcode, stdout, stderr=None): | 31 def __init__(self, command, cwd, retcode, stdout, stderr=None): |
| 35 OSError.__init__(self, command, cwd, retcode, stdout, stderr) | 32 OSError.__init__(self, command, cwd, retcode, stdout, stderr) |
| 36 self.command = command | 33 self.command = command |
| 37 self.cwd = cwd | 34 self.cwd = cwd |
| (...skipping 311 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 349 config_path = FindFileUpwards(config_file, path) | 346 config_path = FindFileUpwards(config_file, path) |
| 350 | 347 |
| 351 if not config_path: | 348 if not config_path: |
| 352 print "Can't find", config_file | 349 print "Can't find", config_file |
| 353 return None | 350 return None |
| 354 | 351 |
| 355 env = {} | 352 env = {} |
| 356 execfile(config_path, env) | 353 execfile(config_path, env) |
| 357 config_dir = os.path.dirname(config_path) | 354 config_dir = os.path.dirname(config_path) |
| 358 return config_dir, env['entries'] | 355 return config_dir, env['entries'] |
| 359 | |
| 360 | |
| 361 class ThreadPool: | |
| 362 """A thread pool class that lets one schedule jobs on many worker threads.""" | |
| 363 | |
| 364 def __init__(self, threads=1): | |
| 365 self._threads = threads | |
| 366 self._queue = Queue.Queue() | |
| 367 self._jobs_left = 0 | |
| 368 self._condition = threading.Condition() | |
| 369 self._workers = [] | |
| 370 | |
| 371 class Worker(threading.Thread): | |
| 372 """Internal worker class that executes jobs from the ThreadPool queue.""" | |
| 373 | |
| 374 def __init__(self, pool): | |
| 375 threading.Thread.__init__(self) | |
| 376 self._pool = pool | |
| 377 self._done = False | |
| 378 self.exceptions = [] | |
| 379 | |
| 380 def Done(self): | |
| 381 """Terminates the worker threads.""" | |
| 382 self._done = True | |
| 383 | |
| 384 def run(self): | |
| 385 """Executes jobs from the pool's queue.""" | |
| 386 while not self._done: | |
| 387 f = self._pool._queue.get() | |
| 388 try: | |
| 389 try: | |
| 390 f(self) | |
| 391 except Exception, e: | |
| 392 # Catch all exceptions, otherwise we can't join the thread. Print | |
| 393 # the backtrace now, but keep the exception so that we can raise it | |
| 394 # on the main thread. | |
| 395 type, value, tb = sys.exc_info() | |
| 396 traceback.print_exception(type, value, tb) | |
| 397 self.exceptions.append(e) | |
| 398 finally: | |
| 399 self._pool._JobDone() | |
| 400 | |
| 401 def _AddJobToQueue(self, job): | |
| 402 self._condition.acquire() | |
| 403 self._queue.put(job) | |
| 404 self._jobs_left += 1 | |
| 405 self._condition.release() | |
| 406 | |
| 407 def _JobDone(self): | |
| 408 self._condition.acquire() | |
| 409 try: | |
| 410 assert self._jobs_left | |
| 411 self._jobs_left -= 1 | |
| 412 if self._jobs_left == 0: | |
| 413 self._condition.notify() | |
| 414 finally: | |
| 415 self._condition.release() | |
| 416 | |
| 417 def _JoinQueue(self): | |
| 418 self._condition.acquire() | |
| 419 try: | |
| 420 while self._jobs_left: | |
| 421 self._condition.wait() | |
| 422 finally: | |
| 423 self._condition.release() | |
| 424 | |
| 425 def Start(self): | |
| 426 """Starts the thread pool. Spawns worker threads.""" | |
| 427 assert not self._workers | |
| 428 for i in xrange(0, self._threads): | |
| 429 worker = self.Worker(self) | |
| 430 self._workers.append(worker) | |
| 431 worker.start() | |
| 432 | |
| 433 def Stop(self): | |
| 434 """Stops the thread pool. Joins all worker threads.""" | |
| 435 assert self._workers | |
| 436 for i in xrange(0, len(self._workers)): | |
| 437 wrapped = lambda thread: thread.Done() | |
| 438 self._AddJobToQueue(wrapped) | |
| 439 self._JoinQueue() | |
| 440 for worker in self._workers: | |
| 441 worker.join() | |
| 442 try: | |
| 443 for worker in self._workers: | |
| 444 for e in worker.exceptions: | |
| 445 # If we collected exceptions, raise them now. | |
| 446 raise e | |
| 447 finally: | |
| 448 self._workers = [] | |
| 449 | |
| 450 def AddJob(self, function): | |
| 451 """Adds a job to the queue. | |
| 452 | |
| 453 A job is a simple closure, that will get executed on one of the worker | |
| 454 threads.""" | |
| 455 wrapped = lambda worker: function() | |
| 456 self._AddJobToQueue(wrapped) | |
| 457 | |
| 458 def WaitJobs(self): | |
| 459 """Waits for all jobs to be completed.""" | |
| 460 assert self._workers | |
| 461 self._JoinQueue() | |
| OLD | NEW |