OLD | NEW |
---|---|
1 # Copyright (c) 2009, Google Inc. All rights reserved. | 1 # Copyright (c) 2009, Google Inc. All rights reserved. |
2 # Copyright (c) 2009 Apple Inc. All rights reserved. | 2 # Copyright (c) 2009 Apple Inc. All rights reserved. |
3 # | 3 # |
4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
6 # met: | 6 # met: |
7 # | 7 # |
8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
(...skipping 453 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
464 else: | 464 else: |
465 string_args = self._stringify_args(args) | 465 string_args = self._stringify_args(args) |
466 return subprocess.Popen(string_args, **kwargs) | 466 return subprocess.Popen(string_args, **kwargs) |
467 | 467 |
468 def call(self, args, **kwargs): | 468 def call(self, args, **kwargs): |
469 return subprocess.call(self._stringify_args(args), **kwargs) | 469 return subprocess.call(self._stringify_args(args), **kwargs) |
470 | 470 |
471 def run_in_parallel(self, command_lines_and_cwds, processes=None): | 471 def run_in_parallel(self, command_lines_and_cwds, processes=None): |
472 """Runs a list of (cmd_line list, cwd string) tuples in parallel and ret urns a list of (retcode, stdout, stderr) tuples.""" | 472 """Runs a list of (cmd_line list, cwd string) tuples in parallel and ret urns a list of (retcode, stdout, stderr) tuples.""" |
473 assert len(command_lines_and_cwds) | 473 assert len(command_lines_and_cwds) |
474 return self.map(_run_command_thunk, command_lines_and_cwds, processes) | |
474 | 475 |
475 if sys.platform in ('cygwin', 'win32'): | 476 def map(self, thunk, arglist, processes=None): |
476 return map(_run_command_thunk, command_lines_and_cwds) | 477 if sys.platform in ('cygwin', 'win32') or len(arglist) == 1: |
477 pool = multiprocessing.Pool(processes=processes) | 478 return map(thunk, arglist) |
478 results = pool.map(_run_command_thunk, command_lines_and_cwds) | 479 pool = multiprocessing.Pool(processes=(processes or multiprocessing.cpu_ count())) |
479 pool.close() | 480 try: |
480 pool.join() | 481 return pool.map(thunk, arglist) |
eseidel
2014/09/09 23:10:55
This may be incompatible with keyboard interrupts:
Dirk Pranke
2014/09/09 23:32:22
Could be. I can switch this to a fancier pool if n
| |
481 return results | 482 finally: |
483 pool.close() | |
484 pool.join() | |
482 | 485 |
483 | 486 |
484 def _run_command_thunk(cmd_line_and_cwd): | 487 def _run_command_thunk(cmd_line_and_cwd): |
485 # Note that this needs to be a bare module (and hence Picklable) method to w ork with multiprocessing.Pool. | 488 # Note that this needs to be a bare module (and hence Picklable) method to w ork with multiprocessing.Pool. |
486 (cmd_line, cwd) = cmd_line_and_cwd | 489 (cmd_line, cwd) = cmd_line_and_cwd |
487 proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=su bprocess.PIPE) | 490 proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=su bprocess.PIPE) |
488 stdout, stderr = proc.communicate() | 491 stdout, stderr = proc.communicate() |
489 return (proc.returncode, stdout, stderr) | 492 return (proc.returncode, stdout, stderr) |
OLD | NEW |