| OLD | NEW |
| 1 # Copyright (c) 2010 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Generic utils.""" | 5 """Generic utils.""" |
| 6 | 6 |
| 7 import errno | 7 import errno |
| 8 import logging | 8 import logging |
| 9 import os | 9 import os |
| 10 import Queue | 10 import Queue |
| 11 import re | 11 import re |
| 12 import stat | 12 import stat |
| 13 import subprocess | |
| 14 import sys | 13 import sys |
| 15 import threading | 14 import threading |
| 16 import time | 15 import time |
| 17 | 16 |
| 18 import subprocess2 | 17 import subprocess2 |
| 19 | 18 |
| 20 | 19 |
| 21 def hack_subprocess(): | |
| 22 """subprocess functions may throw exceptions when used in multiple threads. | |
| 23 | |
| 24 See http://bugs.python.org/issue1731717 for more information. | |
| 25 """ | |
| 26 subprocess._cleanup = lambda: None | |
| 27 | |
| 28 | |
| 29 class Error(Exception): | 20 class Error(Exception): |
| 30 """gclient exception class.""" | 21 """gclient exception class.""" |
| 31 pass | 22 pass |
| 32 | 23 |
| 33 | 24 |
| 34 class CheckCallError(subprocess2.CalledProcessError, Error): | |
| 35 """CheckCall() returned non-0.""" | |
| 36 def __init__(self, cmd, cwd, returncode, stdout, stderr=None): | |
| 37 subprocess2.CalledProcessError.__init__( | |
| 38 self, returncode, cmd, cwd, stdout, stderr) | |
| 39 Error.__init__(self, cmd) | |
| 40 | |
| 41 def __str__(self): | |
| 42 return subprocess2.CalledProcessError.__str__(self) | |
| 43 | |
| 44 | |
| 45 def Popen(args, **kwargs): | |
| 46 """Calls subprocess.Popen() with hacks to work around certain behaviors. | |
| 47 | |
| 48 Ensure English outpout for svn and make it work reliably on Windows. | |
| 49 """ | |
| 50 logging.debug(u'%s, cwd=%s' % (u' '.join(args), kwargs.get('cwd', ''))) | |
| 51 if not 'env' in kwargs: | |
| 52 # It's easier to parse the stdout if it is always in English. | |
| 53 kwargs['env'] = os.environ.copy() | |
| 54 kwargs['env']['LANGUAGE'] = 'en' | |
| 55 if not 'shell' in kwargs: | |
| 56 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the | |
| 57 # executable, but shell=True makes subprocess on Linux fail when it's called | |
| 58 # with a list because it only tries to execute the first item in the list. | |
| 59 kwargs['shell'] = (sys.platform=='win32') | |
| 60 try: | |
| 61 return subprocess.Popen(args, **kwargs) | |
| 62 except OSError, e: | |
| 63 if e.errno == errno.EAGAIN and sys.platform == 'cygwin': | |
| 64 raise Error( | |
| 65 'Visit ' | |
| 66 'http://code.google.com/p/chromium/wiki/CygwinDllRemappingFailure to ' | |
| 67 'learn how to fix this error; you need to rebase your cygwin dlls') | |
| 68 raise | |
| 69 | |
| 70 | |
| 71 def CheckCall(command, print_error=True, **kwargs): | |
| 72 """Similar subprocess.check_call() but redirects stdout and | |
| 73 returns (stdout, stderr). | |
| 74 | |
| 75 Works on python 2.4 | |
| 76 """ | |
| 77 try: | |
| 78 stderr = None | |
| 79 if not print_error: | |
| 80 stderr = subprocess.PIPE | |
| 81 process = Popen(command, stdout=subprocess.PIPE, stderr=stderr, **kwargs) | |
| 82 std_out, std_err = process.communicate() | |
| 83 except OSError, e: | |
| 84 raise CheckCallError(command, kwargs.get('cwd', None), e.errno, None) | |
| 85 if process.returncode: | |
| 86 raise CheckCallError(command, kwargs.get('cwd', None), process.returncode, | |
| 87 std_out, std_err) | |
| 88 return std_out, std_err | |
| 89 | |
| 90 | |
| 91 def SplitUrlRevision(url): | 25 def SplitUrlRevision(url): |
| 92 """Splits url and returns a two-tuple: url, rev""" | 26 """Splits url and returns a two-tuple: url, rev""" |
| 93 if url.startswith('ssh:'): | 27 if url.startswith('ssh:'): |
| 94 # Make sure ssh://user-name@example.com/~/test.git@stable works | 28 # Make sure ssh://user-name@example.com/~/test.git@stable works |
| 95 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' | 29 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' |
| 96 components = re.search(regex, url).groups() | 30 components = re.search(regex, url).groups() |
| 97 else: | 31 else: |
| 98 components = url.split('@', 1) | 32 components = url.split('@', 1) |
| 99 if len(components) == 1: | 33 if len(components) == 1: |
| 100 components += [None] | 34 components += [None] |
| (...skipping 291 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 392 of the subprocess's output. Each line has the trailing newline | 326 of the subprocess's output. Each line has the trailing newline |
| 393 character trimmed. | 327 character trimmed. |
| 394 stdout: Can be any bufferable output. | 328 stdout: Can be any bufferable output. |
| 395 | 329 |
| 396 stderr is always redirected to stdout. | 330 stderr is always redirected to stdout. |
| 397 """ | 331 """ |
| 398 assert print_stdout or filter_fn | 332 assert print_stdout or filter_fn |
| 399 stdout = stdout or sys.stdout | 333 stdout = stdout or sys.stdout |
| 400 filter_fn = filter_fn or (lambda x: None) | 334 filter_fn = filter_fn or (lambda x: None) |
| 401 assert not 'stderr' in kwargs | 335 assert not 'stderr' in kwargs |
| 402 kid = Popen(args, bufsize=0, | 336 kid = subprocess2.Popen( |
| 403 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | 337 args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, |
| 404 **kwargs) | 338 **kwargs) |
| 405 | 339 |
| 406 # Do a flush of stdout before we begin reading from the subprocess's stdout | 340 # Do a flush of stdout before we begin reading from the subprocess's stdout |
| 407 stdout.flush() | 341 stdout.flush() |
| 408 | 342 |
| 409 # Also, we need to forward stdout to prevent weird re-ordering of output. | 343 # Also, we need to forward stdout to prevent weird re-ordering of output. |
| 410 # This has to be done on a per byte basis to make sure it is not buffered: | 344 # This has to be done on a per byte basis to make sure it is not buffered: |
| 411 # normally buffering is done for each line, but if svn requests input, no | 345 # normally buffering is done for each line, but if svn requests input, no |
| 412 # end-of-line character is output after the prompt and it would not show up. | 346 # end-of-line character is output after the prompt and it would not show up. |
| 413 in_byte = kid.stdout.read(1) | 347 in_byte = kid.stdout.read(1) |
| 414 if in_byte: | 348 if in_byte: |
| 415 if call_filter_on_first_line: | 349 if call_filter_on_first_line: |
| 416 filter_fn(None) | 350 filter_fn(None) |
| 417 in_line = '' | 351 in_line = '' |
| 418 while in_byte: | 352 while in_byte: |
| 419 if in_byte != '\r': | 353 if in_byte != '\r': |
| 420 if print_stdout: | 354 if print_stdout: |
| 421 stdout.write(in_byte) | 355 stdout.write(in_byte) |
| 422 if in_byte != '\n': | 356 if in_byte != '\n': |
| 423 in_line += in_byte | 357 in_line += in_byte |
| 424 else: | 358 else: |
| 425 filter_fn(in_line) | 359 filter_fn(in_line) |
| 426 in_line = '' | 360 in_line = '' |
| 427 in_byte = kid.stdout.read(1) | 361 in_byte = kid.stdout.read(1) |
| 428 # Flush the rest of buffered output. This is only an issue with | 362 # Flush the rest of buffered output. This is only an issue with |
| 429 # stdout/stderr not ending with a \n. | 363 # stdout/stderr not ending with a \n. |
| 430 if len(in_line): | 364 if len(in_line): |
| 431 filter_fn(in_line) | 365 filter_fn(in_line) |
| 432 rv = kid.wait() | 366 rv = kid.wait() |
| 433 if rv: | 367 if rv: |
| 434 raise CheckCallError(args, kwargs.get('cwd', None), rv, None) | 368 raise subprocess2.CalledProcessError( |
| 369 rv, args, kwargs.get('cwd', None), None, None) |
| 435 return 0 | 370 return 0 |
| 436 | 371 |
| 437 | 372 |
| 438 def FindGclientRoot(from_dir, filename='.gclient'): | 373 def FindGclientRoot(from_dir, filename='.gclient'): |
| 439 """Tries to find the gclient root.""" | 374 """Tries to find the gclient root.""" |
| 440 real_from_dir = os.path.realpath(from_dir) | 375 real_from_dir = os.path.realpath(from_dir) |
| 441 path = real_from_dir | 376 path = real_from_dir |
| 442 while not os.path.exists(os.path.join(path, filename)): | 377 while not os.path.exists(os.path.join(path, filename)): |
| 443 split_path = os.path.split(path) | 378 split_path = os.path.split(path) |
| 444 if not split_path[1]: | 379 if not split_path[1]: |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 536 | 471 |
| 537 In gclient's case, Dependencies sometime needs to be run out of order due to | 472 In gclient's case, Dependencies sometime needs to be run out of order due to |
| 538 From() keyword. This class manages that all the required dependencies are run | 473 From() keyword. This class manages that all the required dependencies are run |
| 539 before running each one. | 474 before running each one. |
| 540 | 475 |
| 541 Methods of this class are thread safe. | 476 Methods of this class are thread safe. |
| 542 """ | 477 """ |
| 543 def __init__(self, jobs, progress): | 478 def __init__(self, jobs, progress): |
| 544 """jobs specifies the number of concurrent tasks to allow. progress is a | 479 """jobs specifies the number of concurrent tasks to allow. progress is a |
| 545 Progress instance.""" | 480 Progress instance.""" |
| 546 hack_subprocess() | |
| 547 # Set when a thread is done or a new item is enqueued. | 481 # Set when a thread is done or a new item is enqueued. |
| 548 self.ready_cond = threading.Condition() | 482 self.ready_cond = threading.Condition() |
| 549 # Maximum number of concurrent tasks. | 483 # Maximum number of concurrent tasks. |
| 550 self.jobs = jobs | 484 self.jobs = jobs |
| 551 # List of WorkItem, for gclient, these are Dependency instances. | 485 # List of WorkItem, for gclient, these are Dependency instances. |
| 552 self.queued = [] | 486 self.queued = [] |
| 553 # List of strings representing each Dependency.name that was run. | 487 # List of strings representing each Dependency.name that was run. |
| 554 self.ran = [] | 488 self.ran = [] |
| 555 # List of items currently running. | 489 # List of items currently running. |
| 556 self.running = [] | 490 self.running = [] |
| (...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 680 logging.info('Caught exception in thread %s' % self.item.name) | 614 logging.info('Caught exception in thread %s' % self.item.name) |
| 681 logging.info(str(sys.exc_info())) | 615 logging.info(str(sys.exc_info())) |
| 682 work_queue.exceptions.put(sys.exc_info()) | 616 work_queue.exceptions.put(sys.exc_info()) |
| 683 logging.info('Task %s done' % self.item.name) | 617 logging.info('Task %s done' % self.item.name) |
| 684 | 618 |
| 685 work_queue.ready_cond.acquire() | 619 work_queue.ready_cond.acquire() |
| 686 try: | 620 try: |
| 687 work_queue.ready_cond.notifyAll() | 621 work_queue.ready_cond.notifyAll() |
| 688 finally: | 622 finally: |
| 689 work_queue.ready_cond.release() | 623 work_queue.ready_cond.release() |
| OLD | NEW |