OLD | NEW |
1 # Copyright (c) 2010 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Generic utils.""" | 5 """Generic utils.""" |
6 | 6 |
7 import errno | 7 import errno |
8 import logging | 8 import logging |
9 import os | 9 import os |
10 import Queue | 10 import Queue |
11 import re | 11 import re |
12 import stat | 12 import stat |
13 import subprocess | |
14 import sys | 13 import sys |
15 import threading | 14 import threading |
16 import time | 15 import time |
17 | 16 |
18 import subprocess2 | 17 import subprocess2 |
19 | 18 |
20 | 19 |
21 def hack_subprocess(): | |
22 """subprocess functions may throw exceptions when used in multiple threads. | |
23 | |
24 See http://bugs.python.org/issue1731717 for more information. | |
25 """ | |
26 subprocess._cleanup = lambda: None | |
27 | |
28 | |
29 class Error(Exception): | 20 class Error(Exception): |
30 """gclient exception class.""" | 21 """gclient exception class.""" |
31 pass | 22 pass |
32 | 23 |
33 | 24 |
34 class CheckCallError(subprocess2.CalledProcessError, Error): | |
35 """CheckCall() returned non-0.""" | |
36 def __init__(self, cmd, cwd, returncode, stdout, stderr=None): | |
37 subprocess2.CalledProcessError.__init__( | |
38 self, returncode, cmd, cwd, stdout, stderr) | |
39 Error.__init__(self, cmd) | |
40 | |
41 def __str__(self): | |
42 return subprocess2.CalledProcessError.__str__(self) | |
43 | |
44 | |
45 def Popen(args, **kwargs): | |
46 """Calls subprocess.Popen() with hacks to work around certain behaviors. | |
47 | |
48 Ensure English outpout for svn and make it work reliably on Windows. | |
49 """ | |
50 logging.debug(u'%s, cwd=%s' % (u' '.join(args), kwargs.get('cwd', ''))) | |
51 if not 'env' in kwargs: | |
52 # It's easier to parse the stdout if it is always in English. | |
53 kwargs['env'] = os.environ.copy() | |
54 kwargs['env']['LANGUAGE'] = 'en' | |
55 if not 'shell' in kwargs: | |
56 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the | |
57 # executable, but shell=True makes subprocess on Linux fail when it's called | |
58 # with a list because it only tries to execute the first item in the list. | |
59 kwargs['shell'] = (sys.platform=='win32') | |
60 try: | |
61 return subprocess.Popen(args, **kwargs) | |
62 except OSError, e: | |
63 if e.errno == errno.EAGAIN and sys.platform == 'cygwin': | |
64 raise Error( | |
65 'Visit ' | |
66 'http://code.google.com/p/chromium/wiki/CygwinDllRemappingFailure to ' | |
67 'learn how to fix this error; you need to rebase your cygwin dlls') | |
68 raise | |
69 | |
70 | |
71 def CheckCall(command, print_error=True, **kwargs): | |
72 """Similar subprocess.check_call() but redirects stdout and | |
73 returns (stdout, stderr). | |
74 | |
75 Works on python 2.4 | |
76 """ | |
77 try: | |
78 stderr = None | |
79 if not print_error: | |
80 stderr = subprocess.PIPE | |
81 process = Popen(command, stdout=subprocess.PIPE, stderr=stderr, **kwargs) | |
82 std_out, std_err = process.communicate() | |
83 except OSError, e: | |
84 raise CheckCallError(command, kwargs.get('cwd', None), e.errno, None) | |
85 if process.returncode: | |
86 raise CheckCallError(command, kwargs.get('cwd', None), process.returncode, | |
87 std_out, std_err) | |
88 return std_out, std_err | |
89 | |
90 | |
91 def SplitUrlRevision(url): | 25 def SplitUrlRevision(url): |
92 """Splits url and returns a two-tuple: url, rev""" | 26 """Splits url and returns a two-tuple: url, rev""" |
93 if url.startswith('ssh:'): | 27 if url.startswith('ssh:'): |
94 # Make sure ssh://user-name@example.com/~/test.git@stable works | 28 # Make sure ssh://user-name@example.com/~/test.git@stable works |
95 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' | 29 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' |
96 components = re.search(regex, url).groups() | 30 components = re.search(regex, url).groups() |
97 else: | 31 else: |
98 components = url.split('@', 1) | 32 components = url.split('@', 1) |
99 if len(components) == 1: | 33 if len(components) == 1: |
100 components += [None] | 34 components += [None] |
(...skipping 291 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
392 of the subprocess's output. Each line has the trailing newline | 326 of the subprocess's output. Each line has the trailing newline |
393 character trimmed. | 327 character trimmed. |
394 stdout: Can be any bufferable output. | 328 stdout: Can be any bufferable output. |
395 | 329 |
396 stderr is always redirected to stdout. | 330 stderr is always redirected to stdout. |
397 """ | 331 """ |
398 assert print_stdout or filter_fn | 332 assert print_stdout or filter_fn |
399 stdout = stdout or sys.stdout | 333 stdout = stdout or sys.stdout |
400 filter_fn = filter_fn or (lambda x: None) | 334 filter_fn = filter_fn or (lambda x: None) |
401 assert not 'stderr' in kwargs | 335 assert not 'stderr' in kwargs |
402 kid = Popen(args, bufsize=0, | 336 kid = subprocess2.Popen( |
403 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | 337 args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, |
404 **kwargs) | 338 **kwargs) |
405 | 339 |
406 # Do a flush of stdout before we begin reading from the subprocess's stdout | 340 # Do a flush of stdout before we begin reading from the subprocess's stdout |
407 stdout.flush() | 341 stdout.flush() |
408 | 342 |
409 # Also, we need to forward stdout to prevent weird re-ordering of output. | 343 # Also, we need to forward stdout to prevent weird re-ordering of output. |
410 # This has to be done on a per byte basis to make sure it is not buffered: | 344 # This has to be done on a per byte basis to make sure it is not buffered: |
411 # normally buffering is done for each line, but if svn requests input, no | 345 # normally buffering is done for each line, but if svn requests input, no |
412 # end-of-line character is output after the prompt and it would not show up. | 346 # end-of-line character is output after the prompt and it would not show up. |
413 in_byte = kid.stdout.read(1) | 347 in_byte = kid.stdout.read(1) |
414 if in_byte: | 348 if in_byte: |
415 if call_filter_on_first_line: | 349 if call_filter_on_first_line: |
416 filter_fn(None) | 350 filter_fn(None) |
417 in_line = '' | 351 in_line = '' |
418 while in_byte: | 352 while in_byte: |
419 if in_byte != '\r': | 353 if in_byte != '\r': |
420 if print_stdout: | 354 if print_stdout: |
421 stdout.write(in_byte) | 355 stdout.write(in_byte) |
422 if in_byte != '\n': | 356 if in_byte != '\n': |
423 in_line += in_byte | 357 in_line += in_byte |
424 else: | 358 else: |
425 filter_fn(in_line) | 359 filter_fn(in_line) |
426 in_line = '' | 360 in_line = '' |
427 in_byte = kid.stdout.read(1) | 361 in_byte = kid.stdout.read(1) |
428 # Flush the rest of buffered output. This is only an issue with | 362 # Flush the rest of buffered output. This is only an issue with |
429 # stdout/stderr not ending with a \n. | 363 # stdout/stderr not ending with a \n. |
430 if len(in_line): | 364 if len(in_line): |
431 filter_fn(in_line) | 365 filter_fn(in_line) |
432 rv = kid.wait() | 366 rv = kid.wait() |
433 if rv: | 367 if rv: |
434 raise CheckCallError(args, kwargs.get('cwd', None), rv, None) | 368 raise subprocess2.CalledProcessError( |
| 369 rv, args, kwargs.get('cwd', None), None, None) |
435 return 0 | 370 return 0 |
436 | 371 |
437 | 372 |
438 def FindGclientRoot(from_dir, filename='.gclient'): | 373 def FindGclientRoot(from_dir, filename='.gclient'): |
439 """Tries to find the gclient root.""" | 374 """Tries to find the gclient root.""" |
440 real_from_dir = os.path.realpath(from_dir) | 375 real_from_dir = os.path.realpath(from_dir) |
441 path = real_from_dir | 376 path = real_from_dir |
442 while not os.path.exists(os.path.join(path, filename)): | 377 while not os.path.exists(os.path.join(path, filename)): |
443 split_path = os.path.split(path) | 378 split_path = os.path.split(path) |
444 if not split_path[1]: | 379 if not split_path[1]: |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
536 | 471 |
537 In gclient's case, Dependencies sometime needs to be run out of order due to | 472 In gclient's case, Dependencies sometime needs to be run out of order due to |
538 From() keyword. This class manages that all the required dependencies are run | 473 From() keyword. This class manages that all the required dependencies are run |
539 before running each one. | 474 before running each one. |
540 | 475 |
541 Methods of this class are thread safe. | 476 Methods of this class are thread safe. |
542 """ | 477 """ |
543 def __init__(self, jobs, progress): | 478 def __init__(self, jobs, progress): |
544 """jobs specifies the number of concurrent tasks to allow. progress is a | 479 """jobs specifies the number of concurrent tasks to allow. progress is a |
545 Progress instance.""" | 480 Progress instance.""" |
546 hack_subprocess() | |
547 # Set when a thread is done or a new item is enqueued. | 481 # Set when a thread is done or a new item is enqueued. |
548 self.ready_cond = threading.Condition() | 482 self.ready_cond = threading.Condition() |
549 # Maximum number of concurrent tasks. | 483 # Maximum number of concurrent tasks. |
550 self.jobs = jobs | 484 self.jobs = jobs |
551 # List of WorkItem, for gclient, these are Dependency instances. | 485 # List of WorkItem, for gclient, these are Dependency instances. |
552 self.queued = [] | 486 self.queued = [] |
553 # List of strings representing each Dependency.name that was run. | 487 # List of strings representing each Dependency.name that was run. |
554 self.ran = [] | 488 self.ran = [] |
555 # List of items currently running. | 489 # List of items currently running. |
556 self.running = [] | 490 self.running = [] |
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
680 logging.info('Caught exception in thread %s' % self.item.name) | 614 logging.info('Caught exception in thread %s' % self.item.name) |
681 logging.info(str(sys.exc_info())) | 615 logging.info(str(sys.exc_info())) |
682 work_queue.exceptions.put(sys.exc_info()) | 616 work_queue.exceptions.put(sys.exc_info()) |
683 logging.info('Task %s done' % self.item.name) | 617 logging.info('Task %s done' % self.item.name) |
684 | 618 |
685 work_queue.ready_cond.acquire() | 619 work_queue.ready_cond.acquire() |
686 try: | 620 try: |
687 work_queue.ready_cond.notifyAll() | 621 work_queue.ready_cond.notifyAll() |
688 finally: | 622 finally: |
689 work_queue.ready_cond.release() | 623 work_queue.ready_cond.release() |
OLD | NEW |