OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Generic utils.""" | 5 """Generic utils.""" |
6 | 6 |
7 import codecs | 7 import codecs |
8 import logging | 8 import logging |
9 import os | 9 import os |
10 import pipes | 10 import pipes |
11 import Queue | 11 import Queue |
12 import re | 12 import re |
13 import stat | 13 import stat |
14 import subprocess | 14 import subprocess |
15 import sys | 15 import sys |
16 import tempfile | 16 import tempfile |
17 import threading | 17 import threading |
18 import time | 18 import time |
19 import urlparse | 19 import urlparse |
20 | 20 |
21 import subprocess2 | 21 import subprocess2 |
22 | 22 |
23 | 23 |
| 24 RETRY_MAX = 3 |
| 25 RETRY_INITIAL_SLEEP = 0.5 |
| 26 |
| 27 |
24 class Error(Exception): | 28 class Error(Exception): |
25 """gclient exception class.""" | 29 """gclient exception class.""" |
26 def __init__(self, msg, *args, **kwargs): | 30 def __init__(self, msg, *args, **kwargs): |
27 index = getattr(threading.currentThread(), 'index', 0) | 31 index = getattr(threading.currentThread(), 'index', 0) |
28 if index: | 32 if index: |
29 msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines()) | 33 msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines()) |
30 super(Error, self).__init__(msg, *args, **kwargs) | 34 super(Error, self).__init__(msg, *args, **kwargs) |
31 | 35 |
32 def SplitUrlRevision(url): | 36 def SplitUrlRevision(url): |
33 """Splits url and returns a two-tuple: url, rev""" | 37 """Splits url and returns a two-tuple: url, rev""" |
(...skipping 366 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
400 | 404 |
401 with GCLIENT_CHILDREN_LOCK: | 405 with GCLIENT_CHILDREN_LOCK: |
402 if GCLIENT_CHILDREN: | 406 if GCLIENT_CHILDREN: |
403 print >> sys.stderr, 'Could not kill the following subprocesses:' | 407 print >> sys.stderr, 'Could not kill the following subprocesses:' |
404 for zombie in GCLIENT_CHILDREN: | 408 for zombie in GCLIENT_CHILDREN: |
405 print >> sys.stderr, ' ', zombie.pid | 409 print >> sys.stderr, ' ', zombie.pid |
406 | 410 |
407 | 411 |
408 def CheckCallAndFilter(args, stdout=None, filter_fn=None, | 412 def CheckCallAndFilter(args, stdout=None, filter_fn=None, |
409 print_stdout=None, call_filter_on_first_line=False, | 413 print_stdout=None, call_filter_on_first_line=False, |
410 **kwargs): | 414 retry=False, **kwargs): |
411 """Runs a command and calls back a filter function if needed. | 415 """Runs a command and calls back a filter function if needed. |
412 | 416 |
413 Accepts all subprocess2.Popen() parameters plus: | 417 Accepts all subprocess2.Popen() parameters plus: |
414 print_stdout: If True, the command's stdout is forwarded to stdout. | 418 print_stdout: If True, the command's stdout is forwarded to stdout. |
415 filter_fn: A function taking a single string argument called with each line | 419 filter_fn: A function taking a single string argument called with each line |
416 of the subprocess2's output. Each line has the trailing newline | 420 of the subprocess2's output. Each line has the trailing newline |
417 character trimmed. | 421 character trimmed. |
418 stdout: Can be any bufferable output. | 422 stdout: Can be any bufferable output. |
| 423 retry: If the process exits non-zero, sleep for a brief interval and try |
| 424 again, up to RETRY_MAX times. |
419 | 425 |
420 stderr is always redirected to stdout. | 426 stderr is always redirected to stdout. |
421 """ | 427 """ |
422 assert print_stdout or filter_fn | 428 assert print_stdout or filter_fn |
423 stdout = stdout or sys.stdout | 429 stdout = stdout or sys.stdout |
424 filter_fn = filter_fn or (lambda x: None) | 430 filter_fn = filter_fn or (lambda x: None) |
425 kid = subprocess2.Popen( | |
426 args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, | |
427 **kwargs) | |
428 | 431 |
429 GClientChildren.add(kid) | 432 sleep_interval = RETRY_INITIAL_SLEEP |
| 433 run_cwd = kwargs.get('cwd', os.getcwd()) |
| 434 for _ in xrange(RETRY_MAX + 1): |
| 435 kid = subprocess2.Popen( |
| 436 args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, |
| 437 **kwargs) |
430 | 438 |
431 # Do a flush of stdout before we begin reading from the subprocess2's stdout | 439 GClientChildren.add(kid) |
432 stdout.flush() | |
433 | 440 |
434 # Also, we need to forward stdout to prevent weird re-ordering of output. | 441 # Do a flush of stdout before we begin reading from the subprocess2's stdout |
435 # This has to be done on a per byte basis to make sure it is not buffered: | 442 stdout.flush() |
436 # normally buffering is done for each line, but if svn requests input, no | 443 |
437 # end-of-line character is output after the prompt and it would not show up. | 444 # Also, we need to forward stdout to prevent weird re-ordering of output. |
438 try: | 445 # This has to be done on a per byte basis to make sure it is not buffered: |
439 in_byte = kid.stdout.read(1) | 446 # normally buffering is done for each line, but if svn requests input, no |
440 if in_byte: | 447 # end-of-line character is output after the prompt and it would not show up. |
441 if call_filter_on_first_line: | 448 try: |
442 filter_fn(None) | 449 in_byte = kid.stdout.read(1) |
443 in_line = '' | 450 if in_byte: |
444 while in_byte: | 451 if call_filter_on_first_line: |
445 if in_byte != '\r': | 452 filter_fn(None) |
446 if print_stdout: | 453 in_line = '' |
447 stdout.write(in_byte) | 454 while in_byte: |
448 if in_byte != '\n': | 455 if in_byte != '\r': |
449 in_line += in_byte | 456 if print_stdout: |
| 457 stdout.write(in_byte) |
| 458 if in_byte != '\n': |
| 459 in_line += in_byte |
| 460 else: |
| 461 filter_fn(in_line) |
| 462 in_line = '' |
450 else: | 463 else: |
451 filter_fn(in_line) | 464 filter_fn(in_line) |
452 in_line = '' | 465 in_line = '' |
453 else: | 466 in_byte = kid.stdout.read(1) |
| 467 # Flush the rest of buffered output. This is only an issue with |
| 468 # stdout/stderr not ending with a \n. |
| 469 if len(in_line): |
454 filter_fn(in_line) | 470 filter_fn(in_line) |
455 in_line = '' | 471 rv = kid.wait() |
456 in_byte = kid.stdout.read(1) | |
457 # Flush the rest of buffered output. This is only an issue with | |
458 # stdout/stderr not ending with a \n. | |
459 if len(in_line): | |
460 filter_fn(in_line) | |
461 rv = kid.wait() | |
462 | 472 |
463 # Don't put this in a 'finally,' since the child may still run if we get an | 473 # Don't put this in a 'finally,' since the child may still run if we get |
464 # exception. | 474 # an exception. |
465 GClientChildren.remove(kid) | 475 GClientChildren.remove(kid) |
466 | 476 |
467 except KeyboardInterrupt: | 477 except KeyboardInterrupt: |
468 print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args) | 478 print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args) |
469 raise | 479 raise |
470 | 480 |
471 if rv: | 481 if rv == 0: |
472 raise subprocess2.CalledProcessError( | 482 return 0 |
473 rv, args, kwargs.get('cwd', None), None, None) | 483 if not retry: |
474 return 0 | 484 break |
| 485 print ("WARNING: subprocess '%s' in %s failed; will retry after a short " |
| 486 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd)) |
| 487 sys.sleep(sleep_interval) |
| 488 sleep_interval *= 2 |
| 489 raise subprocess2.CalledProcessError( |
| 490 rv, args, kwargs.get('cwd', None), None, None) |
475 | 491 |
476 | 492 |
477 def FindGclientRoot(from_dir, filename='.gclient'): | 493 def FindGclientRoot(from_dir, filename='.gclient'): |
478 """Tries to find the gclient root.""" | 494 """Tries to find the gclient root.""" |
479 real_from_dir = os.path.realpath(from_dir) | 495 real_from_dir = os.path.realpath(from_dir) |
480 path = real_from_dir | 496 path = real_from_dir |
481 while not os.path.exists(os.path.join(path, filename)): | 497 while not os.path.exists(os.path.join(path, filename)): |
482 split_path = os.path.split(path) | 498 split_path = os.path.split(path) |
483 if not split_path[1]: | 499 if not split_path[1]: |
484 return None | 500 return None |
(...skipping 393 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
878 | 894 |
879 Python on OSX 10.6 raises a NotImplementedError exception. | 895 Python on OSX 10.6 raises a NotImplementedError exception. |
880 """ | 896 """ |
881 try: | 897 try: |
882 import multiprocessing | 898 import multiprocessing |
883 return multiprocessing.cpu_count() | 899 return multiprocessing.cpu_count() |
884 except: # pylint: disable=W0702 | 900 except: # pylint: disable=W0702 |
885 # Mac OS 10.6 only | 901 # Mac OS 10.6 only |
886 # pylint: disable=E1101 | 902 # pylint: disable=E1101 |
887 return int(os.sysconf('SC_NPROCESSORS_ONLN')) | 903 return int(os.sysconf('SC_NPROCESSORS_ONLN')) |
OLD | NEW |