Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(570)

Side by Side Diff: gclient_utils.py

Issue 7846003: Remove CheckCall, CheckCallError and Popen from gclient_utils. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Created 9 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2010 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Generic utils.""" 5 """Generic utils."""
6 6
7 import errno 7 import errno
8 import logging 8 import logging
9 import os 9 import os
10 import Queue 10 import Queue
11 import re 11 import re
12 import stat 12 import stat
13 import subprocess
14 import sys 13 import sys
15 import threading 14 import threading
16 import time 15 import time
17 16
18 import subprocess2 17 import subprocess2
19 18
20 19
21 def hack_subprocess():
22 """subprocess functions may throw exceptions when used in multiple threads.
23
24 See http://bugs.python.org/issue1731717 for more information.
25 """
26 subprocess._cleanup = lambda: None
27
28
29 class Error(Exception): 20 class Error(Exception):
30 """gclient exception class.""" 21 """gclient exception class."""
31 pass 22 pass
32 23
33 24
34 class CheckCallError(subprocess2.CalledProcessError, Error):
35 """CheckCall() returned non-0."""
36 def __init__(self, cmd, cwd, returncode, stdout, stderr=None):
37 subprocess2.CalledProcessError.__init__(
38 self, returncode, cmd, cwd, stdout, stderr)
39 Error.__init__(self, cmd)
40
41 def __str__(self):
42 return subprocess2.CalledProcessError.__str__(self)
43
44
45 def Popen(args, **kwargs):
46 """Calls subprocess.Popen() with hacks to work around certain behaviors.
47
48 Ensure English outpout for svn and make it work reliably on Windows.
49 """
50 logging.debug(u'%s, cwd=%s' % (u' '.join(args), kwargs.get('cwd', '')))
51 if not 'env' in kwargs:
52 # It's easier to parse the stdout if it is always in English.
53 kwargs['env'] = os.environ.copy()
54 kwargs['env']['LANGUAGE'] = 'en'
55 if not 'shell' in kwargs:
56 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
57 # executable, but shell=True makes subprocess on Linux fail when it's called
58 # with a list because it only tries to execute the first item in the list.
59 kwargs['shell'] = (sys.platform=='win32')
60 try:
61 return subprocess.Popen(args, **kwargs)
62 except OSError, e:
63 if e.errno == errno.EAGAIN and sys.platform == 'cygwin':
64 raise Error(
65 'Visit '
66 'http://code.google.com/p/chromium/wiki/CygwinDllRemappingFailure to '
67 'learn how to fix this error; you need to rebase your cygwin dlls')
68 raise
69
70
71 def CheckCall(command, print_error=True, **kwargs):
72 """Similar subprocess.check_call() but redirects stdout and
73 returns (stdout, stderr).
74
75 Works on python 2.4
76 """
77 try:
78 stderr = None
79 if not print_error:
80 stderr = subprocess.PIPE
81 process = Popen(command, stdout=subprocess.PIPE, stderr=stderr, **kwargs)
82 std_out, std_err = process.communicate()
83 except OSError, e:
84 raise CheckCallError(command, kwargs.get('cwd', None), e.errno, None)
85 if process.returncode:
86 raise CheckCallError(command, kwargs.get('cwd', None), process.returncode,
87 std_out, std_err)
88 return std_out, std_err
89
90
91 def SplitUrlRevision(url): 25 def SplitUrlRevision(url):
92 """Splits url and returns a two-tuple: url, rev""" 26 """Splits url and returns a two-tuple: url, rev"""
93 if url.startswith('ssh:'): 27 if url.startswith('ssh:'):
94 # Make sure ssh://user-name@example.com/~/test.git@stable works 28 # Make sure ssh://user-name@example.com/~/test.git@stable works
95 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' 29 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?'
96 components = re.search(regex, url).groups() 30 components = re.search(regex, url).groups()
97 else: 31 else:
98 components = url.split('@', 1) 32 components = url.split('@', 1)
99 if len(components) == 1: 33 if len(components) == 1:
100 components += [None] 34 components += [None]
(...skipping 291 matching lines...) Expand 10 before | Expand all | Expand 10 after
392 of the subprocess's output. Each line has the trailing newline 326 of the subprocess's output. Each line has the trailing newline
393 character trimmed. 327 character trimmed.
394 stdout: Can be any bufferable output. 328 stdout: Can be any bufferable output.
395 329
396 stderr is always redirected to stdout. 330 stderr is always redirected to stdout.
397 """ 331 """
398 assert print_stdout or filter_fn 332 assert print_stdout or filter_fn
399 stdout = stdout or sys.stdout 333 stdout = stdout or sys.stdout
400 filter_fn = filter_fn or (lambda x: None) 334 filter_fn = filter_fn or (lambda x: None)
401 assert not 'stderr' in kwargs 335 assert not 'stderr' in kwargs
402 kid = Popen(args, bufsize=0, 336 kid = subprocess2.Popen(
403 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 337 args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
404 **kwargs) 338 **kwargs)
405 339
406 # Do a flush of stdout before we begin reading from the subprocess's stdout 340 # Do a flush of stdout before we begin reading from the subprocess's stdout
407 stdout.flush() 341 stdout.flush()
408 342
409 # Also, we need to forward stdout to prevent weird re-ordering of output. 343 # Also, we need to forward stdout to prevent weird re-ordering of output.
410 # This has to be done on a per byte basis to make sure it is not buffered: 344 # This has to be done on a per byte basis to make sure it is not buffered:
411 # normally buffering is done for each line, but if svn requests input, no 345 # normally buffering is done for each line, but if svn requests input, no
412 # end-of-line character is output after the prompt and it would not show up. 346 # end-of-line character is output after the prompt and it would not show up.
413 in_byte = kid.stdout.read(1) 347 in_byte = kid.stdout.read(1)
414 if in_byte: 348 if in_byte:
415 if call_filter_on_first_line: 349 if call_filter_on_first_line:
416 filter_fn(None) 350 filter_fn(None)
417 in_line = '' 351 in_line = ''
418 while in_byte: 352 while in_byte:
419 if in_byte != '\r': 353 if in_byte != '\r':
420 if print_stdout: 354 if print_stdout:
421 stdout.write(in_byte) 355 stdout.write(in_byte)
422 if in_byte != '\n': 356 if in_byte != '\n':
423 in_line += in_byte 357 in_line += in_byte
424 else: 358 else:
425 filter_fn(in_line) 359 filter_fn(in_line)
426 in_line = '' 360 in_line = ''
427 in_byte = kid.stdout.read(1) 361 in_byte = kid.stdout.read(1)
428 # Flush the rest of buffered output. This is only an issue with 362 # Flush the rest of buffered output. This is only an issue with
429 # stdout/stderr not ending with a \n. 363 # stdout/stderr not ending with a \n.
430 if len(in_line): 364 if len(in_line):
431 filter_fn(in_line) 365 filter_fn(in_line)
432 rv = kid.wait() 366 rv = kid.wait()
433 if rv: 367 if rv:
434 raise CheckCallError(args, kwargs.get('cwd', None), rv, None) 368 raise subprocess2.CalledProcessError(
369 rv, args, kwargs.get('cwd', None), None, None)
435 return 0 370 return 0
436 371
437 372
438 def FindGclientRoot(from_dir, filename='.gclient'): 373 def FindGclientRoot(from_dir, filename='.gclient'):
439 """Tries to find the gclient root.""" 374 """Tries to find the gclient root."""
440 real_from_dir = os.path.realpath(from_dir) 375 real_from_dir = os.path.realpath(from_dir)
441 path = real_from_dir 376 path = real_from_dir
442 while not os.path.exists(os.path.join(path, filename)): 377 while not os.path.exists(os.path.join(path, filename)):
443 split_path = os.path.split(path) 378 split_path = os.path.split(path)
444 if not split_path[1]: 379 if not split_path[1]:
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after
536 471
537 In gclient's case, Dependencies sometime needs to be run out of order due to 472 In gclient's case, Dependencies sometime needs to be run out of order due to
538 From() keyword. This class manages that all the required dependencies are run 473 From() keyword. This class manages that all the required dependencies are run
539 before running each one. 474 before running each one.
540 475
541 Methods of this class are thread safe. 476 Methods of this class are thread safe.
542 """ 477 """
543 def __init__(self, jobs, progress): 478 def __init__(self, jobs, progress):
544 """jobs specifies the number of concurrent tasks to allow. progress is a 479 """jobs specifies the number of concurrent tasks to allow. progress is a
545 Progress instance.""" 480 Progress instance."""
546 hack_subprocess()
547 # Set when a thread is done or a new item is enqueued. 481 # Set when a thread is done or a new item is enqueued.
548 self.ready_cond = threading.Condition() 482 self.ready_cond = threading.Condition()
549 # Maximum number of concurrent tasks. 483 # Maximum number of concurrent tasks.
550 self.jobs = jobs 484 self.jobs = jobs
551 # List of WorkItem, for gclient, these are Dependency instances. 485 # List of WorkItem, for gclient, these are Dependency instances.
552 self.queued = [] 486 self.queued = []
553 # List of strings representing each Dependency.name that was run. 487 # List of strings representing each Dependency.name that was run.
554 self.ran = [] 488 self.ran = []
555 # List of items currently running. 489 # List of items currently running.
556 self.running = [] 490 self.running = []
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after
680 logging.info('Caught exception in thread %s' % self.item.name) 614 logging.info('Caught exception in thread %s' % self.item.name)
681 logging.info(str(sys.exc_info())) 615 logging.info(str(sys.exc_info()))
682 work_queue.exceptions.put(sys.exc_info()) 616 work_queue.exceptions.put(sys.exc_info())
683 logging.info('Task %s done' % self.item.name) 617 logging.info('Task %s done' % self.item.name)
684 618
685 work_queue.ready_cond.acquire() 619 work_queue.ready_cond.acquire()
686 try: 620 try:
687 work_queue.ready_cond.notifyAll() 621 work_queue.ready_cond.notifyAll()
688 finally: 622 finally:
689 work_queue.ready_cond.release() 623 work_queue.ready_cond.release()
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698