| OLD | NEW |
| 1 # Copyright (c) 2009, Google Inc. All rights reserved. | 1 # Copyright (c) 2009, Google Inc. All rights reserved. |
| 2 # Copyright (c) 2009 Apple Inc. All rights reserved. | 2 # Copyright (c) 2009 Apple Inc. All rights reserved. |
| 3 # | 3 # |
| 4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
| 5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
| 6 # met: | 6 # met: |
| 7 # | 7 # |
| 8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
| 9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
| 10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
| (...skipping 10 matching lines...) Expand all Loading... |
| 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | 29 |
| 30 import csv | 30 import csv |
| 31 import ctypes |
| 31 import errno | 32 import errno |
| 32 import logging | 33 import logging |
| 33 import multiprocessing | 34 import multiprocessing |
| 34 import os | 35 import os |
| 35 import signal | 36 import signal |
| 36 import subprocess | 37 import subprocess |
| 37 import sys | 38 import sys |
| 38 import threading | 39 import threading |
| 39 import time | 40 import time |
| 40 | 41 |
| 41 from webkitpy.common.system.filesystem import FileSystem | |
| 42 | |
| 43 | 42 |
| 44 _log = logging.getLogger(__name__) | 43 _log = logging.getLogger(__name__) |
| 45 | 44 |
| 46 | 45 |
| 47 class ScriptError(Exception): | 46 class ScriptError(Exception): |
| 48 | 47 |
| 49 def __init__(self, | 48 def __init__(self, |
| 50 message=None, | 49 message=None, |
| 51 script_args=None, | 50 script_args=None, |
| 52 exit_code=None, | 51 exit_code=None, |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 87 PIPE = subprocess.PIPE | 86 PIPE = subprocess.PIPE |
| 88 STDOUT = subprocess.STDOUT | 87 STDOUT = subprocess.STDOUT |
| 89 DEVNULL = open(os.devnull, 'wb') | 88 DEVNULL = open(os.devnull, 'wb') |
| 90 | 89 |
| 91 def _should_close_fds(self): | 90 def _should_close_fds(self): |
| 92 # We need to pass close_fds=True to work around Python bug #2320 | 91 # We need to pass close_fds=True to work around Python bug #2320 |
| 93 # (otherwise we can hang when we kill DumpRenderTree when we are running | 92 # (otherwise we can hang when we kill DumpRenderTree when we are running |
| 94 # multiple threads). See http://bugs.python.org/issue2320 . | 93 # multiple threads). See http://bugs.python.org/issue2320 . |
| 95 # Note that close_fds isn't supported on Windows, but this bug only | 94 # Note that close_fds isn't supported on Windows, but this bug only |
| 96 # shows up on Mac and Linux. | 95 # shows up on Mac and Linux. |
| 97 return sys.platform not in ('win32', 'cygwin') | 96 return sys.platform != 'win32' |
| 98 | 97 |
| 99 def cpu_count(self): | 98 def cpu_count(self): |
| 100 return multiprocessing.cpu_count() | 99 return multiprocessing.cpu_count() |
| 101 | 100 |
| 102 def kill_process(self, pid): | 101 def kill_process(self, pid): |
| 103 """Attempts to kill the given pid. | 102 """Attempts to kill the given pid. |
| 103 |
| 104 Will fail silently if pid does not exist or insufficient permissions. | 104 Will fail silently if pid does not exist or insufficient permissions. |
| 105 """ | 105 """ |
| 106 # According to http://docs.python.org/library/os.html |
| 107 # os.kill isn't available on Windows. |
| 106 if sys.platform == 'win32': | 108 if sys.platform == 'win32': |
| 107 # We only use taskkill.exe on windows (not cygwin) because subproces
s.pid | |
| 108 # is a CYGWIN pid and taskkill.exe expects a windows pid. | |
| 109 # Thankfully os.kill on CYGWIN handles either pid type. | |
| 110 command = ['taskkill.exe', '/f', '/t', '/pid', pid] | 109 command = ['taskkill.exe', '/f', '/t', '/pid', pid] |
| 111 # taskkill will exit 128 if the process is not found. We should log
. | 110 # taskkill will exit 128 if the process is not found. We should log. |
| 112 self.run_command(command, error_handler=self.ignore_error) | 111 self.run_command(command, error_handler=self.ignore_error) |
| 113 return | 112 return |
| 114 | 113 |
| 115 # According to http://docs.python.org/library/os.html | 114 try: |
| 116 # os.kill isn't available on Windows. python 2.5.5 os.kill appears | 115 os.kill(pid, signal.SIGKILL) |
| 117 # to work in cygwin, however it occasionally raises EAGAIN. | 116 os.waitpid(pid, os.WNOHANG) |
| 118 retries_left = 10 if sys.platform == 'cygwin' else 1 | 117 except OSError as error: |
| 119 while retries_left > 0: | 118 if error.errno == errno.ESRCH: |
| 120 try: | 119 # The process does not exist. |
| 121 retries_left -= 1 | 120 return |
| 122 os.kill(pid, signal.SIGKILL) | 121 if error.errno == errno.ECHILD: |
| 123 _ = os.waitpid(pid, os.WNOHANG) | 122 # Can't wait on a non-child process, but the kill worked. |
| 124 except OSError as error: | 123 return |
| 125 if error.errno == errno.EAGAIN: | 124 raise |
| 126 if retries_left <= 0: | |
| 127 _log.warning('Failed to kill pid %s. Too many EAGAIN er
rors.', pid) | |
| 128 continue | |
| 129 if error.errno == errno.ESRCH: # The process does not exist. | |
| 130 return | |
| 131 if error.errno == errno.EPIPE: # The process has exited already
on cygwin | |
| 132 return | |
| 133 if error.errno == errno.ECHILD: | |
| 134 # Can't wait on a non-child process, but the kill worked. | |
| 135 return | |
| 136 if error.errno == errno.EACCES and sys.platform == 'cygwin': | |
| 137 # Cygwin python sometimes can't kill native processes. | |
| 138 return | |
| 139 raise | |
| 140 | 125 |
| 141 def _win32_check_running_pid(self, pid): | 126 def _win32_check_running_pid(self, pid): |
| 142 # importing ctypes at the top-level seems to cause weird crashes at | |
| 143 # exit under cygwin on apple's win port. Only win32 needs cygwin, so | |
| 144 # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id
=91682 | |
| 145 import ctypes | |
| 146 | 127 |
| 147 class PROCESSENTRY32(ctypes.Structure): | 128 class PROCESSENTRY32(ctypes.Structure): |
| 148 _fields_ = [('dwSize', ctypes.c_ulong), | 129 _fields_ = [('dwSize', ctypes.c_ulong), |
| 149 ('cntUsage', ctypes.c_ulong), | 130 ('cntUsage', ctypes.c_ulong), |
| 150 ('th32ProcessID', ctypes.c_ulong), | 131 ('th32ProcessID', ctypes.c_ulong), |
| 151 ('th32DefaultHeapID', ctypes.POINTER(ctypes.c_ulong)), | 132 ('th32DefaultHeapID', ctypes.POINTER(ctypes.c_ulong)), |
| 152 ('th32ModuleID', ctypes.c_ulong), | 133 ('th32ModuleID', ctypes.c_ulong), |
| 153 ('cntThreads', ctypes.c_ulong), | 134 ('cntThreads', ctypes.c_ulong), |
| 154 ('th32ParentProcessID', ctypes.c_ulong), | 135 ('th32ParentProcessID', ctypes.c_ulong), |
| 155 ('pcPriClassBase', ctypes.c_ulong), | 136 ('pcPriClassBase', ctypes.c_ulong), |
| (...skipping 29 matching lines...) Expand all Loading... |
| 185 return self._win32_check_running_pid(pid) | 166 return self._win32_check_running_pid(pid) |
| 186 | 167 |
| 187 try: | 168 try: |
| 188 os.kill(pid, 0) | 169 os.kill(pid, 0) |
| 189 return True | 170 return True |
| 190 except OSError: | 171 except OSError: |
| 191 return False | 172 return False |
| 192 | 173 |
| 193 def _running_processes(self): | 174 def _running_processes(self): |
| 194 processes = [] | 175 processes = [] |
| 195 if sys.platform in ('win32', 'cygwin'): | 176 if sys.platform == 'win32': |
| 196 tasklist_process = self.popen(['tasklist', '/fo', 'csv'], | 177 tasklist_process = self.popen(['tasklist', '/fo', 'csv'], |
| 197 stdout=self.PIPE, stderr=self.PIPE) | 178 stdout=self.PIPE, stderr=self.PIPE) |
| 198 stdout, _ = tasklist_process.communicate() | 179 stdout, _ = tasklist_process.communicate() |
| 199 stdout_reader = csv.reader(stdout.splitlines()) | 180 stdout_reader = csv.reader(stdout.splitlines()) |
| 200 for line in stdout_reader: | 181 for line in stdout_reader: |
| 201 processes.append([column for column in line]) | 182 processes.append([column for column in line]) |
| 202 else: | 183 else: |
| 203 ps_process = self.popen(['ps', '-eo', 'pid,comm'], | 184 ps_process = self.popen(['ps', '-eo', 'pid,comm'], |
| 204 stdout=self.PIPE, stderr=self.PIPE) | 185 stdout=self.PIPE, stderr=self.PIPE) |
| 205 stdout, _ = ps_process.communicate() | 186 stdout, _ = ps_process.communicate() |
| (...skipping 15 matching lines...) Expand all Loading... |
| 221 pid = line[1] | 202 pid = line[1] |
| 222 if process_name_filter(process_name): | 203 if process_name_filter(process_name): |
| 223 running_pids.append(int(pid)) | 204 running_pids.append(int(pid)) |
| 224 except (ValueError, IndexError): | 205 except (ValueError, IndexError): |
| 225 pass | 206 pass |
| 226 | 207 |
| 227 return sorted(running_pids) | 208 return sorted(running_pids) |
| 228 | 209 |
| 229 def process_dump(self): | 210 def process_dump(self): |
| 230 ps_process = None | 211 ps_process = None |
| 231 if sys.platform in ('win32', 'cygwin'): | 212 if sys.platform in 'win32': |
| 232 ps_process = self.popen( | 213 ps_process = self.popen( |
| 233 ['wmic', 'process', 'get', | 214 ['wmic', 'process', 'get', |
| 234 'ProcessId,ParentProcessId,CommandLine'], | 215 'ProcessId,ParentProcessId,CommandLine'], |
| 235 stdout=self.PIPE, stderr=self.PIPE) | 216 stdout=self.PIPE, stderr=self.PIPE) |
| 236 else: | 217 else: |
| 237 ps_process = self.popen(['ps', 'aux'], stdout=self.PIPE, stderr=self
.PIPE) | 218 ps_process = self.popen(['ps', 'aux'], stdout=self.PIPE, stderr=self
.PIPE) |
| 238 | 219 |
| 239 stdout, _ = ps_process.communicate() | 220 stdout, _ = ps_process.communicate() |
| 240 return [line.strip() for line in stdout.splitlines()] | 221 return [line.strip() for line in stdout.splitlines()] |
| 241 | 222 |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 376 # current code page. | 357 # current code page. |
| 377 if sys.platform == 'win32' and sys.version < '3': | 358 if sys.platform == 'win32' and sys.version < '3': |
| 378 return 'mbcs' | 359 return 'mbcs' |
| 379 # All other platforms use UTF-8. | 360 # All other platforms use UTF-8. |
| 380 # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands | 361 # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands |
| 381 # which will expect arguments to be encoded using the current code | 362 # which will expect arguments to be encoded using the current code |
| 382 # page. | 363 # page. |
| 383 return 'utf-8' | 364 return 'utf-8' |
| 384 | 365 |
| 385 def _should_encode_child_process_arguments(self): | 366 def _should_encode_child_process_arguments(self): |
| 386 # Cygwin's Python's os.execv doesn't support unicode command | |
| 387 # arguments, and neither does Cygwin's execv itself. | |
| 388 if sys.platform == 'cygwin': | |
| 389 return True | |
| 390 | |
| 391 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW | 367 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW |
| 392 # to launch subprocesses, so we have to encode arguments using the | 368 # to launch subprocesses, so we have to encode arguments using the |
| 393 # current code page. | 369 # current code page. |
| 394 if sys.platform == 'win32' and sys.version < '3': | 370 if sys.platform == 'win32' and sys.version < '3': |
| 395 return True | 371 return True |
| 396 | 372 |
| 397 return False | 373 return False |
| 398 | 374 |
| 399 def _encode_argument_if_needed(self, argument): | 375 def _encode_argument_if_needed(self, argument): |
| 400 if not self._should_encode_child_process_arguments(): | 376 if not self._should_encode_child_process_arguments(): |
| (...skipping 13 matching lines...) Expand all Loading... |
| 414 | 390 |
| 415 def call(self, args, **kwargs): | 391 def call(self, args, **kwargs): |
| 416 return subprocess.call(self._stringify_args(args), **kwargs) | 392 return subprocess.call(self._stringify_args(args), **kwargs) |
| 417 | 393 |
| 418 def run_in_parallel(self, command_lines_and_cwds, processes=None): | 394 def run_in_parallel(self, command_lines_and_cwds, processes=None): |
| 419 """Runs a list of (cmd_line list, cwd string) tuples in parallel and ret
urns a list of (retcode, stdout, stderr) tuples.""" | 395 """Runs a list of (cmd_line list, cwd string) tuples in parallel and ret
urns a list of (retcode, stdout, stderr) tuples.""" |
| 420 assert len(command_lines_and_cwds) | 396 assert len(command_lines_and_cwds) |
| 421 return self.map(_run_command_thunk, command_lines_and_cwds, processes) | 397 return self.map(_run_command_thunk, command_lines_and_cwds, processes) |
| 422 | 398 |
| 423 def map(self, thunk, arglist, processes=None): | 399 def map(self, thunk, arglist, processes=None): |
| 424 if sys.platform in ('cygwin', 'win32') or len(arglist) == 1: | 400 if sys.platform == 'win32' or len(arglist) == 1: |
| 425 return map(thunk, arglist) | 401 return map(thunk, arglist) |
| 426 pool = multiprocessing.Pool(processes=(processes or multiprocessing.cpu_
count())) | 402 pool = multiprocessing.Pool(processes=(processes or multiprocessing.cpu_
count())) |
| 427 try: | 403 try: |
| 428 return pool.map(thunk, arglist) | 404 return pool.map(thunk, arglist) |
| 429 finally: | 405 finally: |
| 430 pool.close() | 406 pool.close() |
| 431 pool.join() | 407 pool.join() |
| 432 | 408 |
| 433 | 409 |
| 434 def _run_command_thunk(cmd_line_and_cwd): | 410 def _run_command_thunk(cmd_line_and_cwd): |
| 435 # Note that this needs to be a bare module (and hence Picklable) method to w
ork with multiprocessing.Pool. | 411 # Note that this needs to be a bare module (and hence Picklable) method to w
ork with multiprocessing.Pool. |
| 436 (cmd_line, cwd) = cmd_line_and_cwd | 412 (cmd_line, cwd) = cmd_line_and_cwd |
| 437 proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=su
bprocess.PIPE) | 413 proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=su
bprocess.PIPE) |
| 438 stdout, stderr = proc.communicate() | 414 stdout, stderr = proc.communicate() |
| 439 return (proc.returncode, stdout, stderr) | 415 return (proc.returncode, stdout, stderr) |
| OLD | NEW |