| OLD | NEW |
| 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Module contains communication methods between cbuildbot instances.""" | 5 """Module contains communication methods between cbuildbot instances.""" |
| 6 | 6 |
| 7 import Queue | 7 import Queue |
| 8 import SocketServer | 8 import SocketServer |
| 9 import os |
| 9 import socket | 10 import socket |
| 10 import sys | 11 import sys |
| 11 import time | 12 import time |
| 12 | 13 |
| 13 from cbuildbot import RunCommand | 14 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) |
| 15 from cros_build_lib import Info, Warning, RunCommand |
| 14 | 16 |
| 15 # Communication port for master to slave communication. | 17 # Communication port for master to slave communication. |
| 16 _COMM_PORT = 32890 | 18 _COMM_PORT = 32890 |
| 17 # TCP Buffer Size. | 19 # TCP Buffer Size. |
| 18 _BUFFER = 4096 | 20 _BUFFER = 4096 |
| 19 # Timeout between checks for new status by either end. | 21 # Timeout between checks for new status by either end. |
| 20 _HEARTBEAT_TIMEOUT = 60 # in sec. | 22 _HEARTBEAT_TIMEOUT = 60 # in sec. |
| 21 # Max Timeout to wait before assuming failure. | 23 # Max Timeout to wait before assuming failure. |
| 22 _MAX_TIMEOUT = 30 * 60 # in sec. | 24 _MAX_TIMEOUT = 30 * 60 # in sec. |
| 23 | 25 |
| (...skipping 21 matching lines...) Expand all Loading... |
| 45 def __init__(self, address, handler, timeout): | 47 def __init__(self, address, handler, timeout): |
| 46 SocketServer.TCPServer.__init__(self, address, handler) | 48 SocketServer.TCPServer.__init__(self, address, handler) |
| 47 self.socket.settimeout(timeout) | 49 self.socket.settimeout(timeout) |
| 48 | 50 |
| 49 | 51 |
| 50 class _SlaveCommandHandler(SocketServer.BaseRequestHandler): | 52 class _SlaveCommandHandler(SocketServer.BaseRequestHandler): |
| 51 """Handles requests from a master pre-flight-queue bot.""" | 53 """Handles requests from a master pre-flight-queue bot.""" |
| 52 | 54 |
| 53 def _HandleCommand(self, command, args): | 55 def _HandleCommand(self, command, args): |
| 54 """Handles command and returns status for master.""" | 56 """Handles command and returns status for master.""" |
| 55 print >> sys.stderr, ('(Slave) - Received command %s with args %s' % | 57 Info('(Slave) - Received command %s with args %s' % (command, args)) |
| 56 (command, args)) | |
| 57 command_to_expect = _command_queue.get() | 58 command_to_expect = _command_queue.get() |
| 58 # Check status also adds an entry on the status queue. | 59 # Check status also adds an entry on the status queue. |
| 59 if command_to_expect == _COMMAND_CHECK_STATUS: | 60 if command_to_expect == _COMMAND_CHECK_STATUS: |
| 60 slave_status = _status_queue.get() | 61 slave_status = _status_queue.get() |
| 61 # Safety check to make sure the server is in a good state. | 62 # Safety check to make sure the server is in a good state. |
| 62 if command_to_expect != command: | 63 if command_to_expect != command: |
| 63 print >> sys.stderr, ( | 64 Warning( |
| 64 '(Slave) - Rejecting command %s. Was expecting %s.' % (command, | 65 '(Slave) - Rejecting command %s. Was expecting %s.' % (command, |
| 65 command_to_expect)) | 66 command_to_expect)) |
| 66 return _STATUS_COMMAND_REJECTED | 67 return _STATUS_COMMAND_REJECTED |
| 67 # Give slave command with optional args. | 68 # Give slave command with optional args. |
| 68 _receive_queue.put(args) | 69 _receive_queue.put(args) |
| 69 if command == _COMMAND_CHECK_STATUS: | 70 if command == _COMMAND_CHECK_STATUS: |
| 70 # Returns status to send. | 71 # Returns status to send. |
| 71 return slave_status | 72 return slave_status |
| 72 | 73 |
| 73 def handle(self): | 74 def handle(self): |
| (...skipping 10 matching lines...) Expand all Loading... |
| 84 for slave_config in configuration.items(): | 85 for slave_config in configuration.items(): |
| 85 if (not slave_config[1]['master'] and | 86 if (not slave_config[1]['master'] and |
| 86 slave_config[1]['important']): | 87 slave_config[1]['important']): |
| 87 slaves.append(slave_config[1]['hostname']) | 88 slaves.append(slave_config[1]['hostname']) |
| 88 return slaves | 89 return slaves |
| 89 | 90 |
| 90 | 91 |
| 91 def _SendCommand(hostname, command, args): | 92 def _SendCommand(hostname, command, args): |
| 92 """Returns response from host or _STATUS_TIMEOUT on error.""" | 93 """Returns response from host or _STATUS_TIMEOUT on error.""" |
| 93 data = '%s\n%s\n' % (command, args) | 94 data = '%s\n%s\n' % (command, args) |
| 94 print '(Master) - Sending %s %s to %s' % (command, args, hostname) | 95 Info('(Master) - Sending %s %s to %s' % (command, args, hostname)) |
| 95 | 96 |
| 96 # Create a socket (SOCK_STREAM means a TCP socket). | 97 # Create a socket (SOCK_STREAM means a TCP socket). |
| 97 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | 98 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 98 | 99 |
| 99 try: | 100 try: |
| 100 # Connect to server and send data | 101 # Connect to server and send data |
| 101 sock.connect((hostname, _COMM_PORT)) | 102 sock.connect((hostname, _COMM_PORT)) |
| 102 sock.send(data) | 103 sock.send(data) |
| 103 | 104 |
| 104 # Receive data from the server and shut down. | 105 # Receive data from the server and shut down. |
| (...skipping 12 matching lines...) Expand all Loading... |
| 117 True as long as no slave reports STATUS_BUILD_FAILED. | 118 True as long as no slave reports STATUS_BUILD_FAILED. |
| 118 | 119 |
| 119 Keyword arguments: | 120 Keyword arguments: |
| 120 slaves_to_check -- Array of hostnames to check. | 121 slaves_to_check -- Array of hostnames to check. |
| 121 | 122 |
| 122 """ | 123 """ |
| 123 slaves_to_remove = [] | 124 slaves_to_remove = [] |
| 124 for slave in slaves_to_check: | 125 for slave in slaves_to_check: |
| 125 status = _SendCommand(slave, _COMMAND_CHECK_STATUS, 'empty') | 126 status = _SendCommand(slave, _COMMAND_CHECK_STATUS, 'empty') |
| 126 if status == STATUS_BUILD_FAILED: | 127 if status == STATUS_BUILD_FAILED: |
| 127 print >> sys.stderr, '(Master) - Slave %s failed' % slave | 128 Warning('(Master) - Slave %s failed' % slave) |
| 128 return False | 129 return False |
| 129 elif status == STATUS_BUILD_COMPLETE: | 130 elif status == STATUS_BUILD_COMPLETE: |
| 130 print >> sys.stderr, '(Master) - Slave %s completed' % slave | 131 Info('(Master) - Slave %s completed' % slave) |
| 131 slaves_to_remove.append(slave) | 132 slaves_to_remove.append(slave) |
| 132 for slave in slaves_to_remove: | 133 for slave in slaves_to_remove: |
| 133 slaves_to_check.remove(slave) | 134 slaves_to_check.remove(slave) |
| 134 return True | 135 return True |
| 135 | 136 |
| 136 | 137 |
| 137 def HaveSlavesCompleted(configuration): | 138 def HaveSlavesCompleted(configuration): |
| 138 """Returns True if all other slaves have succeeded. | 139 """Returns True if all other slaves have succeeded. |
| 139 | 140 |
| 140 Checks other slaves status until either '_MAX_TIMEOUT' has passed, | 141 Checks other slaves status until either '_MAX_TIMEOUT' has passed, |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 178 server = _TCPServerWithReuse(('localhost', _COMM_PORT), | 179 server = _TCPServerWithReuse(('localhost', _COMM_PORT), |
| 179 _SlaveCommandHandler, _HEARTBEAT_TIMEOUT) | 180 _SlaveCommandHandler, _HEARTBEAT_TIMEOUT) |
| 180 timeout = 0 | 181 timeout = 0 |
| 181 response = None | 182 response = None |
| 182 try: | 183 try: |
| 183 while not response and timeout < _MAX_TIMEOUT: | 184 while not response and timeout < _MAX_TIMEOUT: |
| 184 server.handle_request() | 185 server.handle_request() |
| 185 try: | 186 try: |
| 186 response = _receive_queue.get_nowait() | 187 response = _receive_queue.get_nowait() |
| 187 except Queue.Empty: | 188 except Queue.Empty: |
| 188 print >> sys.stderr, ('(Slave) - Waiting for master to accept %s' % ( | 189 Info('(Slave) - Waiting for master to accept %s' % status) |
| 189 status)) | |
| 190 timeout += _HEARTBEAT_TIMEOUT | 190 timeout += _HEARTBEAT_TIMEOUT |
| 191 response = None | 191 response = None |
| 192 except Exception, e: | 192 except Exception, e: |
| 193 print >> sys.stderr, '%s' % e | 193 Warning('%s' % e) |
| 194 server.server_close() | 194 server.server_close() |
| 195 return response != None | 195 return response != None |
| OLD | NEW |