OLD | NEW |
1 """ | 1 """ |
2 KVM test utility functions. | 2 KVM test utility functions. |
3 | 3 |
4 @copyright: 2008-2009 Red Hat Inc. | 4 @copyright: 2008-2009 Red Hat Inc. |
5 """ | 5 """ |
6 | 6 |
7 import time, string, random, socket, os, signal, re, logging, commands, cPickle | 7 import time, string, random, socket, os, signal, re, logging, commands, cPickle |
8 from autotest_lib.client.bin import utils | 8 import fcntl, shelve, ConfigParser |
| 9 from autotest_lib.client.bin import utils, os_dep |
9 from autotest_lib.client.common_lib import error, logging_config | 10 from autotest_lib.client.common_lib import error, logging_config |
10 import kvm_subprocess | 11 import kvm_subprocess |
| 12 try: |
| 13 import koji |
| 14 KOJI_INSTALLED = True |
| 15 except ImportError: |
| 16 KOJI_INSTALLED = False |
| 17 |
| 18 |
| 19 def _lock_file(filename): |
| 20 f = open(filename, "w") |
| 21 fcntl.lockf(f, fcntl.LOCK_EX) |
| 22 return f |
| 23 |
| 24 |
| 25 def _unlock_file(f): |
| 26 fcntl.lockf(f, fcntl.LOCK_UN) |
| 27 f.close() |
11 | 28 |
12 | 29 |
13 def dump_env(obj, filename): | 30 def dump_env(obj, filename): |
14 """ | 31 """ |
15 Dump KVM test environment to a file. | 32 Dump KVM test environment to a file. |
16 | 33 |
17 @param filename: Path to a file where the environment will be dumped to. | 34 @param filename: Path to a file where the environment will be dumped to. |
18 """ | 35 """ |
19 file = open(filename, "w") | 36 file = open(filename, "w") |
20 cPickle.dump(obj, file) | 37 cPickle.dump(obj, file) |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
75 """ | 92 """ |
76 names = dict.get(keyword) | 93 names = dict.get(keyword) |
77 if names: | 94 if names: |
78 return names.split() | 95 return names.split() |
79 else: | 96 else: |
80 return [] | 97 return [] |
81 | 98 |
82 | 99 |
83 # Functions related to MAC/IP addresses | 100 # Functions related to MAC/IP addresses |
84 | 101 |
85 def mac_str_to_int(addr): | 102 def _open_mac_pool(lock_mode): |
86 """ | 103 lock_file = open("/tmp/mac_lock", "w+") |
87 Convert MAC address string to integer. | 104 fcntl.lockf(lock_file, lock_mode) |
88 | 105 pool = shelve.open("/tmp/address_pool") |
89 @param addr: String representing the MAC address. | 106 return pool, lock_file |
90 """ | |
91 return sum(int(s, 16) * 256 ** i | |
92 for i, s in enumerate(reversed(addr.split(":")))) | |
93 | 107 |
94 | 108 |
95 def mac_int_to_str(addr): | 109 def _close_mac_pool(pool, lock_file): |
96 """ | 110 pool.close() |
97 Convert MAC address integer to string. | 111 fcntl.lockf(lock_file, fcntl.LOCK_UN) |
98 | 112 lock_file.close() |
99 @param addr: Integer representing the MAC address. | |
100 """ | |
101 return ":".join("%02x" % (addr >> 8 * i & 0xFF) | |
102 for i in reversed(range(6))) | |
103 | 113 |
104 | 114 |
105 def ip_str_to_int(addr): | 115 def _generate_mac_address_prefix(mac_pool): |
106 """ | 116 """ |
107 Convert IP address string to integer. | 117 Generate a random MAC address prefix and add it to the MAC pool dictionary. |
| 118 If there's a MAC prefix there already, do not update the MAC pool and just |
| 119 return what's in there. By convention we will set KVM autotest MAC |
| 120 addresses to start with 0x9a. |
108 | 121 |
109 @param addr: String representing the IP address. | 122 @param mac_pool: The MAC address pool object. |
| 123 @return: The MAC address prefix. |
110 """ | 124 """ |
111 return sum(int(s) * 256 ** i | 125 if "prefix" in mac_pool: |
112 for i, s in enumerate(reversed(addr.split(".")))) | 126 prefix = mac_pool["prefix"] |
| 127 logging.debug("Used previously generated MAC address prefix for this " |
| 128 "host: %s", prefix) |
| 129 else: |
| 130 r = random.SystemRandom() |
| 131 prefix = "9a:%02x:%02x:%02x:" % (r.randint(0x00, 0xff), |
| 132 r.randint(0x00, 0xff), |
| 133 r.randint(0x00, 0xff)) |
| 134 mac_pool["prefix"] = prefix |
| 135 logging.debug("Generated MAC address prefix for this host: %s", prefix) |
| 136 return prefix |
113 | 137 |
114 | 138 |
115 def ip_int_to_str(addr): | 139 def generate_mac_address(vm_instance, nic_index): |
116 """ | 140 """ |
117 Convert IP address integer to string. | 141 Randomly generate a MAC address and add it to the MAC address pool. |
118 | 142 |
119 @param addr: Integer representing the IP address. | 143 Try to generate a MAC address based on a randomly generated MAC address |
| 144 prefix and add it to a persistent dictionary. |
| 145 key = VM instance + NIC index, value = MAC address |
| 146 e.g. {'20100310-165222-Wt7l:0': '9a:5d:94:6a:9b:f9'} |
| 147 |
| 148 @param vm_instance: The instance attribute of a VM. |
| 149 @param nic_index: The index of the NIC. |
| 150 @return: MAC address string. |
120 """ | 151 """ |
121 return ".".join(str(addr >> 8 * i & 0xFF) | 152 mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) |
122 for i in reversed(range(4))) | 153 key = "%s:%s" % (vm_instance, nic_index) |
| 154 if key in mac_pool: |
| 155 mac = mac_pool[key] |
| 156 else: |
| 157 prefix = _generate_mac_address_prefix(mac_pool) |
| 158 r = random.SystemRandom() |
| 159 while key not in mac_pool: |
| 160 mac = prefix + "%02x:%02x" % (r.randint(0x00, 0xff), |
| 161 r.randint(0x00, 0xff)) |
| 162 if mac in mac_pool.values(): |
| 163 continue |
| 164 mac_pool[key] = mac |
| 165 logging.debug("Generated MAC address for NIC %s: %s", key, mac) |
| 166 _close_mac_pool(mac_pool, lock_file) |
| 167 return mac |
123 | 168 |
124 | 169 |
125 def offset_mac(base, offset): | 170 def free_mac_address(vm_instance, nic_index): |
126 """ | 171 """ |
127 Add offset to a given MAC address. | 172 Remove a MAC address from the address pool. |
128 | 173 |
129 @param base: String representing a MAC address. | 174 @param vm_instance: The instance attribute of a VM. |
130 @param offset: Offset to add to base (integer) | 175 @param nic_index: The index of the NIC. |
131 @return: A string representing the offset MAC address. | |
132 """ | 176 """ |
133 return mac_int_to_str(mac_str_to_int(base) + offset) | 177 mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) |
| 178 key = "%s:%s" % (vm_instance, nic_index) |
| 179 if key in mac_pool: |
| 180 logging.debug("Freeing MAC address for NIC %s: %s", key, mac_pool[key]) |
| 181 del mac_pool[key] |
| 182 _close_mac_pool(mac_pool, lock_file) |
134 | 183 |
135 | 184 |
136 def offset_ip(base, offset): | 185 def set_mac_address(vm_instance, nic_index, mac): |
137 """ | 186 """ |
138 Add offset to a given IP address. | 187 Set a MAC address in the pool. |
139 | 188 |
140 @param base: String representing an IP address. | 189 @param vm_instance: The instance attribute of a VM. |
141 @param offset: Offset to add to base (integer) | 190 @param nic_index: The index of the NIC. |
142 @return: A string representing the offset IP address. | |
143 """ | 191 """ |
144 return ip_int_to_str(ip_str_to_int(base) + offset) | 192 mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) |
| 193 mac_pool["%s:%s" % (vm_instance, nic_index)] = mac |
| 194 _close_mac_pool(mac_pool, lock_file) |
145 | 195 |
146 | 196 |
147 def get_mac_ip_pair_from_dict(dict): | 197 def get_mac_address(vm_instance, nic_index): |
148 """ | 198 """ |
149 Fetch a MAC-IP address pair from dict and return it. | 199 Return a MAC address from the pool. |
150 | 200 |
151 The parameters in dict are expected to conform to a certain syntax. | 201 @param vm_instance: The instance attribute of a VM. |
152 Typical usage may be: | 202 @param nic_index: The index of the NIC. |
153 | 203 @return: MAC address string. |
154 address_ranges = r1 r2 r3 | |
155 | |
156 address_range_base_mac_r1 = 55:44:33:22:11:00 | |
157 address_range_base_ip_r1 = 10.0.0.0 | |
158 address_range_size_r1 = 16 | |
159 | |
160 address_range_base_mac_r2 = 55:44:33:22:11:40 | |
161 address_range_base_ip_r2 = 10.0.0.60 | |
162 address_range_size_r2 = 25 | |
163 | |
164 address_range_base_mac_r3 = 55:44:33:22:12:10 | |
165 address_range_base_ip_r3 = 10.0.1.20 | |
166 address_range_size_r3 = 230 | |
167 | |
168 address_index = 0 | |
169 | |
170 All parameters except address_index specify a MAC-IP address pool. The | |
171 pool consists of several MAC-IP address ranges. | |
172 address_index specified the index of the desired MAC-IP pair from the pool. | |
173 | |
174 @param dict: The dictionary from which to fetch the addresses. | |
175 """ | 204 """ |
176 index = int(dict.get("address_index", 0)) | 205 mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_SH) |
177 for mac_range_name in get_sub_dict_names(dict, "address_ranges"): | 206 mac = mac_pool.get("%s:%s" % (vm_instance, nic_index)) |
178 mac_range_params = get_sub_dict(dict, mac_range_name) | 207 _close_mac_pool(mac_pool, lock_file) |
179 mac_base = mac_range_params.get("address_range_base_mac") | 208 return mac |
180 ip_base = mac_range_params.get("address_range_base_ip") | |
181 size = int(mac_range_params.get("address_range_size", 1)) | |
182 if index < size: | |
183 return (mac_base and offset_mac(mac_base, index), | |
184 ip_base and offset_ip(ip_base, index)) | |
185 index -= size | |
186 return (None, None) | |
187 | |
188 | |
189 def get_sub_pool(dict, piece, num_pieces): | |
190 """ | |
191 Split a MAC-IP pool and return a single requested piece. | |
192 | |
193 For example, get_sub_pool(dict, 0, 3) will split the pool in 3 pieces and | |
194 return a dict representing the first piece. | |
195 | |
196 @param dict: A dict that contains pool parameters. | |
197 @param piece: The index of the requested piece. Should range from 0 to | |
198 num_pieces - 1. | |
199 @param num_pieces: The total number of pieces. | |
200 @return: A copy of dict, modified to describe the requested sub-pool. | |
201 """ | |
202 range_dicts = [get_sub_dict(dict, name) for name in | |
203 get_sub_dict_names(dict, "address_ranges")] | |
204 if not range_dicts: | |
205 return dict | |
206 ranges = [[d.get("address_range_base_mac"), | |
207 d.get("address_range_base_ip"), | |
208 int(d.get("address_range_size", 1))] for d in range_dicts] | |
209 total_size = sum(r[2] for r in ranges) | |
210 base = total_size * piece / num_pieces | |
211 size = total_size * (piece + 1) / num_pieces - base | |
212 | |
213 # Find base of current sub-pool | |
214 for i in range(len(ranges)): | |
215 r = ranges[i] | |
216 if base < r[2]: | |
217 r[0] = r[0] and offset_mac(r[0], base) | |
218 r[1] = r[1] and offset_ip(r[1], base) | |
219 r[2] -= base | |
220 break | |
221 base -= r[2] | |
222 | |
223 # Collect ranges up to end of current sub-pool | |
224 new_ranges = [] | |
225 for i in range(i, len(ranges)): | |
226 r = ranges[i] | |
227 new_ranges.append(r) | |
228 if size <= r[2]: | |
229 r[2] = size | |
230 break | |
231 size -= r[2] | |
232 | |
233 # Write new dict | |
234 new_dict = dict.copy() | |
235 new_dict["address_ranges"] = " ".join("r%d" % i for i in | |
236 range(len(new_ranges))) | |
237 for i in range(len(new_ranges)): | |
238 new_dict["address_range_base_mac_r%d" % i] = new_ranges[i][0] | |
239 new_dict["address_range_base_ip_r%d" % i] = new_ranges[i][1] | |
240 new_dict["address_range_size_r%d" % i] = new_ranges[i][2] | |
241 return new_dict | |
242 | 209 |
243 | 210 |
244 def verify_ip_address_ownership(ip, macs, timeout=10.0): | 211 def verify_ip_address_ownership(ip, macs, timeout=10.0): |
245 """ | 212 """ |
246 Use arping and the ARP cache to make sure a given IP address belongs to one | 213 Use arping and the ARP cache to make sure a given IP address belongs to one |
247 of the given MAC addresses. | 214 of the given MAC addresses. |
248 | 215 |
249 @param ip: An IP address. | 216 @param ip: An IP address. |
250 @param macs: A list or tuple of MAC addresses. | 217 @param macs: A list or tuple of MAC addresses. |
251 @return: True iff ip is assigned to a MAC address in macs. | 218 @return: True iff ip is assigned to a MAC address in macs. |
(...skipping 456 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
708 @return: True on success and False on failure. | 675 @return: True on success and False on failure. |
709 """ | 676 """ |
710 command = ("scp -v -o UserKnownHostsFile=/dev/null " | 677 command = ("scp -v -o UserKnownHostsFile=/dev/null " |
711 "-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" % | 678 "-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" % |
712 (port, username, host, remote_path, local_path)) | 679 (port, username, host, remote_path, local_path)) |
713 return remote_scp(command, password, log_filename, timeout) | 680 return remote_scp(command, password, log_filename, timeout) |
714 | 681 |
715 | 682 |
716 # The following are utility functions related to ports. | 683 # The following are utility functions related to ports. |
717 | 684 |
718 def is_port_free(port): | 685 def is_port_free(port, address): |
719 """ | 686 """ |
720 Return True if the given port is available for use. | 687 Return True if the given port is available for use. |
721 | 688 |
722 @param port: Port number | 689 @param port: Port number |
723 """ | 690 """ |
724 try: | 691 try: |
725 s = socket.socket() | 692 s = socket.socket() |
726 #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | 693 #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
727 s.bind(("localhost", port)) | 694 if address == "localhost": |
728 free = True | 695 s.bind(("localhost", port)) |
| 696 free = True |
| 697 else: |
| 698 s.connect((address, port)) |
| 699 free = False |
729 except socket.error: | 700 except socket.error: |
730 free = False | 701 if address == "localhost": |
| 702 free = False |
| 703 else: |
| 704 free = True |
731 s.close() | 705 s.close() |
732 return free | 706 return free |
733 | 707 |
734 | 708 |
735 def find_free_port(start_port, end_port): | 709 def find_free_port(start_port, end_port, address="localhost"): |
736 """ | 710 """ |
737 Return a host free port in the range [start_port, end_port]. | 711 Return a host free port in the range [start_port, end_port]. |
738 | 712 |
739 @param start_port: First port that will be checked. | 713 @param start_port: First port that will be checked. |
740 @param end_port: Port immediately after the last one that will be checked. | 714 @param end_port: Port immediately after the last one that will be checked. |
741 """ | 715 """ |
742 for i in range(start_port, end_port): | 716 for i in range(start_port, end_port): |
743 if is_port_free(i): | 717 if is_port_free(i, address): |
744 return i | 718 return i |
745 return None | 719 return None |
746 | 720 |
747 | 721 |
748 def find_free_ports(start_port, end_port, count): | 722 def find_free_ports(start_port, end_port, count, address="localhost"): |
749 """ | 723 """ |
750 Return count of host free ports in the range [start_port, end_port]. | 724 Return count of host free ports in the range [start_port, end_port]. |
751 | 725 |
752 @count: Initial number of ports known to be free in the range. | 726 @count: Initial number of ports known to be free in the range. |
753 @param start_port: First port that will be checked. | 727 @param start_port: First port that will be checked. |
754 @param end_port: Port immediately after the last one that will be checked. | 728 @param end_port: Port immediately after the last one that will be checked. |
755 """ | 729 """ |
756 ports = [] | 730 ports = [] |
757 i = start_port | 731 i = start_port |
758 while i < end_port and count > 0: | 732 while i < end_port and count > 0: |
759 if is_port_free(i): | 733 if is_port_free(i, address): |
760 ports.append(i) | 734 ports.append(i) |
761 count -= 1 | 735 count -= 1 |
762 i += 1 | 736 i += 1 |
763 return ports | 737 return ports |
764 | 738 |
765 | 739 |
766 # An easy way to log lines to files when the logging system can't be used | 740 # An easy way to log lines to files when the logging system can't be used |
767 | 741 |
768 _open_log_files = {} | 742 _open_log_files = {} |
769 _log_file_dir = "/tmp" | 743 _log_file_dir = "/tmp" |
(...skipping 501 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1271 virtualization host. | 1245 virtualization host. |
1272 """ | 1246 """ |
1273 try: | 1247 try: |
1274 for pci_id in self.dev_drivers: | 1248 for pci_id in self.dev_drivers: |
1275 if not self._release_dev(pci_id): | 1249 if not self._release_dev(pci_id): |
1276 logging.error("Failed to release device %s to host", pci_id) | 1250 logging.error("Failed to release device %s to host", pci_id) |
1277 else: | 1251 else: |
1278 logging.info("Released device %s successfully", pci_id) | 1252 logging.info("Released device %s successfully", pci_id) |
1279 except: | 1253 except: |
1280 return | 1254 return |
| 1255 |
| 1256 |
| 1257 class KojiDownloader(object): |
| 1258 """ |
| 1259 Stablish a connection with the build system, either koji or brew. |
| 1260 |
| 1261 This class provides a convenience methods to retrieve packages hosted on |
| 1262 the build system. |
| 1263 """ |
| 1264 def __init__(self, cmd): |
| 1265 """ |
| 1266 Verifies whether the system has koji or brew installed, then loads |
| 1267 the configuration file that will be used to download the files. |
| 1268 |
| 1269 @param cmd: Command name, either 'brew' or 'koji'. It is important |
| 1270 to figure out the appropriate configuration used by the |
| 1271 downloader. |
| 1272 @param dst_dir: Destination dir for the packages. |
| 1273 """ |
| 1274 if not KOJI_INSTALLED: |
| 1275 raise ValueError('No koji/brew installed on the machine') |
| 1276 |
| 1277 if os.path.isfile(cmd): |
| 1278 koji_cmd = cmd |
| 1279 else: |
| 1280 koji_cmd = os_dep.command(cmd) |
| 1281 |
| 1282 logging.debug("Found %s as the buildsystem interface", koji_cmd) |
| 1283 |
| 1284 config_map = {'/usr/bin/koji': '/etc/koji.conf', |
| 1285 '/usr/bin/brew': '/etc/brewkoji.conf'} |
| 1286 |
| 1287 try: |
| 1288 config_file = config_map[koji_cmd] |
| 1289 except IndexError: |
| 1290 raise ValueError('Could not find config file for %s' % koji_cmd) |
| 1291 |
| 1292 base_name = os.path.basename(koji_cmd) |
| 1293 if os.access(config_file, os.F_OK): |
| 1294 f = open(config_file) |
| 1295 config = ConfigParser.ConfigParser() |
| 1296 config.readfp(f) |
| 1297 f.close() |
| 1298 else: |
| 1299 raise IOError('Configuration file %s missing or with wrong ' |
| 1300 'permissions' % config_file) |
| 1301 |
| 1302 if config.has_section(base_name): |
| 1303 self.koji_options = {} |
| 1304 session_options = {} |
| 1305 server = None |
| 1306 for name, value in config.items(base_name): |
| 1307 if name in ('user', 'password', 'debug_xmlrpc', 'debug'): |
| 1308 session_options[name] = value |
| 1309 self.koji_options[name] = value |
| 1310 self.session = koji.ClientSession(self.koji_options['server'], |
| 1311 session_options) |
| 1312 else: |
| 1313 raise ValueError('Koji config file %s does not have a %s ' |
| 1314 'session' % (config_file, base_name)) |
| 1315 |
| 1316 |
| 1317 def get(self, src_package, dst_dir, rfilter=None, tag=None, build=None, |
| 1318 arch=None): |
| 1319 """ |
| 1320 Download a list of packages from the build system. |
| 1321 |
| 1322 This will download all packages originated from source package [package] |
| 1323 with given [tag] or [build] for the architecture reported by the |
| 1324 machine. |
| 1325 |
| 1326 @param src_package: Source package name. |
| 1327 @param dst_dir: Destination directory for the downloaded packages. |
| 1328 @param rfilter: Regexp filter, only download the packages that match |
| 1329 that particular filter. |
| 1330 @param tag: Build system tag. |
| 1331 @param build: Build system ID. |
| 1332 @param arch: Package arch. Useful when you want to download noarch |
| 1333 packages. |
| 1334 |
| 1335 @return: List of paths with the downloaded rpm packages. |
| 1336 """ |
| 1337 if build and build.isdigit(): |
| 1338 build = int(build) |
| 1339 |
| 1340 if tag and build: |
| 1341 logging.info("Both tag and build parameters provided, ignoring tag " |
| 1342 "parameter...") |
| 1343 |
| 1344 if not tag and not build: |
| 1345 raise ValueError("Koji install selected but neither koji_tag " |
| 1346 "nor koji_build parameters provided. Please " |
| 1347 "provide an appropriate tag or build name.") |
| 1348 |
| 1349 if not build: |
| 1350 builds = self.session.listTagged(tag, latest=True, |
| 1351 package=src_package) |
| 1352 if not builds: |
| 1353 raise ValueError("Tag %s has no builds of %s" % (tag, |
| 1354 src_package)) |
| 1355 info = builds[0] |
| 1356 else: |
| 1357 info = self.session.getBuild(build) |
| 1358 |
| 1359 if info is None: |
| 1360 raise ValueError('No such brew/koji build: %s' % build) |
| 1361 |
| 1362 if arch is None: |
| 1363 arch = utils.get_arch() |
| 1364 |
| 1365 rpms = self.session.listRPMs(buildID=info['id'], |
| 1366 arches=arch) |
| 1367 if not rpms: |
| 1368 raise ValueError("No %s packages available for %s" % |
| 1369 arch, koji.buildLabel(info)) |
| 1370 |
| 1371 rpm_paths = [] |
| 1372 for rpm in rpms: |
| 1373 rpm_name = koji.pathinfo.rpm(rpm) |
| 1374 url = ("%s/%s/%s/%s/%s" % (self.koji_options['pkgurl'], |
| 1375 info['package_name'], |
| 1376 info['version'], info['release'], |
| 1377 rpm_name)) |
| 1378 if rfilter: |
| 1379 filter_regexp = re.compile(rfilter, re.IGNORECASE) |
| 1380 if filter_regexp.match(os.path.basename(rpm_name)): |
| 1381 download = True |
| 1382 else: |
| 1383 download = False |
| 1384 else: |
| 1385 download = True |
| 1386 |
| 1387 if download: |
| 1388 r = utils.get_file(url, |
| 1389 os.path.join(dst_dir, os.path.basename(url))) |
| 1390 rpm_paths.append(r) |
| 1391 |
| 1392 return rpm_paths |
OLD | NEW |