| OLD | NEW |
|---|
| 1 # | 1 """ |
| 2 # Copyright 2008 Google Inc. Released under the GPL v2 | 2 Convenience functions for use by tests or whomever. |
| 3 | 3 |
| 4 import os, pickle, random, re, resource, select, shutil, signal, StringIO | 4 NOTE: this is a mixin library that pulls in functions from several places |
| 5 import socket, struct, subprocess, sys, time, textwrap, urlparse | 5 Note carefully what the precendece order is |
| 6 import warnings, smtplib, logging, urllib2 | |
| 7 from threading import Thread, Event | |
| 8 try: | |
| 9 import hashlib | |
| 10 except ImportError: | |
| 11 import md5, sha | |
| 12 from autotest_lib.client.common_lib import error, logging_manager | |
| 13 | 6 |
| 14 def deprecated(func): | 7 There's no really good way to do this, as this isn't a class we can do |
| 15 """This is a decorator which can be used to mark functions as deprecated. | 8 inheritance with, just a collection of static methods. |
| 16 It will result in a warning being emmitted when the function is used.""" | 9 """ |
| 17 def new_func(*args, **dargs): | |
| 18 warnings.warn("Call to deprecated function %s." % func.__name__, | |
| 19 category=DeprecationWarning) | |
| 20 return func(*args, **dargs) | |
| 21 new_func.__name__ = func.__name__ | |
| 22 new_func.__doc__ = func.__doc__ | |
| 23 new_func.__dict__.update(func.__dict__) | |
| 24 return new_func | |
| 25 | 10 |
| 26 | 11 from autotest_lib.client.common_lib.base_utils import * |
| 27 class _NullStream(object): | 12 if os.path.exists(os.path.join(os.path.dirname(__file__), 'site_utils.py')): |
| 28 def write(self, data): | 13 from autotest_lib.client.common_lib.site_utils import * |
| 29 pass | |
| 30 | |
| 31 | |
| 32 def flush(self): | |
| 33 pass | |
| 34 | |
| 35 | |
| 36 TEE_TO_LOGS = object() | |
| 37 _the_null_stream = _NullStream() | |
| 38 | |
| 39 DEFAULT_STDOUT_LEVEL = logging.DEBUG | |
| 40 DEFAULT_STDERR_LEVEL = logging.ERROR | |
| 41 | |
| 42 # prefixes for logging stdout/stderr of commands | |
| 43 STDOUT_PREFIX = '[stdout] ' | |
| 44 STDERR_PREFIX = '[stderr] ' | |
| 45 | |
| 46 | |
| 47 def get_stream_tee_file(stream, level, prefix=''): | |
| 48 if stream is None: | |
| 49 return _the_null_stream | |
| 50 if stream is TEE_TO_LOGS: | |
| 51 return logging_manager.LoggingFile(level=level, prefix=prefix) | |
| 52 return stream | |
| 53 | |
| 54 | |
| 55 class BgJob(object): | |
| 56 def __init__(self, command, stdout_tee=None, stderr_tee=None, verbose=True, | |
| 57 stdin=None, stderr_level=DEFAULT_STDERR_LEVEL): | |
| 58 self.command = command | |
| 59 self.stdout_tee = get_stream_tee_file(stdout_tee, DEFAULT_STDOUT_LEVEL, | |
| 60 prefix=STDOUT_PREFIX) | |
| 61 self.stderr_tee = get_stream_tee_file(stderr_tee, stderr_level, | |
| 62 prefix=STDERR_PREFIX) | |
| 63 self.result = CmdResult(command) | |
| 64 | |
| 65 # allow for easy stdin input by string, we'll let subprocess create | |
| 66 # a pipe for stdin input and we'll write to it in the wait loop | |
| 67 if isinstance(stdin, basestring): | |
| 68 self.string_stdin = stdin | |
| 69 stdin = subprocess.PIPE | |
| 70 else: | |
| 71 self.string_stdin = None | |
| 72 | |
| 73 if verbose: | |
| 74 logging.debug("Running '%s'" % command) | |
| 75 self.sp = subprocess.Popen(command, stdout=subprocess.PIPE, | |
| 76 stderr=subprocess.PIPE, | |
| 77 preexec_fn=self._reset_sigpipe, shell=True, | |
| 78 executable="/bin/bash", | |
| 79 stdin=stdin) | |
| 80 | |
| 81 | |
| 82 def output_prepare(self, stdout_file=None, stderr_file=None): | |
| 83 self.stdout_file = stdout_file | |
| 84 self.stderr_file = stderr_file | |
| 85 | |
| 86 | |
| 87 def process_output(self, stdout=True, final_read=False): | |
| 88 """output_prepare must be called prior to calling this""" | |
| 89 if stdout: | |
| 90 pipe, buf, tee = self.sp.stdout, self.stdout_file, self.stdout_tee | |
| 91 else: | |
| 92 pipe, buf, tee = self.sp.stderr, self.stderr_file, self.stderr_tee | |
| 93 | |
| 94 if final_read: | |
| 95 # read in all the data we can from pipe and then stop | |
| 96 data = [] | |
| 97 while select.select([pipe], [], [], 0)[0]: | |
| 98 data.append(os.read(pipe.fileno(), 1024)) | |
| 99 if len(data[-1]) == 0: | |
| 100 break | |
| 101 data = "".join(data) | |
| 102 else: | |
| 103 # perform a single read | |
| 104 data = os.read(pipe.fileno(), 1024) | |
| 105 buf.write(data) | |
| 106 tee.write(data) | |
| 107 | |
| 108 | |
| 109 def cleanup(self): | |
| 110 self.stdout_tee.flush() | |
| 111 self.stderr_tee.flush() | |
| 112 self.sp.stdout.close() | |
| 113 self.sp.stderr.close() | |
| 114 self.result.stdout = self.stdout_file.getvalue() | |
| 115 self.result.stderr = self.stderr_file.getvalue() | |
| 116 | |
| 117 | |
| 118 def _reset_sigpipe(self): | |
| 119 signal.signal(signal.SIGPIPE, signal.SIG_DFL) | |
| 120 | |
| 121 | |
| 122 def ip_to_long(ip): | |
| 123 # !L is a long in network byte order | |
| 124 return struct.unpack('!L', socket.inet_aton(ip))[0] | |
| 125 | |
| 126 | |
| 127 def long_to_ip(number): | |
| 128 # See above comment. | |
| 129 return socket.inet_ntoa(struct.pack('!L', number)) | |
| 130 | |
| 131 | |
| 132 def create_subnet_mask(bits): | |
| 133 return (1 << 32) - (1 << 32-bits) | |
| 134 | |
| 135 | |
| 136 def format_ip_with_mask(ip, mask_bits): | |
| 137 masked_ip = ip_to_long(ip) & create_subnet_mask(mask_bits) | |
| 138 return "%s/%s" % (long_to_ip(masked_ip), mask_bits) | |
| 139 | |
| 140 | |
| 141 def normalize_hostname(alias): | |
| 142 ip = socket.gethostbyname(alias) | |
| 143 return socket.gethostbyaddr(ip)[0] | |
| 144 | |
| 145 | |
| 146 def get_ip_local_port_range(): | |
| 147 match = re.match(r'\s*(\d+)\s*(\d+)\s*$', | |
| 148 read_one_line('/proc/sys/net/ipv4/ip_local_port_range')) | |
| 149 return (int(match.group(1)), int(match.group(2))) | |
| 150 | |
| 151 | |
| 152 def set_ip_local_port_range(lower, upper): | |
| 153 write_one_line('/proc/sys/net/ipv4/ip_local_port_range', | |
| 154 '%d %d\n' % (lower, upper)) | |
| 155 | |
| 156 | |
| 157 | |
| 158 def send_email(mail_from, mail_to, subject, body): | |
| 159 """ | |
| 160 Sends an email via smtp | |
| 161 | |
| 162 mail_from: string with email address of sender | |
| 163 mail_to: string or list with email address(es) of recipients | |
| 164 subject: string with subject of email | |
| 165 body: (multi-line) string with body of email | |
| 166 """ | |
| 167 if isinstance(mail_to, str): | |
| 168 mail_to = [mail_to] | |
| 169 msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (mail_from, ','.join(mail_to), | |
| 170 subject, body) | |
| 171 try: | |
| 172 mailer = smtplib.SMTP('localhost') | |
| 173 try: | |
| 174 mailer.sendmail(mail_from, mail_to, msg) | |
| 175 finally: | |
| 176 mailer.quit() | |
| 177 except Exception, e: | |
| 178 # Emails are non-critical, not errors, but don't raise them | |
| 179 print "Sending email failed. Reason: %s" % repr(e) | |
| 180 | |
| 181 | |
| 182 def read_one_line(filename): | |
| 183 return open(filename, 'r').readline().rstrip('\n') | |
| 184 | |
| 185 | |
| 186 def read_file(filename): | |
| 187 f = open(filename) | |
| 188 try: | |
| 189 return f.read() | |
| 190 finally: | |
| 191 f.close() | |
| 192 | |
| 193 | |
| 194 def get_field(data, param, linestart="", sep=" "): | |
| 195 """ | |
| 196 Parse data from string. | |
| 197 @param data: Data to parse. | |
| 198 example: | |
| 199 data: | |
| 200 cpu 324 345 34 5 345 | |
| 201 cpu0 34 11 34 34 33 | |
| 202 ^^^^ | |
| 203 start of line | |
| 204 params 0 1 2 3 4 | |
| 205 @param param: Position of parameter after linestart marker. | |
| 206 @param linestart: String to which start line with parameters. | |
| 207 @param sep: Separator between parameters regular expression. | |
| 208 """ | |
| 209 search = re.compile(r"(?<=^%s)\s*(.*)" % linestart, re.MULTILINE) | |
| 210 find = search.search(data) | |
| 211 if find != None: | |
| 212 return re.split("%s" % sep, find.group(1))[param] | |
| 213 else: | |
| 214 print "There is no line which starts with %s in data." % linestart | |
| 215 return None | |
| 216 | |
| 217 | |
| 218 def write_one_line(filename, line): | |
| 219 open_write_close(filename, line.rstrip('\n') + '\n') | |
| 220 | |
| 221 | |
| 222 def open_write_close(filename, data): | |
| 223 f = open(filename, 'w') | |
| 224 try: | |
| 225 f.write(data) | |
| 226 finally: | |
| 227 f.close() | |
| 228 | |
| 229 | |
| 230 def matrix_to_string(matrix, header=None): | |
| 231 """ | |
| 232 Return a pretty, aligned string representation of a nxm matrix. | |
| 233 | |
| 234 This representation can be used to print any tabular data, such as | |
| 235 database results. It works by scanning the lengths of each element | |
| 236 in each column, and determining the format string dynamically. | |
| 237 | |
| 238 @param matrix: Matrix representation (list with n rows of m elements). | |
| 239 @param header: Optional tuple or list with header elements to be displayed. | |
| 240 """ | |
| 241 if type(header) is list: | |
| 242 header = tuple(header) | |
| 243 lengths = [] | |
| 244 if header: | |
| 245 for column in header: | |
| 246 lengths.append(len(column)) | |
| 247 for row in matrix: | |
| 248 for column in row: | |
| 249 i = row.index(column) | |
| 250 cl = len(column) | |
| 251 try: | |
| 252 ml = lengths[i] | |
| 253 if cl > ml: | |
| 254 lengths[i] = cl | |
| 255 except IndexError: | |
| 256 lengths.append(cl) | |
| 257 | |
| 258 lengths = tuple(lengths) | |
| 259 format_string = "" | |
| 260 for length in lengths: | |
| 261 format_string += "%-" + str(length) + "s " | |
| 262 format_string += "\n" | |
| 263 | |
| 264 matrix_str = "" | |
| 265 if header: | |
| 266 matrix_str += format_string % header | |
| 267 for row in matrix: | |
| 268 matrix_str += format_string % tuple(row) | |
| 269 | |
| 270 return matrix_str | |
| 271 | |
| 272 | |
| 273 def read_keyval(path): | |
| 274 """ | |
| 275 Read a key-value pair format file into a dictionary, and return it. | |
| 276 Takes either a filename or directory name as input. If it's a | |
| 277 directory name, we assume you want the file to be called keyval. | |
| 278 """ | |
| 279 if os.path.isdir(path): | |
| 280 path = os.path.join(path, 'keyval') | |
| 281 keyval = {} | |
| 282 if os.path.exists(path): | |
| 283 for line in open(path): | |
| 284 line = re.sub('#.*', '', line).rstrip() | |
| 285 if not re.search(r'^[-\.\w]+=', line): | |
| 286 raise ValueError('Invalid format line: %s' % line) | |
| 287 key, value = line.split('=', 1) | |
| 288 if re.search('^\d+$', value): | |
| 289 value = int(value) | |
| 290 elif re.search('^(\d+\.)?\d+$', value): | |
| 291 value = float(value) | |
| 292 keyval[key] = value | |
| 293 return keyval | |
| 294 | |
| 295 | |
| 296 def write_keyval(path, dictionary, type_tag=None): | |
| 297 """ | |
| 298 Write a key-value pair format file out to a file. This uses append | |
| 299 mode to open the file, so existing text will not be overwritten or | |
| 300 reparsed. | |
| 301 | |
| 302 If type_tag is None, then the key must be composed of alphanumeric | |
| 303 characters (or dashes+underscores). However, if type-tag is not | |
| 304 null then the keys must also have "{type_tag}" as a suffix. At | |
| 305 the moment the only valid values of type_tag are "attr" and "perf". | |
| 306 """ | |
| 307 if os.path.isdir(path): | |
| 308 path = os.path.join(path, 'keyval') | |
| 309 keyval = open(path, 'a') | |
| 310 | |
| 311 if type_tag is None: | |
| 312 key_regex = re.compile(r'^[-\.\w]+$') | |
| 313 else: | |
| 314 if type_tag not in ('attr', 'perf'): | |
| 315 raise ValueError('Invalid type tag: %s' % type_tag) | |
| 316 escaped_tag = re.escape(type_tag) | |
| 317 key_regex = re.compile(r'^[-\.\w]+\{%s\}$' % escaped_tag) | |
| 318 try: | |
| 319 for key in sorted(dictionary.keys()): | |
| 320 if not key_regex.search(key): | |
| 321 raise ValueError('Invalid key: %s' % key) | |
| 322 keyval.write('%s=%s\n' % (key, dictionary[key])) | |
| 323 finally: | |
| 324 keyval.close() | |
| 325 | |
| 326 | |
| 327 class FileFieldMonitor(object): | |
| 328 """ | |
| 329 Monitors the information from the file and reports it's values. | |
| 330 | |
| 331 It gather the information at start and stop of the measurement or | |
| 332 continuously during the measurement. | |
| 333 """ | |
| 334 class Monitor(Thread): | |
| 335 """ | |
| 336 Internal monitor class to ensure continuous monitor of monitored file. | |
| 337 """ | |
| 338 def __init__(self, master): | |
| 339 """ | |
| 340 @param master: Master class which control Monitor | |
| 341 """ | |
| 342 Thread.__init__(self) | |
| 343 self.master = master | |
| 344 | |
| 345 def run(self): | |
| 346 """ | |
| 347 Start monitor in thread mode | |
| 348 """ | |
| 349 while not self.master.end_event.isSet(): | |
| 350 self.master._get_value(self.master.logging) | |
| 351 time.sleep(self.master.time_step) | |
| 352 | |
| 353 | |
| 354 def __init__(self, status_file, data_to_read, mode_diff, continuously=False, | |
| 355 contlogging=False, separator=" +", time_step=0.1): | |
| 356 """ | |
| 357 Initialize variables. | |
| 358 @param status_file: File contain status. | |
| 359 @param mode_diff: If True make a difference of value, else average. | |
| 360 @param data_to_read: List of tuples with data position. | |
| 361 format: [(start_of_line,position in params)] | |
| 362 example: | |
| 363 data: | |
| 364 cpu 324 345 34 5 345 | |
| 365 cpu0 34 11 34 34 33 | |
| 366 ^^^^ | |
| 367 start of line | |
| 368 params 0 1 2 3 4 | |
| 369 @param mode_diff: True to subtract old value from new value, | |
| 370 False make average of the values. | |
| 371 @parma continuously: Start the monitoring thread using the time_step | |
| 372 as the measurement period. | |
| 373 @param contlogging: Log data in continuous run. | |
| 374 @param separator: Regular expression of separator. | |
| 375 @param time_step: Time period of the monitoring value. | |
| 376 """ | |
| 377 self.end_event = Event() | |
| 378 self.start_time = 0 | |
| 379 self.end_time = 0 | |
| 380 self.test_time = 0 | |
| 381 | |
| 382 self.status_file = status_file | |
| 383 self.separator = separator | |
| 384 self.data_to_read = data_to_read | |
| 385 self.num_of_params = len(self.data_to_read) | |
| 386 self.mode_diff = mode_diff | |
| 387 self.continuously = continuously | |
| 388 self.time_step = time_step | |
| 389 | |
| 390 self.value = [0 for i in range(self.num_of_params)] | |
| 391 self.old_value = [0 for i in range(self.num_of_params)] | |
| 392 self.log = [] | |
| 393 self.logging = contlogging | |
| 394 | |
| 395 self.started = False | |
| 396 self.num_of_get_value = 0 | |
| 397 self.monitor = None | |
| 398 | |
| 399 | |
| 400 def _get_value(self, logging=True): | |
| 401 """ | |
| 402 Return current values. | |
| 403 @param logging: If true log value in memory. There can be problem | |
| 404 with long run. | |
| 405 """ | |
| 406 data = read_file(self.status_file) | |
| 407 value = [] | |
| 408 for i in range(self.num_of_params): | |
| 409 value.append(int(get_field(data, | |
| 410 self.data_to_read[i][1], | |
| 411 self.data_to_read[i][0], | |
| 412 self.separator))) | |
| 413 | |
| 414 if logging: | |
| 415 self.log.append(value) | |
| 416 if not self.mode_diff: | |
| 417 value = map(lambda x, y: x + y, value, self.old_value) | |
| 418 | |
| 419 self.old_value = value | |
| 420 self.num_of_get_value += 1 | |
| 421 return value | |
| 422 | |
| 423 | |
| 424 def start(self): | |
| 425 """ | |
| 426 Start value monitor. | |
| 427 """ | |
| 428 if self.started: | |
| 429 self.stop() | |
| 430 self.old_value = [0 for i in range(self.num_of_params)] | |
| 431 self.num_of_get_value = 0 | |
| 432 self.log = [] | |
| 433 self.end_event.clear() | |
| 434 self.start_time = time.time() | |
| 435 self._get_value() | |
| 436 self.started = True | |
| 437 if (self.continuously): | |
| 438 self.monitor = FileFieldMonitor.Monitor(self) | |
| 439 self.monitor.start() | |
| 440 | |
| 441 | |
| 442 def stop(self): | |
| 443 """ | |
| 444 Stop value monitor. | |
| 445 """ | |
| 446 if self.started: | |
| 447 self.started = False | |
| 448 self.end_time = time.time() | |
| 449 self.test_time = self.end_time - self.start_time | |
| 450 self.value = self._get_value() | |
| 451 if (self.continuously): | |
| 452 self.end_event.set() | |
| 453 self.monitor.join() | |
| 454 if (self.mode_diff): | |
| 455 self.value = map(lambda x, y: x - y, self.log[-1], self.log[0]) | |
| 456 else: | |
| 457 self.value = map(lambda x: x / self.num_of_get_value, | |
| 458 self.value) | |
| 459 | |
| 460 | |
| 461 def get_status(self): | |
| 462 """ | |
| 463 @return: Status of monitored process average value, | |
| 464 time of test and array of monitored values and time step of | |
| 465 continuous run. | |
| 466 """ | |
| 467 if self.started: | |
| 468 self.stop() | |
| 469 if self.mode_diff: | |
| 470 for i in range(len(self.log) - 1): | |
| 471 self.log[i] = (map(lambda x, y: x - y, | |
| 472 self.log[i + 1], self.log[i])) | |
| 473 self.log.pop() | |
| 474 return (self.value, self.test_time, self.log, self.time_step) | |
| 475 | |
| 476 | |
| 477 def is_url(path): | |
| 478 """Return true if path looks like a URL""" | |
| 479 # for now, just handle http and ftp | |
| 480 url_parts = urlparse.urlparse(path) | |
| 481 return (url_parts[0] in ('http', 'ftp')) | |
| 482 | |
| 483 | |
| 484 def urlopen(url, data=None, timeout=5): | |
| 485 """Wrapper to urllib2.urlopen with timeout addition.""" | |
| 486 | |
| 487 # Save old timeout | |
| 488 old_timeout = socket.getdefaulttimeout() | |
| 489 socket.setdefaulttimeout(timeout) | |
| 490 try: | |
| 491 return urllib2.urlopen(url, data=data) | |
| 492 finally: | |
| 493 socket.setdefaulttimeout(old_timeout) | |
| 494 | |
| 495 | |
| 496 def urlretrieve(url, filename, data=None, timeout=300): | |
| 497 """Retrieve a file from given url.""" | |
| 498 logging.debug('Fetching %s -> %s', url, filename) | |
| 499 | |
| 500 src_file = urlopen(url, data=data, timeout=timeout) | |
| 501 try: | |
| 502 dest_file = open(filename, 'wb') | |
| 503 try: | |
| 504 shutil.copyfileobj(src_file, dest_file) | |
| 505 finally: | |
| 506 dest_file.close() | |
| 507 finally: | |
| 508 src_file.close() | |
| 509 | |
| 510 | |
| 511 def hash(type, input=None): | |
| 512 """ | |
| 513 Returns an hash object of type md5 or sha1. This function is implemented in | |
| 514 order to encapsulate hash objects in a way that is compatible with python | |
| 515 2.4 and python 2.6 without warnings. | |
| 516 | |
| 517 Note that even though python 2.6 hashlib supports hash types other than | |
| 518 md5 and sha1, we are artificially limiting the input values in order to | |
| 519 make the function to behave exactly the same among both python | |
| 520 implementations. | |
| 521 | |
| 522 @param input: Optional input string that will be used to update the hash. | |
| 523 """ | |
| 524 if type not in ['md5', 'sha1']: | |
| 525 raise ValueError("Unsupported hash type: %s" % type) | |
| 526 | |
| 527 try: | |
| 528 hash = hashlib.new(type) | |
| 529 except NameError: | |
| 530 if type == 'md5': | |
| 531 hash = md5.new() | |
| 532 elif type == 'sha1': | |
| 533 hash = sha.new() | |
| 534 | |
| 535 if input: | |
| 536 hash.update(input) | |
| 537 | |
| 538 return hash | |
| 539 | |
| 540 | |
| 541 def get_file(src, dest, permissions=None): | |
| 542 """Get a file from src, which can be local or a remote URL""" | |
| 543 if src == dest: | |
| 544 return | |
| 545 | |
| 546 if is_url(src): | |
| 547 urlretrieve(src, dest) | |
| 548 else: | |
| 549 shutil.copyfile(src, dest) | |
| 550 | |
| 551 if permissions: | |
| 552 os.chmod(dest, permissions) | |
| 553 return dest | |
| 554 | |
| 555 | |
| 556 def unmap_url(srcdir, src, destdir='.'): | |
| 557 """ | |
| 558 Receives either a path to a local file or a URL. | |
| 559 returns either the path to the local file, or the fetched URL | |
| 560 | |
| 561 unmap_url('/usr/src', 'foo.tar', '/tmp') | |
| 562 = '/usr/src/foo.tar' | |
| 563 unmap_url('/usr/src', 'http://site/file', '/tmp') | |
| 564 = '/tmp/file' | |
| 565 (after retrieving it) | |
| 566 """ | |
| 567 if is_url(src): | |
| 568 url_parts = urlparse.urlparse(src) | |
| 569 filename = os.path.basename(url_parts[2]) | |
| 570 dest = os.path.join(destdir, filename) | |
| 571 return get_file(src, dest) | |
| 572 else: | |
| 573 return os.path.join(srcdir, src) | |
| 574 | |
| 575 | |
| 576 def update_version(srcdir, preserve_srcdir, new_version, install, | |
| 577 *args, **dargs): | |
| 578 """ | |
| 579 Make sure srcdir is version new_version | |
| 580 | |
| 581 If not, delete it and install() the new version. | |
| 582 | |
| 583 In the preserve_srcdir case, we just check it's up to date, | |
| 584 and if not, we rerun install, without removing srcdir | |
| 585 """ | |
| 586 versionfile = os.path.join(srcdir, '.version') | |
| 587 install_needed = True | |
| 588 | |
| 589 if os.path.exists(versionfile): | |
| 590 old_version = pickle.load(open(versionfile)) | |
| 591 if old_version == new_version: | |
| 592 install_needed = False | |
| 593 | |
| 594 if install_needed: | |
| 595 if not preserve_srcdir and os.path.exists(srcdir): | |
| 596 shutil.rmtree(srcdir) | |
| 597 install(*args, **dargs) | |
| 598 if os.path.exists(srcdir): | |
| 599 pickle.dump(new_version, open(versionfile, 'w')) | |
| 600 | |
| 601 | |
| 602 def get_stderr_level(stderr_is_expected): | |
| 603 if stderr_is_expected: | |
| 604 return DEFAULT_STDOUT_LEVEL | |
| 605 return DEFAULT_STDERR_LEVEL | |
| 606 | |
| 607 | |
| 608 def run(command, timeout=None, ignore_status=False, | |
| 609 stdout_tee=None, stderr_tee=None, verbose=True, stdin=None, | |
| 610 stderr_is_expected=None, args=()): | |
| 611 """ | |
| 612 Run a command on the host. | |
| 613 | |
| 614 @param command: the command line string. | |
| 615 @param timeout: time limit in seconds before attempting to kill the | |
| 616 running process. The run() function will take a few seconds | |
| 617 longer than 'timeout' to complete if it has to kill the process. | |
| 618 @param ignore_status: do not raise an exception, no matter what the exit | |
| 619 code of the command is. | |
| 620 @param stdout_tee: optional file-like object to which stdout data | |
| 621 will be written as it is generated (data will still be stored | |
| 622 in result.stdout). | |
| 623 @param stderr_tee: likewise for stderr. | |
| 624 @param verbose: if True, log the command being run. | |
| 625 @param stdin: stdin to pass to the executed process (can be a file | |
| 626 descriptor, a file object of a real file or a string). | |
| 627 @param args: sequence of strings of arguments to be given to the command | |
| 628 inside " quotes after they have been escaped for that; each | |
| 629 element in the sequence will be given as a separate command | |
| 630 argument | |
| 631 | |
| 632 @return a CmdResult object | |
| 633 | |
| 634 @raise CmdError: the exit code of the command execution was not 0 | |
| 635 """ | |
| 636 if isinstance(args, basestring): | |
| 637 raise TypeError('Got a string for the "args" keyword argument, ' | |
| 638 'need a sequence.') | |
| 639 | |
| 640 for arg in args: | |
| 641 command += ' "%s"' % sh_escape(arg) | |
| 642 if stderr_is_expected is None: | |
| 643 stderr_is_expected = ignore_status | |
| 644 | |
| 645 bg_job = join_bg_jobs( | |
| 646 (BgJob(command, stdout_tee, stderr_tee, verbose, stdin=stdin, | |
| 647 stderr_level=get_stderr_level(stderr_is_expected)),), | |
| 648 timeout)[0] | |
| 649 if not ignore_status and bg_job.result.exit_status: | |
| 650 raise error.CmdError(command, bg_job.result, | |
| 651 "Command returned non-zero exit status") | |
| 652 | |
| 653 return bg_job.result | |
| 654 | |
| 655 | |
| 656 def run_parallel(commands, timeout=None, ignore_status=False, | |
| 657 stdout_tee=None, stderr_tee=None): | |
| 658 """ | |
| 659 Behaves the same as run() with the following exceptions: | |
| 660 | |
| 661 - commands is a list of commands to run in parallel. | |
| 662 - ignore_status toggles whether or not an exception should be raised | |
| 663 on any error. | |
| 664 | |
| 665 @return: a list of CmdResult objects | |
| 666 """ | |
| 667 bg_jobs = [] | |
| 668 for command in commands: | |
| 669 bg_jobs.append(BgJob(command, stdout_tee, stderr_tee, | |
| 670 stderr_level=get_stderr_level(ignore_status))) | |
| 671 | |
| 672 # Updates objects in bg_jobs list with their process information | |
| 673 join_bg_jobs(bg_jobs, timeout) | |
| 674 | |
| 675 for bg_job in bg_jobs: | |
| 676 if not ignore_status and bg_job.result.exit_status: | |
| 677 raise error.CmdError(command, bg_job.result, | |
| 678 "Command returned non-zero exit status") | |
| 679 | |
| 680 return [bg_job.result for bg_job in bg_jobs] | |
| 681 | |
| 682 | |
| 683 @deprecated | |
| 684 def run_bg(command): | |
| 685 """Function deprecated. Please use BgJob class instead.""" | |
| 686 bg_job = BgJob(command) | |
| 687 return bg_job.sp, bg_job.result | |
| 688 | |
| 689 | |
| 690 def join_bg_jobs(bg_jobs, timeout=None): | |
| 691 """Joins the bg_jobs with the current thread. | |
| 692 | |
| 693 Returns the same list of bg_jobs objects that was passed in. | |
| 694 """ | |
| 695 ret, timeout_error = 0, False | |
| 696 for bg_job in bg_jobs: | |
| 697 bg_job.output_prepare(StringIO.StringIO(), StringIO.StringIO()) | |
| 698 | |
| 699 try: | |
| 700 # We are holding ends to stdin, stdout pipes | |
| 701 # hence we need to be sure to close those fds no mater what | |
| 702 start_time = time.time() | |
| 703 timeout_error = _wait_for_commands(bg_jobs, start_time, timeout) | |
| 704 | |
| 705 for bg_job in bg_jobs: | |
| 706 # Process stdout and stderr | |
| 707 bg_job.process_output(stdout=True,final_read=True) | |
| 708 bg_job.process_output(stdout=False,final_read=True) | |
| 709 finally: | |
| 710 # close our ends of the pipes to the sp no matter what | |
| 711 for bg_job in bg_jobs: | |
| 712 bg_job.cleanup() | |
| 713 | |
| 714 if timeout_error: | |
| 715 # TODO: This needs to be fixed to better represent what happens when | |
| 716 # running in parallel. However this is backwards compatable, so it will | |
| 717 # do for the time being. | |
| 718 raise error.CmdError(bg_jobs[0].command, bg_jobs[0].result, | |
| 719 "Command(s) did not complete within %d seconds" | |
| 720 % timeout) | |
| 721 | |
| 722 | |
| 723 return bg_jobs | |
| 724 | |
| 725 | |
| 726 def _wait_for_commands(bg_jobs, start_time, timeout): | |
| 727 # This returns True if it must return due to a timeout, otherwise False. | |
| 728 | |
| 729 # To check for processes which terminate without producing any output | |
| 730 # a 1 second timeout is used in select. | |
| 731 SELECT_TIMEOUT = 1 | |
| 732 | |
| 733 read_list = [] | |
| 734 write_list = [] | |
| 735 reverse_dict = {} | |
| 736 | |
| 737 for bg_job in bg_jobs: | |
| 738 read_list.append(bg_job.sp.stdout) | |
| 739 read_list.append(bg_job.sp.stderr) | |
| 740 reverse_dict[bg_job.sp.stdout] = (bg_job, True) | |
| 741 reverse_dict[bg_job.sp.stderr] = (bg_job, False) | |
| 742 if bg_job.string_stdin is not None: | |
| 743 write_list.append(bg_job.sp.stdin) | |
| 744 reverse_dict[bg_job.sp.stdin] = bg_job | |
| 745 | |
| 746 if timeout: | |
| 747 stop_time = start_time + timeout | |
| 748 time_left = stop_time - time.time() | |
| 749 else: | |
| 750 time_left = None # so that select never times out | |
| 751 | |
| 752 while not timeout or time_left > 0: | |
| 753 # select will return when we may write to stdin or when there is | |
| 754 # stdout/stderr output we can read (including when it is | |
| 755 # EOF, that is the process has terminated). | |
| 756 read_ready, write_ready, _ = select.select(read_list, write_list, [], | |
| 757 SELECT_TIMEOUT) | |
| 758 | |
| 759 # os.read() has to be used instead of | |
| 760 # subproc.stdout.read() which will otherwise block | |
| 761 for file_obj in read_ready: | |
| 762 bg_job, is_stdout = reverse_dict[file_obj] | |
| 763 bg_job.process_output(is_stdout) | |
| 764 | |
| 765 for file_obj in write_ready: | |
| 766 # we can write PIPE_BUF bytes without blocking | |
| 767 # POSIX requires PIPE_BUF is >= 512 | |
| 768 bg_job = reverse_dict[file_obj] | |
| 769 file_obj.write(bg_job.string_stdin[:512]) | |
| 770 bg_job.string_stdin = bg_job.string_stdin[512:] | |
| 771 # no more input data, close stdin, remove it from the select set | |
| 772 if not bg_job.string_stdin: | |
| 773 file_obj.close() | |
| 774 write_list.remove(file_obj) | |
| 775 del reverse_dict[file_obj] | |
| 776 | |
| 777 all_jobs_finished = True | |
| 778 for bg_job in bg_jobs: | |
| 779 if bg_job.result.exit_status is not None: | |
| 780 continue | |
| 781 | |
| 782 bg_job.result.exit_status = bg_job.sp.poll() | |
| 783 if bg_job.result.exit_status is not None: | |
| 784 # process exited, remove its stdout/stdin from the select set | |
| 785 bg_job.result.duration = time.time() - start_time | |
| 786 read_list.remove(bg_job.sp.stdout) | |
| 787 read_list.remove(bg_job.sp.stderr) | |
| 788 del reverse_dict[bg_job.sp.stdout] | |
| 789 del reverse_dict[bg_job.sp.stderr] | |
| 790 else: | |
| 791 all_jobs_finished = False | |
| 792 | |
| 793 if all_jobs_finished: | |
| 794 return False | |
| 795 | |
| 796 if timeout: | |
| 797 time_left = stop_time - time.time() | |
| 798 | |
| 799 # Kill all processes which did not complete prior to timeout | |
| 800 for bg_job in bg_jobs: | |
| 801 if bg_job.result.exit_status is not None: | |
| 802 continue | |
| 803 | |
| 804 logging.warn('run process timeout (%s) fired on: %s', timeout, | |
| 805 bg_job.command) | |
| 806 nuke_subprocess(bg_job.sp) | |
| 807 bg_job.result.exit_status = bg_job.sp.poll() | |
| 808 bg_job.result.duration = time.time() - start_time | |
| 809 | |
| 810 return True | |
| 811 | |
| 812 | |
| 813 def pid_is_alive(pid): | |
| 814 """ | |
| 815 True if process pid exists and is not yet stuck in Zombie state. | |
| 816 Zombies are impossible to move between cgroups, etc. | |
| 817 pid can be integer, or text of integer. | |
| 818 """ | |
| 819 path = '/proc/%s/stat' % pid | |
| 820 | |
| 821 try: | |
| 822 stat = read_one_line(path) | |
| 823 except IOError: | |
| 824 if not os.path.exists(path): | |
| 825 # file went away | |
| 826 return False | |
| 827 raise | |
| 828 | |
| 829 return stat.split()[2] != 'Z' | |
| 830 | |
| 831 | |
| 832 def signal_pid(pid, sig): | |
| 833 """ | |
| 834 Sends a signal to a process id. Returns True if the process terminated | |
| 835 successfully, False otherwise. | |
| 836 """ | |
| 837 try: | |
| 838 os.kill(pid, sig) | |
| 839 except OSError: | |
| 840 # The process may have died before we could kill it. | |
| 841 pass | |
| 842 | |
| 843 for i in range(5): | |
| 844 if not pid_is_alive(pid): | |
| 845 return True | |
| 846 time.sleep(1) | |
| 847 | |
| 848 # The process is still alive | |
| 849 return False | |
| 850 | |
| 851 | |
| 852 def nuke_subprocess(subproc): | |
| 853 # check if the subprocess is still alive, first | |
| 854 if subproc.poll() is not None: | |
| 855 return subproc.poll() | |
| 856 | |
| 857 # the process has not terminated within timeout, | |
| 858 # kill it via an escalating series of signals. | |
| 859 signal_queue = [signal.SIGTERM, signal.SIGKILL] | |
| 860 for sig in signal_queue: | |
| 861 signal_pid(subproc.pid, sig) | |
| 862 if subproc.poll() is not None: | |
| 863 return subproc.poll() | |
| 864 | |
| 865 | |
| 866 def nuke_pid(pid, signal_queue=(signal.SIGTERM, signal.SIGKILL)): | |
| 867 # the process has not terminated within timeout, | |
| 868 # kill it via an escalating series of signals. | |
| 869 for sig in signal_queue: | |
| 870 if signal_pid(pid, sig): | |
| 871 return | |
| 872 | |
| 873 # no signal successfully terminated the process | |
| 874 raise error.AutoservRunError('Could not kill %d' % pid, None) | |
| 875 | |
| 876 | |
| 877 def system(command, timeout=None, ignore_status=False): | |
| 878 """ | |
| 879 Run a command | |
| 880 | |
| 881 @param timeout: timeout in seconds | |
| 882 @param ignore_status: if ignore_status=False, throw an exception if the | |
| 883 command's exit code is non-zero | |
| 884 if ignore_stauts=True, return the exit code. | |
| 885 | |
| 886 @return exit status of command | |
| 887 (note, this will always be zero unless ignore_status=True) | |
| 888 """ | |
| 889 return run(command, timeout=timeout, ignore_status=ignore_status, | |
| 890 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS).exit_status | |
| 891 | |
| 892 | |
| 893 def system_parallel(commands, timeout=None, ignore_status=False): | |
| 894 """This function returns a list of exit statuses for the respective | |
| 895 list of commands.""" | |
| 896 return [bg_jobs.exit_status for bg_jobs in | |
| 897 run_parallel(commands, timeout=timeout, ignore_status=ignore_status, | |
| 898 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] | |
| 899 | |
| 900 | |
| 901 def system_output(command, timeout=None, ignore_status=False, | |
| 902 retain_output=False, args=()): | |
| 903 """ | |
| 904 Run a command and return the stdout output. | |
| 905 | |
| 906 @param command: command string to execute. | |
| 907 @param timeout: time limit in seconds before attempting to kill the | |
| 908 running process. The function will take a few seconds longer | |
| 909 than 'timeout' to complete if it has to kill the process. | |
| 910 @param ignore_status: do not raise an exception, no matter what the exit | |
| 911 code of the command is. | |
| 912 @param retain_output: set to True to make stdout/stderr of the command | |
| 913 output to be also sent to the logging system | |
| 914 @param args: sequence of strings of arguments to be given to the command | |
| 915 inside " quotes after they have been escaped for that; each | |
| 916 element in the sequence will be given as a separate command | |
| 917 argument | |
| 918 | |
| 919 @return a string with the stdout output of the command. | |
| 920 """ | |
| 921 if retain_output: | |
| 922 out = run(command, timeout=timeout, ignore_status=ignore_status, | |
| 923 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS, | |
| 924 args=args).stdout | |
| 925 else: | |
| 926 out = run(command, timeout=timeout, ignore_status=ignore_status, | |
| 927 args=args).stdout | |
| 928 if out[-1:] == '\n': | |
| 929 out = out[:-1] | |
| 930 return out | |
| 931 | |
| 932 | |
| 933 def system_output_parallel(commands, timeout=None, ignore_status=False, | |
| 934 retain_output=False): | |
| 935 if retain_output: | |
| 936 out = [bg_job.stdout for bg_job | |
| 937 in run_parallel(commands, timeout=timeout, | |
| 938 ignore_status=ignore_status, | |
| 939 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] | |
| 940 else: | |
| 941 out = [bg_job.stdout for bg_job in run_parallel(commands, | |
| 942 timeout=timeout, ignore_status=ignore_status)] | |
| 943 for x in out: | |
| 944 if out[-1:] == '\n': out = out[:-1] | |
| 945 return out | |
| 946 | |
| 947 | |
| 948 def strip_unicode(input): | |
| 949 if type(input) == list: | |
| 950 return [strip_unicode(i) for i in input] | |
| 951 elif type(input) == dict: | |
| 952 output = {} | |
| 953 for key in input.keys(): | |
| 954 output[str(key)] = strip_unicode(input[key]) | |
| 955 return output | |
| 956 elif type(input) == unicode: | |
| 957 return str(input) | |
| 958 else: | |
| 959 return input | |
| 960 | |
| 961 | |
| 962 def get_cpu_percentage(function, *args, **dargs): | |
| 963 """Returns a tuple containing the CPU% and return value from function call. | |
| 964 | |
| 965 This function calculates the usage time by taking the difference of | |
| 966 the user and system times both before and after the function call. | |
| 967 """ | |
| 968 child_pre = resource.getrusage(resource.RUSAGE_CHILDREN) | |
| 969 self_pre = resource.getrusage(resource.RUSAGE_SELF) | |
| 970 start = time.time() | |
| 971 to_return = function(*args, **dargs) | |
| 972 elapsed = time.time() - start | |
| 973 self_post = resource.getrusage(resource.RUSAGE_SELF) | |
| 974 child_post = resource.getrusage(resource.RUSAGE_CHILDREN) | |
| 975 | |
| 976 # Calculate CPU Percentage | |
| 977 s_user, s_system = [a - b for a, b in zip(self_post, self_pre)[:2]] | |
| 978 c_user, c_system = [a - b for a, b in zip(child_post, child_pre)[:2]] | |
| 979 cpu_percent = (s_user + c_user + s_system + c_system) / elapsed | |
| 980 | |
| 981 return cpu_percent, to_return | |
| 982 | |
| 983 | |
| 984 class SystemLoad(object): | |
| 985 """ | |
| 986 Get system and/or process values and return average value of load. | |
| 987 """ | |
| 988 def __init__(self, pids, advanced=False, time_step=0.1, cpu_cont=False, | |
| 989 use_log=False): | |
| 990 """ | |
| 991 @param pids: List of pids to be monitored. If pid = 0 whole system will | |
| 992 be monitored. pid == 0 means whole system. | |
| 993 @param advanced: monitor add value for system irq count and softirq | |
| 994 for process minor and maior page fault | |
| 995 @param time_step: Time step for continuous monitoring. | |
| 996 @param cpu_cont: If True monitor CPU load continuously. | |
| 997 @param use_log: If true every monitoring is logged for dump. | |
| 998 """ | |
| 999 self.pids = [] | |
| 1000 self.stats = {} | |
| 1001 for pid in pids: | |
| 1002 if pid == 0: | |
| 1003 cpu = FileFieldMonitor("/proc/stat", | |
| 1004 [("cpu", 0), # User Time | |
| 1005 ("cpu", 2), # System Time | |
| 1006 ("intr", 0), # IRQ Count | |
| 1007 ("softirq", 0)], # Soft IRQ Count | |
| 1008 True, | |
| 1009 cpu_cont, | |
| 1010 use_log, | |
| 1011 " +", | |
| 1012 time_step) | |
| 1013 mem = FileFieldMonitor("/proc/meminfo", | |
| 1014 [("MemTotal:", 0), # Mem Total | |
| 1015 ("MemFree:", 0), # Mem Free | |
| 1016 ("Buffers:", 0), # Buffers | |
| 1017 ("Cached:", 0)], # Cached | |
| 1018 False, | |
| 1019 True, | |
| 1020 use_log, | |
| 1021 " +", | |
| 1022 time_step) | |
| 1023 self.stats[pid] = ["TOTAL", cpu, mem] | |
| 1024 self.pids.append(pid) | |
| 1025 else: | |
| 1026 name = "" | |
| 1027 if (type(pid) is int): | |
| 1028 self.pids.append(pid) | |
| 1029 name = get_process_name(pid) | |
| 1030 else: | |
| 1031 self.pids.append(pid[0]) | |
| 1032 name = pid[1] | |
| 1033 | |
| 1034 cpu = FileFieldMonitor("/proc/%d/stat" % | |
| 1035 self.pids[-1], | |
| 1036 [("", 13), # User Time | |
| 1037 ("", 14), # System Time | |
| 1038 ("", 9), # Minority Page Fault | |
| 1039 ("", 11)], # Majority Page Fault | |
| 1040 True, | |
| 1041 cpu_cont, | |
| 1042 use_log, | |
| 1043 " +", | |
| 1044 time_step) | |
| 1045 mem = FileFieldMonitor("/proc/%d/status" % | |
| 1046 self.pids[-1], | |
| 1047 [("VmSize:", 0), # Virtual Memory Size | |
| 1048 ("VmRSS:", 0), # Resident Set Size | |
| 1049 ("VmPeak:", 0), # Peak VM Size | |
| 1050 ("VmSwap:", 0)], # VM in Swap | |
| 1051 False, | |
| 1052 True, | |
| 1053 use_log, | |
| 1054 " +", | |
| 1055 time_step) | |
| 1056 self.stats[self.pids[-1]] = [name, cpu, mem] | |
| 1057 | |
| 1058 self.advanced = advanced | |
| 1059 | |
| 1060 | |
| 1061 def __str__(self): | |
| 1062 """ | |
| 1063 Define format how to print | |
| 1064 """ | |
| 1065 out = "" | |
| 1066 for pid in self.pids: | |
| 1067 for stat in self.stats[pid][1:]: | |
| 1068 out += str(stat.get_status()) + "\n" | |
| 1069 return out | |
| 1070 | |
| 1071 | |
| 1072 def start(self, pids=[]): | |
| 1073 """ | |
| 1074 Start monitoring of the process system usage. | |
| 1075 @param pids: List of PIDs you intend to control. Use pids=[] to control | |
| 1076 all defined PIDs. | |
| 1077 """ | |
| 1078 if pids == []: | |
| 1079 pids = self.pids | |
| 1080 | |
| 1081 for pid in pids: | |
| 1082 for stat in self.stats[pid][1:]: | |
| 1083 stat.start() | |
| 1084 | |
| 1085 | |
| 1086 def stop(self, pids=[]): | |
| 1087 """ | |
| 1088 Stop monitoring of the process system usage. | |
| 1089 @param pids: List of PIDs you intend to control. Use pids=[] to control | |
| 1090 all defined PIDs. | |
| 1091 """ | |
| 1092 if pids == []: | |
| 1093 pids = self.pids | |
| 1094 | |
| 1095 for pid in pids: | |
| 1096 for stat in self.stats[pid][1:]: | |
| 1097 stat.stop() | |
| 1098 | |
| 1099 | |
| 1100 def dump(self, pids=[]): | |
| 1101 """ | |
| 1102 Get the status of monitoring. | |
| 1103 @param pids: List of PIDs you intend to control. Use pids=[] to control | |
| 1104 all defined PIDs. | |
| 1105 @return: | |
| 1106 tuple([cpu load], [memory load]): | |
| 1107 ([(PID1, (PID1_cpu_meas)), (PID2, (PID2_cpu_meas)), ...], | |
| 1108 [(PID1, (PID1_mem_meas)), (PID2, (PID2_mem_meas)), ...]) | |
| 1109 | |
| 1110 PID1_cpu_meas: | |
| 1111 average_values[], test_time, cont_meas_values[[]], time_step | |
| 1112 PID1_mem_meas: | |
| 1113 average_values[], test_time, cont_meas_values[[]], time_step | |
| 1114 where average_values[] are the measured values (mem_free,swap,...) | |
| 1115 which are described in SystemLoad.__init__()-FileFieldMonitor. | |
| 1116 cont_meas_values[[]] is a list of average_values in the sampling | |
| 1117 times. | |
| 1118 """ | |
| 1119 if pids == []: | |
| 1120 pids = self.pids | |
| 1121 | |
| 1122 cpus = [] | |
| 1123 memory = [] | |
| 1124 for pid in pids: | |
| 1125 stat = (pid, self.stats[pid][1].get_status()) | |
| 1126 cpus.append(stat) | |
| 1127 for pid in pids: | |
| 1128 stat = (pid, self.stats[pid][2].get_status()) | |
| 1129 memory.append(stat) | |
| 1130 | |
| 1131 return (cpus, memory) | |
| 1132 | |
| 1133 | |
| 1134 def get_cpu_status_string(self, pids=[]): | |
| 1135 """ | |
| 1136 Convert status to string array. | |
| 1137 @param pids: List of PIDs you intend to control. Use pids=[] to control | |
| 1138 all defined PIDs. | |
| 1139 @return: String format to table. | |
| 1140 """ | |
| 1141 if pids == []: | |
| 1142 pids = self.pids | |
| 1143 | |
| 1144 headers = ["NAME", | |
| 1145 ("%7s") % "PID", | |
| 1146 ("%5s") % "USER", | |
| 1147 ("%5s") % "SYS", | |
| 1148 ("%5s") % "SUM"] | |
| 1149 if self.advanced: | |
| 1150 headers.extend(["MINFLT/IRQC", | |
| 1151 "MAJFLT/SOFTIRQ"]) | |
| 1152 headers.append(("%11s") % "TIME") | |
| 1153 textstatus = [] | |
| 1154 for pid in pids: | |
| 1155 stat = self.stats[pid][1].get_status() | |
| 1156 time = stat[1] | |
| 1157 stat = stat[0] | |
| 1158 textstatus.append(["%s" % self.stats[pid][0], | |
| 1159 "%7s" % pid, | |
| 1160 "%4.0f%%" % (stat[0] / time), | |
| 1161 "%4.0f%%" % (stat[1] / time), | |
| 1162 "%4.0f%%" % ((stat[0] + stat[1]) / time), | |
| 1163 "%10.3fs" % time]) | |
| 1164 if self.advanced: | |
| 1165 textstatus[-1].insert(-1, "%11d" % stat[2]) | |
| 1166 textstatus[-1].insert(-1, "%14d" % stat[3]) | |
| 1167 | |
| 1168 return matrix_to_string(textstatus, tuple(headers)) | |
| 1169 | |
| 1170 | |
| 1171 def get_mem_status_string(self, pids=[]): | |
| 1172 """ | |
| 1173 Convert status to string array. | |
| 1174 @param pids: List of PIDs you intend to control. Use pids=[] to control | |
| 1175 all defined PIDs. | |
| 1176 @return: String format to table. | |
| 1177 """ | |
| 1178 if pids == []: | |
| 1179 pids = self.pids | |
| 1180 | |
| 1181 headers = ["NAME", | |
| 1182 ("%7s") % "PID", | |
| 1183 ("%8s") % "TOTAL/VMSIZE", | |
| 1184 ("%8s") % "FREE/VMRSS", | |
| 1185 ("%8s") % "BUFFERS/VMPEAK", | |
| 1186 ("%8s") % "CACHED/VMSWAP", | |
| 1187 ("%11s") % "TIME"] | |
| 1188 textstatus = [] | |
| 1189 for pid in pids: | |
| 1190 stat = self.stats[pid][2].get_status() | |
| 1191 time = stat[1] | |
| 1192 stat = stat[0] | |
| 1193 textstatus.append(["%s" % self.stats[pid][0], | |
| 1194 "%7s" % pid, | |
| 1195 "%10dMB" % (stat[0] / 1024), | |
| 1196 "%8dMB" % (stat[1] / 1024), | |
| 1197 "%12dMB" % (stat[2] / 1024), | |
| 1198 "%11dMB" % (stat[3] / 1024), | |
| 1199 "%10.3fs" % time]) | |
| 1200 | |
| 1201 return matrix_to_string(textstatus, tuple(headers)) | |
| 1202 | |
| 1203 | |
| 1204 def get_arch(run_function=run): | |
| 1205 """ | |
| 1206 Get the hardware architecture of the machine. | |
| 1207 run_function is used to execute the commands. It defaults to | |
| 1208 utils.run() but a custom method (if provided) should be of the | |
| 1209 same schema as utils.run. It should return a CmdResult object and | |
| 1210 throw a CmdError exception. | |
| 1211 """ | |
| 1212 arch = run_function('/bin/uname -m').stdout.rstrip() | |
| 1213 if re.match(r'i\d86$', arch): | |
| 1214 arch = 'i386' | |
| 1215 return arch | |
| 1216 | |
| 1217 | |
| 1218 def get_num_logical_cpus_per_socket(run_function=run): | |
| 1219 """ | |
| 1220 Get the number of cores (including hyperthreading) per cpu. | |
| 1221 run_function is used to execute the commands. It defaults to | |
| 1222 utils.run() but a custom method (if provided) should be of the | |
| 1223 same schema as utils.run. It should return a CmdResult object and | |
| 1224 throw a CmdError exception. | |
| 1225 """ | |
| 1226 siblings = run_function('grep "^siblings" /proc/cpuinfo').stdout.rstrip() | |
| 1227 num_siblings = map(int, | |
| 1228 re.findall(r'^siblings\s*:\s*(\d+)\s*$', | |
| 1229 siblings, re.M)) | |
| 1230 if len(num_siblings) == 0: | |
| 1231 raise error.TestError('Unable to find siblings info in /proc/cpuinfo') | |
| 1232 if min(num_siblings) != max(num_siblings): | |
| 1233 raise error.TestError('Number of siblings differ %r' % | |
| 1234 num_siblings) | |
| 1235 return num_siblings[0] | |
| 1236 | |
| 1237 | |
| 1238 def merge_trees(src, dest): | |
| 1239 """ | |
| 1240 Merges a source directory tree at 'src' into a destination tree at | |
| 1241 'dest'. If a path is a file in both trees than the file in the source | |
| 1242 tree is APPENDED to the one in the destination tree. If a path is | |
| 1243 a directory in both trees then the directories are recursively merged | |
| 1244 with this function. In any other case, the function will skip the | |
| 1245 paths that cannot be merged (instead of failing). | |
| 1246 """ | |
| 1247 if not os.path.exists(src): | |
| 1248 return # exists only in dest | |
| 1249 elif not os.path.exists(dest): | |
| 1250 if os.path.isfile(src): | |
| 1251 shutil.copy2(src, dest) # file only in src | |
| 1252 else: | |
| 1253 shutil.copytree(src, dest, symlinks=True) # dir only in src | |
| 1254 return | |
| 1255 elif os.path.isfile(src) and os.path.isfile(dest): | |
| 1256 # src & dest are files in both trees, append src to dest | |
| 1257 destfile = open(dest, "a") | |
| 1258 try: | |
| 1259 srcfile = open(src) | |
| 1260 try: | |
| 1261 destfile.write(srcfile.read()) | |
| 1262 finally: | |
| 1263 srcfile.close() | |
| 1264 finally: | |
| 1265 destfile.close() | |
| 1266 elif os.path.isdir(src) and os.path.isdir(dest): | |
| 1267 # src & dest are directories in both trees, so recursively merge | |
| 1268 for name in os.listdir(src): | |
| 1269 merge_trees(os.path.join(src, name), os.path.join(dest, name)) | |
| 1270 else: | |
| 1271 # src & dest both exist, but are incompatible | |
| 1272 return | |
| 1273 | |
| 1274 | |
| 1275 class CmdResult(object): | |
| 1276 """ | |
| 1277 Command execution result. | |
| 1278 | |
| 1279 command: String containing the command line itself | |
| 1280 exit_status: Integer exit code of the process | |
| 1281 stdout: String containing stdout of the process | |
| 1282 stderr: String containing stderr of the process | |
| 1283 duration: Elapsed wall clock time running the process | |
| 1284 """ | |
| 1285 | |
| 1286 | |
| 1287 def __init__(self, command="", stdout="", stderr="", | |
| 1288 exit_status=None, duration=0): | |
| 1289 self.command = command | |
| 1290 self.exit_status = exit_status | |
| 1291 self.stdout = stdout | |
| 1292 self.stderr = stderr | |
| 1293 self.duration = duration | |
| 1294 | |
| 1295 | |
| 1296 def __repr__(self): | |
| 1297 wrapper = textwrap.TextWrapper(width = 78, | |
| 1298 initial_indent="\n ", | |
| 1299 subsequent_indent=" ") | |
| 1300 | |
| 1301 stdout = self.stdout.rstrip() | |
| 1302 if stdout: | |
| 1303 stdout = "\nstdout:\n%s" % stdout | |
| 1304 | |
| 1305 stderr = self.stderr.rstrip() | |
| 1306 if stderr: | |
| 1307 stderr = "\nstderr:\n%s" % stderr | |
| 1308 | |
| 1309 return ("* Command: %s\n" | |
| 1310 "Exit status: %s\n" | |
| 1311 "Duration: %s\n" | |
| 1312 "%s" | |
| 1313 "%s" | |
| 1314 % (wrapper.fill(self.command), self.exit_status, | |
| 1315 self.duration, stdout, stderr)) | |
| 1316 | |
| 1317 | |
| 1318 class run_randomly: | |
| 1319 def __init__(self, run_sequentially=False): | |
| 1320 # Run sequentially is for debugging control files | |
| 1321 self.test_list = [] | |
| 1322 self.run_sequentially = run_sequentially | |
| 1323 | |
| 1324 | |
| 1325 def add(self, *args, **dargs): | |
| 1326 test = (args, dargs) | |
| 1327 self.test_list.append(test) | |
| 1328 | |
| 1329 | |
| 1330 def run(self, fn): | |
| 1331 while self.test_list: | |
| 1332 test_index = random.randint(0, len(self.test_list)-1) | |
| 1333 if self.run_sequentially: | |
| 1334 test_index = 0 | |
| 1335 (args, dargs) = self.test_list.pop(test_index) | |
| 1336 fn(*args, **dargs) | |
| 1337 | |
| 1338 | |
| 1339 def import_site_module(path, module, dummy=None, modulefile=None): | |
| 1340 """ | |
| 1341 Try to import the site specific module if it exists. | |
| 1342 | |
| 1343 @param path full filename of the source file calling this (ie __file__) | |
| 1344 @param module full module name | |
| 1345 @param dummy dummy value to return in case there is no symbol to import | |
| 1346 @param modulefile module filename | |
| 1347 | |
| 1348 @return site specific module or dummy | |
| 1349 | |
| 1350 @raises ImportError if the site file exists but imports fails | |
| 1351 """ | |
| 1352 short_module = module[module.rfind(".") + 1:] | |
| 1353 | |
| 1354 if not modulefile: | |
| 1355 modulefile = short_module + ".py" | |
| 1356 | |
| 1357 if os.path.exists(os.path.join(os.path.dirname(path), modulefile)): | |
| 1358 return __import__(module, {}, {}, [short_module]) | |
| 1359 return dummy | |
| 1360 | |
| 1361 | |
| 1362 def import_site_symbol(path, module, name, dummy=None, modulefile=None): | |
| 1363 """ | |
| 1364 Try to import site specific symbol from site specific file if it exists | |
| 1365 | |
| 1366 @param path full filename of the source file calling this (ie __file__) | |
| 1367 @param module full module name | |
| 1368 @param name symbol name to be imported from the site file | |
| 1369 @param dummy dummy value to return in case there is no symbol to import | |
| 1370 @param modulefile module filename | |
| 1371 | |
| 1372 @return site specific symbol or dummy | |
| 1373 | |
| 1374 @raises ImportError if the site file exists but imports fails | |
| 1375 """ | |
| 1376 module = import_site_module(path, module, modulefile=modulefile) | |
| 1377 if not module: | |
| 1378 return dummy | |
| 1379 | |
| 1380 # special unique value to tell us if the symbol can't be imported | |
| 1381 cant_import = object() | |
| 1382 | |
| 1383 obj = getattr(module, name, cant_import) | |
| 1384 if obj is cant_import: | |
| 1385 logging.debug("unable to import site symbol '%s', using non-site " | |
| 1386 "implementation", name) | |
| 1387 return dummy | |
| 1388 | |
| 1389 return obj | |
| 1390 | |
| 1391 | |
| 1392 def import_site_class(path, module, classname, baseclass, modulefile=None): | |
| 1393 """ | |
| 1394 Try to import site specific class from site specific file if it exists | |
| 1395 | |
| 1396 Args: | |
| 1397 path: full filename of the source file calling this (ie __file__) | |
| 1398 module: full module name | |
| 1399 classname: class name to be loaded from site file | |
| 1400 baseclass: base class object to return when no site file present or | |
| 1401 to mixin when site class exists but is not inherited from baseclass | |
| 1402 modulefile: module filename | |
| 1403 | |
| 1404 Returns: baseclass if site specific class does not exist, the site specific | |
| 1405 class if it exists and is inherited from baseclass or a mixin of the | |
| 1406 site specific class and baseclass when the site specific class exists | |
| 1407 and is not inherited from baseclass | |
| 1408 | |
| 1409 Raises: ImportError if the site file exists but imports fails | |
| 1410 """ | |
| 1411 | |
| 1412 res = import_site_symbol(path, module, classname, None, modulefile) | |
| 1413 if res: | |
| 1414 if not issubclass(res, baseclass): | |
| 1415 # if not a subclass of baseclass then mix in baseclass with the | |
| 1416 # site specific class object and return the result | |
| 1417 res = type(classname, (res, baseclass), {}) | |
| 1418 else: | |
| 1419 res = baseclass | |
| 1420 | |
| 1421 return res | |
| 1422 | |
| 1423 | |
| 1424 def import_site_function(path, module, funcname, dummy, modulefile=None): | |
| 1425 """ | |
| 1426 Try to import site specific function from site specific file if it exists | |
| 1427 | |
| 1428 Args: | |
| 1429 path: full filename of the source file calling this (ie __file__) | |
| 1430 module: full module name | |
| 1431 funcname: function name to be imported from site file | |
| 1432 dummy: dummy function to return in case there is no function to import | |
| 1433 modulefile: module filename | |
| 1434 | |
| 1435 Returns: site specific function object or dummy | |
| 1436 | |
| 1437 Raises: ImportError if the site file exists but imports fails | |
| 1438 """ | |
| 1439 | |
| 1440 return import_site_symbol(path, module, funcname, dummy, modulefile) | |
| 1441 | |
| 1442 | |
| 1443 def _get_pid_path(program_name): | |
| 1444 my_path = os.path.dirname(__file__) | |
| 1445 return os.path.abspath(os.path.join(my_path, "..", "..", | |
| 1446 "%s.pid" % program_name)) | |
| 1447 | |
| 1448 | |
| 1449 def write_pid(program_name): | |
| 1450 """ | |
| 1451 Try to drop <program_name>.pid in the main autotest directory. | |
| 1452 | |
| 1453 Args: | |
| 1454 program_name: prefix for file name | |
| 1455 """ | |
| 1456 pidfile = open(_get_pid_path(program_name), "w") | |
| 1457 try: | |
| 1458 pidfile.write("%s\n" % os.getpid()) | |
| 1459 finally: | |
| 1460 pidfile.close() | |
| 1461 | |
| 1462 | |
| 1463 def delete_pid_file_if_exists(program_name): | |
| 1464 """ | |
| 1465 Tries to remove <program_name>.pid from the main autotest directory. | |
| 1466 """ | |
| 1467 pidfile_path = _get_pid_path(program_name) | |
| 1468 | |
| 1469 try: | |
| 1470 os.remove(pidfile_path) | |
| 1471 except OSError: | |
| 1472 if not os.path.exists(pidfile_path): | |
| 1473 return | |
| 1474 raise | |
| 1475 | |
| 1476 | |
| 1477 def get_pid_from_file(program_name): | |
| 1478 """ | |
| 1479 Reads the pid from <program_name>.pid in the autotest directory. | |
| 1480 | |
| 1481 @param program_name the name of the program | |
| 1482 @return the pid if the file exists, None otherwise. | |
| 1483 """ | |
| 1484 pidfile_path = _get_pid_path(program_name) | |
| 1485 if not os.path.exists(pidfile_path): | |
| 1486 return None | |
| 1487 | |
| 1488 pidfile = open(_get_pid_path(program_name), 'r') | |
| 1489 | |
| 1490 try: | |
| 1491 try: | |
| 1492 pid = int(pidfile.readline()) | |
| 1493 except IOError: | |
| 1494 if not os.path.exists(pidfile_path): | |
| 1495 return None | |
| 1496 raise | |
| 1497 finally: | |
| 1498 pidfile.close() | |
| 1499 | |
| 1500 return pid | |
| 1501 | |
| 1502 | |
| 1503 def get_process_name(pid): | |
| 1504 """ | |
| 1505 Get process name from PID. | |
| 1506 @param pid: PID of process. | |
| 1507 """ | |
| 1508 return get_field(read_file("/proc/%d/stat" % pid), 1)[1:-1] | |
| 1509 | |
| 1510 | |
| 1511 def program_is_alive(program_name): | |
| 1512 """ | |
| 1513 Checks if the process is alive and not in Zombie state. | |
| 1514 | |
| 1515 @param program_name the name of the program | |
| 1516 @return True if still alive, False otherwise | |
| 1517 """ | |
| 1518 pid = get_pid_from_file(program_name) | |
| 1519 if pid is None: | |
| 1520 return False | |
| 1521 return pid_is_alive(pid) | |
| 1522 | |
| 1523 | |
| 1524 def signal_program(program_name, sig=signal.SIGTERM): | |
| 1525 """ | |
| 1526 Sends a signal to the process listed in <program_name>.pid | |
| 1527 | |
| 1528 @param program_name the name of the program | |
| 1529 @param sig signal to send | |
| 1530 """ | |
| 1531 pid = get_pid_from_file(program_name) | |
| 1532 if pid: | |
| 1533 signal_pid(pid, sig) | |
| 1534 | |
| 1535 | |
| 1536 def get_relative_path(path, reference): | |
| 1537 """Given 2 absolute paths "path" and "reference", compute the path of | |
| 1538 "path" as relative to the directory "reference". | |
| 1539 | |
| 1540 @param path the absolute path to convert to a relative path | |
| 1541 @param reference an absolute directory path to which the relative | |
| 1542 path will be computed | |
| 1543 """ | |
| 1544 # normalize the paths (remove double slashes, etc) | |
| 1545 assert(os.path.isabs(path)) | |
| 1546 assert(os.path.isabs(reference)) | |
| 1547 | |
| 1548 path = os.path.normpath(path) | |
| 1549 reference = os.path.normpath(reference) | |
| 1550 | |
| 1551 # we could use os.path.split() but it splits from the end | |
| 1552 path_list = path.split(os.path.sep)[1:] | |
| 1553 ref_list = reference.split(os.path.sep)[1:] | |
| 1554 | |
| 1555 # find the longest leading common path | |
| 1556 for i in xrange(min(len(path_list), len(ref_list))): | |
| 1557 if path_list[i] != ref_list[i]: | |
| 1558 # decrement i so when exiting this loop either by no match or by | |
| 1559 # end of range we are one step behind | |
| 1560 i -= 1 | |
| 1561 break | |
| 1562 i += 1 | |
| 1563 # drop the common part of the paths, not interested in that anymore | |
| 1564 del path_list[:i] | |
| 1565 | |
| 1566 # for each uncommon component in the reference prepend a ".." | |
| 1567 path_list[:0] = ['..'] * (len(ref_list) - i) | |
| 1568 | |
| 1569 return os.path.join(*path_list) | |
| 1570 | |
| 1571 | |
| 1572 def sh_escape(command): | |
| 1573 """ | |
| 1574 Escape special characters from a command so that it can be passed | |
| 1575 as a double quoted (" ") string in a (ba)sh command. | |
| 1576 | |
| 1577 Args: | |
| 1578 command: the command string to escape. | |
| 1579 | |
| 1580 Returns: | |
| 1581 The escaped command string. The required englobing double | |
| 1582 quotes are NOT added and so should be added at some point by | |
| 1583 the caller. | |
| 1584 | |
| 1585 See also: http://www.tldp.org/LDP/abs/html/escapingsection.html | |
| 1586 """ | |
| 1587 command = command.replace("\\", "\\\\") | |
| 1588 command = command.replace("$", r'\$') | |
| 1589 command = command.replace('"', r'\"') | |
| 1590 command = command.replace('`', r'\`') | |
| 1591 return command | |
| 1592 | |
| 1593 | |
| 1594 def configure(extra=None, configure='./configure'): | |
| 1595 """ | |
| 1596 Run configure passing in the correct host, build, and target options. | |
| 1597 | |
| 1598 @param extra: extra command line arguments to pass to configure | |
| 1599 @param configure: which configure script to use | |
| 1600 """ | |
| 1601 args = [] | |
| 1602 if 'CHOST' in os.environ: | |
| 1603 args.append('--host=' + os.environ['CHOST']) | |
| 1604 if 'CBUILD' in os.environ: | |
| 1605 args.append('--build=' + os.environ['CBUILD']) | |
| 1606 if 'CTARGET' in os.environ: | |
| 1607 args.append('--target=' + os.environ['CTARGET']) | |
| 1608 if extra: | |
| 1609 args.append(extra) | |
| 1610 | |
| 1611 system('%s %s' % (configure, ' '.join(args))) | |
| 1612 | |
| 1613 | |
| 1614 def make(extra='', make='make', timeout=None, ignore_status=False): | |
| 1615 """ | |
| 1616 Run make, adding MAKEOPTS to the list of options. | |
| 1617 | |
| 1618 @param extra: extra command line arguments to pass to make. | |
| 1619 """ | |
| 1620 cmd = '%s %s %s' % (make, os.environ.get('MAKEOPTS', ''), extra) | |
| 1621 return system(cmd, timeout=timeout, ignore_status=ignore_status) | |
| 1622 | |
| 1623 | |
| 1624 def compare_versions(ver1, ver2): | |
| 1625 """Version number comparison between ver1 and ver2 strings. | |
| 1626 | |
| 1627 >>> compare_tuple("1", "2") | |
| 1628 -1 | |
| 1629 >>> compare_tuple("foo-1.1", "foo-1.2") | |
| 1630 -1 | |
| 1631 >>> compare_tuple("1.2", "1.2a") | |
| 1632 -1 | |
| 1633 >>> compare_tuple("1.2b", "1.2a") | |
| 1634 1 | |
| 1635 >>> compare_tuple("1.3.5.3a", "1.3.5.3b") | |
| 1636 -1 | |
| 1637 | |
| 1638 Args: | |
| 1639 ver1: version string | |
| 1640 ver2: version string | |
| 1641 | |
| 1642 Returns: | |
| 1643 int: 1 if ver1 > ver2 | |
| 1644 0 if ver1 == ver2 | |
| 1645 -1 if ver1 < ver2 | |
| 1646 """ | |
| 1647 ax = re.split('[.-]', ver1) | |
| 1648 ay = re.split('[.-]', ver2) | |
| 1649 while len(ax) > 0 and len(ay) > 0: | |
| 1650 cx = ax.pop(0) | |
| 1651 cy = ay.pop(0) | |
| 1652 maxlen = max(len(cx), len(cy)) | |
| 1653 c = cmp(cx.zfill(maxlen), cy.zfill(maxlen)) | |
| 1654 if c != 0: | |
| 1655 return c | |
| 1656 return cmp(len(ax), len(ay)) | |
| 1657 | |
| 1658 | |
| 1659 def args_to_dict(args): | |
| 1660 """Convert autoserv extra arguments in the form of key=val or key:val to a | |
| 1661 dictionary. Each argument key is converted to lowercase dictionary key. | |
| 1662 | |
| 1663 Args: | |
| 1664 args - list of autoserv extra arguments. | |
| 1665 | |
| 1666 Returns: | |
| 1667 dictionary | |
| 1668 """ | |
| 1669 arg_re = re.compile(r'(\w+)[:=](.*)$') | |
| 1670 dict = {} | |
| 1671 for arg in args: | |
| 1672 match = arg_re.match(arg) | |
| 1673 if match: | |
| 1674 dict[match.group(1).lower()] = match.group(2) | |
| 1675 else: | |
| 1676 logging.warning("args_to_dict: argument '%s' doesn't match " | |
| 1677 "'%s' pattern. Ignored." % (arg, arg_re.pattern)) | |
| 1678 return dict | |
| 1679 | |
| 1680 | |
| 1681 def get_unused_port(): | |
| 1682 """ | |
| 1683 Finds a semi-random available port. A race condition is still | |
| 1684 possible after the port number is returned, if another process | |
| 1685 happens to bind it. | |
| 1686 | |
| 1687 Returns: | |
| 1688 A port number that is unused on both TCP and UDP. | |
| 1689 """ | |
| 1690 | |
| 1691 def try_bind(port, socket_type, socket_proto): | |
| 1692 s = socket.socket(socket.AF_INET, socket_type, socket_proto) | |
| 1693 try: | |
| 1694 try: | |
| 1695 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
| 1696 s.bind(('', port)) | |
| 1697 return s.getsockname()[1] | |
| 1698 except socket.error: | |
| 1699 return None | |
| 1700 finally: | |
| 1701 s.close() | |
| 1702 | |
| 1703 # On the 2.6 kernel, calling try_bind() on UDP socket returns the | |
| 1704 # same port over and over. So always try TCP first. | |
| 1705 while True: | |
| 1706 # Ask the OS for an unused port. | |
| 1707 port = try_bind(0, socket.SOCK_STREAM, socket.IPPROTO_TCP) | |
| 1708 # Check if this port is unused on the other protocol. | |
| 1709 if port and try_bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP): | |
| 1710 return port | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 6246035:
Merge remote branch 'cros/upstream' into master (Closed)
Base URL: ssh://git@gitrw.chromium.org:9222/autotest.git@master