Index: client/common_lib/utils.py |
diff --git a/client/common_lib/utils.py b/client/common_lib/utils.py |
index 47f3cb4c2e11d32a7331f6d12fbc448a6ee09b7f..382f79d50c32c64652cb81d617aa47e2c00eff14 100644 |
--- a/client/common_lib/utils.py |
+++ b/client/common_lib/utils.py |
@@ -1,1710 +1,13 @@ |
-# |
-# Copyright 2008 Google Inc. Released under the GPL v2 |
+""" |
+Convenience functions for use by tests or whomever. |
-import os, pickle, random, re, resource, select, shutil, signal, StringIO |
-import socket, struct, subprocess, sys, time, textwrap, urlparse |
-import warnings, smtplib, logging, urllib2 |
-from threading import Thread, Event |
-try: |
- import hashlib |
-except ImportError: |
- import md5, sha |
-from autotest_lib.client.common_lib import error, logging_manager |
+NOTE: this is a mixin library that pulls in functions from several places |
+Note carefully what the precendece order is |
-def deprecated(func): |
- """This is a decorator which can be used to mark functions as deprecated. |
- It will result in a warning being emmitted when the function is used.""" |
- def new_func(*args, **dargs): |
- warnings.warn("Call to deprecated function %s." % func.__name__, |
- category=DeprecationWarning) |
- return func(*args, **dargs) |
- new_func.__name__ = func.__name__ |
- new_func.__doc__ = func.__doc__ |
- new_func.__dict__.update(func.__dict__) |
- return new_func |
+There's no really good way to do this, as this isn't a class we can do |
+inheritance with, just a collection of static methods. |
+""" |
- |
-class _NullStream(object): |
- def write(self, data): |
- pass |
- |
- |
- def flush(self): |
- pass |
- |
- |
-TEE_TO_LOGS = object() |
-_the_null_stream = _NullStream() |
- |
-DEFAULT_STDOUT_LEVEL = logging.DEBUG |
-DEFAULT_STDERR_LEVEL = logging.ERROR |
- |
-# prefixes for logging stdout/stderr of commands |
-STDOUT_PREFIX = '[stdout] ' |
-STDERR_PREFIX = '[stderr] ' |
- |
- |
-def get_stream_tee_file(stream, level, prefix=''): |
- if stream is None: |
- return _the_null_stream |
- if stream is TEE_TO_LOGS: |
- return logging_manager.LoggingFile(level=level, prefix=prefix) |
- return stream |
- |
- |
-class BgJob(object): |
- def __init__(self, command, stdout_tee=None, stderr_tee=None, verbose=True, |
- stdin=None, stderr_level=DEFAULT_STDERR_LEVEL): |
- self.command = command |
- self.stdout_tee = get_stream_tee_file(stdout_tee, DEFAULT_STDOUT_LEVEL, |
- prefix=STDOUT_PREFIX) |
- self.stderr_tee = get_stream_tee_file(stderr_tee, stderr_level, |
- prefix=STDERR_PREFIX) |
- self.result = CmdResult(command) |
- |
- # allow for easy stdin input by string, we'll let subprocess create |
- # a pipe for stdin input and we'll write to it in the wait loop |
- if isinstance(stdin, basestring): |
- self.string_stdin = stdin |
- stdin = subprocess.PIPE |
- else: |
- self.string_stdin = None |
- |
- if verbose: |
- logging.debug("Running '%s'" % command) |
- self.sp = subprocess.Popen(command, stdout=subprocess.PIPE, |
- stderr=subprocess.PIPE, |
- preexec_fn=self._reset_sigpipe, shell=True, |
- executable="/bin/bash", |
- stdin=stdin) |
- |
- |
- def output_prepare(self, stdout_file=None, stderr_file=None): |
- self.stdout_file = stdout_file |
- self.stderr_file = stderr_file |
- |
- |
- def process_output(self, stdout=True, final_read=False): |
- """output_prepare must be called prior to calling this""" |
- if stdout: |
- pipe, buf, tee = self.sp.stdout, self.stdout_file, self.stdout_tee |
- else: |
- pipe, buf, tee = self.sp.stderr, self.stderr_file, self.stderr_tee |
- |
- if final_read: |
- # read in all the data we can from pipe and then stop |
- data = [] |
- while select.select([pipe], [], [], 0)[0]: |
- data.append(os.read(pipe.fileno(), 1024)) |
- if len(data[-1]) == 0: |
- break |
- data = "".join(data) |
- else: |
- # perform a single read |
- data = os.read(pipe.fileno(), 1024) |
- buf.write(data) |
- tee.write(data) |
- |
- |
- def cleanup(self): |
- self.stdout_tee.flush() |
- self.stderr_tee.flush() |
- self.sp.stdout.close() |
- self.sp.stderr.close() |
- self.result.stdout = self.stdout_file.getvalue() |
- self.result.stderr = self.stderr_file.getvalue() |
- |
- |
- def _reset_sigpipe(self): |
- signal.signal(signal.SIGPIPE, signal.SIG_DFL) |
- |
- |
-def ip_to_long(ip): |
- # !L is a long in network byte order |
- return struct.unpack('!L', socket.inet_aton(ip))[0] |
- |
- |
-def long_to_ip(number): |
- # See above comment. |
- return socket.inet_ntoa(struct.pack('!L', number)) |
- |
- |
-def create_subnet_mask(bits): |
- return (1 << 32) - (1 << 32-bits) |
- |
- |
-def format_ip_with_mask(ip, mask_bits): |
- masked_ip = ip_to_long(ip) & create_subnet_mask(mask_bits) |
- return "%s/%s" % (long_to_ip(masked_ip), mask_bits) |
- |
- |
-def normalize_hostname(alias): |
- ip = socket.gethostbyname(alias) |
- return socket.gethostbyaddr(ip)[0] |
- |
- |
-def get_ip_local_port_range(): |
- match = re.match(r'\s*(\d+)\s*(\d+)\s*$', |
- read_one_line('/proc/sys/net/ipv4/ip_local_port_range')) |
- return (int(match.group(1)), int(match.group(2))) |
- |
- |
-def set_ip_local_port_range(lower, upper): |
- write_one_line('/proc/sys/net/ipv4/ip_local_port_range', |
- '%d %d\n' % (lower, upper)) |
- |
- |
- |
-def send_email(mail_from, mail_to, subject, body): |
- """ |
- Sends an email via smtp |
- |
- mail_from: string with email address of sender |
- mail_to: string or list with email address(es) of recipients |
- subject: string with subject of email |
- body: (multi-line) string with body of email |
- """ |
- if isinstance(mail_to, str): |
- mail_to = [mail_to] |
- msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (mail_from, ','.join(mail_to), |
- subject, body) |
- try: |
- mailer = smtplib.SMTP('localhost') |
- try: |
- mailer.sendmail(mail_from, mail_to, msg) |
- finally: |
- mailer.quit() |
- except Exception, e: |
- # Emails are non-critical, not errors, but don't raise them |
- print "Sending email failed. Reason: %s" % repr(e) |
- |
- |
-def read_one_line(filename): |
- return open(filename, 'r').readline().rstrip('\n') |
- |
- |
-def read_file(filename): |
- f = open(filename) |
- try: |
- return f.read() |
- finally: |
- f.close() |
- |
- |
-def get_field(data, param, linestart="", sep=" "): |
- """ |
- Parse data from string. |
- @param data: Data to parse. |
- example: |
- data: |
- cpu 324 345 34 5 345 |
- cpu0 34 11 34 34 33 |
- ^^^^ |
- start of line |
- params 0 1 2 3 4 |
- @param param: Position of parameter after linestart marker. |
- @param linestart: String to which start line with parameters. |
- @param sep: Separator between parameters regular expression. |
- """ |
- search = re.compile(r"(?<=^%s)\s*(.*)" % linestart, re.MULTILINE) |
- find = search.search(data) |
- if find != None: |
- return re.split("%s" % sep, find.group(1))[param] |
- else: |
- print "There is no line which starts with %s in data." % linestart |
- return None |
- |
- |
-def write_one_line(filename, line): |
- open_write_close(filename, line.rstrip('\n') + '\n') |
- |
- |
-def open_write_close(filename, data): |
- f = open(filename, 'w') |
- try: |
- f.write(data) |
- finally: |
- f.close() |
- |
- |
-def matrix_to_string(matrix, header=None): |
- """ |
- Return a pretty, aligned string representation of a nxm matrix. |
- |
- This representation can be used to print any tabular data, such as |
- database results. It works by scanning the lengths of each element |
- in each column, and determining the format string dynamically. |
- |
- @param matrix: Matrix representation (list with n rows of m elements). |
- @param header: Optional tuple or list with header elements to be displayed. |
- """ |
- if type(header) is list: |
- header = tuple(header) |
- lengths = [] |
- if header: |
- for column in header: |
- lengths.append(len(column)) |
- for row in matrix: |
- for column in row: |
- i = row.index(column) |
- cl = len(column) |
- try: |
- ml = lengths[i] |
- if cl > ml: |
- lengths[i] = cl |
- except IndexError: |
- lengths.append(cl) |
- |
- lengths = tuple(lengths) |
- format_string = "" |
- for length in lengths: |
- format_string += "%-" + str(length) + "s " |
- format_string += "\n" |
- |
- matrix_str = "" |
- if header: |
- matrix_str += format_string % header |
- for row in matrix: |
- matrix_str += format_string % tuple(row) |
- |
- return matrix_str |
- |
- |
-def read_keyval(path): |
- """ |
- Read a key-value pair format file into a dictionary, and return it. |
- Takes either a filename or directory name as input. If it's a |
- directory name, we assume you want the file to be called keyval. |
- """ |
- if os.path.isdir(path): |
- path = os.path.join(path, 'keyval') |
- keyval = {} |
- if os.path.exists(path): |
- for line in open(path): |
- line = re.sub('#.*', '', line).rstrip() |
- if not re.search(r'^[-\.\w]+=', line): |
- raise ValueError('Invalid format line: %s' % line) |
- key, value = line.split('=', 1) |
- if re.search('^\d+$', value): |
- value = int(value) |
- elif re.search('^(\d+\.)?\d+$', value): |
- value = float(value) |
- keyval[key] = value |
- return keyval |
- |
- |
-def write_keyval(path, dictionary, type_tag=None): |
- """ |
- Write a key-value pair format file out to a file. This uses append |
- mode to open the file, so existing text will not be overwritten or |
- reparsed. |
- |
- If type_tag is None, then the key must be composed of alphanumeric |
- characters (or dashes+underscores). However, if type-tag is not |
- null then the keys must also have "{type_tag}" as a suffix. At |
- the moment the only valid values of type_tag are "attr" and "perf". |
- """ |
- if os.path.isdir(path): |
- path = os.path.join(path, 'keyval') |
- keyval = open(path, 'a') |
- |
- if type_tag is None: |
- key_regex = re.compile(r'^[-\.\w]+$') |
- else: |
- if type_tag not in ('attr', 'perf'): |
- raise ValueError('Invalid type tag: %s' % type_tag) |
- escaped_tag = re.escape(type_tag) |
- key_regex = re.compile(r'^[-\.\w]+\{%s\}$' % escaped_tag) |
- try: |
- for key in sorted(dictionary.keys()): |
- if not key_regex.search(key): |
- raise ValueError('Invalid key: %s' % key) |
- keyval.write('%s=%s\n' % (key, dictionary[key])) |
- finally: |
- keyval.close() |
- |
- |
-class FileFieldMonitor(object): |
- """ |
- Monitors the information from the file and reports it's values. |
- |
- It gather the information at start and stop of the measurement or |
- continuously during the measurement. |
- """ |
- class Monitor(Thread): |
- """ |
- Internal monitor class to ensure continuous monitor of monitored file. |
- """ |
- def __init__(self, master): |
- """ |
- @param master: Master class which control Monitor |
- """ |
- Thread.__init__(self) |
- self.master = master |
- |
- def run(self): |
- """ |
- Start monitor in thread mode |
- """ |
- while not self.master.end_event.isSet(): |
- self.master._get_value(self.master.logging) |
- time.sleep(self.master.time_step) |
- |
- |
- def __init__(self, status_file, data_to_read, mode_diff, continuously=False, |
- contlogging=False, separator=" +", time_step=0.1): |
- """ |
- Initialize variables. |
- @param status_file: File contain status. |
- @param mode_diff: If True make a difference of value, else average. |
- @param data_to_read: List of tuples with data position. |
- format: [(start_of_line,position in params)] |
- example: |
- data: |
- cpu 324 345 34 5 345 |
- cpu0 34 11 34 34 33 |
- ^^^^ |
- start of line |
- params 0 1 2 3 4 |
- @param mode_diff: True to subtract old value from new value, |
- False make average of the values. |
- @parma continuously: Start the monitoring thread using the time_step |
- as the measurement period. |
- @param contlogging: Log data in continuous run. |
- @param separator: Regular expression of separator. |
- @param time_step: Time period of the monitoring value. |
- """ |
- self.end_event = Event() |
- self.start_time = 0 |
- self.end_time = 0 |
- self.test_time = 0 |
- |
- self.status_file = status_file |
- self.separator = separator |
- self.data_to_read = data_to_read |
- self.num_of_params = len(self.data_to_read) |
- self.mode_diff = mode_diff |
- self.continuously = continuously |
- self.time_step = time_step |
- |
- self.value = [0 for i in range(self.num_of_params)] |
- self.old_value = [0 for i in range(self.num_of_params)] |
- self.log = [] |
- self.logging = contlogging |
- |
- self.started = False |
- self.num_of_get_value = 0 |
- self.monitor = None |
- |
- |
- def _get_value(self, logging=True): |
- """ |
- Return current values. |
- @param logging: If true log value in memory. There can be problem |
- with long run. |
- """ |
- data = read_file(self.status_file) |
- value = [] |
- for i in range(self.num_of_params): |
- value.append(int(get_field(data, |
- self.data_to_read[i][1], |
- self.data_to_read[i][0], |
- self.separator))) |
- |
- if logging: |
- self.log.append(value) |
- if not self.mode_diff: |
- value = map(lambda x, y: x + y, value, self.old_value) |
- |
- self.old_value = value |
- self.num_of_get_value += 1 |
- return value |
- |
- |
- def start(self): |
- """ |
- Start value monitor. |
- """ |
- if self.started: |
- self.stop() |
- self.old_value = [0 for i in range(self.num_of_params)] |
- self.num_of_get_value = 0 |
- self.log = [] |
- self.end_event.clear() |
- self.start_time = time.time() |
- self._get_value() |
- self.started = True |
- if (self.continuously): |
- self.monitor = FileFieldMonitor.Monitor(self) |
- self.monitor.start() |
- |
- |
- def stop(self): |
- """ |
- Stop value monitor. |
- """ |
- if self.started: |
- self.started = False |
- self.end_time = time.time() |
- self.test_time = self.end_time - self.start_time |
- self.value = self._get_value() |
- if (self.continuously): |
- self.end_event.set() |
- self.monitor.join() |
- if (self.mode_diff): |
- self.value = map(lambda x, y: x - y, self.log[-1], self.log[0]) |
- else: |
- self.value = map(lambda x: x / self.num_of_get_value, |
- self.value) |
- |
- |
- def get_status(self): |
- """ |
- @return: Status of monitored process average value, |
- time of test and array of monitored values and time step of |
- continuous run. |
- """ |
- if self.started: |
- self.stop() |
- if self.mode_diff: |
- for i in range(len(self.log) - 1): |
- self.log[i] = (map(lambda x, y: x - y, |
- self.log[i + 1], self.log[i])) |
- self.log.pop() |
- return (self.value, self.test_time, self.log, self.time_step) |
- |
- |
-def is_url(path): |
- """Return true if path looks like a URL""" |
- # for now, just handle http and ftp |
- url_parts = urlparse.urlparse(path) |
- return (url_parts[0] in ('http', 'ftp')) |
- |
- |
-def urlopen(url, data=None, timeout=5): |
- """Wrapper to urllib2.urlopen with timeout addition.""" |
- |
- # Save old timeout |
- old_timeout = socket.getdefaulttimeout() |
- socket.setdefaulttimeout(timeout) |
- try: |
- return urllib2.urlopen(url, data=data) |
- finally: |
- socket.setdefaulttimeout(old_timeout) |
- |
- |
-def urlretrieve(url, filename, data=None, timeout=300): |
- """Retrieve a file from given url.""" |
- logging.debug('Fetching %s -> %s', url, filename) |
- |
- src_file = urlopen(url, data=data, timeout=timeout) |
- try: |
- dest_file = open(filename, 'wb') |
- try: |
- shutil.copyfileobj(src_file, dest_file) |
- finally: |
- dest_file.close() |
- finally: |
- src_file.close() |
- |
- |
-def hash(type, input=None): |
- """ |
- Returns an hash object of type md5 or sha1. This function is implemented in |
- order to encapsulate hash objects in a way that is compatible with python |
- 2.4 and python 2.6 without warnings. |
- |
- Note that even though python 2.6 hashlib supports hash types other than |
- md5 and sha1, we are artificially limiting the input values in order to |
- make the function to behave exactly the same among both python |
- implementations. |
- |
- @param input: Optional input string that will be used to update the hash. |
- """ |
- if type not in ['md5', 'sha1']: |
- raise ValueError("Unsupported hash type: %s" % type) |
- |
- try: |
- hash = hashlib.new(type) |
- except NameError: |
- if type == 'md5': |
- hash = md5.new() |
- elif type == 'sha1': |
- hash = sha.new() |
- |
- if input: |
- hash.update(input) |
- |
- return hash |
- |
- |
-def get_file(src, dest, permissions=None): |
- """Get a file from src, which can be local or a remote URL""" |
- if src == dest: |
- return |
- |
- if is_url(src): |
- urlretrieve(src, dest) |
- else: |
- shutil.copyfile(src, dest) |
- |
- if permissions: |
- os.chmod(dest, permissions) |
- return dest |
- |
- |
-def unmap_url(srcdir, src, destdir='.'): |
- """ |
- Receives either a path to a local file or a URL. |
- returns either the path to the local file, or the fetched URL |
- |
- unmap_url('/usr/src', 'foo.tar', '/tmp') |
- = '/usr/src/foo.tar' |
- unmap_url('/usr/src', 'http://site/file', '/tmp') |
- = '/tmp/file' |
- (after retrieving it) |
- """ |
- if is_url(src): |
- url_parts = urlparse.urlparse(src) |
- filename = os.path.basename(url_parts[2]) |
- dest = os.path.join(destdir, filename) |
- return get_file(src, dest) |
- else: |
- return os.path.join(srcdir, src) |
- |
- |
-def update_version(srcdir, preserve_srcdir, new_version, install, |
- *args, **dargs): |
- """ |
- Make sure srcdir is version new_version |
- |
- If not, delete it and install() the new version. |
- |
- In the preserve_srcdir case, we just check it's up to date, |
- and if not, we rerun install, without removing srcdir |
- """ |
- versionfile = os.path.join(srcdir, '.version') |
- install_needed = True |
- |
- if os.path.exists(versionfile): |
- old_version = pickle.load(open(versionfile)) |
- if old_version == new_version: |
- install_needed = False |
- |
- if install_needed: |
- if not preserve_srcdir and os.path.exists(srcdir): |
- shutil.rmtree(srcdir) |
- install(*args, **dargs) |
- if os.path.exists(srcdir): |
- pickle.dump(new_version, open(versionfile, 'w')) |
- |
- |
-def get_stderr_level(stderr_is_expected): |
- if stderr_is_expected: |
- return DEFAULT_STDOUT_LEVEL |
- return DEFAULT_STDERR_LEVEL |
- |
- |
-def run(command, timeout=None, ignore_status=False, |
- stdout_tee=None, stderr_tee=None, verbose=True, stdin=None, |
- stderr_is_expected=None, args=()): |
- """ |
- Run a command on the host. |
- |
- @param command: the command line string. |
- @param timeout: time limit in seconds before attempting to kill the |
- running process. The run() function will take a few seconds |
- longer than 'timeout' to complete if it has to kill the process. |
- @param ignore_status: do not raise an exception, no matter what the exit |
- code of the command is. |
- @param stdout_tee: optional file-like object to which stdout data |
- will be written as it is generated (data will still be stored |
- in result.stdout). |
- @param stderr_tee: likewise for stderr. |
- @param verbose: if True, log the command being run. |
- @param stdin: stdin to pass to the executed process (can be a file |
- descriptor, a file object of a real file or a string). |
- @param args: sequence of strings of arguments to be given to the command |
- inside " quotes after they have been escaped for that; each |
- element in the sequence will be given as a separate command |
- argument |
- |
- @return a CmdResult object |
- |
- @raise CmdError: the exit code of the command execution was not 0 |
- """ |
- if isinstance(args, basestring): |
- raise TypeError('Got a string for the "args" keyword argument, ' |
- 'need a sequence.') |
- |
- for arg in args: |
- command += ' "%s"' % sh_escape(arg) |
- if stderr_is_expected is None: |
- stderr_is_expected = ignore_status |
- |
- bg_job = join_bg_jobs( |
- (BgJob(command, stdout_tee, stderr_tee, verbose, stdin=stdin, |
- stderr_level=get_stderr_level(stderr_is_expected)),), |
- timeout)[0] |
- if not ignore_status and bg_job.result.exit_status: |
- raise error.CmdError(command, bg_job.result, |
- "Command returned non-zero exit status") |
- |
- return bg_job.result |
- |
- |
-def run_parallel(commands, timeout=None, ignore_status=False, |
- stdout_tee=None, stderr_tee=None): |
- """ |
- Behaves the same as run() with the following exceptions: |
- |
- - commands is a list of commands to run in parallel. |
- - ignore_status toggles whether or not an exception should be raised |
- on any error. |
- |
- @return: a list of CmdResult objects |
- """ |
- bg_jobs = [] |
- for command in commands: |
- bg_jobs.append(BgJob(command, stdout_tee, stderr_tee, |
- stderr_level=get_stderr_level(ignore_status))) |
- |
- # Updates objects in bg_jobs list with their process information |
- join_bg_jobs(bg_jobs, timeout) |
- |
- for bg_job in bg_jobs: |
- if not ignore_status and bg_job.result.exit_status: |
- raise error.CmdError(command, bg_job.result, |
- "Command returned non-zero exit status") |
- |
- return [bg_job.result for bg_job in bg_jobs] |
- |
- |
-@deprecated |
-def run_bg(command): |
- """Function deprecated. Please use BgJob class instead.""" |
- bg_job = BgJob(command) |
- return bg_job.sp, bg_job.result |
- |
- |
-def join_bg_jobs(bg_jobs, timeout=None): |
- """Joins the bg_jobs with the current thread. |
- |
- Returns the same list of bg_jobs objects that was passed in. |
- """ |
- ret, timeout_error = 0, False |
- for bg_job in bg_jobs: |
- bg_job.output_prepare(StringIO.StringIO(), StringIO.StringIO()) |
- |
- try: |
- # We are holding ends to stdin, stdout pipes |
- # hence we need to be sure to close those fds no mater what |
- start_time = time.time() |
- timeout_error = _wait_for_commands(bg_jobs, start_time, timeout) |
- |
- for bg_job in bg_jobs: |
- # Process stdout and stderr |
- bg_job.process_output(stdout=True,final_read=True) |
- bg_job.process_output(stdout=False,final_read=True) |
- finally: |
- # close our ends of the pipes to the sp no matter what |
- for bg_job in bg_jobs: |
- bg_job.cleanup() |
- |
- if timeout_error: |
- # TODO: This needs to be fixed to better represent what happens when |
- # running in parallel. However this is backwards compatable, so it will |
- # do for the time being. |
- raise error.CmdError(bg_jobs[0].command, bg_jobs[0].result, |
- "Command(s) did not complete within %d seconds" |
- % timeout) |
- |
- |
- return bg_jobs |
- |
- |
-def _wait_for_commands(bg_jobs, start_time, timeout): |
- # This returns True if it must return due to a timeout, otherwise False. |
- |
- # To check for processes which terminate without producing any output |
- # a 1 second timeout is used in select. |
- SELECT_TIMEOUT = 1 |
- |
- read_list = [] |
- write_list = [] |
- reverse_dict = {} |
- |
- for bg_job in bg_jobs: |
- read_list.append(bg_job.sp.stdout) |
- read_list.append(bg_job.sp.stderr) |
- reverse_dict[bg_job.sp.stdout] = (bg_job, True) |
- reverse_dict[bg_job.sp.stderr] = (bg_job, False) |
- if bg_job.string_stdin is not None: |
- write_list.append(bg_job.sp.stdin) |
- reverse_dict[bg_job.sp.stdin] = bg_job |
- |
- if timeout: |
- stop_time = start_time + timeout |
- time_left = stop_time - time.time() |
- else: |
- time_left = None # so that select never times out |
- |
- while not timeout or time_left > 0: |
- # select will return when we may write to stdin or when there is |
- # stdout/stderr output we can read (including when it is |
- # EOF, that is the process has terminated). |
- read_ready, write_ready, _ = select.select(read_list, write_list, [], |
- SELECT_TIMEOUT) |
- |
- # os.read() has to be used instead of |
- # subproc.stdout.read() which will otherwise block |
- for file_obj in read_ready: |
- bg_job, is_stdout = reverse_dict[file_obj] |
- bg_job.process_output(is_stdout) |
- |
- for file_obj in write_ready: |
- # we can write PIPE_BUF bytes without blocking |
- # POSIX requires PIPE_BUF is >= 512 |
- bg_job = reverse_dict[file_obj] |
- file_obj.write(bg_job.string_stdin[:512]) |
- bg_job.string_stdin = bg_job.string_stdin[512:] |
- # no more input data, close stdin, remove it from the select set |
- if not bg_job.string_stdin: |
- file_obj.close() |
- write_list.remove(file_obj) |
- del reverse_dict[file_obj] |
- |
- all_jobs_finished = True |
- for bg_job in bg_jobs: |
- if bg_job.result.exit_status is not None: |
- continue |
- |
- bg_job.result.exit_status = bg_job.sp.poll() |
- if bg_job.result.exit_status is not None: |
- # process exited, remove its stdout/stdin from the select set |
- bg_job.result.duration = time.time() - start_time |
- read_list.remove(bg_job.sp.stdout) |
- read_list.remove(bg_job.sp.stderr) |
- del reverse_dict[bg_job.sp.stdout] |
- del reverse_dict[bg_job.sp.stderr] |
- else: |
- all_jobs_finished = False |
- |
- if all_jobs_finished: |
- return False |
- |
- if timeout: |
- time_left = stop_time - time.time() |
- |
- # Kill all processes which did not complete prior to timeout |
- for bg_job in bg_jobs: |
- if bg_job.result.exit_status is not None: |
- continue |
- |
- logging.warn('run process timeout (%s) fired on: %s', timeout, |
- bg_job.command) |
- nuke_subprocess(bg_job.sp) |
- bg_job.result.exit_status = bg_job.sp.poll() |
- bg_job.result.duration = time.time() - start_time |
- |
- return True |
- |
- |
-def pid_is_alive(pid): |
- """ |
- True if process pid exists and is not yet stuck in Zombie state. |
- Zombies are impossible to move between cgroups, etc. |
- pid can be integer, or text of integer. |
- """ |
- path = '/proc/%s/stat' % pid |
- |
- try: |
- stat = read_one_line(path) |
- except IOError: |
- if not os.path.exists(path): |
- # file went away |
- return False |
- raise |
- |
- return stat.split()[2] != 'Z' |
- |
- |
-def signal_pid(pid, sig): |
- """ |
- Sends a signal to a process id. Returns True if the process terminated |
- successfully, False otherwise. |
- """ |
- try: |
- os.kill(pid, sig) |
- except OSError: |
- # The process may have died before we could kill it. |
- pass |
- |
- for i in range(5): |
- if not pid_is_alive(pid): |
- return True |
- time.sleep(1) |
- |
- # The process is still alive |
- return False |
- |
- |
-def nuke_subprocess(subproc): |
- # check if the subprocess is still alive, first |
- if subproc.poll() is not None: |
- return subproc.poll() |
- |
- # the process has not terminated within timeout, |
- # kill it via an escalating series of signals. |
- signal_queue = [signal.SIGTERM, signal.SIGKILL] |
- for sig in signal_queue: |
- signal_pid(subproc.pid, sig) |
- if subproc.poll() is not None: |
- return subproc.poll() |
- |
- |
-def nuke_pid(pid, signal_queue=(signal.SIGTERM, signal.SIGKILL)): |
- # the process has not terminated within timeout, |
- # kill it via an escalating series of signals. |
- for sig in signal_queue: |
- if signal_pid(pid, sig): |
- return |
- |
- # no signal successfully terminated the process |
- raise error.AutoservRunError('Could not kill %d' % pid, None) |
- |
- |
-def system(command, timeout=None, ignore_status=False): |
- """ |
- Run a command |
- |
- @param timeout: timeout in seconds |
- @param ignore_status: if ignore_status=False, throw an exception if the |
- command's exit code is non-zero |
- if ignore_stauts=True, return the exit code. |
- |
- @return exit status of command |
- (note, this will always be zero unless ignore_status=True) |
- """ |
- return run(command, timeout=timeout, ignore_status=ignore_status, |
- stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS).exit_status |
- |
- |
-def system_parallel(commands, timeout=None, ignore_status=False): |
- """This function returns a list of exit statuses for the respective |
- list of commands.""" |
- return [bg_jobs.exit_status for bg_jobs in |
- run_parallel(commands, timeout=timeout, ignore_status=ignore_status, |
- stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] |
- |
- |
-def system_output(command, timeout=None, ignore_status=False, |
- retain_output=False, args=()): |
- """ |
- Run a command and return the stdout output. |
- |
- @param command: command string to execute. |
- @param timeout: time limit in seconds before attempting to kill the |
- running process. The function will take a few seconds longer |
- than 'timeout' to complete if it has to kill the process. |
- @param ignore_status: do not raise an exception, no matter what the exit |
- code of the command is. |
- @param retain_output: set to True to make stdout/stderr of the command |
- output to be also sent to the logging system |
- @param args: sequence of strings of arguments to be given to the command |
- inside " quotes after they have been escaped for that; each |
- element in the sequence will be given as a separate command |
- argument |
- |
- @return a string with the stdout output of the command. |
- """ |
- if retain_output: |
- out = run(command, timeout=timeout, ignore_status=ignore_status, |
- stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS, |
- args=args).stdout |
- else: |
- out = run(command, timeout=timeout, ignore_status=ignore_status, |
- args=args).stdout |
- if out[-1:] == '\n': |
- out = out[:-1] |
- return out |
- |
- |
-def system_output_parallel(commands, timeout=None, ignore_status=False, |
- retain_output=False): |
- if retain_output: |
- out = [bg_job.stdout for bg_job |
- in run_parallel(commands, timeout=timeout, |
- ignore_status=ignore_status, |
- stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] |
- else: |
- out = [bg_job.stdout for bg_job in run_parallel(commands, |
- timeout=timeout, ignore_status=ignore_status)] |
- for x in out: |
- if out[-1:] == '\n': out = out[:-1] |
- return out |
- |
- |
-def strip_unicode(input): |
- if type(input) == list: |
- return [strip_unicode(i) for i in input] |
- elif type(input) == dict: |
- output = {} |
- for key in input.keys(): |
- output[str(key)] = strip_unicode(input[key]) |
- return output |
- elif type(input) == unicode: |
- return str(input) |
- else: |
- return input |
- |
- |
-def get_cpu_percentage(function, *args, **dargs): |
- """Returns a tuple containing the CPU% and return value from function call. |
- |
- This function calculates the usage time by taking the difference of |
- the user and system times both before and after the function call. |
- """ |
- child_pre = resource.getrusage(resource.RUSAGE_CHILDREN) |
- self_pre = resource.getrusage(resource.RUSAGE_SELF) |
- start = time.time() |
- to_return = function(*args, **dargs) |
- elapsed = time.time() - start |
- self_post = resource.getrusage(resource.RUSAGE_SELF) |
- child_post = resource.getrusage(resource.RUSAGE_CHILDREN) |
- |
- # Calculate CPU Percentage |
- s_user, s_system = [a - b for a, b in zip(self_post, self_pre)[:2]] |
- c_user, c_system = [a - b for a, b in zip(child_post, child_pre)[:2]] |
- cpu_percent = (s_user + c_user + s_system + c_system) / elapsed |
- |
- return cpu_percent, to_return |
- |
- |
-class SystemLoad(object): |
- """ |
- Get system and/or process values and return average value of load. |
- """ |
- def __init__(self, pids, advanced=False, time_step=0.1, cpu_cont=False, |
- use_log=False): |
- """ |
- @param pids: List of pids to be monitored. If pid = 0 whole system will |
- be monitored. pid == 0 means whole system. |
- @param advanced: monitor add value for system irq count and softirq |
- for process minor and maior page fault |
- @param time_step: Time step for continuous monitoring. |
- @param cpu_cont: If True monitor CPU load continuously. |
- @param use_log: If true every monitoring is logged for dump. |
- """ |
- self.pids = [] |
- self.stats = {} |
- for pid in pids: |
- if pid == 0: |
- cpu = FileFieldMonitor("/proc/stat", |
- [("cpu", 0), # User Time |
- ("cpu", 2), # System Time |
- ("intr", 0), # IRQ Count |
- ("softirq", 0)], # Soft IRQ Count |
- True, |
- cpu_cont, |
- use_log, |
- " +", |
- time_step) |
- mem = FileFieldMonitor("/proc/meminfo", |
- [("MemTotal:", 0), # Mem Total |
- ("MemFree:", 0), # Mem Free |
- ("Buffers:", 0), # Buffers |
- ("Cached:", 0)], # Cached |
- False, |
- True, |
- use_log, |
- " +", |
- time_step) |
- self.stats[pid] = ["TOTAL", cpu, mem] |
- self.pids.append(pid) |
- else: |
- name = "" |
- if (type(pid) is int): |
- self.pids.append(pid) |
- name = get_process_name(pid) |
- else: |
- self.pids.append(pid[0]) |
- name = pid[1] |
- |
- cpu = FileFieldMonitor("/proc/%d/stat" % |
- self.pids[-1], |
- [("", 13), # User Time |
- ("", 14), # System Time |
- ("", 9), # Minority Page Fault |
- ("", 11)], # Majority Page Fault |
- True, |
- cpu_cont, |
- use_log, |
- " +", |
- time_step) |
- mem = FileFieldMonitor("/proc/%d/status" % |
- self.pids[-1], |
- [("VmSize:", 0), # Virtual Memory Size |
- ("VmRSS:", 0), # Resident Set Size |
- ("VmPeak:", 0), # Peak VM Size |
- ("VmSwap:", 0)], # VM in Swap |
- False, |
- True, |
- use_log, |
- " +", |
- time_step) |
- self.stats[self.pids[-1]] = [name, cpu, mem] |
- |
- self.advanced = advanced |
- |
- |
- def __str__(self): |
- """ |
- Define format how to print |
- """ |
- out = "" |
- for pid in self.pids: |
- for stat in self.stats[pid][1:]: |
- out += str(stat.get_status()) + "\n" |
- return out |
- |
- |
- def start(self, pids=[]): |
- """ |
- Start monitoring of the process system usage. |
- @param pids: List of PIDs you intend to control. Use pids=[] to control |
- all defined PIDs. |
- """ |
- if pids == []: |
- pids = self.pids |
- |
- for pid in pids: |
- for stat in self.stats[pid][1:]: |
- stat.start() |
- |
- |
- def stop(self, pids=[]): |
- """ |
- Stop monitoring of the process system usage. |
- @param pids: List of PIDs you intend to control. Use pids=[] to control |
- all defined PIDs. |
- """ |
- if pids == []: |
- pids = self.pids |
- |
- for pid in pids: |
- for stat in self.stats[pid][1:]: |
- stat.stop() |
- |
- |
- def dump(self, pids=[]): |
- """ |
- Get the status of monitoring. |
- @param pids: List of PIDs you intend to control. Use pids=[] to control |
- all defined PIDs. |
- @return: |
- tuple([cpu load], [memory load]): |
- ([(PID1, (PID1_cpu_meas)), (PID2, (PID2_cpu_meas)), ...], |
- [(PID1, (PID1_mem_meas)), (PID2, (PID2_mem_meas)), ...]) |
- |
- PID1_cpu_meas: |
- average_values[], test_time, cont_meas_values[[]], time_step |
- PID1_mem_meas: |
- average_values[], test_time, cont_meas_values[[]], time_step |
- where average_values[] are the measured values (mem_free,swap,...) |
- which are described in SystemLoad.__init__()-FileFieldMonitor. |
- cont_meas_values[[]] is a list of average_values in the sampling |
- times. |
- """ |
- if pids == []: |
- pids = self.pids |
- |
- cpus = [] |
- memory = [] |
- for pid in pids: |
- stat = (pid, self.stats[pid][1].get_status()) |
- cpus.append(stat) |
- for pid in pids: |
- stat = (pid, self.stats[pid][2].get_status()) |
- memory.append(stat) |
- |
- return (cpus, memory) |
- |
- |
- def get_cpu_status_string(self, pids=[]): |
- """ |
- Convert status to string array. |
- @param pids: List of PIDs you intend to control. Use pids=[] to control |
- all defined PIDs. |
- @return: String format to table. |
- """ |
- if pids == []: |
- pids = self.pids |
- |
- headers = ["NAME", |
- ("%7s") % "PID", |
- ("%5s") % "USER", |
- ("%5s") % "SYS", |
- ("%5s") % "SUM"] |
- if self.advanced: |
- headers.extend(["MINFLT/IRQC", |
- "MAJFLT/SOFTIRQ"]) |
- headers.append(("%11s") % "TIME") |
- textstatus = [] |
- for pid in pids: |
- stat = self.stats[pid][1].get_status() |
- time = stat[1] |
- stat = stat[0] |
- textstatus.append(["%s" % self.stats[pid][0], |
- "%7s" % pid, |
- "%4.0f%%" % (stat[0] / time), |
- "%4.0f%%" % (stat[1] / time), |
- "%4.0f%%" % ((stat[0] + stat[1]) / time), |
- "%10.3fs" % time]) |
- if self.advanced: |
- textstatus[-1].insert(-1, "%11d" % stat[2]) |
- textstatus[-1].insert(-1, "%14d" % stat[3]) |
- |
- return matrix_to_string(textstatus, tuple(headers)) |
- |
- |
- def get_mem_status_string(self, pids=[]): |
- """ |
- Convert status to string array. |
- @param pids: List of PIDs you intend to control. Use pids=[] to control |
- all defined PIDs. |
- @return: String format to table. |
- """ |
- if pids == []: |
- pids = self.pids |
- |
- headers = ["NAME", |
- ("%7s") % "PID", |
- ("%8s") % "TOTAL/VMSIZE", |
- ("%8s") % "FREE/VMRSS", |
- ("%8s") % "BUFFERS/VMPEAK", |
- ("%8s") % "CACHED/VMSWAP", |
- ("%11s") % "TIME"] |
- textstatus = [] |
- for pid in pids: |
- stat = self.stats[pid][2].get_status() |
- time = stat[1] |
- stat = stat[0] |
- textstatus.append(["%s" % self.stats[pid][0], |
- "%7s" % pid, |
- "%10dMB" % (stat[0] / 1024), |
- "%8dMB" % (stat[1] / 1024), |
- "%12dMB" % (stat[2] / 1024), |
- "%11dMB" % (stat[3] / 1024), |
- "%10.3fs" % time]) |
- |
- return matrix_to_string(textstatus, tuple(headers)) |
- |
- |
-def get_arch(run_function=run): |
- """ |
- Get the hardware architecture of the machine. |
- run_function is used to execute the commands. It defaults to |
- utils.run() but a custom method (if provided) should be of the |
- same schema as utils.run. It should return a CmdResult object and |
- throw a CmdError exception. |
- """ |
- arch = run_function('/bin/uname -m').stdout.rstrip() |
- if re.match(r'i\d86$', arch): |
- arch = 'i386' |
- return arch |
- |
- |
-def get_num_logical_cpus_per_socket(run_function=run): |
- """ |
- Get the number of cores (including hyperthreading) per cpu. |
- run_function is used to execute the commands. It defaults to |
- utils.run() but a custom method (if provided) should be of the |
- same schema as utils.run. It should return a CmdResult object and |
- throw a CmdError exception. |
- """ |
- siblings = run_function('grep "^siblings" /proc/cpuinfo').stdout.rstrip() |
- num_siblings = map(int, |
- re.findall(r'^siblings\s*:\s*(\d+)\s*$', |
- siblings, re.M)) |
- if len(num_siblings) == 0: |
- raise error.TestError('Unable to find siblings info in /proc/cpuinfo') |
- if min(num_siblings) != max(num_siblings): |
- raise error.TestError('Number of siblings differ %r' % |
- num_siblings) |
- return num_siblings[0] |
- |
- |
-def merge_trees(src, dest): |
- """ |
- Merges a source directory tree at 'src' into a destination tree at |
- 'dest'. If a path is a file in both trees than the file in the source |
- tree is APPENDED to the one in the destination tree. If a path is |
- a directory in both trees then the directories are recursively merged |
- with this function. In any other case, the function will skip the |
- paths that cannot be merged (instead of failing). |
- """ |
- if not os.path.exists(src): |
- return # exists only in dest |
- elif not os.path.exists(dest): |
- if os.path.isfile(src): |
- shutil.copy2(src, dest) # file only in src |
- else: |
- shutil.copytree(src, dest, symlinks=True) # dir only in src |
- return |
- elif os.path.isfile(src) and os.path.isfile(dest): |
- # src & dest are files in both trees, append src to dest |
- destfile = open(dest, "a") |
- try: |
- srcfile = open(src) |
- try: |
- destfile.write(srcfile.read()) |
- finally: |
- srcfile.close() |
- finally: |
- destfile.close() |
- elif os.path.isdir(src) and os.path.isdir(dest): |
- # src & dest are directories in both trees, so recursively merge |
- for name in os.listdir(src): |
- merge_trees(os.path.join(src, name), os.path.join(dest, name)) |
- else: |
- # src & dest both exist, but are incompatible |
- return |
- |
- |
-class CmdResult(object): |
- """ |
- Command execution result. |
- |
- command: String containing the command line itself |
- exit_status: Integer exit code of the process |
- stdout: String containing stdout of the process |
- stderr: String containing stderr of the process |
- duration: Elapsed wall clock time running the process |
- """ |
- |
- |
- def __init__(self, command="", stdout="", stderr="", |
- exit_status=None, duration=0): |
- self.command = command |
- self.exit_status = exit_status |
- self.stdout = stdout |
- self.stderr = stderr |
- self.duration = duration |
- |
- |
- def __repr__(self): |
- wrapper = textwrap.TextWrapper(width = 78, |
- initial_indent="\n ", |
- subsequent_indent=" ") |
- |
- stdout = self.stdout.rstrip() |
- if stdout: |
- stdout = "\nstdout:\n%s" % stdout |
- |
- stderr = self.stderr.rstrip() |
- if stderr: |
- stderr = "\nstderr:\n%s" % stderr |
- |
- return ("* Command: %s\n" |
- "Exit status: %s\n" |
- "Duration: %s\n" |
- "%s" |
- "%s" |
- % (wrapper.fill(self.command), self.exit_status, |
- self.duration, stdout, stderr)) |
- |
- |
-class run_randomly: |
- def __init__(self, run_sequentially=False): |
- # Run sequentially is for debugging control files |
- self.test_list = [] |
- self.run_sequentially = run_sequentially |
- |
- |
- def add(self, *args, **dargs): |
- test = (args, dargs) |
- self.test_list.append(test) |
- |
- |
- def run(self, fn): |
- while self.test_list: |
- test_index = random.randint(0, len(self.test_list)-1) |
- if self.run_sequentially: |
- test_index = 0 |
- (args, dargs) = self.test_list.pop(test_index) |
- fn(*args, **dargs) |
- |
- |
-def import_site_module(path, module, dummy=None, modulefile=None): |
- """ |
- Try to import the site specific module if it exists. |
- |
- @param path full filename of the source file calling this (ie __file__) |
- @param module full module name |
- @param dummy dummy value to return in case there is no symbol to import |
- @param modulefile module filename |
- |
- @return site specific module or dummy |
- |
- @raises ImportError if the site file exists but imports fails |
- """ |
- short_module = module[module.rfind(".") + 1:] |
- |
- if not modulefile: |
- modulefile = short_module + ".py" |
- |
- if os.path.exists(os.path.join(os.path.dirname(path), modulefile)): |
- return __import__(module, {}, {}, [short_module]) |
- return dummy |
- |
- |
-def import_site_symbol(path, module, name, dummy=None, modulefile=None): |
- """ |
- Try to import site specific symbol from site specific file if it exists |
- |
- @param path full filename of the source file calling this (ie __file__) |
- @param module full module name |
- @param name symbol name to be imported from the site file |
- @param dummy dummy value to return in case there is no symbol to import |
- @param modulefile module filename |
- |
- @return site specific symbol or dummy |
- |
- @raises ImportError if the site file exists but imports fails |
- """ |
- module = import_site_module(path, module, modulefile=modulefile) |
- if not module: |
- return dummy |
- |
- # special unique value to tell us if the symbol can't be imported |
- cant_import = object() |
- |
- obj = getattr(module, name, cant_import) |
- if obj is cant_import: |
- logging.debug("unable to import site symbol '%s', using non-site " |
- "implementation", name) |
- return dummy |
- |
- return obj |
- |
- |
-def import_site_class(path, module, classname, baseclass, modulefile=None): |
- """ |
- Try to import site specific class from site specific file if it exists |
- |
- Args: |
- path: full filename of the source file calling this (ie __file__) |
- module: full module name |
- classname: class name to be loaded from site file |
- baseclass: base class object to return when no site file present or |
- to mixin when site class exists but is not inherited from baseclass |
- modulefile: module filename |
- |
- Returns: baseclass if site specific class does not exist, the site specific |
- class if it exists and is inherited from baseclass or a mixin of the |
- site specific class and baseclass when the site specific class exists |
- and is not inherited from baseclass |
- |
- Raises: ImportError if the site file exists but imports fails |
- """ |
- |
- res = import_site_symbol(path, module, classname, None, modulefile) |
- if res: |
- if not issubclass(res, baseclass): |
- # if not a subclass of baseclass then mix in baseclass with the |
- # site specific class object and return the result |
- res = type(classname, (res, baseclass), {}) |
- else: |
- res = baseclass |
- |
- return res |
- |
- |
-def import_site_function(path, module, funcname, dummy, modulefile=None): |
- """ |
- Try to import site specific function from site specific file if it exists |
- |
- Args: |
- path: full filename of the source file calling this (ie __file__) |
- module: full module name |
- funcname: function name to be imported from site file |
- dummy: dummy function to return in case there is no function to import |
- modulefile: module filename |
- |
- Returns: site specific function object or dummy |
- |
- Raises: ImportError if the site file exists but imports fails |
- """ |
- |
- return import_site_symbol(path, module, funcname, dummy, modulefile) |
- |
- |
-def _get_pid_path(program_name): |
- my_path = os.path.dirname(__file__) |
- return os.path.abspath(os.path.join(my_path, "..", "..", |
- "%s.pid" % program_name)) |
- |
- |
-def write_pid(program_name): |
- """ |
- Try to drop <program_name>.pid in the main autotest directory. |
- |
- Args: |
- program_name: prefix for file name |
- """ |
- pidfile = open(_get_pid_path(program_name), "w") |
- try: |
- pidfile.write("%s\n" % os.getpid()) |
- finally: |
- pidfile.close() |
- |
- |
-def delete_pid_file_if_exists(program_name): |
- """ |
- Tries to remove <program_name>.pid from the main autotest directory. |
- """ |
- pidfile_path = _get_pid_path(program_name) |
- |
- try: |
- os.remove(pidfile_path) |
- except OSError: |
- if not os.path.exists(pidfile_path): |
- return |
- raise |
- |
- |
-def get_pid_from_file(program_name): |
- """ |
- Reads the pid from <program_name>.pid in the autotest directory. |
- |
- @param program_name the name of the program |
- @return the pid if the file exists, None otherwise. |
- """ |
- pidfile_path = _get_pid_path(program_name) |
- if not os.path.exists(pidfile_path): |
- return None |
- |
- pidfile = open(_get_pid_path(program_name), 'r') |
- |
- try: |
- try: |
- pid = int(pidfile.readline()) |
- except IOError: |
- if not os.path.exists(pidfile_path): |
- return None |
- raise |
- finally: |
- pidfile.close() |
- |
- return pid |
- |
- |
-def get_process_name(pid): |
- """ |
- Get process name from PID. |
- @param pid: PID of process. |
- """ |
- return get_field(read_file("/proc/%d/stat" % pid), 1)[1:-1] |
- |
- |
-def program_is_alive(program_name): |
- """ |
- Checks if the process is alive and not in Zombie state. |
- |
- @param program_name the name of the program |
- @return True if still alive, False otherwise |
- """ |
- pid = get_pid_from_file(program_name) |
- if pid is None: |
- return False |
- return pid_is_alive(pid) |
- |
- |
-def signal_program(program_name, sig=signal.SIGTERM): |
- """ |
- Sends a signal to the process listed in <program_name>.pid |
- |
- @param program_name the name of the program |
- @param sig signal to send |
- """ |
- pid = get_pid_from_file(program_name) |
- if pid: |
- signal_pid(pid, sig) |
- |
- |
-def get_relative_path(path, reference): |
- """Given 2 absolute paths "path" and "reference", compute the path of |
- "path" as relative to the directory "reference". |
- |
- @param path the absolute path to convert to a relative path |
- @param reference an absolute directory path to which the relative |
- path will be computed |
- """ |
- # normalize the paths (remove double slashes, etc) |
- assert(os.path.isabs(path)) |
- assert(os.path.isabs(reference)) |
- |
- path = os.path.normpath(path) |
- reference = os.path.normpath(reference) |
- |
- # we could use os.path.split() but it splits from the end |
- path_list = path.split(os.path.sep)[1:] |
- ref_list = reference.split(os.path.sep)[1:] |
- |
- # find the longest leading common path |
- for i in xrange(min(len(path_list), len(ref_list))): |
- if path_list[i] != ref_list[i]: |
- # decrement i so when exiting this loop either by no match or by |
- # end of range we are one step behind |
- i -= 1 |
- break |
- i += 1 |
- # drop the common part of the paths, not interested in that anymore |
- del path_list[:i] |
- |
- # for each uncommon component in the reference prepend a ".." |
- path_list[:0] = ['..'] * (len(ref_list) - i) |
- |
- return os.path.join(*path_list) |
- |
- |
-def sh_escape(command): |
- """ |
- Escape special characters from a command so that it can be passed |
- as a double quoted (" ") string in a (ba)sh command. |
- |
- Args: |
- command: the command string to escape. |
- |
- Returns: |
- The escaped command string. The required englobing double |
- quotes are NOT added and so should be added at some point by |
- the caller. |
- |
- See also: http://www.tldp.org/LDP/abs/html/escapingsection.html |
- """ |
- command = command.replace("\\", "\\\\") |
- command = command.replace("$", r'\$') |
- command = command.replace('"', r'\"') |
- command = command.replace('`', r'\`') |
- return command |
- |
- |
-def configure(extra=None, configure='./configure'): |
- """ |
- Run configure passing in the correct host, build, and target options. |
- |
- @param extra: extra command line arguments to pass to configure |
- @param configure: which configure script to use |
- """ |
- args = [] |
- if 'CHOST' in os.environ: |
- args.append('--host=' + os.environ['CHOST']) |
- if 'CBUILD' in os.environ: |
- args.append('--build=' + os.environ['CBUILD']) |
- if 'CTARGET' in os.environ: |
- args.append('--target=' + os.environ['CTARGET']) |
- if extra: |
- args.append(extra) |
- |
- system('%s %s' % (configure, ' '.join(args))) |
- |
- |
-def make(extra='', make='make', timeout=None, ignore_status=False): |
- """ |
- Run make, adding MAKEOPTS to the list of options. |
- |
- @param extra: extra command line arguments to pass to make. |
- """ |
- cmd = '%s %s %s' % (make, os.environ.get('MAKEOPTS', ''), extra) |
- return system(cmd, timeout=timeout, ignore_status=ignore_status) |
- |
- |
-def compare_versions(ver1, ver2): |
- """Version number comparison between ver1 and ver2 strings. |
- |
- >>> compare_tuple("1", "2") |
- -1 |
- >>> compare_tuple("foo-1.1", "foo-1.2") |
- -1 |
- >>> compare_tuple("1.2", "1.2a") |
- -1 |
- >>> compare_tuple("1.2b", "1.2a") |
- 1 |
- >>> compare_tuple("1.3.5.3a", "1.3.5.3b") |
- -1 |
- |
- Args: |
- ver1: version string |
- ver2: version string |
- |
- Returns: |
- int: 1 if ver1 > ver2 |
- 0 if ver1 == ver2 |
- -1 if ver1 < ver2 |
- """ |
- ax = re.split('[.-]', ver1) |
- ay = re.split('[.-]', ver2) |
- while len(ax) > 0 and len(ay) > 0: |
- cx = ax.pop(0) |
- cy = ay.pop(0) |
- maxlen = max(len(cx), len(cy)) |
- c = cmp(cx.zfill(maxlen), cy.zfill(maxlen)) |
- if c != 0: |
- return c |
- return cmp(len(ax), len(ay)) |
- |
- |
-def args_to_dict(args): |
- """Convert autoserv extra arguments in the form of key=val or key:val to a |
- dictionary. Each argument key is converted to lowercase dictionary key. |
- |
- Args: |
- args - list of autoserv extra arguments. |
- |
- Returns: |
- dictionary |
- """ |
- arg_re = re.compile(r'(\w+)[:=](.*)$') |
- dict = {} |
- for arg in args: |
- match = arg_re.match(arg) |
- if match: |
- dict[match.group(1).lower()] = match.group(2) |
- else: |
- logging.warning("args_to_dict: argument '%s' doesn't match " |
- "'%s' pattern. Ignored." % (arg, arg_re.pattern)) |
- return dict |
- |
- |
-def get_unused_port(): |
- """ |
- Finds a semi-random available port. A race condition is still |
- possible after the port number is returned, if another process |
- happens to bind it. |
- |
- Returns: |
- A port number that is unused on both TCP and UDP. |
- """ |
- |
- def try_bind(port, socket_type, socket_proto): |
- s = socket.socket(socket.AF_INET, socket_type, socket_proto) |
- try: |
- try: |
- s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
- s.bind(('', port)) |
- return s.getsockname()[1] |
- except socket.error: |
- return None |
- finally: |
- s.close() |
- |
- # On the 2.6 kernel, calling try_bind() on UDP socket returns the |
- # same port over and over. So always try TCP first. |
- while True: |
- # Ask the OS for an unused port. |
- port = try_bind(0, socket.SOCK_STREAM, socket.IPPROTO_TCP) |
- # Check if this port is unused on the other protocol. |
- if port and try_bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP): |
- return port |
+from autotest_lib.client.common_lib.base_utils import * |
+if os.path.exists(os.path.join(os.path.dirname(__file__), 'site_utils.py')): |
+ from autotest_lib.client.common_lib.site_utils import * |