Merge remote branch 'cros/upstream' into master

client/tests/kvm/kvm_config.py

 #!/usr/bin/python
 """
 KVM configuration file utility functions.
 
 @copyright: Red Hat 2008-2010
 """
 
 import logging, re, os, sys, optparse, array, traceback, cPickle
 import common
 import kvm_utils
 from autotest_lib.client.common_lib import error
-from autotest_lib.client.common_lib import logging_config, logging_manager
+from autotest_lib.client.common_lib import logging_manager
 
 
 class config:
     """
     Parse an input file or string that follows the KVM Test Config File format
     and generate a list of dicts that will be later used as configuration
     parameters by the KVM tests.
 
     @see: http://www.linux-kvm.org/page/KVM-Autotest/Test_Config_File
     """
 
     def __init__(self, filename=None, debug=True):
         """
         Initialize the list and optionally parse a file.
 
         @param filename: Path of the file that will be taken.
         @param debug: Whether to turn on debugging output.
         """
         self.list = [array.array("H", [4, 4, 4, 4])]
         self.object_cache = []
         self.object_cache_indices = {}
         self.regex_cache = {}
-        self.filename = filename
         self.debug = debug
         if filename:
             self.parse_file(filename)
 
 
     def parse_file(self, filename):
         """
         Parse file.  If it doesn't exist, raise an IOError.
 
         @param filename: Path of the configuration file.
         """
         if not os.path.exists(filename):
             raise IOError("File %s not found" % filename)
-        self.filename = filename
         str = open(filename).read()
-        self.list = self.parse(configreader(str), self.list)
+        self.list = self.parse(configreader(filename, str), self.list)
 
 
     def parse_string(self, str):
         """
         Parse a string.
 
         @param str: String to parse.
         """
-        self.list = self.parse(configreader(str), self.list)
+        self.list = self.parse(configreader('<string>', str, real_file=False), self.list)
 
 
     def fork_and_parse(self, filename=None, str=None):
         """
         Parse a file and/or a string in a separate process to save memory.
 
         Python likes to keep memory to itself even after the objects occupying
         it have been destroyed.  If during a call to parse_file() or
         parse_string() a lot of memory is used, it can only be freed by
         terminating the process.  This function works around the problem by
@@ skipping 219 matching lines @@
                                  "context, %d remain)" %
                                  (len_list, len(list)))
                 continue
 
             # Parse 'variants'
             if line == "variants:":
                 # 'variants' not allowed in restricted mode
                 # (inside an exception or inside subvariants)
                 if restricted:
                     e_msg = "Using variants in this context is not allowed"
-                    raise error.AutotestError(e_msg)
+                    cr.raise_error(e_msg)
                 if self.debug and not restricted:
                     _debug_print(indented_line,
                                  "Entering variants block (%d dicts in "
                                  "current context)" % len_list)
                 list = self.parse_variants(cr, list, subvariants=False,
                                            prev_indent=indent)
                 continue
 
             # Parse 'subvariants' (the block is parsed for each dict
             # separately)
@@ skipping 19 matching lines @@
                                                     prev_indent=indent)
                 list = new_list
                 continue
 
             # Parse 'include' statements
             if words[0] == "include":
                 if len(words) <= 1:
                     continue
                 if self.debug and not restricted:
                     _debug_print(indented_line, "Entering file %s" % words[1])
-                if self.filename:
-                    filename = os.path.join(os.path.dirname(self.filename),
-                                            words[1])
-                    if os.path.exists(filename):
-                        str = open(filename).read()
-                        list = self.parse(configreader(str), list, restricted)
-                        if self.debug and not restricted:
-                            _debug_print("", "Leaving file %s" % words[1])
-                    else:
-                        logging.warning("Cannot include %s -- file not found",
-                                        filename)
-                else:
-                    logging.warning("Cannot include %s because no file is "
-                                    "currently open", words[1])
+
+                cur_filename = cr.real_filename()
+                if cur_filename is None:
+                    cr.raise_error("'include' is valid only when parsing a file")
+
+                filename = os.path.join(os.path.dirname(cur_filename),
+                                        words[1])
+                if not os.path.exists(filename):
+                    cr.raise_error("Cannot include %s -- file not found" % (filename))
+
+                str = open(filename).read()
+                list = self.parse(configreader(filename, str), list, restricted)
+                if self.debug and not restricted:
+                    _debug_print("", "Leaving file %s" % words[1])
+
                 continue
 
             # Parse multi-line exceptions
             # (the block is parsed for each dict separately)
             if line.endswith(":"):
                 if self.debug and not restricted:
                     _debug_print(indented_line,
                                  "Entering multi-line exception block "
                                  "(%d dicts in current context outside "
                                  "exception)" % len_list)
@@ skipping 168 matching lines @@
 
 # configreader
 
 class configreader:
     """
     Preprocess an input string and provide file-like services.
     This is intended as a replacement for the file and StringIO classes,
     whose readline() and/or seek() methods seem to be slow.
     """
 
-    def __init__(self, str):
+    def __init__(self, filename, str, real_file=True):
         """
         Initialize the reader.
 
+        @param filename: the filename we're parsing
         @param str: The string to parse.
+        @param real_file: Indicates if filename represents a real file. Defaults to True.
         """
+        self.filename = filename
+        self.is_real_file = real_file
         self.line_index = 0
         self.lines = []
-        for line in str.splitlines():
+        self.real_number = []
+        for num, line in enumerate(str.splitlines()):
             line = line.rstrip().expandtabs()
             stripped_line = line.strip()
             indent = len(line) - len(stripped_line)
             if (not stripped_line
                 or stripped_line.startswith("#")
                 or stripped_line.startswith("//")):
                 continue
             self.lines.append((line, stripped_line, indent))
+            self.real_number.append(num + 1)
 
 
+    def real_filename(self):
+        """Returns the filename we're reading, in case it is a real file
+
+        @returns the filename we are parsing, or None in case we're not parsing a real file
+        """
+        if self.is_real_file:
+            return self.filename
+
     def get_next_line(self):
         """
         Get the next non-empty, non-comment line in the string.
 
         @param file: File like object.
         @return: (line, stripped_line, indent), where indent is the line's
             indent level or -1 if no line is available.
         """
         try:
             if self.line_index < len(self.lines):
@@ skipping 10 matching lines @@
         """
         return self.line_index
 
 
     def seek(self, index):
         """
         Set the current line index.
         """
         self.line_index = index
 
+    def raise_error(self, msg):
+        """Raise an error related to the last line returned by get_next_line()
+        """
+        if self.line_index == 0: # nothing was read. shouldn't happen, but...
+            line_id = 'BEGIN'
+        elif self.line_index >= len(self.lines): # past EOF
+            line_id = 'EOF'
+        else:
+            # line_index is the _next_ line. get the previous one
+            line_id = str(self.real_number[self.line_index-1])
+        raise error.AutotestError("%s:%s: %s" % (self.filename, line_id, msg))
+
 
 # Array structure:
 # ----------------
 # The first 4 elements contain the indices of the 4 segments.
 # a[0] -- Index of beginning of 'name' segment (always 4).
 # a[1] -- Index of beginning of 'shortname' segment.
 # a[2] -- Index of beginning of 'depend' segment.
 # a[3] -- Index of beginning of 'content' segment.
 # The next elements in the array comprise the aforementioned segments:
 # The 'name' segment begins with a[a[0]] and ends with a[a[1]-1].
@@ skipping 74 matching lines @@
 
 
 if __name__ == "__main__":
     parser = optparse.OptionParser("usage: %prog [options] [filename]")
     parser.add_option('--verbose', dest="debug", action='store_true',
                       help='include debug messages in console output')
 
     options, args = parser.parse_args()
     debug = options.debug
     if args:
-        filename = args[0]
+        filenames = args
     else:
-        filename = os.path.join(os.path.dirname(sys.argv[0]), "tests.cfg")
+        filenames = [os.path.join(os.path.dirname(sys.argv[0]), "tests.cfg")]
 
     # Here we configure the stand alone program to use the autotest
     # logging system.
     logging_manager.configure_logging(kvm_utils.KvmLoggingConfig(),
                                       verbose=debug)
-    dicts = config(filename, debug=debug).get_generator()
+    cfg = config(debug=debug)
+    for fn in filenames:
+        cfg.parse_file(fn)
+    dicts = cfg.get_generator()
     for i, dict in enumerate(dicts):
-        logging.info("Dictionary #%d:", i)
+        print "Dictionary #%d:" % (i)
         keys = dict.keys()
         keys.sort()
         for key in keys:
-            logging.info("    %s = %s", key, dict[key])
+            print "    %s = %s" % (key, dict[key])