Merge autotest upstream from @5318 ~ @5336

client/virt/virt_env_process.py

 import os, time, commands, re, logging, glob, threading, shutil
 from autotest_lib.client.bin import utils
 from autotest_lib.client.common_lib import error
-import kvm_vm, kvm_utils, kvm_subprocess, kvm_monitor, ppm_utils, test_setup
+import aexpect, virt_utils, kvm_monitor, ppm_utils, virt_test_setup
+import virt_vm, kvm_vm
 try:
     import PIL.Image
 except ImportError:
     logging.warning('No python imaging library installed. PPM image '
                     'conversion to JPEG disabled. In order to enable it, '
                     'please install python-imaging or the equivalent for your '
                     'distro.')
 
 
 _screendump_thread = None
 _screendump_thread_termination_event = None
 
 
 def preprocess_image(test, params):
     """
     Preprocess a single QEMU image according to the instructions in params.
 
     @param test: Autotest test object.
     @param params: A dict containing image preprocessing parameters.
     @note: Currently this function just creates an image if requested.
     """
-    image_filename = kvm_vm.get_image_filename(params, test.bindir)
+    image_filename = virt_vm.get_image_filename(params, test.bindir)
 
     create_image = False
 
     if params.get("force_create_image") == "yes":
         logging.debug("'force_create_image' specified; creating image...")
         create_image = True
     elif (params.get("create_image") == "yes" and not
           os.path.exists(image_filename)):
         logging.debug("Creating image...")
         create_image = True
 
-    if create_image and not kvm_vm.create_image(params, test.bindir):
+    if create_image and not virt_vm.create_image(params, test.bindir):
         raise error.TestError("Could not create image")
 
 
 def preprocess_vm(test, params, env, name):
     """
     Preprocess a single VM object according to the instructions in params.
     Start the VM if requested and get a screendump.
 
     @param test: An Autotest test object.
     @param params: A dict containing VM preprocessing parameters.
     @param env: The environment (a dict-like object).
     @param name: The name of the VM object.
     """
     logging.debug("Preprocessing VM '%s'..." % name)
     vm = env.get_vm(name)
     if not vm:
         logging.debug("VM object does not exist; creating it")
-        vm = kvm_vm.VM(name, params, test.bindir, env.get("address_cache"))
+        vm_type = params.get('vm_type')
+        if vm_type == 'kvm':
+            vm = kvm_vm.VM(name, params, test.bindir, env.get("address_cache"))
         env.register_vm(name, vm)
 
     start_vm = False
 
     if params.get("restart_vm") == "yes":
         logging.debug("'restart_vm' specified; (re)starting VM...")
         start_vm = True
     elif params.get("migration_mode"):
         logging.debug("Starting VM in incoming migration mode...")
         start_vm = True
     elif params.get("start_vm") == "yes":
         if not vm.is_alive():
             logging.debug("VM is not alive; starting it...")
             start_vm = True
-        elif vm.make_qemu_command() != vm.make_qemu_command(name, params,
-                                                            test.bindir):
-            logging.debug("VM's qemu command differs from requested one; "
+        if vm.needs_restart(name=name, params=params, basedir=test.bindir):
+            logging.debug("Current VM specs differ from requested one; "
                           "restarting it...")
             start_vm = True
 
     if start_vm:
         # Start the VM (or restart it if it's already up)
         vm.create(name, params, test.bindir,
                   migration_mode=params.get("migration_mode"))
     else:
         # Don't start the VM, just update its params
         vm.params = params
 
     scrdump_filename = os.path.join(test.debugdir, "pre_%s.ppm" % name)
     try:
         if vm.monitor:
             vm.monitor.screendump(scrdump_filename, debug=False)
     except kvm_monitor.MonitorError, e:
         logging.warn(e)
 
 
 def postprocess_image(test, params):
     """
     Postprocess a single QEMU image according to the instructions in params.
 
     @param test: An Autotest test object.
     @param params: A dict containing image postprocessing parameters.
     """
     if params.get("check_image") == "yes":
-        kvm_vm.check_image(params, test.bindir)
+        virt_vm.check_image(params, test.bindir)
     if params.get("remove_image") == "yes":
-        kvm_vm.remove_image(params, test.bindir)
+        virt_vm.remove_image(params, test.bindir)
 
 
 def postprocess_vm(test, params, env, name):
     """
     Postprocess a single VM object according to the instructions in params.
     Kill the VM if requested and get a screendump.
 
     @param test: An Autotest test object.
     @param params: A dict containing VM postprocessing parameters.
     @param env: The environment (a dict-like object).
     @param name: The name of the VM object.
     """
     logging.debug("Postprocessing VM '%s'..." % name)
     vm = env.get_vm(name)
     if not vm:
         return
 
     scrdump_filename = os.path.join(test.debugdir, "post_%s.ppm" % name)
     try:
         if vm.monitor:
             vm.monitor.screendump(scrdump_filename, debug=False)
     except kvm_monitor.MonitorError, e:
         logging.warn(e)
 
     if params.get("kill_vm") == "yes":
         kill_vm_timeout = float(params.get("kill_vm_timeout", 0))
         if kill_vm_timeout:
             logging.debug("'kill_vm' specified; waiting for VM to shut down "
                           "before killing it...")
-            kvm_utils.wait_for(vm.is_dead, kill_vm_timeout, 0, 1)
+            virt_utils.wait_for(vm.is_dead, kill_vm_timeout, 0, 1)
         else:
             logging.debug("'kill_vm' specified; killing VM...")
         vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes")
 
 
 def process_command(test, params, env, command, command_timeout,
                     command_noncritical):
     """
     Pre- or post- custom commands to be executed before/after a test is run
 
@@ skipping 50 matching lines @@
     @param env: The environment (a dict-like object).
     """
     error.context("preprocessing")
     # Start tcpdump if it isn't already running
     if "address_cache" not in env:
         env["address_cache"] = {}
     if "tcpdump" in env and not env["tcpdump"].is_alive():
         env["tcpdump"].close()
         del env["tcpdump"]
     if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes":
-        cmd = "%s -npvi any 'dst port 68'" % kvm_utils.find_command("tcpdump")
+        cmd = "%s -npvi any 'dst port 68'" % virt_utils.find_command("tcpdump")
         logging.debug("Starting tcpdump (%s)...", cmd)
-        env["tcpdump"] = kvm_subprocess.Tail(
+        env["tcpdump"] = aexpect.Tail(
             command=cmd,
             output_func=_update_address_cache,
             output_params=(env["address_cache"],))
-        if kvm_utils.wait_for(lambda: not env["tcpdump"].is_alive(),
+        if virt_utils.wait_for(lambda: not env["tcpdump"].is_alive(),
                               0.1, 0.1, 1.0):
             logging.warn("Could not start tcpdump")
             logging.warn("Status: %s" % env["tcpdump"].get_status())
-            logging.warn("Output:" + kvm_utils.format_str_for_message(
+            logging.warn("Output:" + virt_utils.format_str_for_message(
                 env["tcpdump"].get_output()))
 
     # Destroy and remove VMs that are no longer needed in the environment
     requested_vms = params.objects("vms")
     for key in env.keys():
         vm = env[key]
-        if not kvm_utils.is_vm(vm):
+        if not virt_utils.is_vm(vm):
             continue
         if not vm.name in requested_vms:
             logging.debug("VM '%s' found in environment but not required for "
                           "test; removing it..." % vm.name)
             vm.destroy()
             del env[key]
 
     # Get the KVM kernel module version and write it as a keyval
     logging.debug("Fetching KVM module version...")
     if os.path.exists("/dev/kvm"):
         try:
             kvm_version = open("/sys/module/kvm/version").read().strip()
         except:
             kvm_version = os.uname()[2]
     else:
         kvm_version = "Unknown"
         logging.debug("KVM module not loaded")
     logging.debug("KVM version: %s" % kvm_version)
     test.write_test_keyval({"kvm_version": kvm_version})
 
     # Get the KVM userspace version and write it as a keyval
     logging.debug("Fetching KVM userspace version...")
-    qemu_path = kvm_utils.get_path(test.bindir, params.get("qemu_binary",
+    qemu_path = virt_utils.get_path(test.bindir, params.get("qemu_binary",
                                                            "qemu"))
     version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
     matches = re.findall("[Vv]ersion .*?,", version_line)
     if matches:
         kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
     else:
         kvm_userspace_version = "Unknown"
         logging.debug("Could not fetch KVM userspace version")
     logging.debug("KVM userspace version: %s" % kvm_userspace_version)
     test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
 
     if params.get("setup_hugepages") == "yes":
-        h = test_setup.HugePageConfig(params)
+        h = virt_test_setup.HugePageConfig(params)
         h.setup()
 
     # Execute any pre_commands
     if params.get("pre_command"):
         process_command(test, params, env, params.get("pre_command"),
                         int(params.get("pre_command_timeout", "600")),
                         params.get("pre_command_noncritical") == "yes")
 
     # Preprocess all VMs and images
     process(test, params, env, preprocess_image, preprocess_vm)
@@ skipping 65 matching lines @@
 
     # Kill all unresponsive VMs
     if params.get("kill_unresponsive_vms") == "yes":
         logging.debug("'kill_unresponsive_vms' specified; killing all VMs "
                       "that fail to respond to a remote login request...")
         for vm in env.get_all_vms():
             if vm.is_alive():
                 try:
                     session = vm.login()
                     session.close()
-                except (kvm_utils.LoginError, kvm_vm.VMError), e:
+                except (virt_utils.LoginError, virt_vm.VMError), e:
                     logging.warn(e)
                     vm.destroy(gracefully=False)
 
-    # Kill all kvm_subprocess tail threads
-    kvm_subprocess.kill_tail_threads()
+    # Kill all aexpect tail threads
+    aexpect.kill_tail_threads()
 
     # Terminate tcpdump if no VMs are alive
     living_vms = [vm for vm in env.get_all_vms() if vm.is_alive()]
     if not living_vms and "tcpdump" in env:
         env["tcpdump"].close()
         del env["tcpdump"]
 
     if params.get("setup_hugepages") == "yes":
-        h = test_setup.HugePageConfig(params)
+        h = virt_test_setup.HugePageConfig(params)
         h.cleanup()
 
     # Execute any post_commands
     if params.get("post_command"):
         process_command(test, params, env, params.get("post_command"),
                         int(params.get("post_command_timeout", "600")),
                         params.get("post_command_noncritical") == "yes")
 
 
 def postprocess_on_error(test, params, env):
@@ skipping 21 matching lines @@
                               mac_address, address_cache.get("last_seen"))
             address_cache[mac_address] = address_cache.get("last_seen")
             address_cache["time_%s" % mac_address] = time.time()
             del address_cache["last_seen"]
 
 
 def _take_screendumps(test, params, env):
     global _screendump_thread_termination_event
     temp_dir = test.debugdir
     if params.get("screendump_temp_dir"):
-        temp_dir = kvm_utils.get_path(test.bindir,
+        temp_dir = virt_utils.get_path(test.bindir,
                                       params.get("screendump_temp_dir"))
         try:
             os.makedirs(temp_dir)
         except OSError:
             pass
     temp_filename = os.path.join(temp_dir, "scrdump-%s.ppm" %
-                                 kvm_utils.generate_random_string(6))
+                                 virt_utils.generate_random_string(6))
     delay = float(params.get("screendump_delay", 5))
     quality = int(params.get("screendump_quality", 30))
 
     cache = {}
 
     while True:
         for vm in env.get_all_vms():
             if not vm.is_alive():
                 continue
             try:
@@ skipping 28 matching lines @@
                     image = PIL.Image.open(temp_filename)
                     image.save(screendump_filename, format="JPEG", quality=quality)
                     cache[hash] = screendump_filename
                 except NameError:
                     pass
             os.unlink(temp_filename)
         if _screendump_thread_termination_event.isSet():
             _screendump_thread_termination_event = None
             break
         _screendump_thread_termination_event.wait(delay)