Merge autotest upstream from @5318 ~ @5336

client/virt/kvm_installer.py

-import os, logging, datetime, glob
-import shutil
+import os, logging, datetime, glob, shutil
 from autotest_lib.client.bin import utils, os_dep
 from autotest_lib.client.common_lib import error
-import kvm_utils
-
-
-def check_configure_options(script_path):
-    """
-    Return the list of available options (flags) of a given kvm configure build
-    script.
-
-    @param script: Path to the configure script
-    """
-    abspath = os.path.abspath(script_path)
-    help_raw = utils.system_output('%s --help' % abspath, ignore_status=True)
-    help_output = help_raw.split("\n")
-    option_list = []
-    for line in help_output:
-        cleaned_line = line.lstrip()
-        if cleaned_line.startswith("--"):
-            option = cleaned_line.split()[0]
-            option = option.split("=")[0]
-            option_list.append(option)
-
-    return option_list
+import virt_utils, virt_installer
 
 
 def kill_qemu_processes():
     """
     Kills all qemu processes, also kills all processes holding /dev/kvm down.
     """
     logging.debug("Killing any qemu processes that might be left behind")
     utils.system("pkill qemu", ignore_status=True)
     # Let's double check to see if some other process is holding /dev/kvm
     if os.path.isfile("/dev/kvm"):
         utils.system("fuser -k /dev/kvm", ignore_status=True)
 
 
-def cpu_vendor():
-    vendor = "intel"
-    if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0:
-        vendor = "amd"
-    logging.debug("Detected CPU vendor as '%s'", vendor)
-    return vendor
-
-
 def _unload_kvm_modules(mod_list):
     logging.info("Unloading previously loaded KVM modules")
     for module in reversed(mod_list):
         utils.unload_module(module)
 
 
 def _load_kvm_modules(mod_list, module_dir=None, load_stock=False):
     """
     Just load the KVM modules, without killing Qemu or unloading previous
     modules.
@@ skipping 95 matching lines @@
 
 def install_roms(rom_dir, prefix):
     logging.debug("Path to roms specified. Copying roms to install prefix")
     rom_dst_dir = os.path.join(prefix, 'share', 'qemu')
     for rom_src in glob.glob('%s/*.bin' % rom_dir):
         rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src))
         logging.debug("Copying rom file %s to %s", rom_src, rom_dst)
         shutil.copy(rom_src, rom_dst)
 
 
-def save_build(build_dir, dest_dir):
-    logging.debug('Saving the result of the build on %s', dest_dir)
-    base_name = os.path.basename(build_dir)
-    tarball_name = base_name + '.tar.bz2'
-    os.chdir(os.path.dirname(build_dir))
-    utils.system('tar -cjf %s %s' % (tarball_name, base_name))
-    shutil.move(tarball_name, os.path.join(dest_dir, tarball_name))
-
-
 class KvmInstallException(Exception):
     pass
 
 
 class FailedKvmInstall(KvmInstallException):
     pass
 
 
 class KvmNotInstalled(KvmInstallException):
     pass
@@ skipping 11 matching lines @@
 
         load_modules = params.get('load_modules', 'no')
         if not load_modules or load_modules == 'yes':
             self.should_load_modules = True
         elif load_modules == 'no':
             self.should_load_modules = False
         default_extra_modules = str(None)
         self.extra_modules = eval(params.get("extra_modules",
                                              default_extra_modules))
 
-        self.cpu_vendor = cpu_vendor()
+        self.cpu_vendor = virt_installer.cpu_vendor()
 
         self.srcdir = test.srcdir
         if not os.path.isdir(self.srcdir):
             os.makedirs(self.srcdir)
 
         self.test_bindir = test.bindir
         self.results_dir = test.resultsdir
 
         # KVM build prefix, for the modes that do need it
         prefix = os.path.join(test.bindir, 'build')
@@ skipping 19 matching lines @@
 
     def install_unittests(self):
         userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace")
         test_repo = self.params.get("test_git_repo")
         test_branch = self.params.get("test_branch", "master")
         test_commit = self.params.get("test_commit", None)
         test_lbranch = self.params.get("test_lbranch", "master")
 
         if test_repo:
             test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests")
-            kvm_utils.get_git_branch(test_repo, test_branch, test_srcdir,
+            virt_utils.get_git_branch(test_repo, test_branch, test_srcdir,
                                      test_commit, test_lbranch)
             unittest_cfg = os.path.join(test_srcdir, 'x86',
                                         'unittests.cfg')
             self.test_srcdir = test_srcdir
         else:
             unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86',
                                         'unittests.cfg')
         self.unittest_cfg = None
         if os.path.isfile(unittest_cfg):
             self.unittest_cfg = unittest_cfg
@@ skipping 125 matching lines @@
     def install(self):
         self.install_unittests()
         self._clean_previous_installs()
         self._get_packages()
         self._install_packages()
         create_symlinks(test_bindir=self.test_bindir,
                         bin_list=self.qemu_bin_paths,
                         unittest=self.unittest_prefix)
         self.reload_modules_if_needed()
         if self.save_results:
-            save_build(self.srcdir, self.results_dir)
+            virt_installer.save_build(self.srcdir, self.results_dir)
 
 
 class KojiInstaller(YumInstaller):
     """
     Class that handles installing KVM from the fedora build service, koji.
-
-    It uses yum to install and remove packages. Packages are specified
-    according to the syntax defined in the PkgSpec class.
+    It uses yum to install and remove packages.
     """
     load_stock_modules = True
     def set_install_params(self, test, params):
         """
         Gets parameters and initializes the package downloader.
 
         @param test: kvm test object
         @param params: Dictionary with test arguments
         """
         super(KojiInstaller, self).set_install_params(test, params)
+        default_koji_cmd = '/usr/bin/koji'
+        default_src_pkg = 'qemu'
+        self.src_pkg = params.get("src_pkg", default_src_pkg)
         self.tag = params.get("koji_tag", None)
-        self.koji_cmd = params.get("koji_cmd", None)
-        if self.tag is not None:
-            kvm_utils.set_default_koji_tag(self.tag)
-        self.koji_pkgs = eval(params.get("koji_pkgs", "[]"))
+        self.build = params.get("koji_build", None)
+        self.koji_cmd = params.get("koji_cmd", default_koji_cmd)
 
 
     def _get_packages(self):
         """
         Downloads the specific arch RPMs for the specific build name.
         """
-        koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd)
-        for pkg_text in self.koji_pkgs:
-            pkg = kvm_utils.KojiPkgSpec(pkg_text)
-            if pkg.is_valid():
-                koji_client.get_pkgs(pkg, dst_dir=self.srcdir)
-            else:
-                logging.error('Package specification (%s) is invalid: %s', pkg,
-                              pkg.describe_invalid())
-
-
-    def _clean_previous_installs(self):
-        kill_qemu_processes()
-        removable_packages = " ".join(self._get_rpm_names())
-        utils.system("yum -y remove %s" % removable_packages)
+        downloader = virt_utils.KojiDownloader(cmd=self.koji_cmd)
+        downloader.get(src_package=self.src_pkg, tag=self.tag,
+                            build=self.build, dst_dir=self.srcdir)
 
 
     def install(self):
-        self._clean_previous_installs()
+        super(KojiInstaller, self)._clean_previous_installs()
         self._get_packages()
-        self._install_packages()
+        super(KojiInstaller, self)._install_packages()
         self.install_unittests()
         create_symlinks(test_bindir=self.test_bindir,
                         bin_list=self.qemu_bin_paths,
                         unittest=self.unittest_prefix)
         self.reload_modules_if_needed()
         if self.save_results:
-            save_build(self.srcdir, self.results_dir)
-
-
-    def _get_rpm_names(self):
-        all_rpm_names = []
-        koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd)
-        for pkg_text in self.koji_pkgs:
-            pkg = kvm_utils.KojiPkgSpec(pkg_text)
-            rpm_names = koji_client.get_pkg_rpm_names(pkg)
-            all_rpm_names += rpm_names
-        return all_rpm_names
-
-
-    def _get_rpm_file_names(self):
-        all_rpm_file_names = []
-        koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd)
-        for pkg_text in self.koji_pkgs:
-            pkg = kvm_utils.KojiPkgSpec(pkg_text)
-            rpm_file_names = koji_client.get_pkg_rpm_file_names(pkg)
-            all_rpm_file_names += rpm_file_names
-        return all_rpm_file_names
-
-
-    def _install_packages(self):
-        """
-        Install all downloaded packages.
-        """
-        os.chdir(self.srcdir)
-        rpm_file_names = " ".join(self._get_rpm_file_names())
-        utils.system("yum --nogpgcheck -y localinstall %s" % rpm_file_names)
+            virt_installer.save_build(self.srcdir, self.results_dir)
 
 
 class SourceDirInstaller(BaseInstaller):
     """
     Class that handles building/installing KVM directly from a tarball or
     a single source code dir.
     """
     def set_install_params(self, test, params):
         """
         Initializes class attributes, and retrieves KVM code.
@@ skipping 17 matching lines @@
                                       "control file.")
             else:
                 shutil.copytree(srcdir, self.srcdir)
 
         if self.install_mode == 'release':
             release_tag = params.get("release_tag")
             release_dir = params.get("release_dir")
             release_listing = params.get("release_listing")
             logging.info("Installing KVM from release tarball")
             if not release_tag:
-                release_tag = kvm_utils.get_latest_kvm_release_tag(
+                release_tag = virt_utils.get_latest_kvm_release_tag(
                                                                 release_listing)
             tarball = os.path.join(release_dir, 'kvm', release_tag,
                                    "kvm-%s.tar.gz" % release_tag)
             logging.info("Retrieving release kvm-%s" % release_tag)
             tarball = utils.unmap_url("/", tarball, "/tmp")
 
         elif self.install_mode == 'snapshot':
             logging.info("Installing KVM from snapshot")
             snapshot_dir = params.get("snapshot_dir")
             if not snapshot_dir:
@@ skipping 15 matching lines @@
                 raise error.TestError("KVM Tarball install specified but no"
                                       " tarball provided on control file.")
             logging.info("Installing KVM from a local tarball")
             logging.info("Using tarball %s")
             tarball = utils.unmap_url("/", params.get("tarball"), "/tmp")
 
         if self.install_mode in ['release', 'snapshot', 'localtar']:
             utils.extract_tarball_to_dir(tarball, self.srcdir)
 
         if self.install_mode in ['release', 'snapshot', 'localtar', 'srcdir']:
-            self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir)
-            configure_script = os.path.join(self.srcdir, 'configure')
-            self.configure_options = check_configure_options(configure_script)
+            self.repo_type = virt_utils.check_kvm_source_dir(self.srcdir)
+            p = os.path.join(self.srcdir, 'configure')
+            self.configure_options = virt_installer.check_configure_options(p)
 
 
     def _build(self):
         make_jobs = utils.count_cpus()
         os.chdir(self.srcdir)
         # For testing purposes, it's better to build qemu binaries with
         # debugging symbols, so we can extract more meaningful stack traces.
         cfg = "./configure --prefix=%s" % self.prefix
         if "--disable-strip" in self.configure_options:
             cfg += " --disable-strip"
@@ skipping 67 matching lines @@
         else:
             logging.info("Loading stock KVM modules")
             _load_kvm_modules(mod_list, load_stock=True)
 
 
     def install(self):
         self._build()
         self._install()
         self.reload_modules_if_needed()
         if self.save_results:
-            save_build(self.srcdir, self.results_dir)
+            virt_installer.save_build(self.srcdir, self.results_dir)
 
 
 class GitInstaller(SourceDirInstaller):
     def _pull_code(self):
         """
         Retrieves code from git repositories.
         """
         params = self.params
 
         kernel_repo = params.get("git_repo")
@@ skipping 15 matching lines @@
         kernel_patches = eval(params.get("kernel_patches", "[]"))
         user_patches = eval(params.get("user_patches", "[]"))
         kmod_patches = eval(params.get("user_patches", "[]"))
 
         if not user_repo:
             message = "KVM user git repository path not specified"
             logging.error(message)
             raise error.TestError(message)
 
         userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace")
-        kvm_utils.get_git_branch(user_repo, user_branch, userspace_srcdir,
+        virt_utils.get_git_branch(user_repo, user_branch, userspace_srcdir,
                                  user_commit, user_lbranch)
         self.userspace_srcdir = userspace_srcdir
 
         if user_patches:
             os.chdir(self.userspace_srcdir)
             for patch in user_patches:
                 utils.get_file(patch, os.path.join(self.userspace_srcdir,
                                                    os.path.basename(patch)))
                 utils.system('patch -p1 < %s' % os.path.basename(patch))
 
         if kernel_repo:
             kernel_srcdir = os.path.join(self.srcdir, "kvm")
-            kvm_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir,
+            virt_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir,
                                      kernel_commit, kernel_lbranch)
             self.kernel_srcdir = kernel_srcdir
             if kernel_patches:
                 os.chdir(self.kernel_srcdir)
                 for patch in kernel_patches:
                     utils.get_file(patch, os.path.join(self.userspace_srcdir,
                                                        os.path.basename(patch)))
                     utils.system('patch -p1 < %s' % os.path.basename(patch))
         else:
             self.kernel_srcdir = None
 
         if kmod_repo:
             kmod_srcdir = os.path.join (self.srcdir, "kvm_kmod")
-            kvm_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir,
+            virt_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir,
                                      kmod_commit, kmod_lbranch)
             self.kmod_srcdir = kmod_srcdir
             if kmod_patches:
                 os.chdir(self.kmod_srcdir)
                 for patch in kmod_patches:
                     utils.get_file(patch, os.path.join(self.userspace_srcdir,
                                                        os.path.basename(patch)))
                     utils.system('patch -p1 < %s' % os.path.basename(patch))
         else:
             self.kmod_srcdir = None
 
-        configure_script = os.path.join(self.userspace_srcdir, 'configure')
-        self.configure_options = check_configure_options(configure_script)
+        p = os.path.join(self.userspace_srcdir, 'configure')
+        self.configure_options = virt_installer.check_configure_options(p)
 
 
     def _build(self):
         make_jobs = utils.count_cpus()
         cfg = './configure'
         if self.kmod_srcdir:
             logging.info('Building KVM modules')
             os.chdir(self.kmod_srcdir)
             module_build_steps = [cfg,
                                   'make clean',
@@ skipping 52 matching lines @@
                         bin_list=None,
                         unittest=self.unittest_prefix)
 
 
     def install(self):
         self._pull_code()
         self._build()
         self._install()
         self.reload_modules_if_needed()
         if self.save_results:
-            save_build(self.srcdir, self.results_dir)
+            virt_installer.save_build(self.srcdir, self.results_dir)
 
 
 class PreInstalledKvm(BaseInstaller):
     # load_modules() will use the stock modules:
     load_stock_modules = True
     def install(self):
         logging.info("Expecting KVM to be already installed. Doing nothing")
 
 
 class FailedInstaller:
@@ skipping 31 matching lines @@
     return c
 
 
 def make_installer(params):
     # priority:
     # - 'install_mode' param
     # - 'mode' param
     mode = params.get("install_mode", params.get("mode"))
     klass = _installer_class(mode)
     return klass(mode)