Merge remote branch 'autotest-upstream/master' into autotest-merge

utils/packager.py

 #!/usr/bin/python -u
 
 """
 Utility to upload or remove the packages from the packages repository.
 """
 
 import logging, os, sys, optparse, socket, tempfile, shutil
 import common
 from autotest_lib.client.common_lib import utils as client_utils
 from autotest_lib.client.common_lib import global_config, error
@@ skipping 53 matching lines @@
     parser.add_option("--all", help="Upload all the files locally "
                       "to all the repos specified in global_config.ini. "
                       "(includes the client, tests, deps and profilers)",
                       dest="all", action="store_true", default=False)
 
     options, args = parser.parse_args()
     return options, args
 
 
 # Method to upload or remove package depending on the flag passed to it.
-def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, repo_url,
+def process_packages(pkgmgr, pkg_type, pkg_names, src_dir,
                     remove=False):
     exclude_string = ' .'
     names = [p.strip() for p in pkg_names.split(',')]
     for name in names:
         print "Processing %s ... " % name
         if pkg_type=='client':
             pkg_dir = src_dir
             exclude_string  = get_exclude_string(pkg_dir)
         elif pkg_type=='test':
             # if the package is a test then look whether it is in client/tests
             # or client/site_tests
             pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
         else:
             # for the profilers and deps
             pkg_dir = os.path.join(src_dir, name)
 
         pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
         if not remove:
             # Tar the source and upload
             temp_dir = tempfile.mkdtemp()
             try:
                 try:
                     base_packages.check_diskspace(temp_dir)
                 except error.RepoDiskFullError, e:
                     msg = ("Temporary directory for packages %s does not have "
                            "enough space available: %s" % (temp_dir, e))
                     raise error.RepoDiskFullError(msg)
                 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
                                                   temp_dir, exclude_string)
-                pkgmgr.upload_pkg(tarball_path, repo_url, update_checksum=True)
+                pkgmgr.upload_pkg(tarball_path, update_checksum=True)
             finally:
                 # remove the temporary directory
                 shutil.rmtree(temp_dir)
         else:
-            pkgmgr.remove_pkg(pkg_name, repo_url, remove_checksum=True)
+            pkgmgr.remove_pkg(pkg_name, remove_checksum=True)
         print "Done."
 
 
 def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir):
     """Tar all packages up and return a list of each tar created"""
     tarballs = []
     exclude_string = ' .'
     names = [p.strip() for p in pkg_names.split(',')]
     for name in names:
         print "Processing %s ... " % name
@@ skipping 10 matching lines @@
 
         pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
         tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
                                               temp_dir, exclude_string)
 
         tarballs.append(tarball_path)
 
     return tarballs
 
 
-def process_all_packages(pkgmgr, client_dir, upload_paths, remove=False):
+def process_all_packages(pkgmgr, client_dir, remove=False):
     """Process a full upload of packages as a directory upload."""
     test_dir = os.path.join(client_dir, "tests")
     site_test_dir = os.path.join(client_dir, "site_tests")
     dep_dir = os.path.join(client_dir, "deps")
     prof_dir = os.path.join(client_dir, "profilers")
     # Directory where all are kept
     temp_dir = tempfile.mkdtemp()
     try:
         base_packages.check_diskspace(temp_dir)
     except error.RepoDiskFullError, e:
@@ skipping 22 matching lines @@
     if not remove:
         tar_packages(pkgmgr, 'profiler', profilers, prof_dir, temp_dir)
         tar_packages(pkgmgr, 'dep', deps, dep_dir, temp_dir)
         tar_packages(pkgmgr, 'test', site_tests, client_dir, temp_dir)
         tar_packages(pkgmgr, 'test', tests, client_dir, temp_dir)
         tar_packages(pkgmgr, 'client', 'autotest', client_dir, temp_dir)
         cwd = os.getcwd()
         os.chdir(temp_dir)
         client_utils.system('md5sum * > packages.checksum')
         os.chdir(cwd)
-        for path in upload_paths:
-            print "Uploading to: " + path
-            pkgmgr.upload_pkg(temp_dir, path)
+        pkgmgr.upload_pkg(temp_dir)
         client_utils.run('rm -rf ' + temp_dir)
     else:
-        for repo_url in upload_paths:
-            process_packages(pkgmgr, 'test', tests, client_dir, repo_url,
-                             remove=remove)
-            process_packages(pkgmgr, 'test', site_tests, client_dir, repo_url,
-                             remove=remove)
-            process_packages(pkgmgr, 'client', 'autotest', client_dir, repo_url,
-                             remove=remove)
-            process_packages(pkgmgr, 'dep', deps, dep_dir, repo_url,
-                             remove=remove)
-            process_packages(pkgmgr, 'profiler', profilers, prof_dir, repo_url,
-                             remove=remove)
+        process_packages(pkgmgr, 'test', tests, client_dir,remove=remove)
+        process_packages(pkgmgr, 'test', site_tests, client_dir, remove=remove)
+        process_packages(pkgmgr, 'client', 'autotest', client_dir,
+                         remove=remove)
+        process_packages(pkgmgr, 'dep', deps, dep_dir, remove=remove)
+        process_packages(pkgmgr, 'profiler', profilers, prof_dir,
+                         remove=remove)
 
 
 # Get the list of sub directories present in a directory
 def get_subdir_list(name, client_dir):
     dir_name = os.path.join(client_dir, name)
     return [f for f in
             os.listdir(dir_name)
             if os.path.isdir(os.path.join(dir_name, f)) ]
 
 
@@ skipping 20 matching lines @@
 
     # extract the pkg locations from global config
     repo_urls = c.get_config_value('PACKAGES', 'fetch_location',
                                    type=list, default=[])
     upload_paths = c.get_config_value('PACKAGES', 'upload_location',
                                       type=list, default=[])
     # Having no upload paths basically means you're not using packaging.
     if len(upload_paths) == 0:
         return
 
-    pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls,
-                                     upload_paths=upload_paths,
-                                     run_function_dargs={'timeout':600})
-
     client_dir = os.path.join(autotest_dir, "client")
 
     # Bail out if the client directory does not exist
     if not os.path.exists(client_dir):
         sys.exit(0)
 
     dep_dir = os.path.join(client_dir, "deps")
     prof_dir = os.path.join(client_dir, "profilers")
 
     if len(args)==0 or args[0] not in ['upload','remove']:
         print("Either 'upload' or 'remove' needs to be specified "
               "for the package")
         sys.exit(0)
 
     if args[0]=='upload':
         remove_flag=False
     elif args[0]=='remove':
         remove_flag=True
     else:
         # we should not be getting here
         assert(False)
 
+    if options.repo:
+        upload_path_list = [options.repo]
+    else:
+        upload_path_list = upload_paths
+
+    pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls,
+                                     upload_paths=upload_path_list,
+                                     run_function_dargs={'timeout':600})
+
     if options.all:
-        if options.repo:
-            upload_path_list = [options.repo]
-        else:
-            upload_path_list = upload_paths
-        process_all_packages(pkgmgr, client_dir, upload_path_list,
-                             remove=remove_flag)
+        process_all_packages(pkgmgr, client_dir, remove=remove_flag)
 
     if options.client:
         process_packages(pkgmgr, 'client', 'autotest', client_dir,
-                         options.repo, remove=remove_flag)
+                         remove=remove_flag)
 
     if options.dep:
         process_packages(pkgmgr, 'dep', options.dep, dep_dir,
-                         options.repo, remove=remove_flag)
+                         remove=remove_flag)
 
     if options.test:
         process_packages(pkgmgr, 'test', options.test, client_dir,
-                         options.repo, remove=remove_flag)
+                         remove=remove_flag)
 
     if options.prof:
         process_packages(pkgmgr, 'profiler', options.prof, prof_dir,
-                         options.repo, remove=remove_flag)
+                         remove=remove_flag)
 
     if options.file:
         if remove_flag:
-            pkgmgr.remove_pkg(options.file, options.repo, remove_checksum=True)
+            pkgmgr.remove_pkg(options.file, remove_checksum=True)
         else:
-            pkgmgr.upload_pkg(options.file, options.repo, update_checksum=True)
+            pkgmgr.upload_pkg(options.file, update_checksum=True)
 
 
 if __name__ == "__main__":
     main()