Merge remote branch 'cros/upstream' into tempbranch

scheduler/drone_manager.py

 import os, re, shutil, signal, subprocess, errno, time, heapq, traceback
 import common, logging
 from autotest_lib.client.common_lib import error, global_config
 from autotest_lib.scheduler import email_manager, drone_utility, drones
 from autotest_lib.scheduler import scheduler_config
 
 
 # results on drones will be placed under the drone_installation_directory in a
 # directory with this name
 _DRONE_RESULTS_DIR_SUFFIX = 'results'
@@ skipping 434 matching lines @@
     def _choose_drone_for_execution(self, num_processes, username,
                                     drone_hostnames_allowed):
         # cycle through drones is order of increasing used capacity until
         # we find one that can handle these processes
         checked_drones = []
         usable_drones = []
         drone_to_use = None
         while self._drone_queue:
             drone = heapq.heappop(self._drone_queue).drone
             checked_drones.append(drone)
+            logging.info('Checking drone %s', drone.hostname)
             if not drone.usable_by(username):
                 continue
 
             drone_allowed = (drone_hostnames_allowed is None
                              or drone.hostname in drone_hostnames_allowed)
             if not drone_allowed:
+                logging.debug('Drone %s not allowed: ', drone.hostname)
                 continue
 
             usable_drones.append(drone)
 
             if drone.active_processes + num_processes <= drone.max_processes:
                 drone_to_use = drone
                 break
+            logging.info('Drone %s has %d active + %s requested > %s max',
+                         drone.hostname, drone.active_processes, num_processes,
+                         drone.max_processes)
 
         if not drone_to_use and usable_drones:
             drone_summary = ','.join('%s %s/%s' % (drone.hostname,
                                                    drone.active_processes,
                                                    drone.max_processes)
                                      for drone in usable_drones)
             logging.error('No drone has capacity to handle %d processes (%s) '
                           'for user %s', num_processes, drone_summary, username)
             drone_to_use = self._least_loaded_drone(usable_drones)
 
@@ skipping 217 matching lines @@
 
 def instance():
     if _the_instance is None:
         _set_instance(DroneManager())
     return _the_instance
 
 
 def _set_instance(instance): # usable for testing
     global _the_instance
     _the_instance = instance