Merge remote branch 'cros/upstream' into tempbranch

scheduler/monitor_db_cleanup.py

 """
 Autotest AFE Cleanup used by the scheduler
 """
 
 
-import datetime, time, logging
+import datetime, time, logging, random
 from autotest_lib.database import database_connection
 from autotest_lib.frontend.afe import models
 from autotest_lib.scheduler import email_manager, scheduler_config
 from autotest_lib.client.common_lib import host_protections
 
 
 class PeriodicCleanup(object):
 
 
     def __init__(self, db, clean_interval, run_at_initialize=False):
@@ skipping 138 matching lines @@
 
 
     def _should_reverify_hosts_now(self):
         reverify_period_sec = (scheduler_config.config.reverify_period_minutes
                                * 60)
         if reverify_period_sec == 0:
             return False
         return (self._last_reverify_time + reverify_period_sec) <= time.time()
 
 
+    def _choose_subset_of_hosts_to_reverify(self, hosts):
+        """Given hosts needing verification, return a subset to reverify."""
+        max_at_once = scheduler_config.config.reverify_max_hosts_at_once
+        if (max_at_once > 0 and len(hosts) > max_at_once):
+            return random.sample(hosts, max_at_once)
+        return sorted(hosts)
+
+
     def _reverify_dead_hosts(self):
         if not self._should_reverify_hosts_now():
             return
 
         self._last_reverify_time = time.time()
         logging.info('Checking for dead hosts to reverify')
         hosts = models.Host.objects.filter(
                 status=models.Host.Status.REPAIR_FAILED,
                 locked=False,
                 invalid=False)
         hosts = hosts.exclude(
                 protection=host_protections.Protection.DO_NOT_VERIFY)
         if not hosts:
             return
 
-        logging.info('Reverifying dead hosts %s'
-                     % ', '.join(host.hostname for host in hosts))
+        hosts = list(hosts)
+        total_hosts = len(hosts)
+        hosts = self._choose_subset_of_hosts_to_reverify(hosts)
+        logging.info('Reverifying dead hosts (%d of %d) %s', len(hosts),
+                     total_hosts, ', '.join(host.hostname for host in hosts))
         for host in hosts:
             models.SpecialTask.schedule_special_task(
                     host=host, task=models.SpecialTask.Task.VERIFY)
 
 
 class TwentyFourHourUpkeep(PeriodicCleanup):
     """Cleanup that runs at the startup of monitor_db and every subsequent
        twenty four hours.
     """
 
@@ skipping 91 matching lines @@
                      for row in rows]
             self._send_inconsistency_message(subject, lines)
 
 
     def _send_inconsistency_message(self, subject, lines):
         logging.error(subject)
         message = '\n'.join(lines)
         if len(message) > 5000:
             message = message[:5000] + '\n(truncated)\n'
         email_manager.manager.enqueue_notify_email(subject, message)