Merge remote branch 'cros/upstream' into tempbranch

scheduler/monitor_db_cleanup.py

Index: scheduler/monitor_db_cleanup.py
diff --git a/scheduler/monitor_db_cleanup.py b/scheduler/monitor_db_cleanup.py
index 60b02f76ab2ebbd10b3c85413bd2fdc12e89fd81..d8bab116c6d145c647106780d5f0144269765ae9 100644
--- a/scheduler/monitor_db_cleanup.py
+++ b/scheduler/monitor_db_cleanup.py
@@ -3,7 +3,7 @@ Autotest AFE Cleanup used by the scheduler
 """
 
 
-import datetime, time, logging
+import datetime, time, logging, random
 from autotest_lib.database import database_connection
 from autotest_lib.frontend.afe import models
 from autotest_lib.scheduler import email_manager, scheduler_config
@@ -162,6 +162,14 @@ class UserCleanup(PeriodicCleanup):
         return (self._last_reverify_time + reverify_period_sec) <= time.time()
 
 
+    def _choose_subset_of_hosts_to_reverify(self, hosts):
+        """Given hosts needing verification, return a subset to reverify."""
+        max_at_once = scheduler_config.config.reverify_max_hosts_at_once
+        if (max_at_once > 0 and len(hosts) > max_at_once):
+            return random.sample(hosts, max_at_once)
+        return sorted(hosts)
+
+
     def _reverify_dead_hosts(self):
         if not self._should_reverify_hosts_now():
             return
@@ -177,8 +185,11 @@ class UserCleanup(PeriodicCleanup):
         if not hosts:
             return
 
-        logging.info('Reverifying dead hosts %s'
-                     % ', '.join(host.hostname for host in hosts))
+        hosts = list(hosts)
+        total_hosts = len(hosts)
+        hosts = self._choose_subset_of_hosts_to_reverify(hosts)
+        logging.info('Reverifying dead hosts (%d of %d) %s', len(hosts),
+                     total_hosts, ', '.join(host.hostname for host in hosts))
         for host in hosts:
             models.SpecialTask.schedule_special_task(
                     host=host, task=models.SpecialTask.Task.VERIFY)