Add endpoint and cron job to aggregate all dimensions and values

appengine/swarming/handlers_backend.py

Index: appengine/swarming/handlers_backend.py
diff --git a/appengine/swarming/handlers_backend.py b/appengine/swarming/handlers_backend.py
index bbcdc6bc90678b32459613a04a3983d496827665..f97f9f6ae39e2927d14c938db3f32afa6a9c0cca 100644
--- a/appengine/swarming/handlers_backend.py
+++ b/appengine/swarming/handlers_backend.py
@@ -12,9 +12,12 @@ from google.appengine.api import app_identity
 from google.appengine.api import datastore_errors
 from google.appengine.api import taskqueue
 
+from components import utils
+
 import mapreduce_jobs
 from components import decorators
 from components import machine_provider
+from server import bot_management
 from server import config
 from server import lease_management
 from server import stats
@@ -97,6 +100,31 @@ class CronMachineProviderCleanUpHandler(webapp2.RequestHandler):
     lease_management.clean_up_bots()
 
 
+class CronBotsDimensionAggregationHandler(webapp2.RequestHandler):
+  """Aggregates all bots dimensions (except id) in the fleet."""
+
+  @decorators.require_cronjob
+  def get(self):
+    seen = {}
+    now = utils.utcnow()
+    for b in bot_management.BotInfo.query():
+      for i in b.dimensions_flat:
+        k, v = i.split(':', 1)
+        if k != 'id':
+          seen.setdefault(k, set()).add(v)
+    dims = [bot_management.DimensionValues(

[M-A Ruel, 2016/08/05 18:06:00]

unusual alignment. I'd prefer:

dims = [
  bot_management.DimensionValues(dimension=k, values=sorted(values))
  for k, values in sorted(seen.iteritems())
]

IMHO this is an order of magnitude more readable.

[kjlubick, 2016/08/05 18:28:42]

Done.

+        dimension=k,
+        values=sorted(values))
+        for k, values in sorted(seen.iteritems())]
+
+    logging.info('Saw dimensions %s', dims)
+    aggregate=bot_management.DimensionAggregation(

[M-A Ruel, 2016/08/05 18:06:00]

technically, you do not need a named variable. I'll just construct and call
.put() directly on the unnamed variable.

[kjlubick, 2016/08/05 18:28:42]

Done.

+      key=bot_management.DimensionAggregation.KEY,

[M-A Ruel, 2016/08/05 18:06:00]

+4

[kjlubick, 2016/08/05 18:28:42]

Done.

+      dimensions = dims,

[M-A Ruel, 2016/08/05 18:06:00]

no spaces

[kjlubick, 2016/08/05 18:28:42]

Done.

+      ts = now)
+    aggregate.put()
+
+
 class CronMachineProviderPubSubHandler(webapp2.RequestHandler):
   """Listens for Pub/Sub communication from Machine Provider."""
 
@@ -170,6 +198,8 @@ def get_routes():
 
     ('/internal/cron/stats/update', stats.InternalStatsUpdateHandler),
     ('/internal/cron/trigger_cleanup_data', CronTriggerCleanupDataHandler),
+    ('/internal/cron/aggregate_bots_dimensions',
+        CronBotsDimensionAggregationHandler),
     ('/internal/cron/machine_provider', CronMachineProviderBotHandler),
     ('/internal/cron/machine_provider_cleanup',
         CronMachineProviderCleanUpHandler),