Add endpoint and cron job to aggregate all dimensions and values

appengine/swarming/handlers_backend.py

Index: appengine/swarming/handlers_backend.py
diff --git a/appengine/swarming/handlers_backend.py b/appengine/swarming/handlers_backend.py
index bbcdc6bc90678b32459613a04a3983d496827665..7baaec12eee65288626e85ca482f6c191c492d24 100644
--- a/appengine/swarming/handlers_backend.py
+++ b/appengine/swarming/handlers_backend.py
@@ -15,6 +15,7 @@ from google.appengine.api import taskqueue
 import mapreduce_jobs
 from components import decorators
 from components import machine_provider
+from server import bot_management
 from server import config
 from server import lease_management
 from server import stats
@@ -96,6 +97,28 @@ class CronMachineProviderCleanUpHandler(webapp2.RequestHandler):
 
     lease_management.clean_up_bots()
 
+class CronDimensionAggregationHandler(webapp2.RequestHandler):

[M-A Ruel, 2016/08/04 21:01:15]

2 lines between file level symbols

[kjlubick, 2016/08/05 14:17:26]

Done.

+  """Cleans up leftover BotInfo entities."""
+
+  @decorators.require_cronjob
+  def get(self):
+    q = bot_management.BotInfo.query()

[M-A Ruel, 2016/08/04 21:01:15]

inline this

[kjlubick, 2016/08/05 14:17:26]

Done.

+    seen = {}
+    for b in q:
+      for i in b.dimensions_flat:
+        k, v = i.split(':', 1)

[M-A Ruel, 2016/08/04 21:01:15]

skip 'id' otherwise this won't scale.

[kjlubick, 2016/08/05 14:17:26]

Done.

+        seen.setdefault(k, {})[v] = True

[M-A Ruel, 2016/08/04 21:01:15]

seen.setdefault(K, set()).add(v)

[kjlubick, 2016/08/05 14:17:26]

Done.

+    dims = []
+    for k, values in seen.iteritems():

[M-A Ruel, 2016/08/04 21:01:14]

sorted()

[kjlubick, 2016/08/05 14:17:26]

Done.

+      dims.append(bot_management.DimensionValues(
+        dimension=k,
+        values=values.keys()))

[M-A Ruel, 2016/08/04 21:01:15]

sorted()

[kjlubick, 2016/08/05 14:17:26]

Done.

+    logging.info('Saw dimensions %s' % dims)

[M-A Ruel, 2016/08/04 21:01:15]

s/%/,/

[kjlubick, 2016/08/05 14:17:26]

Done.

+    aggregate = bot_management.DimensionAggregation(
+      id="current",
+      dimensions=dims)
+    aggregate.put()
+
 
 class CronMachineProviderPubSubHandler(webapp2.RequestHandler):
   """Listens for Pub/Sub communication from Machine Provider."""
@@ -170,6 +193,7 @@ def get_routes():
 
     ('/internal/cron/stats/update', stats.InternalStatsUpdateHandler),
     ('/internal/cron/trigger_cleanup_data', CronTriggerCleanupDataHandler),
+    ('/internal/cron/aggregate_dimensions', CronDimensionAggregationHandler),

[M-A Ruel, 2016/08/04 21:01:15]

include "bot" in the URL to make this clear this is not about tasks

[kjlubick, 2016/08/05 14:17:26]

Done.

     ('/internal/cron/machine_provider', CronMachineProviderBotHandler),
     ('/internal/cron/machine_provider_cleanup',
         CronMachineProviderCleanUpHandler),