Add endpoint and cron job to aggregate all dimensions and values

appengine/swarming/handlers_backend.py

 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Main entry point for Swarming backend handlers."""
 
 import json
 import logging
 
 import webapp2
 from google.appengine.api import app_identity
 from google.appengine.api import datastore_errors
 from google.appengine.api import taskqueue
 
 import mapreduce_jobs
 from components import decorators
 from components import machine_provider
+from server import bot_management
 from server import config
 from server import lease_management
 from server import stats
 from server import task_scheduler
 
 
 class CronBotDiedHandler(webapp2.RequestHandler):
   @decorators.require_cronjob
   def get(self):
     try:
@@ skipping 62 matching lines @@
 
   @decorators.require_cronjob
   def get(self):
     if not config.settings().mp.enabled:
       logging.info('MP support is disabled')
       return
 
     lease_management.clean_up_bots()
 
 
+class CronBotDimensionAggregationHandler(webapp2.RequestHandler):
+  """Aggregate all bot dimensions (except id) in the fleet."""

[M-A Ruel, 2016/08/05 14:36:05]

Aggregates
bots

[kjlubick, 2016/08/05 17:38:58]

Done.

+
+  @decorators.require_cronjob
+  def get(self):
+    seen = {}

[M-A Ruel, 2016/08/05 14:36:04]

add:
now = utils.utcnow()

[kjlubick, 2016/08/05 17:38:58]

Done.

+    for b in bot_management.BotInfo.query():
+      for i in b.dimensions_flat:
+        k, v = i.split(':', 1)
+        if k == 'id':

[M-A Ruel, 2016/08/05 14:36:04]

if k != 'id':
  seen.setdefault(k, set()).add(v)

saves you one line. OMG :P

[kjlubick, 2016/08/05 17:38:57]

Done.  The electrons spared rejoiced.

+          continue
+        seen.setdefault(k, set()).add(v)
+    dims = []
+    for k, values in sorted(seen.iteritems()):
+      dims.append(bot_management.DimensionValues(
+        dimension=k,
+        values=sorted(list(values))))
+    logging.info('Saw dimensions %s', dims)
+    aggregate = bot_management.DimensionAggregation(
+      id='current',
+      dimensions=dims)

[M-A Ruel, 2016/08/05 14:36:04]

and save ts=now here so it's at when the query started, not when the entity was
saved, which is an important distinction.

[kjlubick, 2016/08/05 17:38:58]

Done.

+    aggregate.put()
+
+
 class CronMachineProviderPubSubHandler(webapp2.RequestHandler):
   """Listens for Pub/Sub communication from Machine Provider."""
 
   @decorators.require_cronjob
   def get(self):
     if not config.settings().mp.enabled:
       logging.info('MP support is disabled')
       return
 
     taskqueue.add(
@@ skipping 53 matching lines @@
     # Cron jobs.
     # TODO(maruel): Rename cron.yaml job url. Doing so is a bit annoying since
     # the app version has to be running an already compatible version already.
     ('/internal/cron/abort_bot_died', CronBotDiedHandler),
     ('/internal/cron/handle_bot_died', CronBotDiedHandler),
     ('/internal/cron/abort_expired_task_to_run',
         CronAbortExpiredShardToRunHandler),
 
     ('/internal/cron/stats/update', stats.InternalStatsUpdateHandler),
     ('/internal/cron/trigger_cleanup_data', CronTriggerCleanupDataHandler),
+    ('/internal/cron/aggregate_bot_dimensions',
+        CronBotDimensionAggregationHandler),
     ('/internal/cron/machine_provider', CronMachineProviderBotHandler),
     ('/internal/cron/machine_provider_cleanup',
         CronMachineProviderCleanUpHandler),
     ('/internal/cron/machine_provider_pubsub',
         CronMachineProviderPubSubHandler),
 
     # Task queues.
     ('/internal/taskqueue/cleanup_data', TaskCleanupDataHandler),
     (r'/internal/taskqueue/pubsub/<task_id:[0-9a-f]+>', TaskSendPubSubMessage),
     ('/internal/taskqueue/pubsub/machine_provider',
         TaskMachineProviderPubSubHandler),
 
     # Mapreduce related urls.
     (r'/internal/taskqueue/mapreduce/launch/<job_id:[^\/]+>',
       InternalLaunchMapReduceJobWorkerHandler),
   ]
   return [webapp2.Route(*a) for a in routes]