Add endpoint and cron job to aggregate all dimensions and values

appengine/swarming/handlers_endpoints.py

 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """This module defines Swarming Server endpoints handlers."""
 
 import datetime
 import logging
 
 from google.appengine.api import datastore_errors
@@ skipping 626 matching lines @@
 
   @gae_ts_mon.instrument_endpoint()
   @auth.endpoints_method(
       swarming_rpcs.BotsRequest, swarming_rpcs.BotsCount,
       http_method='GET')
   @auth.require(acl.is_privileged_user)
   def count(self, request):
     """Counts number of bots with given dimensions."""
     logging.info('%s', request)
     now = utils.utcnow()
-    q = bot_management.BotInfo.query().order()
+    q = bot_management.BotInfo.query()
     for d in request.dimensions:
       parts = d.split(':', 1)
       if len(parts) != 2 or any(i.strip() != i or not i for i in parts):
         raise endpoints.BadRequestException('Invalid dimensions: %s' % d)
       q = q.filter(bot_management.BotInfo.dimensions_flat == d)
     f_count = q.count_async()
     dt = datetime.timedelta(seconds=config.settings().bot_death_timeout_secs)
     timeout = now - dt
     f_dead = q.filter(
         bot_management.BotInfo.last_seen_ts < timeout).count_async()
     f_quarantined = q.filter(
         bot_management.BotInfo.quarantined == True).count_async()
     f_busy = q.filter(bot_management.BotInfo.is_busy == True).count_async()
     return swarming_rpcs.BotsCount(
         count=f_count.get_result(),
         quarantined=f_quarantined.get_result(),
         dead=f_dead.get_result(),
         busy=f_busy.get_result(),
         now=now)
 
+  @gae_ts_mon.instrument_endpoint()
+  @auth.endpoints_method(
+      message_types.VoidMessage, swarming_rpcs.BotsDimensions,
+      http_method='GET')
+  @auth.require(acl.is_privileged_user)
+  def dimensions(self, request):
+    """Returns the cached set of dimensions currently in use in the fleet."""
+    logging.warn('%s', request)

[M-A Ruel, 2016/08/04 21:01:15]

info

[kjlubick, 2016/08/05 14:17:26]

Done.

+    agg = ndb.Key(bot_management.DimensionAggregation, "current")

[M-A Ruel, 2016/08/04 21:01:15]

single quote

[kjlubick, 2016/08/05 14:17:26]

Done.

+    dims = agg.get()
+    logging.info(dims)

[M-A Ruel, 2016/08/04 21:01:15]

don't

[kjlubick, 2016/08/05 14:17:26]

Done.

+    fd = []

[M-A Ruel, 2016/08/04 21:01:15]

use list comprehension

[kjlubick, 2016/08/05 14:17:26]

Done.

+    for d in dims.dimensions:
+      fd.append(swarming_rpcs.StringListPair(
+        key=d.dimension,
+        value=d.values))
+    logging.info(fd)

[M-A Ruel, 2016/08/04 21:01:15]

remove

[kjlubick, 2016/08/05 14:17:26]

Done.

+    return swarming_rpcs.BotsDimensions(
+      fleet_dimensions=fd)

[M-A Ruel, 2016/08/04 21:01:15]

bot_dimensions?

[kjlubick, 2016/08/05 14:17:26]

Well, it's not the dimensions for just one bot.  bots_dimensions sounds weird. 
I'll keep it as fleet_dimensions, I think.

[M-A Ruel, 2016/08/05 14:36:04]

bots_dimensions is better. We don't use the word "fleet" anywhere else.
Consistency is more important.

On the other hand I'm fine with replacing 'bots' with 'fleet' but that would
require more refactoring.

+
 
 def get_routes():
   return (
     endpoints_webapp2.api_routes(SwarmingServerService) +
     endpoints_webapp2.api_routes(SwarmingTaskService) +
     endpoints_webapp2.api_routes(SwarmingTasksService) +
     endpoints_webapp2.api_routes(SwarmingBotService) +
     endpoints_webapp2.api_routes(SwarmingBotsService) +
     # components.config endpoints for validation and configuring of luci-config
     # service URL.
     endpoints_webapp2.api_routes(config.ConfigApi))