Add endpoint and cron job to aggregate all dimensions and values

appengine/swarming/server/bot_management.py

 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Swarming bot management, e.g. list of known bots and their state.
 
           +---------+
           |BotRoot  |
           |id=bot_id|
           +---------+
@@ skipping 193 matching lines @@
 
   Parent is BotRoot. Key id is 'settings'.
 
   This entity must always be updated in a transaction.
   """
   # If set to True, no task is handed out to this bot due to the bot being in a
   # broken situation.
   quarantined = ndb.BooleanProperty()
 
 
+class DimensionValues(ndb.Model):
+  dimension = ndb.StringProperty()

[M-A Ruel, 2016/08/05 14:36:05]

key = ndb.StringProperty()

[kjlubick, 2016/08/05 17:38:58]

"Don't name a property "key." This name is reserved for a special property used
to store the Model key. Though it may work locally, a property named "key" will
prevent deployment to App Engine."
https://cloud.google.com/appengine/docs/python/ndb/entity-property-reference

[M-A Ruel, 2016/08/05 18:06:00]

Ugh, I forgot about that.

+  values = ndb.StringProperty(repeated=True)
+
+
+class DimensionAggregation(ndb.Model):
+  """Has all dimensions that are currently in use."""
+  dimensions = ndb.LocalStructuredProperty(DimensionValues, repeated=True)
+
+  ts = ndb.DateTimeProperty(auto_now_add=True)

[M-A Ruel, 2016/08/05 14:36:05]

ts = ndb.DateTimeProperty()


KEY = ndb.Key(DimensionAggregation, 'current')


then refer to DimensionAggregation.KEY.get() everywhere, instead of copy-pasting
'current'.

[kjlubick, 2016/08/05 17:38:58]

Done.

+
+
 ### Private APIs.
 
 
 ### Public APIs.
 
 
 def dimensions_to_flat(dimensions):
   out = []
   for k, values in dimensions.iteritems():
     for v in values:
@@ skipping 129 matching lines @@
   Returns:
     Tuple (True to restart, text message explaining the reason).
   """
   # Periodically reboot bots to workaround OS level leaks (especially on Win).
   running_time = state.get('running_time', 0)
   assert isinstance(running_time, (int, float))
   period = get_bot_reboot_period(bot_id, state)
   if period and running_time > period:
     return True, 'Periodic reboot: running longer than %ds' % period
   return False, ''