Allow botlist API call to respond to quarantined: and is_dead:

appengine/swarming/handlers_endpoints.py

 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """This module defines Swarming Server endpoints handlers."""
 
 import datetime
 import logging
 
 from google.appengine.api import datastore_errors
@@ skipping 590 matching lines @@
           message_conversion.task_result_to_rpc(
               r, request.include_performance_stats)
           for r in items
         ],
         now=now)
 
 
 @swarming_api.api_class(resource_name='bots', path='bots')
 class SwarmingBotsService(remote.Service):
   """Bots-related API."""
+
   @gae_ts_mon.instrument_endpoint()
   @auth.endpoints_method(
       swarming_rpcs.BotsRequest, swarming_rpcs.BotList,
       http_method='GET')
   @auth.require(acl.is_privileged_user)
   def list(self, request):
     """Provides list of known bots.
 
     Deleted bots will not be listed.
     """
     logging.info('%s', request)
     now = utils.utcnow()
-    q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
-    for d in request.dimensions:
-      if not ':' in d:
-        raise endpoints.BadRequestException('Invalid dimensions')
-      parts = d.split(':', 1)
-      if len(parts) != 2 or any(i.strip() != i or not i for i in parts):
-        raise endpoints.BadRequestException('Invalid dimensions')
-      q = q.filter(bot_management.BotInfo.dimensions_flat == d)
+    q = bot_management.BotInfo.query()
+    try:
+      q = bot_management.filter_dimensions(q, request.dimensions)
+      q = bot_management.filter_availability(q, request.quarantined,

[M-A Ruel, 2016/08/09 20:23:13]

I don't like this kind of alignment, indent all arguments at +4 or none.

q = bot_management.filter_availability(
    q, request.quarantined, request.is_dead, now)

[kjlubick, 2016/08/09 20:44:44]

Done.

+          request.is_dead, now)
+    except ValueError as e:
+      raise endpoints.BadRequestException('%s' % e)

[M-A Ruel, 2016/08/09 20:23:13]

str(e)

[kjlubick, 2016/08/09 20:44:44]

Done.

+
     bots, cursor = datastore_utils.fetch_page(q, request.limit, request.cursor)
     return swarming_rpcs.BotList(
         cursor=cursor,
         death_timeout=config.settings().bot_death_timeout_secs,
         items=[message_conversion.bot_info_to_rpc(bot, now) for bot in bots],
         now=now)
 
   @gae_ts_mon.instrument_endpoint()
   @auth.endpoints_method(
       swarming_rpcs.BotsRequest, swarming_rpcs.BotsCount,

[M-A Ruel, 2016/08/09 20:23:13]

BotsRequest shouldn't be used here. Only dimensions is needed, nothing else. So
create another BotsCountRequest message. This is not a breaking change _because_
it is a GET method.

[kjlubick, 2016/08/09 20:44:43]

What if I want to count all bots that aren't quarantined or dead, i.e.
is_dead=FALSE&quarantined=FALSE

I think we can still use the BotsRequest here.

[M-A Ruel, 2016/08/09 21:15:32]

It's already part of BoutsCount.

       http_method='GET')
   @auth.require(acl.is_privileged_user)
   def count(self, request):
     """Counts number of bots with given dimensions."""
     logging.info('%s', request)
     now = utils.utcnow()
     q = bot_management.BotInfo.query()
-    for d in request.dimensions:
-      parts = d.split(':', 1)
-      if len(parts) != 2 or any(i.strip() != i or not i for i in parts):
-        raise endpoints.BadRequestException('Invalid dimensions: %s' % d)
-      q = q.filter(bot_management.BotInfo.dimensions_flat == d)
-    f_count = q.count_async()
-    dt = datetime.timedelta(seconds=config.settings().bot_death_timeout_secs)
-    timeout = now - dt
-    f_dead = q.filter(
-        bot_management.BotInfo.last_seen_ts < timeout).count_async()
-    f_quarantined = q.filter(
-        bot_management.BotInfo.quarantined == True).count_async()
-    f_busy = q.filter(bot_management.BotInfo.is_busy == True).count_async()
-    return swarming_rpcs.BotsCount(
-        count=f_count.get_result(),
-        quarantined=f_quarantined.get_result(),
-        dead=f_dead.get_result(),
-        busy=f_busy.get_result(),
-        now=now)
+    try:
+      q = bot_management.filter_dimensions(q, request.dimensions)
+    except ValueError as e:
+      raise endpoints.BadRequestException('%s' % e)
+
+    quarantined = swarming_rpcs.to_bool(request.quarantined)
+    is_dead = swarming_rpcs.to_bool(request.is_dead)
+    if quarantined is None and is_dead is None:
+      f_count = q.count_async()
+      f_dead = (bot_management.filter_availability(q, None, True, now)
+          .count_async())
+      f_quarantined = (bot_management.filter_availability(q, True, None, now)
+          .count_async())
+      f_busy = q.filter(bot_management.BotInfo.is_busy == True).count_async()
+      return swarming_rpcs.BotsCount(
+          count=f_count.get_result(),
+          quarantined=f_quarantined.get_result(),
+          dead=f_dead.get_result(),
+          busy=f_busy.get_result(),
+          now=now)
+    elif quarantined or is_dead:
+      q = bot_management.filter_availability(q, quarantined, is_dead, now)
+      f_count = q.count_async()
+      f_dead = (bot_management.filter_availability(q, None, True, now)
+          .count_async())
+      f_quarantined = (bot_management.filter_availability(q, True, None, now)
+          .count_async())
+      # Dead and quarantined bots, by definition, are not busy.
+      return swarming_rpcs.BotsCount(
+          count=f_count.get_result(),
+          quarantined=f_quarantined.get_result(),
+          dead=f_dead.get_result(),
+          busy=0,
+          now=now)
+    else: # both quarantined and is_dead are false or None
+      f_count = (bot_management.filter_availability(q, quarantined, is_dead,
+          now).count_async())
+      f_busy = q.filter(bot_management.BotInfo.is_busy == True).count_async()
+      return swarming_rpcs.BotsCount(
+          count=f_count.get_result(),
+          quarantined=0,
+          dead=0,
+          busy=f_busy.get_result(),
+          now=now)
+
 
   @gae_ts_mon.instrument_endpoint()
   @auth.endpoints_method(
       message_types.VoidMessage, swarming_rpcs.BotsDimensions,
       http_method='GET')
   @auth.require(acl.is_privileged_user)
   def dimensions(self, _request):
     """Returns the cached set of dimensions currently in use in the fleet."""
     dims = bot_management.DimensionAggregation.KEY.get()
     fd = [
       swarming_rpcs.StringListPair(key=d.dimension, value=d.values)
       for d in dims.dimensions
     ]
     return swarming_rpcs.BotsDimensions(bots_dimensions=fd, ts=dims.ts)
 
 
 def get_routes():
   return (
     endpoints_webapp2.api_routes(SwarmingServerService) +
     endpoints_webapp2.api_routes(SwarmingTaskService) +
     endpoints_webapp2.api_routes(SwarmingTasksService) +
     endpoints_webapp2.api_routes(SwarmingBotService) +
     endpoints_webapp2.api_routes(SwarmingBotsService) +
     # components.config endpoints for validation and configuring of luci-config
     # service URL.
     endpoints_webapp2.api_routes(config.ConfigApi))