Add endpoint and cron job to aggregate all dimensions and values

appengine/swarming/handlers_test.py

 #!/usr/bin/env python
 # coding: utf-8
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
+import datetime
 import itertools
 import json
 import logging
 import os
 import re
 import sys
 import unittest
 import urllib
 
 # Setups environment.
@@ skipping 455 matching lines @@
 
       # Only cron job requests can be gets for this handler.
       response = self.app.get(cron_job_url, status=403)
       self.assertEqual(
           '403 Forbidden\n\nAccess was denied to this resource.\n\n '
           'Only internal cron jobs can do this  ',
           response.body)
     # The actual number doesn't matter, just make sure they are unqueued.
     self.execute_tasks()
 
+
+  def testCronBotsAggregateTask(self):
+    # Tests that the aggregation works
+    now = datetime.datetime(2010, 1, 2, 3, 4, 5)
+    self.mock_now(now)
+
+    bot_management.bot_event(
+        event_type='bot_connected', bot_id='id1',
+        external_ip='8.8.4.4', authenticated_as='bot:whitelisted-ip',
+        dimensions={'foo': ['beta'], 'id': ['id1']}, state={'ram': 65},
+        version='123456789', quarantined=False, task_id=None, task_name=None)
+    bot_management.bot_event(
+        event_type='bot_connected', bot_id='id2',
+        external_ip='8.8.4.4', authenticated_as='bot:whitelisted-ip',
+        dimensions={'foo': ['alpha'], 'id': ['id2']}, state={'ram': 65},
+        version='123456789', quarantined=True, task_id='987', task_name=None)
+
+    self.app.get('/internal/cron/aggregate_bots_dimensions',
+        headers={'X-AppEngine-Cron': 'true'}, status=200)
+    aggregate = bot_management.DimensionAggregation.KEY.get()

[M-A Ruel, 2016/08/05 18:06:01]

actual = bot_management.DimensionAggregation.KEY.get()

[kjlubick, 2016/08/05 18:28:42]

Done.

+    expected = bot_management.DimensionAggregation(
+      key=bot_management.DimensionAggregation.KEY,

[M-A Ruel, 2016/08/05 18:06:01]

+4

[kjlubick, 2016/08/05 18:28:42]

Done.

+      dimensions = [

[M-A Ruel, 2016/08/05 18:06:02]

no spacing

[kjlubick, 2016/08/05 18:28:42]

Done.

+        bot_management.DimensionValues(
+        dimension=u'foo',

[M-A Ruel, 2016/08/05 18:06:02]

alignment is incorrect.

[kjlubick, 2016/08/05 18:28:42]

Done.

+        values=[u'alpha',u'beta'])
+      ],
+      ts = now)

[M-A Ruel, 2016/08/05 18:06:02]

no spacing

[kjlubick, 2016/08/05 18:28:42]

Done.

+    self.assertEqual(expected, aggregate)
+
   def testCronTriggerTask(self):
     triggers = (
       '/internal/cron/trigger_cleanup_data',
     )
 
     for url in triggers:
       response = self.app.get(
           url, headers={'X-AppEngine-Cron': 'true'}, status=200)
       self.assertEqual('Success.', response.body)
       self.assertEqual(1, self.execute_tasks())
@@ skipping 19 matching lines @@
           url, headers={'X-AppEngine-QueueName': 'bogus name'}, status=403)
 
 
 if __name__ == '__main__':
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   logging.basicConfig(
       level=logging.DEBUG if '-v' in sys.argv else logging.CRITICAL,
       format='%(levelname)-7s %(filename)s:%(lineno)3d %(message)s')
   unittest.main()