swarming: add active jobs pending times metric

appengine/swarming/ts_mon_metrics_test.py

 #!/usr/bin/env python
 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import datetime
 import os
 import sys
 import unittest
 
@@ skipping 115 matching lines @@
         'master:test_master',
         'buildername:test_builder',
         'name:some_tests',
     ]
     summary_running = _gen_task_result_summary(self.now, 1, tags=tags)
     summary_running.state = task_result.State.RUNNING
     summary_running.modified_ts = self.now
     summary_running.bot_id = 'test_bot1'
     summary_running.put()
 
-    summary_pending = _gen_task_result_summary(self.now, 2, tags=tags)
+    summary_pending = _gen_task_result_summary(
+        self.now - datetime.timedelta(minutes=5), 2, tags=tags)
     summary_pending.state = task_result.State.PENDING
     summary_pending.modified_ts = self.now
     summary_pending.bot_id = 'test_bot2'
     summary_pending.put()
 
-    summary_pending = _gen_task_result_summary(self.now, 3, tags=tags)
+    summary_pending = _gen_task_result_summary(
+        self.now - datetime.timedelta(minutes=10), 3, tags=tags)
     summary_pending.state = task_result.State.PENDING
     summary_pending.modified_ts = self.now
     summary_pending.bot_id = ''
     summary_pending.put()
 
     _gen_bot_info('bot_ready', self.now).put()
     _gen_bot_info('bot_running', self.now, task_id='deadbeef').put()
     _gen_bot_info('bot_quarantined', self.now, quarantined=True).put()
     _gen_bot_info('bot_dead', self.now - datetime.timedelta(days=365)).put()
     bots_expected = {
@@ skipping 18 matching lines @@
     jobs_target_fields['hostname'] = 'autogen:test_bot2'
     self.assertFalse(ts_mon_metrics.jobs_running.get(
         fields=jobs_fields, target_fields=jobs_target_fields))
     jobs_fields['status'] = 'running'
     self.assertEqual(1, ts_mon_metrics.jobs_active.get(
         fields=jobs_fields, target_fields=ts_mon_metrics.TARGET_FIELDS))
     jobs_fields['status'] = 'pending'
     self.assertEqual(2, ts_mon_metrics.jobs_active.get(
         fields=jobs_fields, target_fields=ts_mon_metrics.TARGET_FIELDS))
 
+    self.assertEqual(900, ts_mon_metrics.jobs_pending_durations.get(
+        fields=jobs_fields, target_fields=ts_mon_metrics.TARGET_FIELDS).sum)
+
     for bot_id, status in bots_expected.iteritems():
       target_fields = dict(ts_mon_metrics.TARGET_FIELDS)
       target_fields['hostname'] = 'autogen:' + bot_id
       self.assertEqual(status, ts_mon_metrics.executors_status.get(
           target_fields=target_fields))
 
       self.assertEqual('bot_id:%s|os:Linux|os:Ubuntu' % bot_id,
                        ts_mon_metrics.executors_pool.get(
                            target_fields=target_fields))
 
 
 if __name__ == '__main__':
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   unittest.main()