swarming: add active jobs pending times metric

appengine/swarming/ts_mon_metrics_test.py

Index: appengine/swarming/ts_mon_metrics_test.py
diff --git a/appengine/swarming/ts_mon_metrics_test.py b/appengine/swarming/ts_mon_metrics_test.py
index c14107ed90423508c1ff236400cd1aca7c0dcdb0..30e59d887b2192be055f7c595e0c4ec896d2afa6 100755
--- a/appengine/swarming/ts_mon_metrics_test.py
+++ b/appengine/swarming/ts_mon_metrics_test.py
@@ -133,13 +133,15 @@ class TestMetrics(test_case.TestCase):
     summary_running.bot_id = 'test_bot1'
     summary_running.put()
 
-    summary_pending = _gen_task_result_summary(self.now, 2, tags=tags)
+    summary_pending = _gen_task_result_summary(
+        self.now - datetime.timedelta(minutes=5), 2, tags=tags)
     summary_pending.state = task_result.State.PENDING
     summary_pending.modified_ts = self.now
     summary_pending.bot_id = 'test_bot2'
     summary_pending.put()
 
-    summary_pending = _gen_task_result_summary(self.now, 3, tags=tags)
+    summary_pending = _gen_task_result_summary(
+        self.now - datetime.timedelta(minutes=10), 3, tags=tags)
     summary_pending.state = task_result.State.PENDING
     summary_pending.modified_ts = self.now
     summary_pending.bot_id = ''
@@ -178,6 +180,9 @@ class TestMetrics(test_case.TestCase):
     self.assertEqual(2, ts_mon_metrics.jobs_active.get(
         fields=jobs_fields, target_fields=ts_mon_metrics.TARGET_FIELDS))
 
+    self.assertEqual(900, ts_mon_metrics.jobs_pending_durations.get(
+        fields=jobs_fields, target_fields=ts_mon_metrics.TARGET_FIELDS).sum)
+
     for bot_id, status in bots_expected.iteritems():
       target_fields = dict(ts_mon_metrics.TARGET_FIELDS)
       target_fields['hostname'] = 'autogen:' + bot_id