Add hung bots to sheriff-o-matic view

Tools/GardeningServer/model/ct-failures.html

 <!--
 Copyright 2014 The Chromium Authors. All rights reserved.
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 -->
 
 <link rel="import" href="../lib/net.html">
 <link rel="import" href="../lib/update-util.html">
 <link rel="import" href="ct-builder-revisions.html">
 <link rel="import" href="ct-failure.html">
 <link rel="import" href="ct-failure-group.html">
+<link rel="import" href="ct-builder-failure-group-data.html">
 <link rel="import" href="ct-sheriff-failure-group-data.html">
 <link rel="import" href="ct-trooper-failure-group-data.html">
 <link rel="import" href="ct-commit-list.html">
 
 <script>
 function CTFailures(commitLog) {
   this.commitLog = commitLog;
-  this.builderLatestRevisions = null;
+  this.builderLatestInfo = null;
   // Maps a tree id to an array of CTFailureGroups within that tree.
   this.failures = null;
   this.lastUpdateDate = null;
 }
 
 (function() {
 'use strict';
 
 // FIXME: This could potentially move onto CTSheriffFailureGroupData as it isn't relevant to
 // trooper failures.
@@ skipping 56 matching lines @@
 };
 
 CTFailures.prototype.update = function() {
   var annotationPromise = CTFailureGroup.fetchAnnotations();
   return Promise.all([annotationPromise, net.json('http://sheriff-o-matic.appspot.com/alerts'),
       net.json('http://trooper-o-matic.appspot.com/alerts')]).then(function(data_array) {
     var annotations = data_array[0];
     var sheriff_data = data_array[1];
     var trooper_data = data_array[2];
 
-    // FIXME: Don't special-case the blink master.
-    this.builderLatestRevisions = new CTBuilderRevisions(sheriff_data.latest_builder_info['chromium.webkit']);
+    this.builderLatestInfo = {};
+
+    Object.keys(sheriff_data.latest_builder_info, function (master, data) {
+      this.builderLatestInfo[master] = data
+    }.bind(this));
     var newFailures = {};
     this.lastUpdateDate = new Date(sheriff_data.date * 1000);
     this._mungeAlerts(sheriff_data.alerts);
     // FIXME: Change builder_alerts to expose the alerts as a map instead of an array.
     var alertsByKey = {}
     sheriff_data.alerts.forEach(function(alert) {
       alertsByKey[alert.key] = alert;
     });
     // Update |failures| with the appropriate CTFailureGroup's for each
     // tree.
     sheriff_data.range_groups.forEach(function(rangeGroup) {
       this._processFailuresForRangeGroup(newFailures, rangeGroup, alertsByKey, annotations);
     }.bind(this));
     // Sort failure groups so that newer failures are shown at the top
     // of the UI.
+
     Object.keys(newFailures, function (tree, failuresByTree) {
       failuresByTree.sort(this._failureByTreeListComparator.bind(this, tree));
     }.bind(this));
+
+    this._checkBuildersForFailures(newFailures);
     this.failures = updateUtil.updateLeft(this.failures, newFailures);
     this._processTrooperFailures(trooper_data);
   }.bind(this));
 };
 
 CTFailures.prototype._failureComparator = function(a, b) {
   if (a.step > b.step)
     return 1;
   if (a.step < b.step)
     return -1
@@ skipping 68 matching lines @@
     })
     treeFailures.sort(this._failureComparator);
 
     if (!newFailures[tree])
       newFailures[tree] = [];
     var commitList = new CTCommitList(this.commitLog, rangeGroup.likely_revisions);
     newFailures[tree].push(new CTFailureGroup(rangeGroup.sort_key, new CTSheriffFailureGroupData(treeFailures, commitList)));
   }.bind(this));
 };
 
+CTFailures.prototype._checkBuildersForFailures = function(newFailures) {
+  var timeThreshold = {
+    "idle": null, // FIXME: We should alert if a bot is idle with jobs queued
+    "building": 3 * 60 * 60,
+    "offline": 0.5 * 60 * 60
+  };
+  Object.keys(this.builderLatestInfo, function(master, builders) {
+    // FIXME: Don't special case blink and chromium.webkit
+    if (master == "chromium.webkit") {
+      var tree = "blink";
+      Object.keys(builders, function(builder, builderInfo) {
+        var timeSinceLastUpdate = (this.lastUpdateDate.valueOf() / 1000)  - builderInfo.lastUpdateTime;
+        var alert;
+        // This alert logic should live on the buildbot.
+        if (timeSinceLastUpdate > timeThreshold[builderInfo.state] && builderInfo.state != "idle")
+          alert = (timeSinceLastUpdate / (60 * 60)).toFixed(2) + " hours ";
+
+        var resultsByBuilder = { };
+        resultsByBuilder[builder] = {
+          "actual": "UNKNOWN",
+          "masterUrl": "http://build.chromium.org/p/" + master
+        }
+        if (alert) {
+          if (!newFailures[tree])
+            newFailures[tree] = [];
+          var key = master + "::" + builder + "::" + builderInfo.state;
+          var failure = new CTFailure(builderInfo.state, alert, resultsByBuilder);
+          var data = new CTBuilderFailureGroupData(failure, builder, "http://build.chromium.org/p/" + master + "/builders/" + builder);
+          newFailures[tree].push(new CTFailureGroup(key, data, 'builders'));
+        }
+      }.bind(this));
+    }
+  }.bind(this));
+};
+
 })();
+
 </script>