Add hung bots to sheriff-o-matic view

Tools/GardeningServer/model/ct-failures.html

 <!--
 Copyright 2014 The Chromium Authors. All rights reserved.
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 -->
 
 <link rel="import" href="../lib/net.html">
 <link rel="import" href="../lib/update-util.html">
 <link rel="import" href="ct-builder-revisions.html">
 <link rel="import" href="ct-failure.html">
 <link rel="import" href="ct-failure-group.html">
+<link rel="import" href="ct-builder-failure-group-data.html">
 <link rel="import" href="ct-sheriff-failure-group-data.html">
 <link rel="import" href="ct-trooper-failure-group-data.html">
 <link rel="import" href="ct-commit-list.html">
 
 <script>
 function CTFailures(commitLog) {
   this.commitLog = commitLog;
-  this.builderLatestRevisions = null;
+  this.builderLatestInfo = null;
   // Maps a tree id to an array of CTFailureGroups within that tree.
   this.failures = null;
   this.lastUpdateDate = null;
 }
 
 (function() {
 'use strict';
 
 // FIXME: This could potentially move onto CTSheriffFailureGroupData as it isn't relevant to
 // trooper failures.
@@ skipping 56 matching lines @@
 };
 
 CTFailures.prototype.update = function() {
   var annotationPromise = CTFailureGroup.fetchAnnotations();
   return Promise.all([annotationPromise, net.json('http://sheriff-o-matic.appspot.com/alerts'),
       net.json('http://trooper-o-matic.appspot.com/alerts')]).then(function(data_array) {
     var annotations = data_array[0];
     var sheriff_data = data_array[1];
     var trooper_data = data_array[2];
 
-    // FIXME: Don't special-case the blink master.
-    this.builderLatestRevisions = new CTBuilderRevisions(sheriff_data.latest_builder_info['chromium.webkit']);
+    this.builderLatestInfo = {};
+
+    Object.keys(sheriff_data.latest_builder_info, function (master, data) {
+      this.builderLatestInfo[master] = new CTBuilderRevisions(data);

[ojan, 2014/09/10 00:40:03]

Do we need CTBuilderRevisions at all anymore? Can we just pass
latest_builder_info to _checkBuildersForFailures and not store anything else on
CTFailures?

lgtm for this patch assuming you add the FIXMEs and fix this bit.

+    }.bind(this));
     var newFailures = {};
     this.lastUpdateDate = new Date(sheriff_data.date * 1000);
     this._mungeAlerts(sheriff_data.alerts);
     // FIXME: Change builder_alerts to expose the alerts as a map instead of an array.
     var alertsByKey = {}
     sheriff_data.alerts.forEach(function(alert) {
       alertsByKey[alert.key] = alert;
     });
     // Update |failures| with the appropriate CTFailureGroup's for each
     // tree.
     sheriff_data.range_groups.forEach(function(rangeGroup) {
       this._processFailuresForRangeGroup(newFailures, rangeGroup, alertsByKey, annotations);
     }.bind(this));
     // Sort failure groups so that newer failures are shown at the top
     // of the UI.
+
     Object.keys(newFailures, function (tree, failuresByTree) {
       failuresByTree.sort(this._failureByTreeListComparator.bind(this, tree));
     }.bind(this));
+
+    this._checkBuildersForFailures(newFailures);
     this.failures = updateUtil.updateLeft(this.failures, newFailures);
     this._processTrooperFailures(trooper_data);
   }.bind(this));
 };
 
 CTFailures.prototype._failureComparator = function(a, b) {
   if (a.step > b.step)
     return 1;
   if (a.step < b.step)
     return -1
@@ skipping 68 matching lines @@
     })
     treeFailures.sort(this._failureComparator);
 
     if (!newFailures[tree])
       newFailures[tree] = [];
     var commitList = new CTCommitList(this.commitLog, rangeGroup.likely_revisions);
     newFailures[tree].push(new CTFailureGroup(rangeGroup.sort_key, new CTSheriffFailureGroupData(treeFailures, commitList)));
   }.bind(this));
 };
 
+CTFailures.prototype._checkBuildersForFailures = function(newFailures) {
+  var timeThreshold = {
+    "idle": null, // FIXME: We should alert if a bot is idle with jobs queued
+    "building": 3 * 60 * 60,
+    "offline": 0.5 * 60 * 60
+  };
+  Object.keys(this.builderLatestInfo, function(master, builders) {

[ojan, 2014/09/10 00:40:03]

In retrospect, we should move all this logic to builder_alerts and just have the
JSON contain the actual alerting things. That way, in theory, people can write
other tools that do something with the alerts. This would also solve the problem
of matching the master name to a tree since we have this data in builder_alerts.

You don't need to do that in this patch if you don't want to though. If you
wanted...you could have this patch form the JSON in the form you want it from
builder_alerts and then in a followup patch move the generation of that to
builder_alerts.

+    // FIXME: Don't special case blink and chromium.webkit
+    if (master == "chromium.webkit") {
+      var tree = "blink";
+      Object.keys(builders, function(builder, builderInfo) {
+        var timeSinceLastUpdate = (this.lastUpdateDate.valueOf() / 1000)  - builderInfo.lastUpdateTime;
+        var alert;
+        // If we add more failing criteria, this should be made into a loop over functions that handle the different errors.
+        if (timeSinceLastUpdate > timeThreshold[builderInfo.state] && builderInfo.state != "idle")
+          alert = (timeSinceLastUpdate / (60 * 60)).toFixed(2) + " hours ";
+
+        var resultsByBuilder = { };
+        resultsByBuilder[builder] = {
+          "actual": "UNKNOWN",
+          "masterUrl": "http://build.chromium.org/p/" + master
+        }
+        if (alert) {
+          if (!newFailures[tree])
+            newFailures[tree] = [];
+          var key = master + "::" + builder + "::" + builderInfo.state;
+          var failure = new CTFailure(builderInfo.state, alert, resultsByBuilder);
+          var data = new CTBuilderFailureGroupData(failure, builder, "http://build.chromium.org/p/" + master + "/builders/" + builder);
+          newFailures[tree].push(new CTFailureGroup(key, data, 'builders'));
+        }
+      }.bind(this));
+    }
+  }.bind(this));
+};
+
 })();
+
 </script>