Make sure that email alerts are sent on bisect fyi jobs failures.

dashboard/dashboard/update_bug_with_results.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """URL endpoint for a cron job to update bugs after bisects."""
 
 import datetime
 import json
 import logging
 import re
 import traceback
 
 from google.appengine.api import mail
 from google.appengine.ext import ndb
 
 from dashboard import bisect_fyi
 from dashboard import bisect_report
+from dashboard import buildbucket_service
 from dashboard import datastore_hooks
 from dashboard import email_template
 from dashboard import issue_tracker_service
 from dashboard import layered_cache
 from dashboard import quick_logger
 from dashboard import request_handler
 from dashboard import utils
 from dashboard.models import anomaly
 from dashboard.models import bug_data
 from dashboard.models import try_job
@@ skipping 10 matching lines @@
 === Auto-CCing suspected CL author %(author)s ===
 
 Hi %(author)s, the bisect results pointed to your CL below as possibly
 causing a regression. Please have a look at this info and see whether
 your CL be related.
 
 """
 
 _CONFIDENCE_LEVEL_TO_CC_AUTHOR = 95
 
+_BUILD_FAILURE_REASON = {
+    'BUILD_FAILURE': 'the build has failed',
+    'INFRA_FAILURE': 'the build has failed due to infrastructure failure.',
+    'BUILDBUCKET_FAILURE': 'the buildbucket service failure.',
+    'INVALID_BUILD_DEFINITION': 'incorrect bisect configuation.',
+    'CANCELED_EXPLICITLY': 'the build was canceled explicitly.',
+    'TIMEOUT': 'the build was canceled by buildbot on timeout.',
+}
+
+class BisectJobFailure(Exception):
+  pass
+
 
 class BugUpdateFailure(Exception):
   pass
 
 
 class UpdateBugWithResultsHandler(request_handler.RequestHandler):
   """URL endpoint for a cron job to update bugs after bisects."""
 
   def get(self):
     """The get handler method is called from a cron job.
@@ skipping 34 matching lines @@
     job: A TryJob entity, which represents one bisect try job.
     issue_tracker: An issue_tracker_service.IssueTrackerService instance.
   """
   if _IsStale(job):
     job.SetStaled()
     # TODO(chrisphan): Add a staled TryJob log.
     # TODO(chrisphan): Do we want to send a FYI Bisect email here?
     return
 
   results_data = job.results_data
-  if not results_data or results_data['status'] not in [COMPLETED, FAILED]:
+  # Skip this check for bisect fyi jobs, because if the job is fails due to
+  # bisect recipe or infra failures then an alert message should be sent to the
+  # team.
+  if ((not results_data or results_data['status'] not in [COMPLETED, FAILED])
+      and job.job_type != 'bisect-fyi'):
     return
 
   if job.job_type == 'perf-try':
     _SendPerfTryJobEmail(job)
   elif job.job_type == 'bisect-fyi':
     _CheckFYIBisectJob(job, issue_tracker)
   else:
     _CheckBisectJob(job, issue_tracker)
 
   if results_data['status'] == COMPLETED:
     job.SetCompleted()
   else:
     job.SetFailed()
 
 
 def _CheckBisectJob(job, issue_tracker):
   results_data = job.results_data
   has_partial_result = ('revision_data' in results_data and
                         results_data['revision_data'])
   if results_data['status'] == FAILED and not has_partial_result:
     return
   _PostResult(job, issue_tracker)
 
 
 def _CheckFYIBisectJob(job, issue_tracker):
   try:
+    if not _IsBisectJobCompleted(job):
+      return
+    error_message = bisect_fyi.VerifyBisectFYIResults(job)
     _PostResult(job, issue_tracker)
-    error_message = bisect_fyi.VerifyBisectFYIResults(job)
     if not bisect_fyi.IsBugUpdated(job, issue_tracker):
       error_message += '\nFailed to update bug with bisect results.'
+  except BisectJobFailure as e:
+    error_message = 'Bisect job failed because, %s' % e
   except BugUpdateFailure as e:
     error_message = 'Failed to update bug with bisect results: %s' % e
   if job.results_data['status'] == FAILED or error_message:
+    job.SetFailed()
     _SendFYIBisectEmail(job, error_message)
 
 
 def _SendPerfTryJobEmail(job):
   """Sends an email to the user who started the perf try job."""
   if not job.email:
     return
   email_report = email_template.GetPerfTryJobEmailReport(job)
   if not email_report:
     return
@@ skipping 179 matching lines @@
     return
   formatter = quick_logger.Formatter()
   logger = quick_logger.QuickLogger('bisect_result', job.bug_id, formatter)
   if job.log_record_id:
     logger.Log(report, record_id=job.log_record_id)
     logger.Save()
   else:
     job.log_record_id = logger.Log(report)
     logger.Save()
     job.put()
+
+
+def _IsBisectJobCompleted(job):
+  return _ValidateBuildbucketResponse(
+      buildbucket_service.GetJobStatus(job.buildbucket_job_id), job)
+
+
+def _ValidateBuildbucketResponse(job_info, job):
+  """Checks and validates the response from the buildbucket service for bisect.
+
+  Args:
+    job_info: A dictionary containing the response from the buildbucket service.
+
+  Returns:
+    True if bisect job is completed successfully and False for pending job.
+
+  Raises:
+    BisectJobFailure: When job is completed but build is failed or cancelled.
+  """
+  job_info = job_info['build']
+  json_response = json.dumps(job_info)
+  if not job_info:
+    raise BisectJobFailure('No response from Buildbucket.')
+
+  if job_info.get('status') in ['SCHEDULED', 'STARTED']:
+    return False
+
+  if not job.results_data:
+    job.results_data = {}
+  job.results_data['buildbot_log_url'] = str(job_info.get('url'))
+
+  if job_info.get('result') is None:
+    raise BisectJobFailure('No "result" in try job results. '
+                           'Buildbucket response: %s' % json_response)
+
+  # There are various failure and cancellation reasons for a buildbucket
+  # job to fail as listed in https://goto.google.com/bb_status.
+  if (job_info.get('status') == 'COMPLETED' and
+      job_info.get('result') != 'SUCCESS'):
+    reason = (job_info.get('cancelation_reason') or
+              job_info.get('failure_reason'))
+    raise BisectJobFailure(_BUILD_FAILURE_REASON.get(reason))
+  return True