Checking for failure in requested builds.

scripts/slave/recipe_modules/auto_bisect/revision_state.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """An interface for holding state and result of revisions in a bisect job.
 
 When implementing support for tests other than perf, one should extend this
 class so that the bisect module and recipe can use it.
 
 See perf_revision_state for an example.
 """
 
 import hashlib
 import json
 import math
 import os
 import tempfile
 import re
 import uuid
 
 from . import depot_config
 
 # These relate to how to increase the number of repetitions during re-test
 MINIMUM_SAMPLE_SIZE = 5
 INCREASE_FACTOR = 1.5
-
+# Buildbot job result codes.
+# See http://docs.buildbot.net/current/developer/results.html
+SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION = range(5)
 
 class RevisionState(object):
   """Abstracts the state of a single revision on a bisect job."""
 
   # Possible values for the status attribute of RevisionState:
   (
       NEW,  # A revision_state object that has just been initialized.
       BUILDING,  # Requested a build for this revision, waiting for it.
       TESTING,  # A test job for this revision was triggered, waiting for it.
       TESTED,  # The test job completed with non-failing results.
@@ skipping 47 matching lines @@
       self.deps = dict(base_revision.deps)
       self.deps[self.depot_name] = self.commit_hash
     else:
       self.needs_patch = False
     self.build_url = self.bisector.get_platform_gs_prefix() + self._gs_suffix()
     self.values = []
     self.mean_value = None
     self.std_dev = None
     self.repeat_count = MINIMUM_SAMPLE_SIZE
     self._test_config = None
+    self.build_number = None
 
   @property
   def tested(self):
     return self.status in (RevisionState.TESTED,)
 
   @property
   def in_progress(self):
     return self.status in (RevisionState.BUILDING, RevisionState.TESTING,
                            RevisionState.NEED_MORE_DATA)
 
@@ skipping 108 matching lines @@
 
   def update_status(self):
     """Checks on the pending jobs and updates status accordingly.
 
     This method will check for the build to complete and then trigger the test,
     or will wait for the test as appropriate.
 
     To wait for the test we try to get the buildbot job url from GS, and if
     available, we query the status of such job.
     """
-    if self.status == RevisionState.BUILDING and self._is_build_archived():
-      self.start_job()
+    if self.status == RevisionState.BUILDING:
+      if self._is_build_archived():
+        self.start_job()
+      elif self._is_build_failed():
+        self.status = RevisionState.FAILED
     elif (self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA)
           and self._results_available()):
       # If we have already decided whether the revision is good or bad we
       # shouldn't check again
       check_revision_goodness = not(self.good or self.bad)
       self._read_test_results(
           check_revision_goodness=check_revision_goodness)
       # We assume _read_test_results may have changed the status to a broken
       # state such as FAILED or ABORTED.
       if self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA):
         self.status = RevisionState.TESTED
 
   def _is_build_archived(self):
     """Checks if the revision is already built and archived."""
     if not self.build_archived:
       api = self.bisector.api
       self.build_archived = api.gsutil_file_exists(self.build_url)
 
     if self.bisector.dummy_builds:
       self.build_archived = self.in_progress
 
     return self.build_archived
 
+  def _fetch_build_info(self, base_url, build_number):
+    api = self.bisector.api
+    build_url = '%s/builds/%s?as_text=1' % (base_url, build_number)
+    fetch_result = api.m.url.fetch_to_file(
+       build_url, None, step_name='fetch build details',
+       stdout=api.m.raw_io.output())
+    return json.loads(fetch_result.stdout or '{}')
+
+  def _is_build_failed(self):
+    api = self.bisector.api
+    current_build = None
+    base_url = '%sjson/builders/%s'% (
+        # If this variable is not set assume local dev master.
+        os.environ.get('BUILDBOT_URL', 'http://localhost:8041/'),
+        self.bisector.get_builder_bot_for_this_platform())
+    if self.build_number is None:
+      try:
+        # Get all the current builds.
+        builder_state_url = base_url + '?as_text=1'
+        fetch_step = api.m.url.fetch_to_file(
+            builder_state_url, None,
+            step_name='fetch builder state',
+            stdout=api.m.raw_io.output())
+        builder_state = fetch_step.stdout
+        builder_state = json.loads(builder_state or '{}')
+        for build_number in builder_state.get('cachedBuilds', []):
+          build = self._fetch_build_info(base_url, build_number)
+          build_properties = dict(build.get('properties', []))
+          if build_properties.get('build_archive_url') == self.build_url:
+            self.build_number = build_number
+            current_build = build
+            break
+      except api.m.step.StepFailure:  # pragma: no cover
+        # If we cannot get json from buildbot, we cannot determine if a build is
+        # failed, hence we consider it in progress until it times out.
+        return False
+    if self.build_number is None:
+      # The build hasn't started yet, therefore it's not failed.
+      return False
+    if not current_build:
+      current_build = self._fetch_build_info(base_url, self.build_number)
+    return current_build.get('results') in [FAILURE, SKIPPED, EXCEPTION]
+
   def _results_available(self):
     """Checks if the results for the test job have been uploaded."""
     api = self.bisector.api
     result = api.gsutil_file_exists(self.test_results_url)
     if self.bisector.dummy_builds:
       return self.in_progress
     return result  # pragma: no cover
 
   def _gs_suffix(self):
     """Provides the expected right half of the build filename.
@@ skipping 53 matching lines @@
     file_name = str(api.m.path['tmp_base'].join(build_name + '.diff'))
     api.m.file.write('Saving diff patch for ' + self.commit_hash,
                      file_name, self.deps_patch + self.deps_sha_patch)
     return file_name
 
   def _request_build(self):
     """Posts a request to buildbot to build this revision and archive it."""
     # TODO: Rewrite using the trigger module.
     api = self.bisector.api
     bot_name = self.bisector.get_builder_bot_for_this_platform()
-    if self.bisector.dummy_builds:
+    if self.bisector.bisect_config.get('dummy_job_names'):
       self.job_name = self.commit_hash + '-build'
     else:  # pragma: no cover
       self.job_name = uuid.uuid4().hex
     if self.needs_patch:
       self.patch_file = self._write_deps_patch_file(
           self.job_name)
     else:
       self.patch_file = '/dev/null'
 
     # To allow multiple nested levels, we go to the topmost revision.
@@ skipping 45 matching lines @@
     self._test_config = result
     return result
 
   def _do_test(self):
     """Triggers tests for a revision, either locally or via try job.
 
     If local testing is enabled (i.e. director/tester merged) then
     the test will be run on the same machine. Otherwise, this posts
     a request to buildbot to download and perf-test this build.
     """
-    if self.bisector.dummy_builds:
+    if self.bisector.bisect_config.get('dummy_job_names'):
       self.job_name = self.commit_hash + '-test'
     else:  # pragma: no cover
       self.job_name = uuid.uuid4().hex
     api = self.bisector.api
     top_revision = self
     while top_revision.base_revision:  # pragma: no cover
       top_revision = top_revision.base_revision
     perf_test_properties = {
         'builder_name': self.bisector.get_perf_tester_name(),
         'properties': {
@@ skipping 116 matching lines @@
         key=lambda x: len(x.values))
     if (len(self.bisector.last_tested_revision.values) ==
         next_revision_to_test.values):
       self.bisector.last_tested_revision.retest()
     else:
       next_revision_to_test.retest()
 
   def __repr__(self):
     return ('RevisionState(rev=%s, values=%r, mean_value=%r, std_dev=%r)' % (
         self.revision_string(), self.values, self.mean_value, self.std_dev))