[Findit] Capture and store latest buildbucket response with each query

appengine/findit/waterfall/monitor_try_job_pipeline.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import datetime
 import json
 import time
 
 from google.appengine.ext import ndb
 
@@ skipping 188 matching lines @@
     allowed_response_error_times = max_error_times
 
     # TODO(chanli): Make sure total wait time equals to timeout_hours
     # regardless of retries.
     deadline = time.time() + timeout_hours * 60 * 60
     try_job_data = WfTryJobData.Get(try_job_id)
     already_set_started = False
     start_time = None
     while True:
       error, build = buildbucket_client.GetTryJobs([try_job_id])[0]
+
       if error:
         if allowed_response_error_times > 0:
           allowed_response_error_times -= 1
           pipeline_wait_seconds += default_pipeline_wait_seconds
         else:  # pragma: no cover
           # Buildbucket has responded error more than 5 times, retry pipeline.
           _UpdateTryJobMetadata(try_job_data, start_time, build, error, False)
           raise pipeline.Retry(
               'Error "%s" occurred. Reason: "%s"' % (error.message,
                                                      error.reason))
@@ skipping 27 matching lines @@
 
           already_set_started = True
 
       if time.time() > deadline:  # pragma: no cover
         _UpdateTryJobMetadata(try_job_data, start_time, build, error, True)
         # Explicitly abort the whole pipeline.
         raise pipeline.Abort(
             'Try job %s timed out after %d hours.' % (
                 try_job_id, timeout_hours))
 
+      # Ensure last_buildbucket_response is always the most recent
+      # whenever available during intermediate queries.
+      if build and build.response:
+        try_job_data.last_buildbucket_response = build.response

[stgao, 2016/08/19 21:44:36]

nit: not update datastore if the current response is the same as last one,
because we want to avoid too many write if possible.

+        try_job_data.put()
+
       time.sleep(pipeline_wait_seconds)  # pragma: no cover