[Findit] Adding new fields to try job metadata and updating as soon as possible

appengine/findit/waterfall/monitor_try_job_pipeline.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import datetime
 import json
 import time
 
 from common.pipeline_wrapper import BasePipeline
 from common.pipeline_wrapper import pipeline
@@ skipping 92 matching lines @@
                 'reason': MonitorTryJobPipeline.UNKNOWN
             },
             try_job_error.UNKNOWN)
 
     return None, None
 
   @staticmethod
   def _UpdateTryJobMetadata(try_job_data, start_time, buildbucket_build,
                             buildbucket_error, timed_out):
     buildbucket_response = {}
+
     if buildbucket_build:
-      try_job_data.request_time = MonitorTryJobPipeline._MicrosecondsToDatetime(
-          buildbucket_build.request_time)
+      try_job_data.request_time = (
+          try_job_data.request_time or
+          MonitorTryJobPipeline._MicrosecondsToDatetime(
+              buildbucket_build.request_time))
       # If start_time is unavailable, fallback to request_time.
       try_job_data.start_time = start_time or try_job_data.request_time
       try_job_data.end_time = MonitorTryJobPipeline._MicrosecondsToDatetime(
           buildbucket_build.end_time)
       try_job_data.number_of_commits_analyzed = len(
           buildbucket_build.report.get('result', {}))
-      try_job_data.try_job_url = buildbucket_build.url
       try_job_data.regression_range_size = buildbucket_build.report.get(
           'metadata', {}).get('regression_range_size')
-      try_job_data.last_buildbucket_response = buildbucket_build.response
+      try_job_data.try_job_url = (
+          try_job_data.try_job_url or buildbucket_build.url)

[stgao, 2016/04/27 17:34:33]

Why we don't use the one from buildbucket?

[lijeffrey, 2016/04/27 22:21:03]

In case it's already set from line 229 when the build status first changes to
STARTED, though it shouldn't make a difference. I'll change it back to just
buildbucket_build.url

       buildbucket_response = buildbucket_build.response
+      try_job_data.last_buildbucket_response = buildbucket_response
 
     error_dict, error_code = MonitorTryJobPipeline._GetError(
         buildbucket_response, buildbucket_error, timed_out)
 
     if error_dict:
       try_job_data.error = error_dict
       try_job_data.error_code = error_code
 
     try_job_data.put()
 
@@ skipping 36 matching lines @@
     default_pipeline_wait_seconds = waterfall_config.GetTryJobSettings().get(
         'server_query_interval_seconds')
     max_error_times = waterfall_config.GetTryJobSettings().get(
         'allowed_response_error_times')
     pipeline_wait_seconds = default_pipeline_wait_seconds
     allowed_response_error_times = max_error_times
 
     # TODO(chanli): Make sure total wait time equals to timeout_hours
     # regardless of retries.
     deadline = time.time() + timeout_hours * 60 * 60
-    try_job_data = (WfTryJobData.Get(try_job_id) or
-                    WfTryJobData.Create(try_job_id))
-    try_job_data.master_name = master_name
-    try_job_data.builder_name = builder_name
-    try_job_data.build_number = build_number
-    try_job_data.try_job_type = try_job_type
-
+    try_job_data = WfTryJobData.Get(try_job_id)
     already_set_started = False
     start_time = None
     while True:
       error, build = buildbucket_client.GetTryJobs([try_job_id])[0]
       if error:
         if allowed_response_error_times > 0:
           allowed_response_error_times -= 1
           pipeline_wait_seconds += default_pipeline_wait_seconds
         else:  # pragma: no cover
           # Buildbucket has responded error more than 5 times, retry pipeline.
@@ skipping 15 matching lines @@
           allowed_response_error_times = max_error_times
           pipeline_wait_seconds = default_pipeline_wait_seconds
         if build.status == BuildbucketBuild.STARTED and not already_set_started:
           # It is possible this branch is skipped if a fast build goes from
           # 'SCHEDULED' to 'COMPLETED' between queries, so start_time may be
           # unavailable.
           start_time = self._MicrosecondsToDatetime(build.updated_time)
           self._UpdateTryJobResult(
               BuildbucketBuild.STARTED, master_name, builder_name, build_number,
               try_job_type, try_job_id, build.url)
+
+          # Update as much try job metadata as soon as possible to avoid data
+          # loss in case of errors.
+          try_job_data.start_time = start_time
+          try_job_data.request_time = (
+              MonitorTryJobPipeline._MicrosecondsToDatetime(build.request_time))
+          try_job_data.try_job_url = build.url
+          try_job_data.put()
+
           already_set_started = True
 
       if time.time() > deadline:  # pragma: no cover
         self._UpdateTryJobMetadata(try_job_data, start_time, build, error, True)
         # Explicitly abort the whole pipeline.
         raise pipeline.Abort(
             'Try job %s timed out after %d hours.' % (
                 try_job_id, timeout_hours))
 
       time.sleep(pipeline_wait_seconds)  # pragma: no cover