[Findit] Refactoring WfTryJobData into BaseTryJobData, WfTryJobData, and FlakeTryJobData

appengine/findit/waterfall/monitor_try_job_pipeline.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import time
 
 from google.appengine.ext import ndb
 
 from common.pipeline_wrapper import BasePipeline
 from common.pipeline_wrapper import pipeline
 from common.waterfall import buildbucket_client
 from common.waterfall import failure_type
 from common.waterfall import try_job_error
 from common.waterfall.buildbucket_client import BuildbucketBuild
 from libs import time_util
 from model import analysis_status
+from model.flake.flake_try_job_data import FlakeTryJobData
 from model.wf_try_job_data import WfTryJobData
 from waterfall import waterfall_config
 
 
 def _GetError(buildbucket_response, buildbucket_error, timed_out):
   """Determines whether or not a try job error occurred.
 
   Args:
     buildbucket_response: A dict of the json response from buildbucket.
     buildbucket_error: A BuildBucketError object returned from the call to
@@ skipping 72 matching lines @@
           {
               'message': 'No result report was found.',
               'reason': MonitorTryJobPipeline.UNKNOWN
           },
           try_job_error.UNKNOWN
       )
 
   return None, None
 
 
-def _UpdateTryJobMetadata(try_job_data, start_time, buildbucket_build,
-                          buildbucket_error, timed_out):
+def _UpdateTryJobMetadata(try_job_data, try_job_type, start_time,
+                          buildbucket_build, buildbucket_error, timed_out):
   buildbucket_response = {}
 
   if buildbucket_build:
     try_job_data.request_time = (
         try_job_data.request_time or
         time_util.MicrosecondsToDatetime(buildbucket_build.request_time))
     # If start_time is unavailable, fallback to request_time.
     try_job_data.start_time = start_time or try_job_data.request_time
     try_job_data.end_time = time_util.MicrosecondsToDatetime(
         buildbucket_build.end_time)
-    try_job_data.number_of_commits_analyzed = len(
-        buildbucket_build.report.get('result', {}))
-    try_job_data.regression_range_size = buildbucket_build.report.get(
-        'metadata', {}).get('regression_range_size')
+
+    if try_job_type != failure_type.FLAKY_TEST:  # pragma: no branch
+      try_job_data.number_of_commits_analyzed = len(
+          buildbucket_build.report.get('result', {}))
+      try_job_data.regression_range_size = buildbucket_build.report.get(
+          'metadata', {}).get('regression_range_size')
+
     try_job_data.try_job_url = buildbucket_build.url
     buildbucket_response = buildbucket_build.response
     try_job_data.last_buildbucket_response = buildbucket_response
 
   error_dict, error_code = _GetError(
       buildbucket_response, buildbucket_error, timed_out)
 
   if error_dict:
     try_job_data.error = error_dict
     try_job_data.error_code = error_code
@@ skipping 93 matching lines @@
 
     timeout_hours = waterfall_config.GetTryJobSettings().get(
         'job_timeout_hours')
     default_pipeline_wait_seconds = waterfall_config.GetTryJobSettings().get(
         'server_query_interval_seconds')
     max_error_times = waterfall_config.GetTryJobSettings().get(
         'allowed_response_error_times')
     pipeline_wait_seconds = default_pipeline_wait_seconds
     allowed_response_error_times = max_error_times
 
+    if try_job_type == failure_type.FLAKY_TEST:
+      try_job_data = FlakeTryJobData.Get(try_job_id)
+    else:
+      try_job_data = WfTryJobData.Get(try_job_id)
+
     # TODO(chanli): Make sure total wait time equals to timeout_hours
     # regardless of retries.
     deadline = time.time() + timeout_hours * 60 * 60
-    try_job_data = WfTryJobData.Get(try_job_id)
     already_set_started = False
     start_time = None
     while True:
       error, build = buildbucket_client.GetTryJobs([try_job_id])[0]
 
       if error:
         if allowed_response_error_times > 0:
           allowed_response_error_times -= 1
           pipeline_wait_seconds += default_pipeline_wait_seconds
         else:  # pragma: no cover
           # Buildbucket has responded error more than 5 times, retry pipeline.
-          _UpdateTryJobMetadata(try_job_data, start_time, build, error, False)
+          _UpdateTryJobMetadata(
+              try_job_data, try_job_type, start_time, build, error, False)
           raise pipeline.Retry(
               'Error "%s" occurred. Reason: "%s"' % (error.message,
                                                      error.reason))
       elif build.status == BuildbucketBuild.COMPLETED:
-        _UpdateTryJobMetadata(try_job_data, start_time, build, error, False)
+        _UpdateTryJobMetadata(
+            try_job_data, try_job_type, start_time, build, error, False)
         result_to_update = self._UpdateTryJobResult(
             urlsafe_try_job_key, try_job_type, try_job_id, build.url,
             BuildbucketBuild.COMPLETED, build.report)
         return result_to_update[-1]
       else:
         if allowed_response_error_times < max_error_times:
           # Recovers from errors.
           allowed_response_error_times = max_error_times
           pipeline_wait_seconds = default_pipeline_wait_seconds
         if build.status == BuildbucketBuild.STARTED and not already_set_started:
           # It is possible this branch is skipped if a fast build goes from
           # 'SCHEDULED' to 'COMPLETED' between queries, so start_time may be
           # unavailable.
           start_time = time_util.MicrosecondsToDatetime(build.updated_time)
           self._UpdateTryJobResult(
               urlsafe_try_job_key, try_job_type, try_job_id, build.url,
               BuildbucketBuild.STARTED)
 
           # Update as much try job metadata as soon as possible to avoid data
           # loss in case of errors.
           try_job_data.start_time = start_time
           try_job_data.request_time = (
               time_util.MicrosecondsToDatetime(build.request_time))
           try_job_data.try_job_url = build.url
           try_job_data.put()
 
           already_set_started = True
 
       if time.time() > deadline:  # pragma: no cover
-        _UpdateTryJobMetadata(try_job_data, start_time, build, error, True)
+        _UpdateTryJobMetadata(
+            try_job_data, try_job_type, start_time, build, error, True)
         # Explicitly abort the whole pipeline.
         raise pipeline.Abort(
             'Try job %s timed out after %d hours.' % (
                 try_job_id, timeout_hours))
 
       # Ensure last_buildbucket_response is always the most recent
       # whenever available during intermediate queries.
       _UpdateLastBuildbucketResponse(try_job_data, build)
 
       time.sleep(pipeline_wait_seconds)  # pragma: no cover