[Findit] Bailing out if build data is too old and moving relevant settings to config

appengine/findit/waterfall/build_util.py

Index: appengine/findit/waterfall/build_util.py
diff --git a/appengine/findit/waterfall/build_util.py b/appengine/findit/waterfall/build_util.py
index dce784e7b6357e5a5465b4d228f73b166da05050..8a4ff92d9fe585590bc97b8ce2abfc53c3b3088d 100644
--- a/appengine/findit/waterfall/build_util.py
+++ b/appengine/findit/waterfall/build_util.py
@@ -8,15 +8,19 @@ from common.http_client_appengine import HttpClientAppengine as HttpClient
 from model.wf_build import WfBuild
 from waterfall import buildbot
 from waterfall import lock_util
+from waterfall import waterfall_config
 
 
 HTTP_CLIENT_LOGGING_ERRORS = HttpClient()
 HTTP_CLIENT_NO_404_ERROR = HttpClient(no_error_logging_statuses=[404])
+CHROME_BUILD_EXTRACT = 'CBE'
+BUILDBOT_MASTER = 'BM'
 
 
 def _BuildDataNeedUpdating(build):
-  return (not build.data or (not build.completed and
-      (datetime.utcnow() - build.last_crawled_time).total_seconds() >= 300))
+  return (not build.data or (
+      not build.completed and (
+          datetime.utcnow() - build.last_crawled_time).total_seconds() >= 300))
 
 
 def DownloadBuildData(master_name, builder_name, build_number):
@@ -27,18 +31,25 @@ def DownloadBuildData(master_name, builder_name, build_number):
 
   # Cache the data to avoid pulling from master again.
   if _BuildDataNeedUpdating(build):
-    # Retrieve build data from build archive first.
-    build.data = buildbot.GetBuildDataFromArchive(
-        master_name, builder_name, build_number, HTTP_CLIENT_NO_404_ERROR)
+    use_cbe = waterfall_config.GetDownloadBuildDataSettings().get(
+        'use_chrome_build_extract')
 
-    if build.data is None:
-      if not lock_util.WaitUntilDownloadAllowed(
+    if use_cbe:
+      # Retrieve build data from build archive first.
+      build.data = buildbot.GetBuildDataFromArchive(
+          master_name, builder_name, build_number, HTTP_CLIENT_NO_404_ERROR)
+
+      if build.data:

[chanli, 2016/05/20 23:53:54]

Nit: how about changing code from ln42 to 55 to:
if use_cbe:
  build.data = buildbot.GetBuildDataFromArchive(
      master_name, builder_name, build_number, HTTP_CLIENT_NO_404_ERROR)

  if build.data is not None:
     build.data_source = CHROME_BUILD_EXTRACT
  else:
    if not lock_util.WaitUntilDownloadAllowed(master_name):
      return None
    build.data = buildbot.GetBuildDataFromBuildMaster(
        master_name, builder_name, build_number, HTTP_CLIENT_LOGGING_ERRORS)
     build.data_source = BUILDBOT_MASTER
    build.last_crawled_time = datetime.utcnow()
    build.put()

[lijeffrey, 2016/05/21 01:50:20]

we still want to fallback to buildmaster even if use_cbe is True but does not
return results. The code as it is also allows for 1 write for build.put()

+        build.data_source = CHROME_BUILD_EXTRACT
+      elif not lock_util.WaitUntilDownloadAllowed(
           master_name):  # pragma: no cover
         return None
 
+    if not build.data:
       # Retrieve build data from build master.
       build.data = buildbot.GetBuildDataFromBuildMaster(
           master_name, builder_name, build_number, HTTP_CLIENT_LOGGING_ERRORS)
+      build.data_source = BUILDBOT_MASTER
 
     build.last_crawled_time = datetime.utcnow()
     build.put()