[Findit] Use step level analysis to exclude flaky test failures.

appengine/findit/waterfall/buildbot.py

Index: appengine/findit/waterfall/buildbot.py
diff --git a/appengine/findit/waterfall/buildbot.py b/appengine/findit/waterfall/buildbot.py
index 28d39db4ab0f9ee9d9348adb02abe0da31398105..840d8fd21d364654b28657bbd007a86510031336 100644
--- a/appengine/findit/waterfall/buildbot.py
+++ b/appengine/findit/waterfall/buildbot.py
@@ -6,6 +6,9 @@ from datetime import datetime
 import json
 import re
 import urllib
+import gzip

[stgao, 2015/05/21 00:29:56]

not in order.

+
+import cloudstorage as gcs
 
 from waterfall.build_info import BuildInfo
 
@@ -97,7 +100,12 @@ def CreateStdioLogUrl(master_name, builder_name, build_number, step_name):
               master_name, builder_name, build_number, step_name)
 
 
-def GetBuildDataFromBuildMaster(master_name, 
+def CreateStepLogPath(master_name, builder_name, build_number, step_name):

[stgao, 2015/05/21 00:29:56]

"GtestResult" might be more clear than "StepLog". Same for others below.

+  return ('/chrome-gtest-results/buildbot/%s/%s/%s/%s.json.gz') % (
+              master_name, builder_name, build_number, step_name)
+
+
+def GetBuildDataFromBuildMaster(master_name,
                                 builder_name, build_number, http_client):
   """Returns the json-format data of the build from buildbot json API."""
   status_code, data = http_client.Get(
@@ -108,7 +116,7 @@ def GetBuildDataFromBuildMaster(master_name,
     return data
 
 
-def GetBuildDataFromArchive(master_name, 
+def GetBuildDataFromArchive(master_name,
                             builder_name, build_number, http_client):
   """Returns the json-format data of the build from build archive."""
   status_code, data = http_client.Get(
@@ -130,6 +138,21 @@ def GetStepStdio(master_name, builder_name, build_number,
     return data
 
 
+def GetGsStepLog(master_name,
+                 builder_name, build_number, step_name):  # pragma: no cover
+  """Returns the archived Step Log file."""
+  try:
+    step_log_file = gcs.open(CreateStepLogPath(

[stgao, 2015/05/21 00:29:56]

Let us use "with" for the two opens here. It is cleaner.

Please refer to contextlib.closing in
https://chromium.googlesource.com/infra/infra/+/master/appengine/sheriff_o_ma...

+        master_name, builder_name, build_number, step_name))
+    unzipped_step_log_file = gzip.open(step_log_file )
+    log_file_content = step_log_file.read()
+    unzipped_step_log_file.close()
+    step_log_file.close()
+    return log_file_content
+  except gcs.NotFoundError:
+    return None
+
+
 def GetStepResult(step_data_json):
   """Returns the result of a step."""
   result = step_data_json.get('results')