[Findit] Use step level analysis to exclude flaky test failures.

appengine/findit/waterfall/buildbot.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import contextlib
 from datetime import datetime
+import gzip
 import json
 import re
 import urllib
 
+import cloudstorage as gcs
+
 from waterfall.build_info import BuildInfo
 
 _MASTER_URL_PATTERN = re.compile(r'^https?://build\.chromium\.org/p/([^/]+)'
                                   '(/.*)?$')
 
 _BUILD_URL_PATTERN = re.compile(r'^https?://build\.chromium\.org/p/([^/]+)/'
                                  'builders/([^/]+)/builds/([\d]+)(/.*)?$')
 
 _STEP_URL_PATTERN = re.compile(r'^https?://build\.chromium\.org/p/([^/]+)/'
                                  'builders/([^/]+)/builds/([\d]+)/steps/'
@@ skipping 70 matching lines @@
 
 
 def CreateStdioLogUrl(master_name, builder_name, build_number, step_name):
   builder_name = urllib.quote(builder_name)
   step_name = urllib.quote(step_name)
   return ('https://build.chromium.org/p/%s/builders/%s/builds/%s/'
           'steps/%s/logs/stdio/text') % (
               master_name, builder_name, build_number, step_name)
 
 
-def GetBuildDataFromBuildMaster(master_name, 
+def CreateGtestResultPath(master_name, builder_name, build_number, step_name):
+  return ('/chrome-gtest-results/buildbot/%s/%s/%s/%s.json.gz') % (
+              master_name, builder_name, build_number, step_name)
+
+
+def GetBuildDataFromBuildMaster(master_name,
                                 builder_name, build_number, http_client):
   """Returns the json-format data of the build from buildbot json API."""
   status_code, data = http_client.Get(
       CreateBuildUrl(master_name, builder_name, build_number, json_api=True))
   if status_code != 200:
     return None
   else:
     return data
 
 
-def GetBuildDataFromArchive(master_name, 
+def GetBuildDataFromArchive(master_name,
                             builder_name, build_number, http_client):
   """Returns the json-format data of the build from build archive."""
   status_code, data = http_client.Get(
       CreateArchivedBuildUrl(master_name, builder_name, build_number))
   if status_code != 200:
     return None
   else:
     return data
 
 
 def GetStepStdio(master_name, builder_name, build_number,
                  step_name, http_client):
   """Returns the raw string of stdio of the specified step."""
   status_code, data = http_client.Get(
       CreateStdioLogUrl(master_name, builder_name, build_number, step_name))
   if status_code != 200:
     return None
   else:
     return data
 
 
+def GetGtestResultLog(
+    master_name, builder_name, build_number, step_name):  # pragma: no cover
+  """Returns the content of the gtest json results for the gtest-based step."""
+  try:
+    archivedLogPath = CreateGtestResultPath(

[stgao, 2015/05/27 01:44:46]

style nit: naming_style.

+        master_name, builder_name, build_number, step_name)
+    with contextlib.closing(gcs.open(archivedLogPath)) as gtest_result_file:
+      with gzip.open(gtest_result_file) as unzipped_gtest_result_file:
+        return unzipped_gtest_result_file.read()
+  except gcs.NotFoundError:
+    return None
+
+
 def GetStepResult(step_data_json):
   """Returns the result of a step."""
   result = step_data_json.get('results')
   if result is None and step_data_json.get('isFinished'):
     # Without parameter filter=0 in the http request to the buildbot json api,
     # the value of the result of a passed step won't be present.
     return SUCCESS
 
   while isinstance(result, list):
     result = result[0]
@@ skipping 53 matching lines @@
       # Skip steps that haven't started yet or are still running.
       continue
 
     step_result = GetStepResult(step_data)
     if step_result in (SUCCESS, WARNINGS):
       build_info.passed_steps.append(step_name)
     elif step_result == FAILURE:
       build_info.failed_steps.append(step_name)
 
   return build_info