[Findit] Use step level analysis to exclude flaky test failures.

appengine/findit/waterfall/test/extract_signal_pipeline_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
+import json
+
 from pipeline_utils.appengine_third_party_pipeline_src_pipeline import handlers
 from testing_utils import testing
 
 from model.wf_step import WfStep
 from waterfall import buildbot
 from waterfall import extractors
 from waterfall.extract_signal_pipeline import ExtractSignalPipeline
 
 
 class ExtractSignalPipelineTest(testing.AppengineTestCase):
@@ skipping 73 matching lines @@
         master_name, builder_name, build_number, step_name)
     with self.mock_urlfetch() as urlfetch:
       urlfetch.register_handler(step_log_url, self.ABC_TEST_FAILURE_LOG)
 
     pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
     pipeline.start()
     self.execute_queued_tasks()
 
     step = WfStep.Create(master_name, builder_name, build_number, step_name)
     self.assertIsNotNone(step)
+
+  def _GetStepLog(self, master_name, builder_name, build_number, step_name):
+    file_name = os.path.join(
+        os.path.dirname(__file__), 'data',
+        '%s_%s_%d_%s.json' % (master_name, 
+                              builder_name, build_number, step_name))
+    with open(file_name, 'r') as f:
+      return f.read()
+
+  def testGetTestLevelFailures(self):
+    master_name = 'm'
+    builder_name = 'b'
+    build_number = 123
+    step_name = 'abc_test'
+
+    expected_failure_log = {
+      "Unittest2.Subtest1":[
+        {
+          "elapsed_time_ms": 56,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u2s1.cc:567: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        },
+        {
+          "elapsed_time_ms": 58,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u2s1.cc:567: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        },
+        {
+          "elapsed_time_ms": 60,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u2s1.cc:567: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        },
+        {
+          "elapsed_time_ms": 60,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u2s1.cc:567: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        }
+      ],
+      "Unittest3.Subtest2":[
+        {
+          "elapsed_time_ms": 80,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u3s2.cc:110: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        },
+        {
+          "elapsed_time_ms": 58,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u3s2.cc:110: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        },
+        {
+          "elapsed_time_ms": 60,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u3s2.cc:110: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        },
+        {
+          "elapsed_time_ms": 60,
+          "losless_snippet": True,
+          "output_snippet": "a/b/u3s2.cc:110: Failure",
+          "status": "FAILURE",
+          "output_snippet_base64": "WyBSVU4gICAgICBdIExheWVyQW5pbWF0aW9"
+        }
+      ]
+    }
+
+    step_log = self._GetStepLog(master_name,
+                                builder_name, build_number, step_name)
+
+    failed_test_log = ExtractSignalPipeline._GetTestLevelFailures(step_log)
+    self.assertEqual(expected_failure_log, failed_test_log)
+
+  def testGetSignalFromStepLog(self):
+    master_name = 'm'
+    builder_name = 'b'
+    build_number = 123
+    step_name = 'abc_test'
+
+    # Mock both stdiolog and step log to test whether Findit will
+    # go to step log first when both logs exist.
+    step_log_url = buildbot.CreateStdioLogUrl(
+        master_name, builder_name, build_number, step_name)
+    with self.mock_urlfetch() as urlfetch:
+      urlfetch.register_handler(step_log_url, self.ABC_TEST_FAILURE_LOG)
+
+    def _MockGetGsStepLog(master_name, builder_name, build_number, step_name):
+      return self._GetStepLog(master_name,
+                              builder_name, build_number, step_name)
+    self.mock(buildbot, 'GetGsStepLog', _MockGetGsStepLog)
+
+    pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
+    signals = pipeline.run(self.FAILURE_INFO)
+
+    step = WfStep.Get(master_name, builder_name, build_number, step_name)
+
+    expected_files = {
+      'a/b/u2s1.cc': [567],
+      'a/b/u3s2.cc': [110]
+    }
+
+    self.assertIsNotNone(step)
+    self.assertIsNotNone(step.log_data)
+    self.assertEqual(expected_files, signals['abc_test']['files'])