[Findit] Use step level analysis to exclude flaky test failures.

appengine/findit/waterfall/test/extract_signal_pipeline_test.py

Index: appengine/findit/waterfall/test/extract_signal_pipeline_test.py
diff --git a/appengine/findit/waterfall/test/extract_signal_pipeline_test.py b/appengine/findit/waterfall/test/extract_signal_pipeline_test.py
index 59c6bbe5d716a22d8c761ec5afeda3aec3370722..a122a0d8b51073744ca2399ed1bf8e9961b9e1c6 100644
--- a/appengine/findit/waterfall/test/extract_signal_pipeline_test.py
+++ b/appengine/findit/waterfall/test/extract_signal_pipeline_test.py
@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
+import json

[stgao, 2015/05/22 22:49:43]

style: order

[chanli, 2015/05/22 23:26:33]

json is actually not used.

Done.

+
 from pipeline_utils.appengine_third_party_pipeline_src_pipeline import handlers
 from testing_utils import testing
 
@@ -95,3 +98,68 @@ class ExtractSignalPipelineTest(testing.AppengineTestCase):
 
     step = WfStep.Create(master_name, builder_name, build_number, step_name)
     self.assertIsNotNone(step)
+
+  def _GetGtestResultLog(self,
+                         master_name, builder_name, build_number, step_name):
+    file_name = os.path.join(
+        os.path.dirname(__file__), 'data',
+        '%s_%s_%d_%s.json' % (master_name,
+                              builder_name, build_number, step_name))
+    with open(file_name, 'r') as f:
+      return f.read()
+
+  def testGetTestLevelFailures(self):
+    master_name = 'm'
+    builder_name = 'b'
+    build_number = 123
+    step_name = 'abc_test'
+
+    expected_failure_log = ('\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure'
+                            '\\n[2]: 2594735000 bogo-microseconds\\n\n'
+                            '\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure\n'
+                            '\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure\n'
+                            '\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                           )
+
+    step_log = self._GetGtestResultLog(master_name,
+                                builder_name, build_number, step_name)
+
+    failed_test_log = ExtractSignalPipeline._GetReliableTestFailureLog(step_log)
+    self.assertEqual(expected_failure_log, failed_test_log)
+
+  def testGetSignalFromStepLog(self):
+    master_name = 'm'
+    builder_name = 'b'
+    build_number = 123
+    step_name = 'abc_test'
+
+    # Mock both stdiolog and step log to test whether Findit will
+    # go to step log first when both logs exist.
+    step_log_url = buildbot.CreateStdioLogUrl(
+        master_name, builder_name, build_number, step_name)
+    with self.mock_urlfetch() as urlfetch:
+      urlfetch.register_handler(step_log_url, self.ABC_TEST_FAILURE_LOG)
+
+    def _MockGetGtestResultLog(master_name,
+                               builder_name, build_number, step_name):
+      return self._GetGtestResultLog(master_name,
+                              builder_name, build_number, step_name)
+    self.mock(buildbot, 'GetGtestResultLog', _MockGetGtestResultLog)
+
+    pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
+    signals = pipeline.run(self.FAILURE_INFO)
+
+    step = WfStep.Get(master_name, builder_name, build_number, step_name)
+
+    expected_files = {
+        'a/b/u2s1.cc': [567],
+        'a/b/u3s2.cc': [110]
+    }
+
+    self.assertIsNotNone(step)
+    self.assertIsNotNone(step.log_data)
+    self.assertEqual(expected_files, signals['abc_test']['files'])