[Findit] Use step level analysis to exclude flaky test failures.

appengine/findit/waterfall/test/extract_signal_pipeline_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
+import json

[stgao, 2015/05/22 22:49:43]

style: order

[chanli, 2015/05/22 23:26:33]

json is actually not used.

Done.

+
 from pipeline_utils.appengine_third_party_pipeline_src_pipeline import handlers
 from testing_utils import testing
 
 from model.wf_step import WfStep
 from waterfall import buildbot
 from waterfall import extractors
 from waterfall.extract_signal_pipeline import ExtractSignalPipeline
 
 
 class ExtractSignalPipelineTest(testing.AppengineTestCase):
@@ skipping 73 matching lines @@
         master_name, builder_name, build_number, step_name)
     with self.mock_urlfetch() as urlfetch:
       urlfetch.register_handler(step_log_url, self.ABC_TEST_FAILURE_LOG)
 
     pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
     pipeline.start()
     self.execute_queued_tasks()
 
     step = WfStep.Create(master_name, builder_name, build_number, step_name)
     self.assertIsNotNone(step)
+
+  def _GetGtestResultLog(self,
+                         master_name, builder_name, build_number, step_name):
+    file_name = os.path.join(
+        os.path.dirname(__file__), 'data',
+        '%s_%s_%d_%s.json' % (master_name,
+                              builder_name, build_number, step_name))
+    with open(file_name, 'r') as f:
+      return f.read()
+
+  def testGetTestLevelFailures(self):
+    master_name = 'm'
+    builder_name = 'b'
+    build_number = 123
+    step_name = 'abc_test'
+
+    expected_failure_log = ('\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure'
+                            '\\n[2]: 2594735000 bogo-microseconds\\n\n'
+                            '\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure\n'
+                            '\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure\n'
+                            '\'Unittest2.Subtest1\': a/b/u2s1.cc:567: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                            '\'Unittest3.Subtest2\': a/b/u3s2.cc:110: Failure\n'
+                           )
+
+    step_log = self._GetGtestResultLog(master_name,
+                                builder_name, build_number, step_name)
+
+    failed_test_log = ExtractSignalPipeline._GetReliableTestFailureLog(step_log)
+    self.assertEqual(expected_failure_log, failed_test_log)
+
+  def testGetSignalFromStepLog(self):
+    master_name = 'm'
+    builder_name = 'b'
+    build_number = 123
+    step_name = 'abc_test'
+
+    # Mock both stdiolog and step log to test whether Findit will
+    # go to step log first when both logs exist.
+    step_log_url = buildbot.CreateStdioLogUrl(
+        master_name, builder_name, build_number, step_name)
+    with self.mock_urlfetch() as urlfetch:
+      urlfetch.register_handler(step_log_url, self.ABC_TEST_FAILURE_LOG)
+
+    def _MockGetGtestResultLog(master_name,
+                               builder_name, build_number, step_name):
+      return self._GetGtestResultLog(master_name,
+                              builder_name, build_number, step_name)
+    self.mock(buildbot, 'GetGtestResultLog', _MockGetGtestResultLog)
+
+    pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
+    signals = pipeline.run(self.FAILURE_INFO)
+
+    step = WfStep.Get(master_name, builder_name, build_number, step_name)
+
+    expected_files = {
+        'a/b/u2s1.cc': [567],
+        'a/b/u3s2.cc': [110]
+    }
+
+    self.assertIsNotNone(step)
+    self.assertIsNotNone(step.log_data)
+    self.assertEqual(expected_files, signals['abc_test']['files'])