Implemented parsing of the ignored failing tests spec and ignoring respective failures.

scripts/common/unittests/gtest_utils_test.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Unit tests for classes in gtest_command.py."""
 
+import os
+import tempfile
 import unittest
 
 import test_env  # pylint: disable=W0611
 
+from common import chromium_utils
+from common import find_depot_tools  # pylint: disable=W0611
 from common import gtest_utils
 
+from testing_support import auto_stub
 
 FAILURES = ['NavigationControllerTest.Reload',
             'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0',
             'BadTest.TimesOut',
             'MoreBadTest.TimesOutAndFails',
             'SomeOtherTest.SwitchTypes',
             'SomeOtherTest.FAILS_ThisTestTimesOut']
 
 FAILS_FAILURES = ['SomeOtherTest.FAILS_Bar']
 FLAKY_FAILURES = ['SomeOtherTest.FLAKY_Baz']
@@ skipping 406 matching lines @@
 Tests took 31 seconds.
 
 
 ================================================================
 End output from shard index 0 (machine tag: swarm12.c, id: swarm12). Return 1
 ================================================================
 
 """
 
 
-class TestGTestLogParserTests(unittest.TestCase):
+class TestGTestLogParserTests(auto_stub.TestCase):
 
   def testGTestLogParserNoSharing(self):
     # Tests for log parsing without sharding.
     parser = gtest_utils.GTestLogParser()
     for line in TEST_DATA.splitlines():
       parser.ProcessLine(line)
 
     self.assertEqual(0, len(parser.ParsingErrors()))
     self.assertFalse(parser.RunningTests())
 
@@ skipping 180 matching lines @@
         ],
         parser.FailureDescription('PickleTest.EncodeDecode'))
 
   def testNestedGtests(self):
     parser = gtest_utils.GTestLogParser()
     for line in TEST_DATA_NESTED_RUNS.splitlines():
       parser.ProcessLine(line)
     self.assertEqual(['Foo.Bar'], parser.FailedTests(True, True))
 
 
-class TestGTestJSONParserTests(unittest.TestCase):
+class TestGTestJSONParserTests(auto_stub.TestCase):
   def testPassedTests(self):
     parser = gtest_utils.GTestJSONParser()
     parser.ProcessJSONData({
       'disabled_tests': [],
       'per_iteration_data': [
         {
           'Test.One': [{'status': 'SUCCESS', 'output_snippet': ''}],
           'Test.Two': [{'status': 'SUCCESS', 'output_snippet': ''}],
         }
       ]
@@ skipping 50 matching lines @@
           'Test.One': [{'status': 'SUCCESS', 'output_snippet': ''}],
         }
       ]
     })
 
     self.assertEqual(['Test.One'], parser.PassedTests())
     self.assertEqual([], parser.FailedTests())
     self.assertEqual(0, parser.FlakyTests())
     self.assertEqual(1, parser.DisabledTests())
 
+  def testIngoredFailedTests(self):
+    TEST_IGNORED_FAILED_TESTS_SPEC = """
+      # A comment.
+
+      http://crbug.com/12345 [ OS_WIN  , OS_LINUX] Test.One
+      http://crbug.com/12345 [OS_WIN CPU_64_BITS MODE_RELEASE] Test.Two/2
+      http://crbug.com/12345 [ OS_MACOSX, OS_WIN CPU_64_BITS ] Perf/Test.Three
+      http://crbug.com/12345 [ invalid.platform.spec ] Test.Four
+      http://crbug.com/12345 [ OS_WIN CPU_32_BITS MODE_RELEASE ] Test.Five
+      invalid line
+    """
+
+    _, spec_filename = tempfile.mkstemp()
+    spec_fd = open(spec_filename, 'w')
+    spec_fd.write(TEST_IGNORED_FAILED_TESTS_SPEC)
+    spec_fd.close()
+
+    self.mock(chromium_utils, 'FindUpward', lambda *_: spec_filename)
+    parser = gtest_utils.GTestJSONParser()
+
+    try:
+      parser.ProcessJSONData({
+        'disabled_tests': ['Test.Six'],
+        'per_iteration_data': [
+          {
+            'Test.One': [{'status': 'FAILURE', 'output_snippet': ''}],
+            'Test.Two/2': [{'status': 'FAILURE', 'output_snippet': ''}],
+            'Perf/Test.Three': [{'status': 'FAILURE', 'output_snippet': ''}],
+            'Test.Four': [{'status': 'FAILURE', 'output_snippet': ''}],
+            'Test.Five': [{'status': 'FAILURE', 'output_snippet': ''}],
+          }
+        ],
+        'global_tags': ['OS_WIN', 'CPU_64_BITS', 'MODE_RELEASE', 'OTHER_FLAG']
+      }, '/fake/path/to/build')
+    finally:
+      os.remove(spec_filename)

[eseidel, 2014/07/09 16:07:02]

It's really unfortunate that this test touches the filesystem.  That can cause
tests to flake or fail in unexpected ways.

[Sergiy Byelozyorov, 2014/07/09 17:23:42]

I wish I could mock 'open' function with self.mock. Then there wouldn't need to
be doing that. But it seems to be a common practice to just use files for that.
tempfile.mkstemp promises to create the file in the safest way possible, so this
should not be flaky.

+
+    self.assertEqual(['Test.Five', 'Test.Four'], parser.FailedTests())
+    self.assertEqual(['Perf/Test.Three', 'Test.One', 'Test.Two/2'],
+                     parser.IgnoredFailedTests())
+
+  # pylint: disable=R0201
+  def testDoesNotThrowExceptionOnMissingIgnoredFailedTestsFile(self):
+    parser = gtest_utils.GTestJSONParser()
+    parser.ProcessJSONData({'per_iteration_data': []}, tempfile.gettempdir())
+
   def testCompressList(self):
     CompressList = gtest_utils.CompressList
     self.assertEqual(['foo'],          CompressList([1, 2, 3, 4], 0, 'foo'))
     self.assertEqual(['foo', 4],       CompressList([1, 2, 3, 4], 1, 'foo'))
     self.assertEqual([1, 'foo', 4],    CompressList([1, 2, 3, 4], 2, 'foo'))
     self.assertEqual([1, 'foo', 3, 4], CompressList([1, 2, 3, 4], 3, 'foo'))
     self.assertEqual([1, 2, 3, 4],     CompressList([1, 2, 3, 4], 4, 'foo'))
     self.assertEqual([1, 2, 3, 4],     CompressList([1, 2, 3, 4], 5, 'foo'))
     self.assertEqual([1, 2, 3, 4],     CompressList([1, 2, 3, 4], 6, 'foo'))
 
 
 if __name__ == '__main__':
   unittest.main()