| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2009 Google Inc. All Rights Reserved. | |
| 4 # | |
| 5 # Redistribution and use in source and binary forms, with or without | |
| 6 # modification, are permitted provided that the following conditions are | |
| 7 # met: | |
| 8 # | |
| 9 # * Redistributions of source code must retain the above copyright | |
| 10 # notice, this list of conditions and the following disclaimer. | |
| 11 # * Redistributions in binary form must reproduce the above | |
| 12 # copyright notice, this list of conditions and the following disclaimer | |
| 13 # in the documentation and/or other materials provided with the | |
| 14 # distribution. | |
| 15 # * Neither the name of Google Inc. nor the names of its | |
| 16 # contributors may be used to endorse or promote products derived from | |
| 17 # this software without specific prior written permission. | |
| 18 # | |
| 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 30 | |
| 31 """Verifies that test shuffling works.""" | |
| 32 | |
| 33 __author__ = 'wan@google.com (Zhanyong Wan)' | |
| 34 | |
| 35 import os | |
| 36 import gtest_test_utils | |
| 37 | |
| 38 # Command to run the gtest_shuffle_test_ program. | |
| 39 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') | |
| 40 | |
| 41 # The environment variables for test sharding. | |
| 42 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' | |
| 43 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' | |
| 44 | |
| 45 TEST_FILTER = 'A*.A:A*.B:C*' | |
| 46 | |
| 47 ALL_TESTS = [] | |
| 48 ACTIVE_TESTS = [] | |
| 49 FILTERED_TESTS = [] | |
| 50 SHARDED_TESTS = [] | |
| 51 | |
| 52 SHUFFLED_ALL_TESTS = [] | |
| 53 SHUFFLED_ACTIVE_TESTS = [] | |
| 54 SHUFFLED_FILTERED_TESTS = [] | |
| 55 SHUFFLED_SHARDED_TESTS = [] | |
| 56 | |
| 57 | |
| 58 def AlsoRunDisabledTestsFlag(): | |
| 59 return '--gtest_also_run_disabled_tests' | |
| 60 | |
| 61 | |
| 62 def FilterFlag(test_filter): | |
| 63 return '--gtest_filter=%s' % (test_filter,) | |
| 64 | |
| 65 | |
| 66 def RepeatFlag(n): | |
| 67 return '--gtest_repeat=%s' % (n,) | |
| 68 | |
| 69 | |
| 70 def ShuffleFlag(): | |
| 71 return '--gtest_shuffle' | |
| 72 | |
| 73 | |
| 74 def RandomSeedFlag(n): | |
| 75 return '--gtest_random_seed=%s' % (n,) | |
| 76 | |
| 77 | |
| 78 def RunAndReturnOutput(extra_env, args): | |
| 79 """Runs the test program and returns its output.""" | |
| 80 | |
| 81 environ_copy = os.environ.copy() | |
| 82 environ_copy.update(extra_env) | |
| 83 | |
| 84 return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output | |
| 85 | |
| 86 | |
| 87 def GetTestsForAllIterations(extra_env, args): | |
| 88 """Runs the test program and returns a list of test lists. | |
| 89 | |
| 90 Args: | |
| 91 extra_env: a map from environment variables to their values | |
| 92 args: command line flags to pass to gtest_shuffle_test_ | |
| 93 | |
| 94 Returns: | |
| 95 A list where the i-th element is the list of tests run in the i-th | |
| 96 test iteration. | |
| 97 """ | |
| 98 | |
| 99 test_iterations = [] | |
| 100 for line in RunAndReturnOutput(extra_env, args).split('\n'): | |
| 101 if line.startswith('----'): | |
| 102 tests = [] | |
| 103 test_iterations.append(tests) | |
| 104 elif line.strip(): | |
| 105 tests.append(line.strip()) # 'TestCaseName.TestName' | |
| 106 | |
| 107 return test_iterations | |
| 108 | |
| 109 | |
| 110 def GetTestCases(tests): | |
| 111 """Returns a list of test cases in the given full test names. | |
| 112 | |
| 113 Args: | |
| 114 tests: a list of full test names | |
| 115 | |
| 116 Returns: | |
| 117 A list of test cases from 'tests', in their original order. | |
| 118 Consecutive duplicates are removed. | |
| 119 """ | |
| 120 | |
| 121 test_cases = [] | |
| 122 for test in tests: | |
| 123 test_case = test.split('.')[0] | |
| 124 if not test_case in test_cases: | |
| 125 test_cases.append(test_case) | |
| 126 | |
| 127 return test_cases | |
| 128 | |
| 129 | |
| 130 def CalculateTestLists(): | |
| 131 """Calculates the list of tests run under different flags.""" | |
| 132 | |
| 133 if not ALL_TESTS: | |
| 134 ALL_TESTS.extend( | |
| 135 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) | |
| 136 | |
| 137 if not ACTIVE_TESTS: | |
| 138 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) | |
| 139 | |
| 140 if not FILTERED_TESTS: | |
| 141 FILTERED_TESTS.extend( | |
| 142 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) | |
| 143 | |
| 144 if not SHARDED_TESTS: | |
| 145 SHARDED_TESTS.extend( | |
| 146 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 147 SHARD_INDEX_ENV_VAR: '1'}, | |
| 148 [])[0]) | |
| 149 | |
| 150 if not SHUFFLED_ALL_TESTS: | |
| 151 SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( | |
| 152 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
| 153 | |
| 154 if not SHUFFLED_ACTIVE_TESTS: | |
| 155 SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( | |
| 156 {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
| 157 | |
| 158 if not SHUFFLED_FILTERED_TESTS: | |
| 159 SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( | |
| 160 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) | |
| 161 | |
| 162 if not SHUFFLED_SHARDED_TESTS: | |
| 163 SHUFFLED_SHARDED_TESTS.extend( | |
| 164 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 165 SHARD_INDEX_ENV_VAR: '1'}, | |
| 166 [ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
| 167 | |
| 168 | |
| 169 class GTestShuffleUnitTest(gtest_test_utils.TestCase): | |
| 170 """Tests test shuffling.""" | |
| 171 | |
| 172 def setUp(self): | |
| 173 CalculateTestLists() | |
| 174 | |
| 175 def testShufflePreservesNumberOfTests(self): | |
| 176 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) | |
| 177 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) | |
| 178 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) | |
| 179 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) | |
| 180 | |
| 181 def testShuffleChangesTestOrder(self): | |
| 182 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) | |
| 183 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) | |
| 184 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, | |
| 185 SHUFFLED_FILTERED_TESTS) | |
| 186 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, | |
| 187 SHUFFLED_SHARDED_TESTS) | |
| 188 | |
| 189 def testShuffleChangesTestCaseOrder(self): | |
| 190 self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), | |
| 191 GetTestCases(SHUFFLED_ALL_TESTS)) | |
| 192 self.assert_( | |
| 193 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), | |
| 194 GetTestCases(SHUFFLED_ACTIVE_TESTS)) | |
| 195 self.assert_( | |
| 196 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), | |
| 197 GetTestCases(SHUFFLED_FILTERED_TESTS)) | |
| 198 self.assert_( | |
| 199 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), | |
| 200 GetTestCases(SHUFFLED_SHARDED_TESTS)) | |
| 201 | |
| 202 def testShuffleDoesNotRepeatTest(self): | |
| 203 for test in SHUFFLED_ALL_TESTS: | |
| 204 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), | |
| 205 '%s appears more than once' % (test,)) | |
| 206 for test in SHUFFLED_ACTIVE_TESTS: | |
| 207 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), | |
| 208 '%s appears more than once' % (test,)) | |
| 209 for test in SHUFFLED_FILTERED_TESTS: | |
| 210 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), | |
| 211 '%s appears more than once' % (test,)) | |
| 212 for test in SHUFFLED_SHARDED_TESTS: | |
| 213 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), | |
| 214 '%s appears more than once' % (test,)) | |
| 215 | |
| 216 def testShuffleDoesNotCreateNewTest(self): | |
| 217 for test in SHUFFLED_ALL_TESTS: | |
| 218 self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) | |
| 219 for test in SHUFFLED_ACTIVE_TESTS: | |
| 220 self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) | |
| 221 for test in SHUFFLED_FILTERED_TESTS: | |
| 222 self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) | |
| 223 for test in SHUFFLED_SHARDED_TESTS: | |
| 224 self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) | |
| 225 | |
| 226 def testShuffleIncludesAllTests(self): | |
| 227 for test in ALL_TESTS: | |
| 228 self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) | |
| 229 for test in ACTIVE_TESTS: | |
| 230 self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) | |
| 231 for test in FILTERED_TESTS: | |
| 232 self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) | |
| 233 for test in SHARDED_TESTS: | |
| 234 self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) | |
| 235 | |
| 236 def testShuffleLeavesDeathTestsAtFront(self): | |
| 237 non_death_test_found = False | |
| 238 for test in SHUFFLED_ACTIVE_TESTS: | |
| 239 if 'DeathTest.' in test: | |
| 240 self.assert_(not non_death_test_found, | |
| 241 '%s appears after a non-death test' % (test,)) | |
| 242 else: | |
| 243 non_death_test_found = True | |
| 244 | |
| 245 def _VerifyTestCasesDoNotInterleave(self, tests): | |
| 246 test_cases = [] | |
| 247 for test in tests: | |
| 248 [test_case, _] = test.split('.') | |
| 249 if test_cases and test_cases[-1] != test_case: | |
| 250 test_cases.append(test_case) | |
| 251 self.assertEqual(1, test_cases.count(test_case), | |
| 252 'Test case %s is not grouped together in %s' % | |
| 253 (test_case, tests)) | |
| 254 | |
| 255 def testShuffleDoesNotInterleaveTestCases(self): | |
| 256 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) | |
| 257 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) | |
| 258 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) | |
| 259 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) | |
| 260 | |
| 261 def testShuffleRestoresOrderAfterEachIteration(self): | |
| 262 # Get the test lists in all 3 iterations, using random seed 1, 2, | |
| 263 # and 3 respectively. Google Test picks a different seed in each | |
| 264 # iteration, and this test depends on the current implementation | |
| 265 # picking successive numbers. This dependency is not ideal, but | |
| 266 # makes the test much easier to write. | |
| 267 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( | |
| 268 GetTestsForAllIterations( | |
| 269 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) | |
| 270 | |
| 271 # Make sure running the tests with random seed 1 gets the same | |
| 272 # order as in iteration 1 above. | |
| 273 [tests_with_seed1] = GetTestsForAllIterations( | |
| 274 {}, [ShuffleFlag(), RandomSeedFlag(1)]) | |
| 275 self.assertEqual(tests_in_iteration1, tests_with_seed1) | |
| 276 | |
| 277 # Make sure running the tests with random seed 2 gets the same | |
| 278 # order as in iteration 2 above. Success means that Google Test | |
| 279 # correctly restores the test order before re-shuffling at the | |
| 280 # beginning of iteration 2. | |
| 281 [tests_with_seed2] = GetTestsForAllIterations( | |
| 282 {}, [ShuffleFlag(), RandomSeedFlag(2)]) | |
| 283 self.assertEqual(tests_in_iteration2, tests_with_seed2) | |
| 284 | |
| 285 # Make sure running the tests with random seed 3 gets the same | |
| 286 # order as in iteration 3 above. Success means that Google Test | |
| 287 # correctly restores the test order before re-shuffling at the | |
| 288 # beginning of iteration 3. | |
| 289 [tests_with_seed3] = GetTestsForAllIterations( | |
| 290 {}, [ShuffleFlag(), RandomSeedFlag(3)]) | |
| 291 self.assertEqual(tests_in_iteration3, tests_with_seed3) | |
| 292 | |
| 293 def testShuffleGeneratesNewOrderInEachIteration(self): | |
| 294 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( | |
| 295 GetTestsForAllIterations( | |
| 296 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) | |
| 297 | |
| 298 self.assert_(tests_in_iteration1 != tests_in_iteration2, | |
| 299 tests_in_iteration1) | |
| 300 self.assert_(tests_in_iteration1 != tests_in_iteration3, | |
| 301 tests_in_iteration1) | |
| 302 self.assert_(tests_in_iteration2 != tests_in_iteration3, | |
| 303 tests_in_iteration2) | |
| 304 | |
| 305 def testShuffleShardedTestsPreservesPartition(self): | |
| 306 # If we run M tests on N shards, the same M tests should be run in | |
| 307 # total, regardless of the random seeds used by the shards. | |
| 308 [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 309 SHARD_INDEX_ENV_VAR: '0'}, | |
| 310 [ShuffleFlag(), RandomSeedFlag(1)]) | |
| 311 [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 312 SHARD_INDEX_ENV_VAR: '1'}, | |
| 313 [ShuffleFlag(), RandomSeedFlag(20)]) | |
| 314 [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 315 SHARD_INDEX_ENV_VAR: '2'}, | |
| 316 [ShuffleFlag(), RandomSeedFlag(25)]) | |
| 317 sorted_sharded_tests = tests1 + tests2 + tests3 | |
| 318 sorted_sharded_tests.sort() | |
| 319 sorted_active_tests = [] | |
| 320 sorted_active_tests.extend(ACTIVE_TESTS) | |
| 321 sorted_active_tests.sort() | |
| 322 self.assertEqual(sorted_active_tests, sorted_sharded_tests) | |
| 323 | |
| 324 if __name__ == '__main__': | |
| 325 gtest_test_utils.Main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 47873006:
libvpx: Pull from upstream (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/deps/third_party/libvpx/