| Index: Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
|
| diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py b/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
|
| index 99d22462ab7c51e14d4af5695413d81893b9bfb1..084bcff1af03dbb9eb4206052d670de3b139324a 100644
|
| --- a/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
|
| +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
|
| @@ -79,8 +79,20 @@ def summarized_results(port, expected, passing, flaky, only_include_failing=Fals
|
| initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow)
|
| else:
|
| initial_results.add(get_result('passes/text.html', test_expectations.TIMEOUT, run_time=1), expected, test_is_slow)
|
| - initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO, run_time=0.049), expected, test_is_slow)
|
| - initial_results.add(get_result('failures/expected/timeout.html', test_expectations.CRASH, run_time=0.05), expected, test_is_slow)
|
| + initial_results.add(
|
| + get_result(
|
| + 'failures/expected/audio.html',
|
| + test_expectations.AUDIO,
|
| + run_time=0.049),
|
| + expected,
|
| + test_is_slow)
|
| + initial_results.add(
|
| + get_result(
|
| + 'failures/expected/timeout.html',
|
| + test_expectations.CRASH,
|
| + run_time=0.05),
|
| + expected,
|
| + test_is_slow)
|
| initial_results.add(get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
|
| initial_results.add(get_result('failures/expected/leak.html', test_expectations.TIMEOUT), expected, test_is_slow)
|
|
|
| @@ -96,16 +108,19 @@ def summarized_results(port, expected, passing, flaky, only_include_failing=Fals
|
| else:
|
| retry_results = None
|
|
|
| - return test_run_results.summarize_results(port, initial_results.expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_include_failing=only_include_failing)
|
| + return test_run_results.summarize_results(port, initial_results.expectations, initial_results,
|
| + retry_results, enabled_pixel_tests_in_retry=False, only_include_failing=only_include_failing)
|
|
|
|
|
| class InterpretTestFailuresTest(unittest.TestCase):
|
| +
|
| def setUp(self):
|
| host = MockHost()
|
| self.port = host.port_factory.get(port_name='test')
|
|
|
| def test_interpret_test_failures(self):
|
| - test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
|
| + test_dict = test_run_results._interpret_test_failures(
|
| + [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
|
| self.assertEqual(len(test_dict), 0)
|
|
|
| test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()])
|
| @@ -122,6 +137,7 @@ class InterpretTestFailuresTest(unittest.TestCase):
|
|
|
|
|
| class SummarizedResultsTest(unittest.TestCase):
|
| +
|
| def setUp(self):
|
| host = MockHost(initialize_scm_by_default=False)
|
| self.port = host.port_factory.get(port_name='test')
|
| @@ -132,13 +148,61 @@ class SummarizedResultsTest(unittest.TestCase):
|
|
|
| def test_num_failures_by_type(self):
|
| summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
|
| - self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1})
|
| + self.assertEquals(summary['num_failures_by_type'],
|
| + {'CRASH': 1,
|
| + 'MISSING': 0,
|
| + 'TEXT': 0,
|
| + 'IMAGE': 0,
|
| + 'NEEDSREBASELINE': 0,
|
| + 'NEEDSMANUALREBASELINE': 0,
|
| + 'PASS': 0,
|
| + 'REBASELINE': 0,
|
| + 'SKIP': 0,
|
| + 'SLOW': 0,
|
| + 'TIMEOUT': 3,
|
| + 'IMAGE+TEXT': 0,
|
| + 'LEAK': 0,
|
| + 'FAIL': 0,
|
| + 'AUDIO': 1,
|
| + 'WONTFIX': 1})
|
|
|
| summary = summarized_results(self.port, expected=True, passing=False, flaky=False)
|
| - self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0})
|
| + self.assertEquals(summary['num_failures_by_type'],
|
| + {'CRASH': 1,
|
| + 'MISSING': 0,
|
| + 'TEXT': 0,
|
| + 'IMAGE': 0,
|
| + 'NEEDSREBASELINE': 0,
|
| + 'NEEDSMANUALREBASELINE': 0,
|
| + 'PASS': 1,
|
| + 'REBASELINE': 0,
|
| + 'SKIP': 0,
|
| + 'SLOW': 0,
|
| + 'TIMEOUT': 1,
|
| + 'IMAGE+TEXT': 0,
|
| + 'LEAK': 1,
|
| + 'FAIL': 0,
|
| + 'AUDIO': 1,
|
| + 'WONTFIX': 0})
|
|
|
| summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
|
| - self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0})
|
| + self.assertEquals(summary['num_failures_by_type'],
|
| + {'CRASH': 0,
|
| + 'MISSING': 0,
|
| + 'TEXT': 0,
|
| + 'IMAGE': 0,
|
| + 'NEEDSREBASELINE': 0,
|
| + 'NEEDSMANUALREBASELINE': 0,
|
| + 'PASS': 5,
|
| + 'REBASELINE': 0,
|
| + 'SKIP': 1,
|
| + 'SLOW': 0,
|
| + 'TIMEOUT': 0,
|
| + 'IMAGE+TEXT': 0,
|
| + 'LEAK': 0,
|
| + 'FAIL': 0,
|
| + 'AUDIO': 0,
|
| + 'WONTFIX': 0})
|
|
|
| def test_svn_revision(self):
|
| self.port._options.builder_name = 'dummy builder'
|
| @@ -201,6 +265,12 @@ class SummarizedResultsTest(unittest.TestCase):
|
| initial_results.add(get_result('failures/expected/image.html', test_expectations.TIMEOUT, run_time=1), False, False)
|
| retry_results = test_run_results.TestRunResults(expectations, len(tests))
|
| retry_results.add(get_result('failures/expected/image.html', test_expectations.PASS, run_time=0.1), False, False)
|
| - summary = test_run_results.summarize_results(self.port, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_failing=True)
|
| + summary = test_run_results.summarize_results(
|
| + self.port,
|
| + expectations,
|
| + initial_results,
|
| + retry_results,
|
| + enabled_pixel_tests_in_retry=True,
|
| + only_include_failing=True)
|
| self.assertEquals(summary['num_regressions'], 0)
|
| self.assertEquals(summary['num_passes'], 1)
|
|
|