OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 | 6 |
7 from telemetry.page import page_set | 7 from telemetry.page import page_set |
8 from telemetry.results import base_test_results_unittest | 8 from telemetry.results import base_test_results_unittest |
9 from telemetry.results import page_test_results | 9 from telemetry.results import page_test_results |
10 from telemetry.value import failure | 10 from telemetry.value import failure |
11 from telemetry.value import skip | 11 from telemetry.value import skip |
| 12 from telemetry.value import histogram |
| 13 from telemetry.value import scalar |
12 | 14 |
13 class NonPrintingPageTestResults(page_test_results.PageTestResults): | |
14 def __init__(self): | |
15 super(NonPrintingPageTestResults, self).__init__() | |
16 | |
17 def _PrintPerfResult(self, *args): | |
18 pass | |
19 | 15 |
20 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): | 16 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): |
21 def setUp(self): | 17 def setUp(self): |
22 self.page_set = page_set.PageSet(file_path=os.path.dirname(__file__)) | 18 self.page_set = page_set.PageSet(file_path=os.path.dirname(__file__)) |
23 self.page_set.AddPageWithDefaultRunNavigate("http://www.bar.com/") | 19 self.page_set.AddPageWithDefaultRunNavigate("http://www.bar.com/") |
24 self.page_set.AddPageWithDefaultRunNavigate("http://www.baz.com/") | 20 self.page_set.AddPageWithDefaultRunNavigate("http://www.baz.com/") |
25 self.page_set.AddPageWithDefaultRunNavigate("http://www.foo.com/") | 21 self.page_set.AddPageWithDefaultRunNavigate("http://www.foo.com/") |
26 | 22 |
27 @property | 23 @property |
28 def pages(self): | 24 def pages(self): |
29 return self.page_set.pages | 25 return self.page_set.pages |
30 | 26 |
31 def testFailures(self): | 27 def testFailures(self): |
32 results = NonPrintingPageTestResults() | 28 results = page_test_results.PageTestResults() |
33 results.WillRunPage(self.pages[0]) | 29 results.WillRunPage(self.pages[0]) |
34 results.AddValue( | 30 results.AddValue( |
35 failure.FailureValue(self.pages[0], self.CreateException())) | 31 failure.FailureValue(self.pages[0], self.CreateException())) |
36 results.DidRunPage(self.pages[0]) | 32 results.DidRunPage(self.pages[0]) |
37 | 33 |
38 results.WillRunPage(self.pages[1]) | 34 results.WillRunPage(self.pages[1]) |
39 results.DidRunPage(self.pages[1]) | 35 results.DidRunPage(self.pages[1]) |
40 | 36 |
41 self.assertEqual(set([self.pages[0]]), results.pages_that_failed) | 37 self.assertEqual(set([self.pages[0]]), results.pages_that_failed) |
42 self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded) | 38 self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded) |
43 | 39 |
44 self.assertEqual(2, len(results.all_page_runs)) | 40 self.assertEqual(2, len(results.all_page_runs)) |
45 self.assertTrue(results.all_page_runs[0].failed) | 41 self.assertTrue(results.all_page_runs[0].failed) |
46 self.assertTrue(results.all_page_runs[1].ok) | 42 self.assertTrue(results.all_page_runs[1].ok) |
47 | 43 |
48 def testSkips(self): | 44 def testSkips(self): |
49 results = NonPrintingPageTestResults() | 45 results = page_test_results.PageTestResults() |
50 results.WillRunPage(self.pages[0]) | 46 results.WillRunPage(self.pages[0]) |
51 results.AddValue(skip.SkipValue(self.pages[0], 'testing reason')) | 47 results.AddValue(skip.SkipValue(self.pages[0], 'testing reason')) |
52 results.DidRunPage(self.pages[0]) | 48 results.DidRunPage(self.pages[0]) |
53 | 49 |
54 results.WillRunPage(self.pages[1]) | 50 results.WillRunPage(self.pages[1]) |
55 results.DidRunPage(self.pages[1]) | 51 results.DidRunPage(self.pages[1]) |
56 | 52 |
57 self.assertTrue(results.all_page_runs[0].skipped) | 53 self.assertTrue(results.all_page_runs[0].skipped) |
58 self.assertEqual(self.pages[0], results.all_page_runs[0].page) | 54 self.assertEqual(self.pages[0], results.all_page_runs[0].page) |
59 self.assertEqual(set([self.pages[0], self.pages[1]]), | 55 self.assertEqual(set([self.pages[0], self.pages[1]]), |
60 results.pages_that_succeeded) | 56 results.pages_that_succeeded) |
61 | 57 |
62 self.assertEqual(2, len(results.all_page_runs)) | 58 self.assertEqual(2, len(results.all_page_runs)) |
63 self.assertTrue(results.all_page_runs[0].skipped) | 59 self.assertTrue(results.all_page_runs[0].skipped) |
64 self.assertTrue(results.all_page_runs[1].ok) | 60 self.assertTrue(results.all_page_runs[1].ok) |
| 61 |
| 62 def test_basic(self): |
| 63 results = page_test_results.PageTestResults() |
| 64 results.WillRunPage(self.pages[0]) |
| 65 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) |
| 66 results.DidRunPage(self.pages[0]) |
| 67 |
| 68 results.WillRunPage(self.pages[1]) |
| 69 results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 3)) |
| 70 results.DidRunPage(self.pages[1]) |
| 71 |
| 72 results.PrintSummary() |
| 73 |
| 74 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') |
| 75 self.assertEquals(1, len(values)) |
| 76 v = values[0] |
| 77 self.assertEquals(v.name, 'a') |
| 78 self.assertEquals(v.page, self.pages[0]) |
| 79 |
| 80 values = results.FindAllPageSpecificValuesNamed('a') |
| 81 assert len(values) == 2 |
| 82 |
| 83 def test_url_is_invalid_value(self): |
| 84 results = page_test_results.PageTestResults() |
| 85 results.WillRunPage(self.pages[0]) |
| 86 self.assertRaises( |
| 87 AssertionError, |
| 88 lambda: results.AddValue(scalar.ScalarValue( |
| 89 self.pages[0], 'url', 'string', 'foo'))) |
| 90 |
| 91 def test_add_summary_value_with_page_specified(self): |
| 92 results = page_test_results.PageTestResults() |
| 93 results.WillRunPage(self.pages[0]) |
| 94 self.assertRaises( |
| 95 AssertionError, |
| 96 lambda: results.AddSummaryValue(scalar.ScalarValue(self.pages[0], |
| 97 'a', 'units', 3))) |
| 98 |
| 99 def test_unit_change(self): |
| 100 results = page_test_results.PageTestResults() |
| 101 results.WillRunPage(self.pages[0]) |
| 102 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) |
| 103 results.DidRunPage(self.pages[0]) |
| 104 |
| 105 results.WillRunPage(self.pages[1]) |
| 106 self.assertRaises( |
| 107 AssertionError, |
| 108 lambda: results.AddValue(scalar.ScalarValue( |
| 109 self.pages[1], 'a', 'foobgrobbers', 3))) |
| 110 |
| 111 def test_type_change(self): |
| 112 results = page_test_results.PageTestResults() |
| 113 results.WillRunPage(self.pages[0]) |
| 114 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) |
| 115 results.DidRunPage(self.pages[0]) |
| 116 |
| 117 results.WillRunPage(self.pages[1]) |
| 118 self.assertRaises( |
| 119 AssertionError, |
| 120 lambda: results.AddValue(histogram.HistogramValue( |
| 121 self.pages[1], 'a', 'seconds', |
| 122 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}'))) |
| 123 |
| 124 def test_get_pages_that_succeeded_all_pages_fail(self): |
| 125 results = page_test_results.PageTestResults() |
| 126 results.WillRunPage(self.pages[0]) |
| 127 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) |
| 128 results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) |
| 129 results.DidRunPage(self.pages[0]) |
| 130 |
| 131 results.WillRunPage(self.pages[1]) |
| 132 results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 7)) |
| 133 results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) |
| 134 results.DidRunPage(self.pages[1]) |
| 135 |
| 136 results.PrintSummary() |
| 137 self.assertEquals(0, len(results.pages_that_succeeded)) |
| 138 |
| 139 def test_get_successful_page_values_merged_no_failures(self): |
| 140 results = page_test_results.PageTestResults() |
| 141 results.WillRunPage(self.pages[0]) |
| 142 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) |
| 143 self.assertEquals(1, len(results.all_page_specific_values)) |
| 144 results.DidRunPage(self.pages[0]) |
| 145 |
| 146 def test_get_all_values_for_successful_pages(self): |
| 147 results = page_test_results.PageTestResults() |
| 148 results.WillRunPage(self.pages[0]) |
| 149 value1 = scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3) |
| 150 results.AddValue(value1) |
| 151 results.DidRunPage(self.pages[0]) |
| 152 |
| 153 results.WillRunPage(self.pages[1]) |
| 154 value2 = scalar.ScalarValue(self.pages[1], 'a', 'seconds', 3) |
| 155 results.AddValue(value2) |
| 156 results.DidRunPage(self.pages[1]) |
| 157 |
| 158 results.WillRunPage(self.pages[2]) |
| 159 value3 = scalar.ScalarValue(self.pages[2], 'a', 'seconds', 3) |
| 160 results.AddValue(value3) |
| 161 results.DidRunPage(self.pages[2]) |
| 162 |
| 163 self.assertEquals( |
| 164 [value1, value2, value3], results.all_page_specific_values) |
| 165 |
| 166 def test_get_all_values_for_successful_pages_one_page_fails(self): |
| 167 results = page_test_results.PageTestResults() |
| 168 results.WillRunPage(self.pages[0]) |
| 169 value1 = scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3) |
| 170 results.AddValue(value1) |
| 171 results.DidRunPage(self.pages[0]) |
| 172 |
| 173 results.WillRunPage(self.pages[1]) |
| 174 value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') |
| 175 results.AddValue(value2) |
| 176 results.DidRunPage(self.pages[1]) |
| 177 |
| 178 results.WillRunPage(self.pages[2]) |
| 179 value3 = scalar.ScalarValue(self.pages[2], 'a', 'seconds', 3) |
| 180 results.AddValue(value3) |
| 181 results.DidRunPage(self.pages[2]) |
| 182 |
| 183 self.assertEquals( |
| 184 [value1, value2, value3], results.all_page_specific_values) |
OLD | NEW |