OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 | 6 |
7 from telemetry import page as page_module | 7 from telemetry import page as page_module |
8 from telemetry.page import page_set | 8 from telemetry.page import page_set |
9 from telemetry.results import base_test_results_unittest | 9 from telemetry.results import base_test_results_unittest |
10 from telemetry.results import page_test_results | 10 from telemetry.results import page_test_results |
11 from telemetry.timeline import trace_data | 11 from telemetry.timeline import trace_data |
12 from telemetry.value import failure | 12 from telemetry.value import failure |
13 from telemetry.value import histogram | 13 from telemetry.value import histogram |
| 14 from telemetry.value import improvement_direction |
14 from telemetry.value import scalar | 15 from telemetry.value import scalar |
15 from telemetry.value import skip | 16 from telemetry.value import skip |
16 from telemetry.value import trace | 17 from telemetry.value import trace |
17 | 18 |
18 | 19 |
19 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): | 20 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): |
20 def setUp(self): | 21 def setUp(self): |
21 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) | 22 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) |
22 ps.AddUserStory(page_module.Page("http://www.bar.com/", ps, ps.base_dir)) | 23 ps.AddUserStory(page_module.Page("http://www.bar.com/", ps, ps.base_dir)) |
23 ps.AddUserStory(page_module.Page("http://www.baz.com/", ps, ps.base_dir)) | 24 ps.AddUserStory(page_module.Page("http://www.baz.com/", ps, ps.base_dir)) |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
59 self.assertEqual(set([self.pages[0], self.pages[1]]), | 60 self.assertEqual(set([self.pages[0], self.pages[1]]), |
60 results.pages_that_succeeded) | 61 results.pages_that_succeeded) |
61 | 62 |
62 self.assertEqual(2, len(results.all_page_runs)) | 63 self.assertEqual(2, len(results.all_page_runs)) |
63 self.assertTrue(results.all_page_runs[0].skipped) | 64 self.assertTrue(results.all_page_runs[0].skipped) |
64 self.assertTrue(results.all_page_runs[1].ok) | 65 self.assertTrue(results.all_page_runs[1].ok) |
65 | 66 |
66 def test_basic(self): | 67 def test_basic(self): |
67 results = page_test_results.PageTestResults() | 68 results = page_test_results.PageTestResults() |
68 results.WillRunPage(self.pages[0]) | 69 results.WillRunPage(self.pages[0]) |
69 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 70 results.AddValue(scalar.ScalarValue( |
| 71 self.pages[0], 'a', 'seconds', 3, |
| 72 improvement_direction=improvement_direction.DOWN)) |
70 results.DidRunPage(self.pages[0]) | 73 results.DidRunPage(self.pages[0]) |
71 | 74 |
72 results.WillRunPage(self.pages[1]) | 75 results.WillRunPage(self.pages[1]) |
73 results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 3)) | 76 results.AddValue(scalar.ScalarValue( |
| 77 self.pages[1], 'a', 'seconds', 3, |
| 78 improvement_direction=improvement_direction.DOWN)) |
74 results.DidRunPage(self.pages[1]) | 79 results.DidRunPage(self.pages[1]) |
75 | 80 |
76 results.PrintSummary() | 81 results.PrintSummary() |
77 | 82 |
78 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') | 83 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') |
79 self.assertEquals(1, len(values)) | 84 self.assertEquals(1, len(values)) |
80 v = values[0] | 85 v = values[0] |
81 self.assertEquals(v.name, 'a') | 86 self.assertEquals(v.name, 'a') |
82 self.assertEquals(v.page, self.pages[0]) | 87 self.assertEquals(v.page, self.pages[0]) |
83 | 88 |
84 values = results.FindAllPageSpecificValuesNamed('a') | 89 values = results.FindAllPageSpecificValuesNamed('a') |
85 assert len(values) == 2 | 90 assert len(values) == 2 |
86 | 91 |
87 def test_url_is_invalid_value(self): | 92 def test_url_is_invalid_value(self): |
88 results = page_test_results.PageTestResults() | 93 results = page_test_results.PageTestResults() |
89 results.WillRunPage(self.pages[0]) | 94 results.WillRunPage(self.pages[0]) |
90 self.assertRaises( | 95 self.assertRaises( |
91 AssertionError, | 96 AssertionError, |
92 lambda: results.AddValue(scalar.ScalarValue( | 97 lambda: results.AddValue(scalar.ScalarValue( |
93 self.pages[0], 'url', 'string', 'foo'))) | 98 self.pages[0], 'url', 'string', 'foo', |
| 99 improvement_direction=improvement_direction.DOWN))) |
94 | 100 |
95 def test_add_summary_value_with_page_specified(self): | 101 def test_add_summary_value_with_page_specified(self): |
96 results = page_test_results.PageTestResults() | 102 results = page_test_results.PageTestResults() |
97 results.WillRunPage(self.pages[0]) | 103 results.WillRunPage(self.pages[0]) |
98 self.assertRaises( | 104 self.assertRaises( |
99 AssertionError, | 105 AssertionError, |
100 lambda: results.AddSummaryValue(scalar.ScalarValue(self.pages[0], | 106 lambda: results.AddSummaryValue(scalar.ScalarValue( |
101 'a', 'units', 3))) | 107 self.pages[0], 'a', 'units', 3, |
| 108 improvement_direction=improvement_direction.DOWN))) |
102 | 109 |
103 def test_unit_change(self): | 110 def test_unit_change(self): |
104 results = page_test_results.PageTestResults() | 111 results = page_test_results.PageTestResults() |
105 results.WillRunPage(self.pages[0]) | 112 results.WillRunPage(self.pages[0]) |
106 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 113 results.AddValue(scalar.ScalarValue( |
| 114 self.pages[0], 'a', 'seconds', 3, |
| 115 improvement_direction=improvement_direction.DOWN)) |
107 results.DidRunPage(self.pages[0]) | 116 results.DidRunPage(self.pages[0]) |
108 | 117 |
109 results.WillRunPage(self.pages[1]) | 118 results.WillRunPage(self.pages[1]) |
110 self.assertRaises( | 119 self.assertRaises( |
111 AssertionError, | 120 AssertionError, |
112 lambda: results.AddValue(scalar.ScalarValue( | 121 lambda: results.AddValue(scalar.ScalarValue( |
113 self.pages[1], 'a', 'foobgrobbers', 3))) | 122 self.pages[1], 'a', 'foobgrobbers', 3, |
| 123 improvement_direction=improvement_direction.DOWN))) |
114 | 124 |
115 def test_type_change(self): | 125 def test_type_change(self): |
116 results = page_test_results.PageTestResults() | 126 results = page_test_results.PageTestResults() |
117 results.WillRunPage(self.pages[0]) | 127 results.WillRunPage(self.pages[0]) |
118 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 128 results.AddValue(scalar.ScalarValue( |
| 129 self.pages[0], 'a', 'seconds', 3, |
| 130 improvement_direction=improvement_direction.DOWN)) |
119 results.DidRunPage(self.pages[0]) | 131 results.DidRunPage(self.pages[0]) |
120 | 132 |
121 results.WillRunPage(self.pages[1]) | 133 results.WillRunPage(self.pages[1]) |
122 self.assertRaises( | 134 self.assertRaises( |
123 AssertionError, | 135 AssertionError, |
124 lambda: results.AddValue(histogram.HistogramValue( | 136 lambda: results.AddValue(histogram.HistogramValue( |
125 self.pages[1], 'a', 'seconds', | 137 self.pages[1], 'a', 'seconds', |
126 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}'))) | 138 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', |
| 139 improvement_direction=improvement_direction.DOWN))) |
127 | 140 |
128 def test_get_pages_that_succeeded_all_pages_fail(self): | 141 def test_get_pages_that_succeeded_all_pages_fail(self): |
129 results = page_test_results.PageTestResults() | 142 results = page_test_results.PageTestResults() |
130 results.WillRunPage(self.pages[0]) | 143 results.WillRunPage(self.pages[0]) |
131 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 144 results.AddValue(scalar.ScalarValue( |
| 145 self.pages[0], 'a', 'seconds', 3, |
| 146 improvement_direction=improvement_direction.DOWN)) |
132 results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) | 147 results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) |
133 results.DidRunPage(self.pages[0]) | 148 results.DidRunPage(self.pages[0]) |
134 | 149 |
135 results.WillRunPage(self.pages[1]) | 150 results.WillRunPage(self.pages[1]) |
136 results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 7)) | 151 results.AddValue(scalar.ScalarValue( |
| 152 self.pages[1], 'a', 'seconds', 7, |
| 153 improvement_direction=improvement_direction.DOWN)) |
137 results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) | 154 results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) |
138 results.DidRunPage(self.pages[1]) | 155 results.DidRunPage(self.pages[1]) |
139 | 156 |
140 results.PrintSummary() | 157 results.PrintSummary() |
141 self.assertEquals(0, len(results.pages_that_succeeded)) | 158 self.assertEquals(0, len(results.pages_that_succeeded)) |
142 | 159 |
143 def test_get_successful_page_values_merged_no_failures(self): | 160 def test_get_successful_page_values_merged_no_failures(self): |
144 results = page_test_results.PageTestResults() | 161 results = page_test_results.PageTestResults() |
145 results.WillRunPage(self.pages[0]) | 162 results.WillRunPage(self.pages[0]) |
146 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 163 results.AddValue(scalar.ScalarValue( |
| 164 self.pages[0], 'a', 'seconds', 3, |
| 165 improvement_direction=improvement_direction.DOWN)) |
147 self.assertEquals(1, len(results.all_page_specific_values)) | 166 self.assertEquals(1, len(results.all_page_specific_values)) |
148 results.DidRunPage(self.pages[0]) | 167 results.DidRunPage(self.pages[0]) |
149 | 168 |
150 def test_get_all_values_for_successful_pages(self): | 169 def test_get_all_values_for_successful_pages(self): |
151 results = page_test_results.PageTestResults() | 170 results = page_test_results.PageTestResults() |
152 results.WillRunPage(self.pages[0]) | 171 results.WillRunPage(self.pages[0]) |
153 value1 = scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3) | 172 value1 = scalar.ScalarValue( |
| 173 self.pages[0], 'a', 'seconds', 3, |
| 174 improvement_direction=improvement_direction.DOWN) |
154 results.AddValue(value1) | 175 results.AddValue(value1) |
155 results.DidRunPage(self.pages[0]) | 176 results.DidRunPage(self.pages[0]) |
156 | 177 |
157 results.WillRunPage(self.pages[1]) | 178 results.WillRunPage(self.pages[1]) |
158 value2 = scalar.ScalarValue(self.pages[1], 'a', 'seconds', 3) | 179 value2 = scalar.ScalarValue( |
| 180 self.pages[1], 'a', 'seconds', 3, |
| 181 improvement_direction=improvement_direction.DOWN) |
159 results.AddValue(value2) | 182 results.AddValue(value2) |
160 results.DidRunPage(self.pages[1]) | 183 results.DidRunPage(self.pages[1]) |
161 | 184 |
162 results.WillRunPage(self.pages[2]) | 185 results.WillRunPage(self.pages[2]) |
163 value3 = scalar.ScalarValue(self.pages[2], 'a', 'seconds', 3) | 186 value3 = scalar.ScalarValue( |
| 187 self.pages[2], 'a', 'seconds', 3, |
| 188 improvement_direction=improvement_direction.DOWN) |
164 results.AddValue(value3) | 189 results.AddValue(value3) |
165 results.DidRunPage(self.pages[2]) | 190 results.DidRunPage(self.pages[2]) |
166 | 191 |
167 self.assertEquals( | 192 self.assertEquals( |
168 [value1, value2, value3], results.all_page_specific_values) | 193 [value1, value2, value3], results.all_page_specific_values) |
169 | 194 |
170 def test_get_all_values_for_successful_pages_one_page_fails(self): | 195 def test_get_all_values_for_successful_pages_one_page_fails(self): |
171 results = page_test_results.PageTestResults() | 196 results = page_test_results.PageTestResults() |
172 results.WillRunPage(self.pages[0]) | 197 results.WillRunPage(self.pages[0]) |
173 value1 = scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3) | 198 value1 = scalar.ScalarValue( |
| 199 self.pages[0], 'a', 'seconds', 3, |
| 200 improvement_direction=improvement_direction.DOWN) |
174 results.AddValue(value1) | 201 results.AddValue(value1) |
175 results.DidRunPage(self.pages[0]) | 202 results.DidRunPage(self.pages[0]) |
176 | 203 |
177 results.WillRunPage(self.pages[1]) | 204 results.WillRunPage(self.pages[1]) |
178 value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') | 205 value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') |
179 results.AddValue(value2) | 206 results.AddValue(value2) |
180 results.DidRunPage(self.pages[1]) | 207 results.DidRunPage(self.pages[1]) |
181 | 208 |
182 results.WillRunPage(self.pages[2]) | 209 results.WillRunPage(self.pages[2]) |
183 value3 = scalar.ScalarValue(self.pages[2], 'a', 'seconds', 3) | 210 value3 = scalar.ScalarValue( |
| 211 self.pages[2], 'a', 'seconds', 3, |
| 212 improvement_direction=improvement_direction.DOWN) |
184 results.AddValue(value3) | 213 results.AddValue(value3) |
185 results.DidRunPage(self.pages[2]) | 214 results.DidRunPage(self.pages[2]) |
186 | 215 |
187 self.assertEquals( | 216 self.assertEquals( |
188 [value1, value2, value3], results.all_page_specific_values) | 217 [value1, value2, value3], results.all_page_specific_values) |
189 | 218 |
190 def testTraceValue(self): | 219 def testTraceValue(self): |
191 results = page_test_results.PageTestResults() | 220 results = page_test_results.PageTestResults() |
192 results.WillRunPage(self.pages[0]) | 221 results.WillRunPage(self.pages[0]) |
193 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 1}))) | 222 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 1}))) |
194 results.DidRunPage(self.pages[0]) | 223 results.DidRunPage(self.pages[0]) |
195 | 224 |
196 results.WillRunPage(self.pages[1]) | 225 results.WillRunPage(self.pages[1]) |
197 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 2}))) | 226 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 2}))) |
198 results.DidRunPage(self.pages[1]) | 227 results.DidRunPage(self.pages[1]) |
199 | 228 |
200 results.PrintSummary() | 229 results.PrintSummary() |
201 | 230 |
202 values = results.FindAllTraceValues() | 231 values = results.FindAllTraceValues() |
203 self.assertEquals(2, len(values)) | 232 self.assertEquals(2, len(values)) |
OLD | NEW |