| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import os | 5 import os |
| 6 | 6 |
| 7 from telemetry import page as page_module | 7 from telemetry import page as page_module |
| 8 from telemetry.page import page_set | 8 from telemetry.page import page_set |
| 9 from telemetry.results import base_test_results_unittest | 9 from telemetry.results import base_test_results_unittest |
| 10 from telemetry.results import page_test_results | 10 from telemetry.results import page_test_results |
| 11 from telemetry.timeline import trace_data | 11 from telemetry.timeline import trace_data |
| 12 from telemetry.value import failure | 12 from telemetry.value import failure |
| 13 from telemetry.value import histogram | 13 from telemetry.value import histogram |
| 14 from telemetry.value import improvement_direction |
| 14 from telemetry.value import scalar | 15 from telemetry.value import scalar |
| 15 from telemetry.value import skip | 16 from telemetry.value import skip |
| 16 from telemetry.value import trace | 17 from telemetry.value import trace |
| 17 | 18 |
| 18 | 19 |
| 19 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): | 20 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): |
| 20 def setUp(self): | 21 def setUp(self): |
| 21 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) | 22 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) |
| 22 ps.AddUserStory(page_module.Page("http://www.bar.com/", ps, ps.base_dir)) | 23 ps.AddUserStory(page_module.Page("http://www.bar.com/", ps, ps.base_dir)) |
| 23 ps.AddUserStory(page_module.Page("http://www.baz.com/", ps, ps.base_dir)) | 24 ps.AddUserStory(page_module.Page("http://www.baz.com/", ps, ps.base_dir)) |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 59 self.assertEqual(set([self.pages[0], self.pages[1]]), | 60 self.assertEqual(set([self.pages[0], self.pages[1]]), |
| 60 results.pages_that_succeeded) | 61 results.pages_that_succeeded) |
| 61 | 62 |
| 62 self.assertEqual(2, len(results.all_page_runs)) | 63 self.assertEqual(2, len(results.all_page_runs)) |
| 63 self.assertTrue(results.all_page_runs[0].skipped) | 64 self.assertTrue(results.all_page_runs[0].skipped) |
| 64 self.assertTrue(results.all_page_runs[1].ok) | 65 self.assertTrue(results.all_page_runs[1].ok) |
| 65 | 66 |
| 66 def test_basic(self): | 67 def test_basic(self): |
| 67 results = page_test_results.PageTestResults() | 68 results = page_test_results.PageTestResults() |
| 68 results.WillRunPage(self.pages[0]) | 69 results.WillRunPage(self.pages[0]) |
| 69 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 70 results.AddValue(scalar.ScalarValue( |
| 71 self.pages[0], 'a', 'seconds', 3, |
| 72 improvement_direction=improvement_direction.DOWN)) |
| 70 results.DidRunPage(self.pages[0]) | 73 results.DidRunPage(self.pages[0]) |
| 71 | 74 |
| 72 results.WillRunPage(self.pages[1]) | 75 results.WillRunPage(self.pages[1]) |
| 73 results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 3)) | 76 results.AddValue(scalar.ScalarValue( |
| 77 self.pages[1], 'a', 'seconds', 3, |
| 78 improvement_direction=improvement_direction.DOWN)) |
| 74 results.DidRunPage(self.pages[1]) | 79 results.DidRunPage(self.pages[1]) |
| 75 | 80 |
| 76 results.PrintSummary() | 81 results.PrintSummary() |
| 77 | 82 |
| 78 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') | 83 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') |
| 79 self.assertEquals(1, len(values)) | 84 self.assertEquals(1, len(values)) |
| 80 v = values[0] | 85 v = values[0] |
| 81 self.assertEquals(v.name, 'a') | 86 self.assertEquals(v.name, 'a') |
| 82 self.assertEquals(v.page, self.pages[0]) | 87 self.assertEquals(v.page, self.pages[0]) |
| 83 | 88 |
| 84 values = results.FindAllPageSpecificValuesNamed('a') | 89 values = results.FindAllPageSpecificValuesNamed('a') |
| 85 assert len(values) == 2 | 90 assert len(values) == 2 |
| 86 | 91 |
| 87 def test_url_is_invalid_value(self): | 92 def test_url_is_invalid_value(self): |
| 88 results = page_test_results.PageTestResults() | 93 results = page_test_results.PageTestResults() |
| 89 results.WillRunPage(self.pages[0]) | 94 results.WillRunPage(self.pages[0]) |
| 90 self.assertRaises( | 95 self.assertRaises( |
| 91 AssertionError, | 96 AssertionError, |
| 92 lambda: results.AddValue(scalar.ScalarValue( | 97 lambda: results.AddValue(scalar.ScalarValue( |
| 93 self.pages[0], 'url', 'string', 'foo'))) | 98 self.pages[0], 'url', 'string', 'foo', |
| 99 improvement_direction=improvement_direction.DOWN))) |
| 94 | 100 |
| 95 def test_add_summary_value_with_page_specified(self): | 101 def test_add_summary_value_with_page_specified(self): |
| 96 results = page_test_results.PageTestResults() | 102 results = page_test_results.PageTestResults() |
| 97 results.WillRunPage(self.pages[0]) | 103 results.WillRunPage(self.pages[0]) |
| 98 self.assertRaises( | 104 self.assertRaises( |
| 99 AssertionError, | 105 AssertionError, |
| 100 lambda: results.AddSummaryValue(scalar.ScalarValue(self.pages[0], | 106 lambda: results.AddSummaryValue(scalar.ScalarValue( |
| 101 'a', 'units', 3))) | 107 self.pages[0], 'a', 'units', 3, |
| 108 improvement_direction=improvement_direction.DOWN))) |
| 102 | 109 |
| 103 def test_unit_change(self): | 110 def test_unit_change(self): |
| 104 results = page_test_results.PageTestResults() | 111 results = page_test_results.PageTestResults() |
| 105 results.WillRunPage(self.pages[0]) | 112 results.WillRunPage(self.pages[0]) |
| 106 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 113 results.AddValue(scalar.ScalarValue( |
| 114 self.pages[0], 'a', 'seconds', 3, |
| 115 improvement_direction=improvement_direction.DOWN)) |
| 107 results.DidRunPage(self.pages[0]) | 116 results.DidRunPage(self.pages[0]) |
| 108 | 117 |
| 109 results.WillRunPage(self.pages[1]) | 118 results.WillRunPage(self.pages[1]) |
| 110 self.assertRaises( | 119 self.assertRaises( |
| 111 AssertionError, | 120 AssertionError, |
| 112 lambda: results.AddValue(scalar.ScalarValue( | 121 lambda: results.AddValue(scalar.ScalarValue( |
| 113 self.pages[1], 'a', 'foobgrobbers', 3))) | 122 self.pages[1], 'a', 'foobgrobbers', 3, |
| 123 improvement_direction=improvement_direction.DOWN))) |
| 114 | 124 |
| 115 def test_type_change(self): | 125 def test_type_change(self): |
| 116 results = page_test_results.PageTestResults() | 126 results = page_test_results.PageTestResults() |
| 117 results.WillRunPage(self.pages[0]) | 127 results.WillRunPage(self.pages[0]) |
| 118 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 128 results.AddValue(scalar.ScalarValue( |
| 129 self.pages[0], 'a', 'seconds', 3, |
| 130 improvement_direction=improvement_direction.DOWN)) |
| 119 results.DidRunPage(self.pages[0]) | 131 results.DidRunPage(self.pages[0]) |
| 120 | 132 |
| 121 results.WillRunPage(self.pages[1]) | 133 results.WillRunPage(self.pages[1]) |
| 122 self.assertRaises( | 134 self.assertRaises( |
| 123 AssertionError, | 135 AssertionError, |
| 124 lambda: results.AddValue(histogram.HistogramValue( | 136 lambda: results.AddValue(histogram.HistogramValue( |
| 125 self.pages[1], 'a', 'seconds', | 137 self.pages[1], 'a', 'seconds', |
| 126 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}'))) | 138 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', |
| 139 improvement_direction=improvement_direction.DOWN))) |
| 127 | 140 |
| 128 def test_get_pages_that_succeeded_all_pages_fail(self): | 141 def test_get_pages_that_succeeded_all_pages_fail(self): |
| 129 results = page_test_results.PageTestResults() | 142 results = page_test_results.PageTestResults() |
| 130 results.WillRunPage(self.pages[0]) | 143 results.WillRunPage(self.pages[0]) |
| 131 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 144 results.AddValue(scalar.ScalarValue( |
| 145 self.pages[0], 'a', 'seconds', 3, |
| 146 improvement_direction=improvement_direction.DOWN)) |
| 132 results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) | 147 results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) |
| 133 results.DidRunPage(self.pages[0]) | 148 results.DidRunPage(self.pages[0]) |
| 134 | 149 |
| 135 results.WillRunPage(self.pages[1]) | 150 results.WillRunPage(self.pages[1]) |
| 136 results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 7)) | 151 results.AddValue(scalar.ScalarValue( |
| 152 self.pages[1], 'a', 'seconds', 7, |
| 153 improvement_direction=improvement_direction.DOWN)) |
| 137 results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) | 154 results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) |
| 138 results.DidRunPage(self.pages[1]) | 155 results.DidRunPage(self.pages[1]) |
| 139 | 156 |
| 140 results.PrintSummary() | 157 results.PrintSummary() |
| 141 self.assertEquals(0, len(results.pages_that_succeeded)) | 158 self.assertEquals(0, len(results.pages_that_succeeded)) |
| 142 | 159 |
| 143 def test_get_successful_page_values_merged_no_failures(self): | 160 def test_get_successful_page_values_merged_no_failures(self): |
| 144 results = page_test_results.PageTestResults() | 161 results = page_test_results.PageTestResults() |
| 145 results.WillRunPage(self.pages[0]) | 162 results.WillRunPage(self.pages[0]) |
| 146 results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) | 163 results.AddValue(scalar.ScalarValue( |
| 164 self.pages[0], 'a', 'seconds', 3, |
| 165 improvement_direction=improvement_direction.DOWN)) |
| 147 self.assertEquals(1, len(results.all_page_specific_values)) | 166 self.assertEquals(1, len(results.all_page_specific_values)) |
| 148 results.DidRunPage(self.pages[0]) | 167 results.DidRunPage(self.pages[0]) |
| 149 | 168 |
| 150 def test_get_all_values_for_successful_pages(self): | 169 def test_get_all_values_for_successful_pages(self): |
| 151 results = page_test_results.PageTestResults() | 170 results = page_test_results.PageTestResults() |
| 152 results.WillRunPage(self.pages[0]) | 171 results.WillRunPage(self.pages[0]) |
| 153 value1 = scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3) | 172 value1 = scalar.ScalarValue( |
| 173 self.pages[0], 'a', 'seconds', 3, |
| 174 improvement_direction=improvement_direction.DOWN) |
| 154 results.AddValue(value1) | 175 results.AddValue(value1) |
| 155 results.DidRunPage(self.pages[0]) | 176 results.DidRunPage(self.pages[0]) |
| 156 | 177 |
| 157 results.WillRunPage(self.pages[1]) | 178 results.WillRunPage(self.pages[1]) |
| 158 value2 = scalar.ScalarValue(self.pages[1], 'a', 'seconds', 3) | 179 value2 = scalar.ScalarValue( |
| 180 self.pages[1], 'a', 'seconds', 3, |
| 181 improvement_direction=improvement_direction.DOWN) |
| 159 results.AddValue(value2) | 182 results.AddValue(value2) |
| 160 results.DidRunPage(self.pages[1]) | 183 results.DidRunPage(self.pages[1]) |
| 161 | 184 |
| 162 results.WillRunPage(self.pages[2]) | 185 results.WillRunPage(self.pages[2]) |
| 163 value3 = scalar.ScalarValue(self.pages[2], 'a', 'seconds', 3) | 186 value3 = scalar.ScalarValue( |
| 187 self.pages[2], 'a', 'seconds', 3, |
| 188 improvement_direction=improvement_direction.DOWN) |
| 164 results.AddValue(value3) | 189 results.AddValue(value3) |
| 165 results.DidRunPage(self.pages[2]) | 190 results.DidRunPage(self.pages[2]) |
| 166 | 191 |
| 167 self.assertEquals( | 192 self.assertEquals( |
| 168 [value1, value2, value3], results.all_page_specific_values) | 193 [value1, value2, value3], results.all_page_specific_values) |
| 169 | 194 |
| 170 def test_get_all_values_for_successful_pages_one_page_fails(self): | 195 def test_get_all_values_for_successful_pages_one_page_fails(self): |
| 171 results = page_test_results.PageTestResults() | 196 results = page_test_results.PageTestResults() |
| 172 results.WillRunPage(self.pages[0]) | 197 results.WillRunPage(self.pages[0]) |
| 173 value1 = scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3) | 198 value1 = scalar.ScalarValue( |
| 199 self.pages[0], 'a', 'seconds', 3, |
| 200 improvement_direction=improvement_direction.DOWN) |
| 174 results.AddValue(value1) | 201 results.AddValue(value1) |
| 175 results.DidRunPage(self.pages[0]) | 202 results.DidRunPage(self.pages[0]) |
| 176 | 203 |
| 177 results.WillRunPage(self.pages[1]) | 204 results.WillRunPage(self.pages[1]) |
| 178 value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') | 205 value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') |
| 179 results.AddValue(value2) | 206 results.AddValue(value2) |
| 180 results.DidRunPage(self.pages[1]) | 207 results.DidRunPage(self.pages[1]) |
| 181 | 208 |
| 182 results.WillRunPage(self.pages[2]) | 209 results.WillRunPage(self.pages[2]) |
| 183 value3 = scalar.ScalarValue(self.pages[2], 'a', 'seconds', 3) | 210 value3 = scalar.ScalarValue( |
| 211 self.pages[2], 'a', 'seconds', 3, |
| 212 improvement_direction=improvement_direction.DOWN) |
| 184 results.AddValue(value3) | 213 results.AddValue(value3) |
| 185 results.DidRunPage(self.pages[2]) | 214 results.DidRunPage(self.pages[2]) |
| 186 | 215 |
| 187 self.assertEquals( | 216 self.assertEquals( |
| 188 [value1, value2, value3], results.all_page_specific_values) | 217 [value1, value2, value3], results.all_page_specific_values) |
| 189 | 218 |
| 190 def testTraceValue(self): | 219 def testTraceValue(self): |
| 191 results = page_test_results.PageTestResults() | 220 results = page_test_results.PageTestResults() |
| 192 results.WillRunPage(self.pages[0]) | 221 results.WillRunPage(self.pages[0]) |
| 193 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 1}))) | 222 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 1}))) |
| 194 results.DidRunPage(self.pages[0]) | 223 results.DidRunPage(self.pages[0]) |
| 195 | 224 |
| 196 results.WillRunPage(self.pages[1]) | 225 results.WillRunPage(self.pages[1]) |
| 197 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 2}))) | 226 results.AddValue(trace.TraceValue(None, trace_data.TraceData({'test' : 2}))) |
| 198 results.DidRunPage(self.pages[1]) | 227 results.DidRunPage(self.pages[1]) |
| 199 | 228 |
| 200 results.PrintSummary() | 229 results.PrintSummary() |
| 201 | 230 |
| 202 values = results.FindAllTraceValues() | 231 values = results.FindAllTraceValues() |
| 203 self.assertEquals(2, len(values)) | 232 self.assertEquals(2, len(values)) |
| OLD | NEW |