OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 | 6 |
7 from telemetry import page as page_module | 7 from telemetry import page as page_module |
8 from telemetry.page import page_set | 8 from telemetry.page import page_set |
9 from telemetry.results import base_test_results_unittest | 9 from telemetry.results import base_test_results_unittest |
10 from telemetry.results import buildbot_output_formatter | 10 from telemetry.results import buildbot_output_formatter |
11 from telemetry.results import page_test_results | 11 from telemetry.results import page_test_results |
12 from telemetry.value import failure | 12 from telemetry.value import failure |
13 from telemetry.value import histogram | 13 from telemetry.value import histogram |
| 14 from telemetry.value import improvement_direction |
14 from telemetry.value import list_of_scalar_values | 15 from telemetry.value import list_of_scalar_values |
15 from telemetry.value import scalar | 16 from telemetry.value import scalar |
16 | 17 |
17 | 18 |
18 def _MakePageSet(): | 19 def _MakePageSet(): |
19 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) | 20 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) |
20 ps.AddUserStory(page_module.Page('http://www.foo.com/', ps, ps.base_dir)) | 21 ps.AddUserStory(page_module.Page('http://www.foo.com/', ps, ps.base_dir)) |
21 ps.AddUserStory(page_module.Page('http://www.bar.com/', ps, ps.base_dir)) | 22 ps.AddUserStory(page_module.Page('http://www.bar.com/', ps, ps.base_dir)) |
22 ps.AddUserStory(page_module.Page('http://www.baz.com/', ps, ps.base_dir)) | 23 ps.AddUserStory(page_module.Page('http://www.baz.com/', ps, ps.base_dir)) |
23 return ps | 24 return ps |
24 | 25 |
25 class BuildbotOutputFormatterTest( | 26 class BuildbotOutputFormatterTest( |
26 base_test_results_unittest.BaseTestResultsUnittest): | 27 base_test_results_unittest.BaseTestResultsUnittest): |
27 def setUp(self): | 28 def setUp(self): |
28 self._test_output_stream = base_test_results_unittest.TestOutputStream() | 29 self._test_output_stream = base_test_results_unittest.TestOutputStream() |
29 | 30 |
30 def test_basic_summary(self): | 31 def test_basic_summary(self): |
31 test_page_set = _MakePageSet() | 32 test_page_set = _MakePageSet() |
32 | 33 |
33 measurement_results = page_test_results.PageTestResults() | 34 measurement_results = page_test_results.PageTestResults() |
34 measurement_results.WillRunPage(test_page_set.pages[0]) | 35 measurement_results.WillRunPage(test_page_set.pages[0]) |
35 measurement_results.AddValue(scalar.ScalarValue( | 36 measurement_results.AddValue(scalar.ScalarValue( |
36 test_page_set.pages[0], 'a', 'seconds', 3)) | 37 test_page_set.pages[0], 'a', 'seconds', 3, |
| 38 improvement_direction=improvement_direction.DOWN)) |
37 measurement_results.DidRunPage(test_page_set.pages[0]) | 39 measurement_results.DidRunPage(test_page_set.pages[0]) |
38 | 40 |
39 measurement_results.WillRunPage(test_page_set.pages[1]) | 41 measurement_results.WillRunPage(test_page_set.pages[1]) |
40 measurement_results.AddValue(scalar.ScalarValue( | 42 measurement_results.AddValue(scalar.ScalarValue( |
41 test_page_set.pages[1], 'a', 'seconds', 7)) | 43 test_page_set.pages[1], 'a', 'seconds', 7, |
| 44 improvement_direction=improvement_direction.DOWN)) |
42 measurement_results.DidRunPage(test_page_set.pages[1]) | 45 measurement_results.DidRunPage(test_page_set.pages[1]) |
43 | 46 |
44 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 47 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
45 self._test_output_stream) | 48 self._test_output_stream) |
46 formatter.Format(measurement_results) | 49 formatter.Format(measurement_results) |
47 | 50 |
48 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n', | 51 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n', |
49 'RESULT a: http___www.foo.com_= 3 seconds\n', | 52 'RESULT a: http___www.foo.com_= 3 seconds\n', |
50 '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' + | 53 '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' + |
51 'Sd a: 2.828427seconds\n', | 54 'Sd a: 2.828427seconds\n', |
52 'RESULT telemetry_page_measurement_results: ' + | 55 'RESULT telemetry_page_measurement_results: ' + |
53 'num_failed= 0 count\n', | 56 'num_failed= 0 count\n', |
54 'RESULT telemetry_page_measurement_results: ' + | 57 'RESULT telemetry_page_measurement_results: ' + |
55 'num_errored= 0 count\n'] | 58 'num_errored= 0 count\n'] |
56 self.assertEquals(expected, self._test_output_stream.output_data) | 59 self.assertEquals(expected, self._test_output_stream.output_data) |
57 | 60 |
58 def test_basic_summary_with_only_one_page(self): | 61 def test_basic_summary_with_only_one_page(self): |
59 test_page_set = _MakePageSet() | 62 test_page_set = _MakePageSet() |
60 | 63 |
61 measurement_results = page_test_results.PageTestResults() | 64 measurement_results = page_test_results.PageTestResults() |
62 measurement_results.WillRunPage(test_page_set.pages[0]) | 65 measurement_results.WillRunPage(test_page_set.pages[0]) |
63 measurement_results.AddValue(scalar.ScalarValue( | 66 measurement_results.AddValue(scalar.ScalarValue( |
64 test_page_set.pages[0], 'a', 'seconds', 3)) | 67 test_page_set.pages[0], 'a', 'seconds', 3, |
| 68 improvement_direction=improvement_direction.DOWN)) |
65 measurement_results.DidRunPage(test_page_set.pages[0]) | 69 measurement_results.DidRunPage(test_page_set.pages[0]) |
66 | 70 |
67 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 71 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
68 self._test_output_stream) | 72 self._test_output_stream) |
69 formatter.Format(measurement_results) | 73 formatter.Format(measurement_results) |
70 | 74 |
71 expected = ['*RESULT a: a= 3 seconds\n', | 75 expected = ['*RESULT a: a= 3 seconds\n', |
72 'RESULT telemetry_page_measurement_results: ' + | 76 'RESULT telemetry_page_measurement_results: ' + |
73 'num_failed= 0 count\n', | 77 'num_failed= 0 count\n', |
74 'RESULT telemetry_page_measurement_results: ' + | 78 'RESULT telemetry_page_measurement_results: ' + |
75 'num_errored= 0 count\n'] | 79 'num_errored= 0 count\n'] |
76 self.assertEquals(expected, self._test_output_stream.output_data) | 80 self.assertEquals(expected, self._test_output_stream.output_data) |
77 | 81 |
78 def test_basic_summary_nonuniform_results(self): | 82 def test_basic_summary_nonuniform_results(self): |
79 test_page_set = _MakePageSet() | 83 test_page_set = _MakePageSet() |
80 | 84 |
81 measurement_results = page_test_results.PageTestResults() | 85 measurement_results = page_test_results.PageTestResults() |
82 measurement_results.WillRunPage(test_page_set.pages[0]) | 86 measurement_results.WillRunPage(test_page_set.pages[0]) |
83 measurement_results.AddValue(scalar.ScalarValue( | 87 measurement_results.AddValue(scalar.ScalarValue( |
84 test_page_set.pages[0], 'a', 'seconds', 3)) | 88 test_page_set.pages[0], 'a', 'seconds', 3, |
| 89 improvement_direction=improvement_direction.DOWN)) |
85 measurement_results.AddValue( | 90 measurement_results.AddValue( |
86 scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 10)) | 91 scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 10, |
| 92 improvement_direction=improvement_direction.DOWN)) |
87 measurement_results.DidRunPage(test_page_set.pages[0]) | 93 measurement_results.DidRunPage(test_page_set.pages[0]) |
88 | 94 |
89 measurement_results.WillRunPage(test_page_set.pages[1]) | 95 measurement_results.WillRunPage(test_page_set.pages[1]) |
90 measurement_results.AddValue(scalar.ScalarValue( | 96 measurement_results.AddValue(scalar.ScalarValue( |
91 test_page_set.pages[1], 'a', 'seconds', 3)) | 97 test_page_set.pages[1], 'a', 'seconds', 3, |
| 98 improvement_direction=improvement_direction.DOWN)) |
92 measurement_results.AddValue(scalar.ScalarValue( | 99 measurement_results.AddValue(scalar.ScalarValue( |
93 test_page_set.pages[1], 'b', 'seconds', 10)) | 100 test_page_set.pages[1], 'b', 'seconds', 10, |
| 101 improvement_direction=improvement_direction.DOWN)) |
94 measurement_results.DidRunPage(test_page_set.pages[1]) | 102 measurement_results.DidRunPage(test_page_set.pages[1]) |
95 | 103 |
96 measurement_results.WillRunPage(test_page_set.pages[2]) | 104 measurement_results.WillRunPage(test_page_set.pages[2]) |
97 measurement_results.AddValue(scalar.ScalarValue( | 105 measurement_results.AddValue(scalar.ScalarValue( |
98 test_page_set.pages[2], 'a', 'seconds', 7)) | 106 test_page_set.pages[2], 'a', 'seconds', 7, |
| 107 improvement_direction=improvement_direction.DOWN)) |
99 # Note, page[2] does not report a 'b' metric. | 108 # Note, page[2] does not report a 'b' metric. |
100 measurement_results.DidRunPage(test_page_set.pages[1]) | 109 measurement_results.DidRunPage(test_page_set.pages[1]) |
101 | 110 |
102 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 111 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
103 self._test_output_stream) | 112 self._test_output_stream) |
104 formatter.Format(measurement_results) | 113 formatter.Format(measurement_results) |
105 | 114 |
106 expected = ['RESULT a: http___www.bar.com_= 3 seconds\n', | 115 expected = ['RESULT a: http___www.bar.com_= 3 seconds\n', |
107 'RESULT a: http___www.baz.com_= 7 seconds\n', | 116 'RESULT a: http___www.baz.com_= 7 seconds\n', |
108 'RESULT a: http___www.foo.com_= 3 seconds\n', | 117 'RESULT a: http___www.foo.com_= 3 seconds\n', |
109 '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' + | 118 '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' + |
110 'Sd a: 2.309401seconds\n', | 119 'Sd a: 2.309401seconds\n', |
111 'RESULT b: http___www.bar.com_= 10 seconds\n', | 120 'RESULT b: http___www.bar.com_= 10 seconds\n', |
112 'RESULT b: http___www.foo.com_= 10 seconds\n', | 121 'RESULT b: http___www.foo.com_= 10 seconds\n', |
113 '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds\n', | 122 '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds\n', |
114 'RESULT telemetry_page_measurement_results: ' + | 123 'RESULT telemetry_page_measurement_results: ' + |
115 'num_failed= 0 count\n', | 124 'num_failed= 0 count\n', |
116 'RESULT telemetry_page_measurement_results: ' + | 125 'RESULT telemetry_page_measurement_results: ' + |
117 'num_errored= 0 count\n'] | 126 'num_errored= 0 count\n'] |
118 self.assertEquals(expected, self._test_output_stream.output_data) | 127 self.assertEquals(expected, self._test_output_stream.output_data) |
119 | 128 |
120 def test_basic_summary_pass_and_fail_page(self): | 129 def test_basic_summary_pass_and_fail_page(self): |
121 """If a page failed, only print summary for individual pages.""" | 130 """If a page failed, only print summary for individual pages.""" |
122 test_page_set = _MakePageSet() | 131 test_page_set = _MakePageSet() |
123 | 132 |
124 measurement_results = page_test_results.PageTestResults() | 133 measurement_results = page_test_results.PageTestResults() |
125 measurement_results.WillRunPage(test_page_set.pages[0]) | 134 measurement_results.WillRunPage(test_page_set.pages[0]) |
126 measurement_results.AddValue(scalar.ScalarValue( | 135 measurement_results.AddValue(scalar.ScalarValue( |
127 test_page_set.pages[0], 'a', 'seconds', 3)) | 136 test_page_set.pages[0], 'a', 'seconds', 3, |
| 137 improvement_direction=improvement_direction.DOWN)) |
128 measurement_results.AddValue( | 138 measurement_results.AddValue( |
129 failure.FailureValue.FromMessage(test_page_set.pages[0], 'message')) | 139 failure.FailureValue.FromMessage(test_page_set.pages[0], 'message')) |
130 measurement_results.DidRunPage(test_page_set.pages[0]) | 140 measurement_results.DidRunPage(test_page_set.pages[0]) |
131 | 141 |
132 measurement_results.WillRunPage(test_page_set.pages[1]) | 142 measurement_results.WillRunPage(test_page_set.pages[1]) |
133 measurement_results.AddValue(scalar.ScalarValue( | 143 measurement_results.AddValue(scalar.ScalarValue( |
134 test_page_set.pages[1], 'a', 'seconds', 7)) | 144 test_page_set.pages[1], 'a', 'seconds', 7, |
| 145 improvement_direction=improvement_direction.DOWN)) |
135 measurement_results.DidRunPage(test_page_set.pages[1]) | 146 measurement_results.DidRunPage(test_page_set.pages[1]) |
136 | 147 |
137 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 148 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
138 self._test_output_stream) | 149 self._test_output_stream) |
139 formatter.Format(measurement_results) | 150 formatter.Format(measurement_results) |
140 | 151 |
141 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n', | 152 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n', |
142 'RESULT a: http___www.foo.com_= 3 seconds\n', | 153 'RESULT a: http___www.foo.com_= 3 seconds\n', |
143 'RESULT telemetry_page_measurement_results: ' + | 154 'RESULT telemetry_page_measurement_results: ' + |
144 'num_failed= 1 count\n', | 155 'num_failed= 1 count\n', |
145 'RESULT telemetry_page_measurement_results: ' + | 156 'RESULT telemetry_page_measurement_results: ' + |
146 'num_errored= 0 count\n'] | 157 'num_errored= 0 count\n'] |
147 self.assertEquals(expected, self._test_output_stream.output_data) | 158 self.assertEquals(expected, self._test_output_stream.output_data) |
148 | 159 |
149 def test_repeated_pageset_one_iteration_one_page_fails(self): | 160 def test_repeated_pageset_one_iteration_one_page_fails(self): |
150 """Page fails on one iteration, no averaged results should print.""" | 161 """Page fails on one iteration, no averaged results should print.""" |
151 test_page_set = _MakePageSet() | 162 test_page_set = _MakePageSet() |
152 | 163 |
153 measurement_results = page_test_results.PageTestResults() | 164 measurement_results = page_test_results.PageTestResults() |
154 measurement_results.WillRunPage(test_page_set.pages[0]) | 165 measurement_results.WillRunPage(test_page_set.pages[0]) |
155 measurement_results.AddValue(scalar.ScalarValue( | 166 measurement_results.AddValue(scalar.ScalarValue( |
156 test_page_set.pages[0], 'a', 'seconds', 3)) | 167 test_page_set.pages[0], 'a', 'seconds', 3, |
| 168 improvement_direction=improvement_direction.DOWN)) |
157 measurement_results.DidRunPage(test_page_set.pages[0]) | 169 measurement_results.DidRunPage(test_page_set.pages[0]) |
158 | 170 |
159 measurement_results.WillRunPage(test_page_set.pages[1]) | 171 measurement_results.WillRunPage(test_page_set.pages[1]) |
160 measurement_results.AddValue(scalar.ScalarValue( | 172 measurement_results.AddValue(scalar.ScalarValue( |
161 test_page_set.pages[1], 'a', 'seconds', 7)) | 173 test_page_set.pages[1], 'a', 'seconds', 7, |
| 174 improvement_direction=improvement_direction.DOWN)) |
162 measurement_results.AddValue( | 175 measurement_results.AddValue( |
163 failure.FailureValue.FromMessage(test_page_set.pages[1], 'message')) | 176 failure.FailureValue.FromMessage(test_page_set.pages[1], 'message')) |
164 measurement_results.DidRunPage(test_page_set.pages[1]) | 177 measurement_results.DidRunPage(test_page_set.pages[1]) |
165 | 178 |
166 measurement_results.WillRunPage(test_page_set.pages[0]) | 179 measurement_results.WillRunPage(test_page_set.pages[0]) |
167 measurement_results.AddValue(scalar.ScalarValue( | 180 measurement_results.AddValue(scalar.ScalarValue( |
168 test_page_set.pages[0], 'a', 'seconds', 4)) | 181 test_page_set.pages[0], 'a', 'seconds', 4, |
| 182 improvement_direction=improvement_direction.DOWN)) |
169 measurement_results.DidRunPage(test_page_set.pages[0]) | 183 measurement_results.DidRunPage(test_page_set.pages[0]) |
170 | 184 |
171 measurement_results.WillRunPage(test_page_set.pages[1]) | 185 measurement_results.WillRunPage(test_page_set.pages[1]) |
172 measurement_results.AddValue(scalar.ScalarValue( | 186 measurement_results.AddValue(scalar.ScalarValue( |
173 test_page_set.pages[1], 'a', 'seconds', 8)) | 187 test_page_set.pages[1], 'a', 'seconds', 8, |
| 188 improvement_direction=improvement_direction.DOWN)) |
174 measurement_results.DidRunPage(test_page_set.pages[1]) | 189 measurement_results.DidRunPage(test_page_set.pages[1]) |
175 | 190 |
176 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 191 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
177 self._test_output_stream) | 192 self._test_output_stream) |
178 formatter.Format(measurement_results) | 193 formatter.Format(measurement_results) |
179 | 194 |
180 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + | 195 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + |
181 'Avg a: 7.500000seconds\n' + | 196 'Avg a: 7.500000seconds\n' + |
182 'Sd a: 0.707107seconds\n', | 197 'Sd a: 0.707107seconds\n', |
183 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + | 198 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + |
184 'Avg a: 3.500000seconds\n' + | 199 'Avg a: 3.500000seconds\n' + |
185 'Sd a: 0.707107seconds\n', | 200 'Sd a: 0.707107seconds\n', |
186 'RESULT telemetry_page_measurement_results: ' + | 201 'RESULT telemetry_page_measurement_results: ' + |
187 'num_failed= 1 count\n', | 202 'num_failed= 1 count\n', |
188 'RESULT telemetry_page_measurement_results: ' + | 203 'RESULT telemetry_page_measurement_results: ' + |
189 'num_errored= 0 count\n'] | 204 'num_errored= 0 count\n'] |
190 self.assertEquals(expected, self._test_output_stream.output_data) | 205 self.assertEquals(expected, self._test_output_stream.output_data) |
191 | 206 |
192 def test_repeated_pageset(self): | 207 def test_repeated_pageset(self): |
193 test_page_set = _MakePageSet() | 208 test_page_set = _MakePageSet() |
194 | 209 |
195 measurement_results = page_test_results.PageTestResults() | 210 measurement_results = page_test_results.PageTestResults() |
196 measurement_results.WillRunPage(test_page_set.pages[0]) | 211 measurement_results.WillRunPage(test_page_set.pages[0]) |
197 measurement_results.AddValue(scalar.ScalarValue( | 212 measurement_results.AddValue(scalar.ScalarValue( |
198 test_page_set.pages[0], 'a', 'seconds', 3)) | 213 test_page_set.pages[0], 'a', 'seconds', 3, |
| 214 improvement_direction=improvement_direction.DOWN)) |
199 measurement_results.DidRunPage(test_page_set.pages[0]) | 215 measurement_results.DidRunPage(test_page_set.pages[0]) |
200 | 216 |
201 measurement_results.WillRunPage(test_page_set.pages[1]) | 217 measurement_results.WillRunPage(test_page_set.pages[1]) |
202 measurement_results.AddValue(scalar.ScalarValue( | 218 measurement_results.AddValue(scalar.ScalarValue( |
203 test_page_set.pages[1], 'a', 'seconds', 7)) | 219 test_page_set.pages[1], 'a', 'seconds', 7, |
| 220 improvement_direction=improvement_direction.DOWN)) |
204 measurement_results.DidRunPage(test_page_set.pages[1]) | 221 measurement_results.DidRunPage(test_page_set.pages[1]) |
205 | 222 |
206 measurement_results.WillRunPage(test_page_set.pages[0]) | 223 measurement_results.WillRunPage(test_page_set.pages[0]) |
207 measurement_results.AddValue(scalar.ScalarValue( | 224 measurement_results.AddValue(scalar.ScalarValue( |
208 test_page_set.pages[0], 'a', 'seconds', 4)) | 225 test_page_set.pages[0], 'a', 'seconds', 4, |
| 226 improvement_direction=improvement_direction.DOWN)) |
209 measurement_results.DidRunPage(test_page_set.pages[0]) | 227 measurement_results.DidRunPage(test_page_set.pages[0]) |
210 | 228 |
211 measurement_results.WillRunPage(test_page_set.pages[1]) | 229 measurement_results.WillRunPage(test_page_set.pages[1]) |
212 measurement_results.AddValue(scalar.ScalarValue( | 230 measurement_results.AddValue(scalar.ScalarValue( |
213 test_page_set.pages[1], 'a', 'seconds', 8)) | 231 test_page_set.pages[1], 'a', 'seconds', 8, |
| 232 improvement_direction=improvement_direction.DOWN)) |
214 measurement_results.DidRunPage(test_page_set.pages[1]) | 233 measurement_results.DidRunPage(test_page_set.pages[1]) |
215 | 234 |
216 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 235 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
217 self._test_output_stream) | 236 self._test_output_stream) |
218 formatter.Format(measurement_results) | 237 formatter.Format(measurement_results) |
219 | 238 |
220 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + | 239 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + |
221 'Avg a: 7.500000seconds\n' + | 240 'Avg a: 7.500000seconds\n' + |
222 'Sd a: 0.707107seconds\n', | 241 'Sd a: 0.707107seconds\n', |
223 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + | 242 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + |
224 'Avg a: 3.500000seconds\n' + | 243 'Avg a: 3.500000seconds\n' + |
225 'Sd a: 0.707107seconds\n', | 244 'Sd a: 0.707107seconds\n', |
226 '*RESULT a: a= [3,7,4,8] seconds\n' + | 245 '*RESULT a: a= [3,7,4,8] seconds\n' + |
227 'Avg a: 5.500000seconds\n' + | 246 'Avg a: 5.500000seconds\n' + |
228 'Sd a: 2.380476seconds\n', | 247 'Sd a: 2.380476seconds\n', |
229 'RESULT telemetry_page_measurement_results: ' + | 248 'RESULT telemetry_page_measurement_results: ' + |
230 'num_failed= 0 count\n', | 249 'num_failed= 0 count\n', |
231 'RESULT telemetry_page_measurement_results: ' + | 250 'RESULT telemetry_page_measurement_results: ' + |
232 'num_errored= 0 count\n' | 251 'num_errored= 0 count\n' |
233 ] | 252 ] |
234 self.assertEquals(expected, self._test_output_stream.output_data) | 253 self.assertEquals(expected, self._test_output_stream.output_data) |
235 | 254 |
236 def test_repeated_pages(self): | 255 def test_repeated_pages(self): |
237 test_page_set = _MakePageSet() | 256 test_page_set = _MakePageSet() |
238 | 257 |
239 measurement_results = page_test_results.PageTestResults() | 258 measurement_results = page_test_results.PageTestResults() |
240 measurement_results.WillRunPage(test_page_set.pages[0]) | 259 measurement_results.WillRunPage(test_page_set.pages[0]) |
241 measurement_results.AddValue(scalar.ScalarValue( | 260 measurement_results.AddValue(scalar.ScalarValue( |
242 test_page_set.pages[0], 'a', 'seconds', 3)) | 261 test_page_set.pages[0], 'a', 'seconds', 3, |
| 262 improvement_direction=improvement_direction.DOWN)) |
243 measurement_results.DidRunPage(test_page_set.pages[0]) | 263 measurement_results.DidRunPage(test_page_set.pages[0]) |
244 | 264 |
245 measurement_results.WillRunPage(test_page_set.pages[0]) | 265 measurement_results.WillRunPage(test_page_set.pages[0]) |
246 measurement_results.AddValue(scalar.ScalarValue( | 266 measurement_results.AddValue(scalar.ScalarValue( |
247 test_page_set.pages[0], 'a', 'seconds', 4)) | 267 test_page_set.pages[0], 'a', 'seconds', 4, |
| 268 improvement_direction=improvement_direction.DOWN)) |
248 measurement_results.DidRunPage(test_page_set.pages[0]) | 269 measurement_results.DidRunPage(test_page_set.pages[0]) |
249 | 270 |
250 measurement_results.WillRunPage(test_page_set.pages[1]) | 271 measurement_results.WillRunPage(test_page_set.pages[1]) |
251 measurement_results.AddValue(scalar.ScalarValue( | 272 measurement_results.AddValue(scalar.ScalarValue( |
252 test_page_set.pages[1], 'a', 'seconds', 7)) | 273 test_page_set.pages[1], 'a', 'seconds', 7, |
| 274 improvement_direction=improvement_direction.DOWN)) |
253 measurement_results.DidRunPage(test_page_set.pages[1]) | 275 measurement_results.DidRunPage(test_page_set.pages[1]) |
254 | 276 |
255 measurement_results.WillRunPage(test_page_set.pages[1]) | 277 measurement_results.WillRunPage(test_page_set.pages[1]) |
256 measurement_results.AddValue(scalar.ScalarValue( | 278 measurement_results.AddValue(scalar.ScalarValue( |
257 test_page_set.pages[1], 'a', 'seconds', 8)) | 279 test_page_set.pages[1], 'a', 'seconds', 8, |
| 280 improvement_direction=improvement_direction.DOWN)) |
258 measurement_results.DidRunPage(test_page_set.pages[1]) | 281 measurement_results.DidRunPage(test_page_set.pages[1]) |
259 | 282 |
260 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 283 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
261 self._test_output_stream) | 284 self._test_output_stream) |
262 formatter.Format(measurement_results) | 285 formatter.Format(measurement_results) |
263 | 286 |
264 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + | 287 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + |
265 'Avg a: 7.500000seconds\n' + | 288 'Avg a: 7.500000seconds\n' + |
266 'Sd a: 0.707107seconds\n', | 289 'Sd a: 0.707107seconds\n', |
267 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + | 290 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + |
268 'Avg a: 3.500000seconds\n' + | 291 'Avg a: 3.500000seconds\n' + |
269 'Sd a: 0.707107seconds\n', | 292 'Sd a: 0.707107seconds\n', |
270 '*RESULT a: a= [3,4,7,8] seconds\n' + | 293 '*RESULT a: a= [3,4,7,8] seconds\n' + |
271 'Avg a: 5.500000seconds\n' + | 294 'Avg a: 5.500000seconds\n' + |
272 'Sd a: 2.380476seconds\n', | 295 'Sd a: 2.380476seconds\n', |
273 'RESULT telemetry_page_measurement_results: ' + | 296 'RESULT telemetry_page_measurement_results: ' + |
274 'num_failed= 0 count\n', | 297 'num_failed= 0 count\n', |
275 'RESULT telemetry_page_measurement_results: ' + | 298 'RESULT telemetry_page_measurement_results: ' + |
276 'num_errored= 0 count\n' | 299 'num_errored= 0 count\n' |
277 ] | 300 ] |
278 self.assertEquals(expected, self._test_output_stream.output_data) | 301 self.assertEquals(expected, self._test_output_stream.output_data) |
279 | 302 |
280 def test_overall_results_trace_tag(self): | 303 def test_overall_results_trace_tag(self): |
281 test_page_set = _MakePageSet() | 304 test_page_set = _MakePageSet() |
282 | 305 |
283 measurement_results = page_test_results.PageTestResults() | 306 measurement_results = page_test_results.PageTestResults() |
284 measurement_results.AddSummaryValue( | 307 measurement_results.AddSummaryValue( |
285 scalar.ScalarValue(None, 'a', 'seconds', 1)) | 308 scalar.ScalarValue(None, 'a', 'seconds', 1, |
| 309 improvement_direction=improvement_direction.DOWN)) |
286 | 310 |
287 measurement_results.WillRunPage(test_page_set.pages[0]) | 311 measurement_results.WillRunPage(test_page_set.pages[0]) |
288 measurement_results.AddValue(scalar.ScalarValue( | 312 measurement_results.AddValue(scalar.ScalarValue( |
289 test_page_set.pages[0], 'b', 'seconds', 2)) | 313 test_page_set.pages[0], 'b', 'seconds', 2, |
| 314 improvement_direction=improvement_direction.DOWN)) |
290 measurement_results.DidRunPage(test_page_set.pages[0]) | 315 measurement_results.DidRunPage(test_page_set.pages[0]) |
291 | 316 |
292 measurement_results.WillRunPage(test_page_set.pages[1]) | 317 measurement_results.WillRunPage(test_page_set.pages[1]) |
293 measurement_results.AddValue(scalar.ScalarValue( | 318 measurement_results.AddValue(scalar.ScalarValue( |
294 test_page_set.pages[1], 'b', 'seconds', 3)) | 319 test_page_set.pages[1], 'b', 'seconds', 3, |
| 320 improvement_direction=improvement_direction.DOWN)) |
295 measurement_results.DidRunPage(test_page_set.pages[1]) | 321 measurement_results.DidRunPage(test_page_set.pages[1]) |
296 | 322 |
297 measurement_results.AddSummaryValue( | 323 measurement_results.AddSummaryValue( |
298 scalar.ScalarValue(None, 'c', 'seconds', 4)) | 324 scalar.ScalarValue(None, 'c', 'seconds', 4, |
| 325 improvement_direction=improvement_direction.DOWN)) |
299 | 326 |
300 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 327 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
301 self._test_output_stream, trace_tag='_ref') | 328 self._test_output_stream, trace_tag='_ref') |
302 formatter.Format(measurement_results) | 329 formatter.Format(measurement_results) |
303 | 330 |
304 expected = [ | 331 expected = [ |
305 '*RESULT b: b_ref= [2,3] seconds\n' + | 332 '*RESULT b: b_ref= [2,3] seconds\n' + |
306 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', | 333 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', |
307 '*RESULT a: a_ref= 1 seconds\n', | 334 '*RESULT a: a_ref= 1 seconds\n', |
308 '*RESULT c: c_ref= 4 seconds\n', | 335 '*RESULT c: c_ref= 4 seconds\n', |
309 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', | 336 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', |
310 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] | 337 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] |
311 self.assertEquals(expected, self._test_output_stream.output_data) | 338 self.assertEquals(expected, self._test_output_stream.output_data) |
312 | 339 |
313 | 340 |
314 def test_overall_results_page_runs_twice(self): | 341 def test_overall_results_page_runs_twice(self): |
315 test_page_set = _MakePageSet() | 342 test_page_set = _MakePageSet() |
316 | 343 |
317 measurement_results = page_test_results.PageTestResults() | 344 measurement_results = page_test_results.PageTestResults() |
318 | 345 |
319 measurement_results.AddSummaryValue( | 346 measurement_results.AddSummaryValue( |
320 scalar.ScalarValue(None, 'a', 'seconds', 1)) | 347 scalar.ScalarValue(None, 'a', 'seconds', 1, |
| 348 improvement_direction=improvement_direction.DOWN)) |
321 | 349 |
322 measurement_results.WillRunPage(test_page_set.pages[0]) | 350 measurement_results.WillRunPage(test_page_set.pages[0]) |
323 measurement_results.AddValue(scalar.ScalarValue( | 351 measurement_results.AddValue(scalar.ScalarValue( |
324 test_page_set.pages[0], 'b', 'seconds', 2)) | 352 test_page_set.pages[0], 'b', 'seconds', 2, |
| 353 improvement_direction=improvement_direction.DOWN)) |
325 measurement_results.DidRunPage(test_page_set.pages[0]) | 354 measurement_results.DidRunPage(test_page_set.pages[0]) |
326 | 355 |
327 measurement_results.WillRunPage(test_page_set.pages[0]) | 356 measurement_results.WillRunPage(test_page_set.pages[0]) |
328 measurement_results.AddValue(scalar.ScalarValue( | 357 measurement_results.AddValue(scalar.ScalarValue( |
329 test_page_set.pages[0], 'b', 'seconds', 3)) | 358 test_page_set.pages[0], 'b', 'seconds', 3, |
| 359 improvement_direction=improvement_direction.DOWN)) |
330 measurement_results.DidRunPage(test_page_set.pages[0]) | 360 measurement_results.DidRunPage(test_page_set.pages[0]) |
331 | 361 |
332 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 362 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
333 self._test_output_stream) | 363 self._test_output_stream) |
334 formatter.Format(measurement_results) | 364 formatter.Format(measurement_results) |
335 | 365 |
336 expected = [ | 366 expected = [ |
337 'RESULT b: http___www.foo.com_= [2,3] seconds\n' + | 367 'RESULT b: http___www.foo.com_= [2,3] seconds\n' + |
338 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', | 368 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', |
339 '*RESULT b: b= [2,3] seconds\n' + | 369 '*RESULT b: b= [2,3] seconds\n' + |
340 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', | 370 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', |
341 '*RESULT a: a= 1 seconds\n', | 371 '*RESULT a: a= 1 seconds\n', |
342 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', | 372 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', |
343 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] | 373 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] |
344 self.assertEquals(expected, self._test_output_stream.output_data) | 374 self.assertEquals(expected, self._test_output_stream.output_data) |
345 | 375 |
346 def test_unimportant_results(self): | 376 def test_unimportant_results(self): |
347 test_page_set = _MakePageSet() | 377 test_page_set = _MakePageSet() |
348 | 378 |
349 measurement_results = page_test_results.PageTestResults() | 379 measurement_results = page_test_results.PageTestResults() |
350 | 380 |
351 measurement_results.AddSummaryValue( | 381 measurement_results.AddSummaryValue( |
352 scalar.ScalarValue(None, 'a', 'seconds', 1, important=False)) | 382 scalar.ScalarValue(None, 'a', 'seconds', 1, important=False, |
| 383 improvement_direction=improvement_direction.DOWN)) |
353 | 384 |
354 measurement_results.WillRunPage(test_page_set.pages[0]) | 385 measurement_results.WillRunPage(test_page_set.pages[0]) |
355 measurement_results.AddValue(scalar.ScalarValue( | 386 measurement_results.AddValue(scalar.ScalarValue( |
356 test_page_set.pages[0], 'b', 'seconds', 2, important=False)) | 387 test_page_set.pages[0], 'b', 'seconds', 2, important=False, |
| 388 improvement_direction=improvement_direction.DOWN)) |
357 measurement_results.DidRunPage(test_page_set.pages[0]) | 389 measurement_results.DidRunPage(test_page_set.pages[0]) |
358 | 390 |
359 measurement_results.WillRunPage(test_page_set.pages[1]) | 391 measurement_results.WillRunPage(test_page_set.pages[1]) |
360 measurement_results.AddValue(scalar.ScalarValue( | 392 measurement_results.AddValue(scalar.ScalarValue( |
361 test_page_set.pages[1], 'b', 'seconds', 3, important=False)) | 393 test_page_set.pages[1], 'b', 'seconds', 3, important=False, |
| 394 improvement_direction=improvement_direction.DOWN)) |
362 measurement_results.DidRunPage(test_page_set.pages[1]) | 395 measurement_results.DidRunPage(test_page_set.pages[1]) |
363 | 396 |
364 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 397 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
365 self._test_output_stream) | 398 self._test_output_stream) |
366 formatter.Format(measurement_results) | 399 formatter.Format(measurement_results) |
367 | 400 |
368 self.assertEquals( | 401 self.assertEquals( |
369 self._test_output_stream.output_data, | 402 self._test_output_stream.output_data, |
370 ['RESULT b: http___www.bar.com_= 3 seconds\n', | 403 ['RESULT b: http___www.bar.com_= 3 seconds\n', |
371 'RESULT b: http___www.foo.com_= 2 seconds\n', | 404 'RESULT b: http___www.foo.com_= 2 seconds\n', |
372 'RESULT b: b= [2,3] seconds\n' + | 405 'RESULT b: b= [2,3] seconds\n' + |
373 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', | 406 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', |
374 'RESULT a: a= 1 seconds\n', | 407 'RESULT a: a= 1 seconds\n', |
375 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', | 408 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', |
376 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']) | 409 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']) |
377 | 410 |
378 def test_list_value(self): | 411 def test_list_value(self): |
379 test_page_set = _MakePageSet() | 412 test_page_set = _MakePageSet() |
380 | 413 |
381 measurement_results = page_test_results.PageTestResults() | 414 measurement_results = page_test_results.PageTestResults() |
382 | 415 |
383 measurement_results.AddSummaryValue( | 416 measurement_results.AddSummaryValue( |
384 list_of_scalar_values.ListOfScalarValues(None, 'a', 'seconds', [1, 1])) | 417 list_of_scalar_values.ListOfScalarValues( |
| 418 None, 'a', 'seconds', [1, 1], |
| 419 improvement_direction=improvement_direction.DOWN)) |
385 | 420 |
386 measurement_results.WillRunPage(test_page_set.pages[0]) | 421 measurement_results.WillRunPage(test_page_set.pages[0]) |
387 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues( | 422 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues( |
388 test_page_set.pages[0], 'b', 'seconds', [2, 2])) | 423 test_page_set.pages[0], 'b', 'seconds', [2, 2], |
| 424 improvement_direction=improvement_direction.DOWN)) |
389 measurement_results.DidRunPage(test_page_set.pages[0]) | 425 measurement_results.DidRunPage(test_page_set.pages[0]) |
390 | 426 |
391 measurement_results.WillRunPage(test_page_set.pages[1]) | 427 measurement_results.WillRunPage(test_page_set.pages[1]) |
392 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues( | 428 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues( |
393 test_page_set.pages[1], 'b', 'seconds', [3, 3])) | 429 test_page_set.pages[1], 'b', 'seconds', [3, 3], |
| 430 improvement_direction=improvement_direction.DOWN)) |
394 measurement_results.DidRunPage(test_page_set.pages[1]) | 431 measurement_results.DidRunPage(test_page_set.pages[1]) |
395 | 432 |
396 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 433 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
397 self._test_output_stream) | 434 self._test_output_stream) |
398 formatter.Format(measurement_results) | 435 formatter.Format(measurement_results) |
399 | 436 |
400 expected = [ | 437 expected = [ |
401 'RESULT b: http___www.bar.com_= [3,3] seconds\n' + | 438 'RESULT b: http___www.bar.com_= [3,3] seconds\n' + |
402 'Avg b: 3.000000seconds\n', | 439 'Avg b: 3.000000seconds\n', |
403 'RESULT b: http___www.foo.com_= [2,2] seconds\n' + | 440 'RESULT b: http___www.foo.com_= [2,2] seconds\n' + |
404 'Avg b: 2.000000seconds\n', | 441 'Avg b: 2.000000seconds\n', |
405 '*RESULT b: b= [2,2,3,3] seconds\nAvg b: 2.500000seconds\n' + | 442 '*RESULT b: b= [2,2,3,3] seconds\nAvg b: 2.500000seconds\n' + |
406 'Sd b: 0.577350seconds\n', | 443 'Sd b: 0.577350seconds\n', |
407 '*RESULT a: a= [1,1] seconds\nAvg a: 1.000000seconds\n', | 444 '*RESULT a: a= [1,1] seconds\nAvg a: 1.000000seconds\n', |
408 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', | 445 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', |
409 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] | 446 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] |
410 self.assertEquals(expected, self._test_output_stream.output_data) | 447 self.assertEquals(expected, self._test_output_stream.output_data) |
411 | 448 |
412 def test_histogram(self): | 449 def test_histogram(self): |
413 test_page_set = _MakePageSet() | 450 test_page_set = _MakePageSet() |
414 | 451 |
415 measurement_results = page_test_results.PageTestResults() | 452 measurement_results = page_test_results.PageTestResults() |
416 measurement_results.WillRunPage(test_page_set.pages[0]) | 453 measurement_results.WillRunPage(test_page_set.pages[0]) |
417 measurement_results.AddValue(histogram.HistogramValue( | 454 measurement_results.AddValue(histogram.HistogramValue( |
418 test_page_set.pages[0], 'a', 'units', | 455 test_page_set.pages[0], 'a', 'units', |
419 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', | 456 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', |
420 important=False)) | 457 important=False, improvement_direction=improvement_direction.UP)) |
421 measurement_results.DidRunPage(test_page_set.pages[0]) | 458 measurement_results.DidRunPage(test_page_set.pages[0]) |
422 | 459 |
423 measurement_results.WillRunPage(test_page_set.pages[1]) | 460 measurement_results.WillRunPage(test_page_set.pages[1]) |
424 measurement_results.AddValue(histogram.HistogramValue( | 461 measurement_results.AddValue(histogram.HistogramValue( |
425 test_page_set.pages[1], 'a', 'units', | 462 test_page_set.pages[1], 'a', 'units', |
426 raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}', | 463 raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}', |
427 important=False)) | 464 important=False, improvement_direction=improvement_direction.DOWN)) |
428 measurement_results.DidRunPage(test_page_set.pages[1]) | 465 measurement_results.DidRunPage(test_page_set.pages[1]) |
429 | 466 |
430 formatter = buildbot_output_formatter.BuildbotOutputFormatter( | 467 formatter = buildbot_output_formatter.BuildbotOutputFormatter( |
431 self._test_output_stream) | 468 self._test_output_stream) |
432 formatter.Format(measurement_results) | 469 formatter.Format(measurement_results) |
433 | 470 |
434 expected = [ | 471 expected = [ |
435 'HISTOGRAM a: http___www.bar.com_= ' + | 472 'HISTOGRAM a: http___www.bar.com_= ' + |
436 '{"buckets": [{"low": 2, "high": 3, "count": 1}]} units\n' + | 473 '{"buckets": [{"low": 2, "high": 3, "count": 1}]} units\n' + |
437 'Avg a: 2.500000units\n', | 474 'Avg a: 2.500000units\n', |
438 'HISTOGRAM a: http___www.foo.com_= ' + | 475 'HISTOGRAM a: http___www.foo.com_= ' + |
439 '{"buckets": [{"low": 1, "high": 2, "count": 1}]} units\n' + | 476 '{"buckets": [{"low": 1, "high": 2, "count": 1}]} units\n' + |
440 'Avg a: 1.500000units\n', | 477 'Avg a: 1.500000units\n', |
441 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', | 478 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', |
442 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] | 479 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] |
443 self.assertEquals(expected, self._test_output_stream.output_data) | 480 self.assertEquals(expected, self._test_output_stream.output_data) |
OLD | NEW |