Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(345)

Side by Side Diff: tools/telemetry/telemetry/results/buildbot_output_formatter_unittest.py

Issue 809393002: Added support for improvement_direction to relevant values, which is propogated to chartjson. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Fix linter issues Created 5 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import os 5 import os
6 6
7 from telemetry import page as page_module 7 from telemetry import page as page_module
8 from telemetry.page import page_set 8 from telemetry.page import page_set
9 from telemetry.results import base_test_results_unittest 9 from telemetry.results import base_test_results_unittest
10 from telemetry.results import buildbot_output_formatter 10 from telemetry.results import buildbot_output_formatter
11 from telemetry.results import page_test_results 11 from telemetry.results import page_test_results
12 from telemetry.value import failure 12 from telemetry.value import failure
13 from telemetry.value import histogram 13 from telemetry.value import histogram
14 from telemetry.value import improvement_direction
14 from telemetry.value import list_of_scalar_values 15 from telemetry.value import list_of_scalar_values
15 from telemetry.value import scalar 16 from telemetry.value import scalar
16 17
17 18
18 def _MakePageSet(): 19 def _MakePageSet():
19 ps = page_set.PageSet(file_path=os.path.dirname(__file__)) 20 ps = page_set.PageSet(file_path=os.path.dirname(__file__))
20 ps.AddUserStory(page_module.Page('http://www.foo.com/', ps, ps.base_dir)) 21 ps.AddUserStory(page_module.Page('http://www.foo.com/', ps, ps.base_dir))
21 ps.AddUserStory(page_module.Page('http://www.bar.com/', ps, ps.base_dir)) 22 ps.AddUserStory(page_module.Page('http://www.bar.com/', ps, ps.base_dir))
22 ps.AddUserStory(page_module.Page('http://www.baz.com/', ps, ps.base_dir)) 23 ps.AddUserStory(page_module.Page('http://www.baz.com/', ps, ps.base_dir))
23 return ps 24 return ps
24 25
25 class BuildbotOutputFormatterTest( 26 class BuildbotOutputFormatterTest(
26 base_test_results_unittest.BaseTestResultsUnittest): 27 base_test_results_unittest.BaseTestResultsUnittest):
27 def setUp(self): 28 def setUp(self):
28 self._test_output_stream = base_test_results_unittest.TestOutputStream() 29 self._test_output_stream = base_test_results_unittest.TestOutputStream()
29 30
30 def test_basic_summary(self): 31 def test_basic_summary(self):
31 test_page_set = _MakePageSet() 32 test_page_set = _MakePageSet()
32 33
33 measurement_results = page_test_results.PageTestResults() 34 measurement_results = page_test_results.PageTestResults()
34 measurement_results.WillRunPage(test_page_set.pages[0]) 35 measurement_results.WillRunPage(test_page_set.pages[0])
35 measurement_results.AddValue(scalar.ScalarValue( 36 measurement_results.AddValue(scalar.ScalarValue(
36 test_page_set.pages[0], 'a', 'seconds', 3)) 37 test_page_set.pages[0], 'a', 'seconds', 3,
38 improvement_direction=improvement_direction.DOWN))
37 measurement_results.DidRunPage(test_page_set.pages[0]) 39 measurement_results.DidRunPage(test_page_set.pages[0])
38 40
39 measurement_results.WillRunPage(test_page_set.pages[1]) 41 measurement_results.WillRunPage(test_page_set.pages[1])
40 measurement_results.AddValue(scalar.ScalarValue( 42 measurement_results.AddValue(scalar.ScalarValue(
41 test_page_set.pages[1], 'a', 'seconds', 7)) 43 test_page_set.pages[1], 'a', 'seconds', 7,
44 improvement_direction=improvement_direction.DOWN))
42 measurement_results.DidRunPage(test_page_set.pages[1]) 45 measurement_results.DidRunPage(test_page_set.pages[1])
43 46
44 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 47 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
45 self._test_output_stream) 48 self._test_output_stream)
46 formatter.Format(measurement_results) 49 formatter.Format(measurement_results)
47 50
48 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n', 51 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n',
49 'RESULT a: http___www.foo.com_= 3 seconds\n', 52 'RESULT a: http___www.foo.com_= 3 seconds\n',
50 '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' + 53 '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' +
51 'Sd a: 2.828427seconds\n', 54 'Sd a: 2.828427seconds\n',
52 'RESULT telemetry_page_measurement_results: ' + 55 'RESULT telemetry_page_measurement_results: ' +
53 'num_failed= 0 count\n', 56 'num_failed= 0 count\n',
54 'RESULT telemetry_page_measurement_results: ' + 57 'RESULT telemetry_page_measurement_results: ' +
55 'num_errored= 0 count\n'] 58 'num_errored= 0 count\n']
56 self.assertEquals(expected, self._test_output_stream.output_data) 59 self.assertEquals(expected, self._test_output_stream.output_data)
57 60
58 def test_basic_summary_with_only_one_page(self): 61 def test_basic_summary_with_only_one_page(self):
59 test_page_set = _MakePageSet() 62 test_page_set = _MakePageSet()
60 63
61 measurement_results = page_test_results.PageTestResults() 64 measurement_results = page_test_results.PageTestResults()
62 measurement_results.WillRunPage(test_page_set.pages[0]) 65 measurement_results.WillRunPage(test_page_set.pages[0])
63 measurement_results.AddValue(scalar.ScalarValue( 66 measurement_results.AddValue(scalar.ScalarValue(
64 test_page_set.pages[0], 'a', 'seconds', 3)) 67 test_page_set.pages[0], 'a', 'seconds', 3,
68 improvement_direction=improvement_direction.DOWN))
65 measurement_results.DidRunPage(test_page_set.pages[0]) 69 measurement_results.DidRunPage(test_page_set.pages[0])
66 70
67 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 71 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
68 self._test_output_stream) 72 self._test_output_stream)
69 formatter.Format(measurement_results) 73 formatter.Format(measurement_results)
70 74
71 expected = ['*RESULT a: a= 3 seconds\n', 75 expected = ['*RESULT a: a= 3 seconds\n',
72 'RESULT telemetry_page_measurement_results: ' + 76 'RESULT telemetry_page_measurement_results: ' +
73 'num_failed= 0 count\n', 77 'num_failed= 0 count\n',
74 'RESULT telemetry_page_measurement_results: ' + 78 'RESULT telemetry_page_measurement_results: ' +
75 'num_errored= 0 count\n'] 79 'num_errored= 0 count\n']
76 self.assertEquals(expected, self._test_output_stream.output_data) 80 self.assertEquals(expected, self._test_output_stream.output_data)
77 81
78 def test_basic_summary_nonuniform_results(self): 82 def test_basic_summary_nonuniform_results(self):
79 test_page_set = _MakePageSet() 83 test_page_set = _MakePageSet()
80 84
81 measurement_results = page_test_results.PageTestResults() 85 measurement_results = page_test_results.PageTestResults()
82 measurement_results.WillRunPage(test_page_set.pages[0]) 86 measurement_results.WillRunPage(test_page_set.pages[0])
83 measurement_results.AddValue(scalar.ScalarValue( 87 measurement_results.AddValue(scalar.ScalarValue(
84 test_page_set.pages[0], 'a', 'seconds', 3)) 88 test_page_set.pages[0], 'a', 'seconds', 3,
89 improvement_direction=improvement_direction.DOWN))
85 measurement_results.AddValue( 90 measurement_results.AddValue(
86 scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 10)) 91 scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 10,
92 improvement_direction=improvement_direction.DOWN))
87 measurement_results.DidRunPage(test_page_set.pages[0]) 93 measurement_results.DidRunPage(test_page_set.pages[0])
88 94
89 measurement_results.WillRunPage(test_page_set.pages[1]) 95 measurement_results.WillRunPage(test_page_set.pages[1])
90 measurement_results.AddValue(scalar.ScalarValue( 96 measurement_results.AddValue(scalar.ScalarValue(
91 test_page_set.pages[1], 'a', 'seconds', 3)) 97 test_page_set.pages[1], 'a', 'seconds', 3,
98 improvement_direction=improvement_direction.DOWN))
92 measurement_results.AddValue(scalar.ScalarValue( 99 measurement_results.AddValue(scalar.ScalarValue(
93 test_page_set.pages[1], 'b', 'seconds', 10)) 100 test_page_set.pages[1], 'b', 'seconds', 10,
101 improvement_direction=improvement_direction.DOWN))
94 measurement_results.DidRunPage(test_page_set.pages[1]) 102 measurement_results.DidRunPage(test_page_set.pages[1])
95 103
96 measurement_results.WillRunPage(test_page_set.pages[2]) 104 measurement_results.WillRunPage(test_page_set.pages[2])
97 measurement_results.AddValue(scalar.ScalarValue( 105 measurement_results.AddValue(scalar.ScalarValue(
98 test_page_set.pages[2], 'a', 'seconds', 7)) 106 test_page_set.pages[2], 'a', 'seconds', 7,
107 improvement_direction=improvement_direction.DOWN))
99 # Note, page[2] does not report a 'b' metric. 108 # Note, page[2] does not report a 'b' metric.
100 measurement_results.DidRunPage(test_page_set.pages[1]) 109 measurement_results.DidRunPage(test_page_set.pages[1])
101 110
102 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 111 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
103 self._test_output_stream) 112 self._test_output_stream)
104 formatter.Format(measurement_results) 113 formatter.Format(measurement_results)
105 114
106 expected = ['RESULT a: http___www.bar.com_= 3 seconds\n', 115 expected = ['RESULT a: http___www.bar.com_= 3 seconds\n',
107 'RESULT a: http___www.baz.com_= 7 seconds\n', 116 'RESULT a: http___www.baz.com_= 7 seconds\n',
108 'RESULT a: http___www.foo.com_= 3 seconds\n', 117 'RESULT a: http___www.foo.com_= 3 seconds\n',
109 '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' + 118 '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' +
110 'Sd a: 2.309401seconds\n', 119 'Sd a: 2.309401seconds\n',
111 'RESULT b: http___www.bar.com_= 10 seconds\n', 120 'RESULT b: http___www.bar.com_= 10 seconds\n',
112 'RESULT b: http___www.foo.com_= 10 seconds\n', 121 'RESULT b: http___www.foo.com_= 10 seconds\n',
113 '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds\n', 122 '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds\n',
114 'RESULT telemetry_page_measurement_results: ' + 123 'RESULT telemetry_page_measurement_results: ' +
115 'num_failed= 0 count\n', 124 'num_failed= 0 count\n',
116 'RESULT telemetry_page_measurement_results: ' + 125 'RESULT telemetry_page_measurement_results: ' +
117 'num_errored= 0 count\n'] 126 'num_errored= 0 count\n']
118 self.assertEquals(expected, self._test_output_stream.output_data) 127 self.assertEquals(expected, self._test_output_stream.output_data)
119 128
120 def test_basic_summary_pass_and_fail_page(self): 129 def test_basic_summary_pass_and_fail_page(self):
121 """If a page failed, only print summary for individual pages.""" 130 """If a page failed, only print summary for individual pages."""
122 test_page_set = _MakePageSet() 131 test_page_set = _MakePageSet()
123 132
124 measurement_results = page_test_results.PageTestResults() 133 measurement_results = page_test_results.PageTestResults()
125 measurement_results.WillRunPage(test_page_set.pages[0]) 134 measurement_results.WillRunPage(test_page_set.pages[0])
126 measurement_results.AddValue(scalar.ScalarValue( 135 measurement_results.AddValue(scalar.ScalarValue(
127 test_page_set.pages[0], 'a', 'seconds', 3)) 136 test_page_set.pages[0], 'a', 'seconds', 3,
137 improvement_direction=improvement_direction.DOWN))
128 measurement_results.AddValue( 138 measurement_results.AddValue(
129 failure.FailureValue.FromMessage(test_page_set.pages[0], 'message')) 139 failure.FailureValue.FromMessage(test_page_set.pages[0], 'message'))
130 measurement_results.DidRunPage(test_page_set.pages[0]) 140 measurement_results.DidRunPage(test_page_set.pages[0])
131 141
132 measurement_results.WillRunPage(test_page_set.pages[1]) 142 measurement_results.WillRunPage(test_page_set.pages[1])
133 measurement_results.AddValue(scalar.ScalarValue( 143 measurement_results.AddValue(scalar.ScalarValue(
134 test_page_set.pages[1], 'a', 'seconds', 7)) 144 test_page_set.pages[1], 'a', 'seconds', 7,
145 improvement_direction=improvement_direction.DOWN))
135 measurement_results.DidRunPage(test_page_set.pages[1]) 146 measurement_results.DidRunPage(test_page_set.pages[1])
136 147
137 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 148 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
138 self._test_output_stream) 149 self._test_output_stream)
139 formatter.Format(measurement_results) 150 formatter.Format(measurement_results)
140 151
141 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n', 152 expected = ['RESULT a: http___www.bar.com_= 7 seconds\n',
142 'RESULT a: http___www.foo.com_= 3 seconds\n', 153 'RESULT a: http___www.foo.com_= 3 seconds\n',
143 'RESULT telemetry_page_measurement_results: ' + 154 'RESULT telemetry_page_measurement_results: ' +
144 'num_failed= 1 count\n', 155 'num_failed= 1 count\n',
145 'RESULT telemetry_page_measurement_results: ' + 156 'RESULT telemetry_page_measurement_results: ' +
146 'num_errored= 0 count\n'] 157 'num_errored= 0 count\n']
147 self.assertEquals(expected, self._test_output_stream.output_data) 158 self.assertEquals(expected, self._test_output_stream.output_data)
148 159
149 def test_repeated_pageset_one_iteration_one_page_fails(self): 160 def test_repeated_pageset_one_iteration_one_page_fails(self):
150 """Page fails on one iteration, no averaged results should print.""" 161 """Page fails on one iteration, no averaged results should print."""
151 test_page_set = _MakePageSet() 162 test_page_set = _MakePageSet()
152 163
153 measurement_results = page_test_results.PageTestResults() 164 measurement_results = page_test_results.PageTestResults()
154 measurement_results.WillRunPage(test_page_set.pages[0]) 165 measurement_results.WillRunPage(test_page_set.pages[0])
155 measurement_results.AddValue(scalar.ScalarValue( 166 measurement_results.AddValue(scalar.ScalarValue(
156 test_page_set.pages[0], 'a', 'seconds', 3)) 167 test_page_set.pages[0], 'a', 'seconds', 3,
168 improvement_direction=improvement_direction.DOWN))
157 measurement_results.DidRunPage(test_page_set.pages[0]) 169 measurement_results.DidRunPage(test_page_set.pages[0])
158 170
159 measurement_results.WillRunPage(test_page_set.pages[1]) 171 measurement_results.WillRunPage(test_page_set.pages[1])
160 measurement_results.AddValue(scalar.ScalarValue( 172 measurement_results.AddValue(scalar.ScalarValue(
161 test_page_set.pages[1], 'a', 'seconds', 7)) 173 test_page_set.pages[1], 'a', 'seconds', 7,
174 improvement_direction=improvement_direction.DOWN))
162 measurement_results.AddValue( 175 measurement_results.AddValue(
163 failure.FailureValue.FromMessage(test_page_set.pages[1], 'message')) 176 failure.FailureValue.FromMessage(test_page_set.pages[1], 'message'))
164 measurement_results.DidRunPage(test_page_set.pages[1]) 177 measurement_results.DidRunPage(test_page_set.pages[1])
165 178
166 measurement_results.WillRunPage(test_page_set.pages[0]) 179 measurement_results.WillRunPage(test_page_set.pages[0])
167 measurement_results.AddValue(scalar.ScalarValue( 180 measurement_results.AddValue(scalar.ScalarValue(
168 test_page_set.pages[0], 'a', 'seconds', 4)) 181 test_page_set.pages[0], 'a', 'seconds', 4,
182 improvement_direction=improvement_direction.DOWN))
169 measurement_results.DidRunPage(test_page_set.pages[0]) 183 measurement_results.DidRunPage(test_page_set.pages[0])
170 184
171 measurement_results.WillRunPage(test_page_set.pages[1]) 185 measurement_results.WillRunPage(test_page_set.pages[1])
172 measurement_results.AddValue(scalar.ScalarValue( 186 measurement_results.AddValue(scalar.ScalarValue(
173 test_page_set.pages[1], 'a', 'seconds', 8)) 187 test_page_set.pages[1], 'a', 'seconds', 8,
188 improvement_direction=improvement_direction.DOWN))
174 measurement_results.DidRunPage(test_page_set.pages[1]) 189 measurement_results.DidRunPage(test_page_set.pages[1])
175 190
176 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 191 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
177 self._test_output_stream) 192 self._test_output_stream)
178 formatter.Format(measurement_results) 193 formatter.Format(measurement_results)
179 194
180 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + 195 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' +
181 'Avg a: 7.500000seconds\n' + 196 'Avg a: 7.500000seconds\n' +
182 'Sd a: 0.707107seconds\n', 197 'Sd a: 0.707107seconds\n',
183 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + 198 'RESULT a: http___www.foo.com_= [3,4] seconds\n' +
184 'Avg a: 3.500000seconds\n' + 199 'Avg a: 3.500000seconds\n' +
185 'Sd a: 0.707107seconds\n', 200 'Sd a: 0.707107seconds\n',
186 'RESULT telemetry_page_measurement_results: ' + 201 'RESULT telemetry_page_measurement_results: ' +
187 'num_failed= 1 count\n', 202 'num_failed= 1 count\n',
188 'RESULT telemetry_page_measurement_results: ' + 203 'RESULT telemetry_page_measurement_results: ' +
189 'num_errored= 0 count\n'] 204 'num_errored= 0 count\n']
190 self.assertEquals(expected, self._test_output_stream.output_data) 205 self.assertEquals(expected, self._test_output_stream.output_data)
191 206
192 def test_repeated_pageset(self): 207 def test_repeated_pageset(self):
193 test_page_set = _MakePageSet() 208 test_page_set = _MakePageSet()
194 209
195 measurement_results = page_test_results.PageTestResults() 210 measurement_results = page_test_results.PageTestResults()
196 measurement_results.WillRunPage(test_page_set.pages[0]) 211 measurement_results.WillRunPage(test_page_set.pages[0])
197 measurement_results.AddValue(scalar.ScalarValue( 212 measurement_results.AddValue(scalar.ScalarValue(
198 test_page_set.pages[0], 'a', 'seconds', 3)) 213 test_page_set.pages[0], 'a', 'seconds', 3,
214 improvement_direction=improvement_direction.DOWN))
199 measurement_results.DidRunPage(test_page_set.pages[0]) 215 measurement_results.DidRunPage(test_page_set.pages[0])
200 216
201 measurement_results.WillRunPage(test_page_set.pages[1]) 217 measurement_results.WillRunPage(test_page_set.pages[1])
202 measurement_results.AddValue(scalar.ScalarValue( 218 measurement_results.AddValue(scalar.ScalarValue(
203 test_page_set.pages[1], 'a', 'seconds', 7)) 219 test_page_set.pages[1], 'a', 'seconds', 7,
220 improvement_direction=improvement_direction.DOWN))
204 measurement_results.DidRunPage(test_page_set.pages[1]) 221 measurement_results.DidRunPage(test_page_set.pages[1])
205 222
206 measurement_results.WillRunPage(test_page_set.pages[0]) 223 measurement_results.WillRunPage(test_page_set.pages[0])
207 measurement_results.AddValue(scalar.ScalarValue( 224 measurement_results.AddValue(scalar.ScalarValue(
208 test_page_set.pages[0], 'a', 'seconds', 4)) 225 test_page_set.pages[0], 'a', 'seconds', 4,
226 improvement_direction=improvement_direction.DOWN))
209 measurement_results.DidRunPage(test_page_set.pages[0]) 227 measurement_results.DidRunPage(test_page_set.pages[0])
210 228
211 measurement_results.WillRunPage(test_page_set.pages[1]) 229 measurement_results.WillRunPage(test_page_set.pages[1])
212 measurement_results.AddValue(scalar.ScalarValue( 230 measurement_results.AddValue(scalar.ScalarValue(
213 test_page_set.pages[1], 'a', 'seconds', 8)) 231 test_page_set.pages[1], 'a', 'seconds', 8,
232 improvement_direction=improvement_direction.DOWN))
214 measurement_results.DidRunPage(test_page_set.pages[1]) 233 measurement_results.DidRunPage(test_page_set.pages[1])
215 234
216 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 235 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
217 self._test_output_stream) 236 self._test_output_stream)
218 formatter.Format(measurement_results) 237 formatter.Format(measurement_results)
219 238
220 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + 239 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' +
221 'Avg a: 7.500000seconds\n' + 240 'Avg a: 7.500000seconds\n' +
222 'Sd a: 0.707107seconds\n', 241 'Sd a: 0.707107seconds\n',
223 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + 242 'RESULT a: http___www.foo.com_= [3,4] seconds\n' +
224 'Avg a: 3.500000seconds\n' + 243 'Avg a: 3.500000seconds\n' +
225 'Sd a: 0.707107seconds\n', 244 'Sd a: 0.707107seconds\n',
226 '*RESULT a: a= [3,7,4,8] seconds\n' + 245 '*RESULT a: a= [3,7,4,8] seconds\n' +
227 'Avg a: 5.500000seconds\n' + 246 'Avg a: 5.500000seconds\n' +
228 'Sd a: 2.380476seconds\n', 247 'Sd a: 2.380476seconds\n',
229 'RESULT telemetry_page_measurement_results: ' + 248 'RESULT telemetry_page_measurement_results: ' +
230 'num_failed= 0 count\n', 249 'num_failed= 0 count\n',
231 'RESULT telemetry_page_measurement_results: ' + 250 'RESULT telemetry_page_measurement_results: ' +
232 'num_errored= 0 count\n' 251 'num_errored= 0 count\n'
233 ] 252 ]
234 self.assertEquals(expected, self._test_output_stream.output_data) 253 self.assertEquals(expected, self._test_output_stream.output_data)
235 254
236 def test_repeated_pages(self): 255 def test_repeated_pages(self):
237 test_page_set = _MakePageSet() 256 test_page_set = _MakePageSet()
238 257
239 measurement_results = page_test_results.PageTestResults() 258 measurement_results = page_test_results.PageTestResults()
240 measurement_results.WillRunPage(test_page_set.pages[0]) 259 measurement_results.WillRunPage(test_page_set.pages[0])
241 measurement_results.AddValue(scalar.ScalarValue( 260 measurement_results.AddValue(scalar.ScalarValue(
242 test_page_set.pages[0], 'a', 'seconds', 3)) 261 test_page_set.pages[0], 'a', 'seconds', 3,
262 improvement_direction=improvement_direction.DOWN))
243 measurement_results.DidRunPage(test_page_set.pages[0]) 263 measurement_results.DidRunPage(test_page_set.pages[0])
244 264
245 measurement_results.WillRunPage(test_page_set.pages[0]) 265 measurement_results.WillRunPage(test_page_set.pages[0])
246 measurement_results.AddValue(scalar.ScalarValue( 266 measurement_results.AddValue(scalar.ScalarValue(
247 test_page_set.pages[0], 'a', 'seconds', 4)) 267 test_page_set.pages[0], 'a', 'seconds', 4,
268 improvement_direction=improvement_direction.DOWN))
248 measurement_results.DidRunPage(test_page_set.pages[0]) 269 measurement_results.DidRunPage(test_page_set.pages[0])
249 270
250 measurement_results.WillRunPage(test_page_set.pages[1]) 271 measurement_results.WillRunPage(test_page_set.pages[1])
251 measurement_results.AddValue(scalar.ScalarValue( 272 measurement_results.AddValue(scalar.ScalarValue(
252 test_page_set.pages[1], 'a', 'seconds', 7)) 273 test_page_set.pages[1], 'a', 'seconds', 7,
274 improvement_direction=improvement_direction.DOWN))
253 measurement_results.DidRunPage(test_page_set.pages[1]) 275 measurement_results.DidRunPage(test_page_set.pages[1])
254 276
255 measurement_results.WillRunPage(test_page_set.pages[1]) 277 measurement_results.WillRunPage(test_page_set.pages[1])
256 measurement_results.AddValue(scalar.ScalarValue( 278 measurement_results.AddValue(scalar.ScalarValue(
257 test_page_set.pages[1], 'a', 'seconds', 8)) 279 test_page_set.pages[1], 'a', 'seconds', 8,
280 improvement_direction=improvement_direction.DOWN))
258 measurement_results.DidRunPage(test_page_set.pages[1]) 281 measurement_results.DidRunPage(test_page_set.pages[1])
259 282
260 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 283 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
261 self._test_output_stream) 284 self._test_output_stream)
262 formatter.Format(measurement_results) 285 formatter.Format(measurement_results)
263 286
264 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' + 287 expected = ['RESULT a: http___www.bar.com_= [7,8] seconds\n' +
265 'Avg a: 7.500000seconds\n' + 288 'Avg a: 7.500000seconds\n' +
266 'Sd a: 0.707107seconds\n', 289 'Sd a: 0.707107seconds\n',
267 'RESULT a: http___www.foo.com_= [3,4] seconds\n' + 290 'RESULT a: http___www.foo.com_= [3,4] seconds\n' +
268 'Avg a: 3.500000seconds\n' + 291 'Avg a: 3.500000seconds\n' +
269 'Sd a: 0.707107seconds\n', 292 'Sd a: 0.707107seconds\n',
270 '*RESULT a: a= [3,4,7,8] seconds\n' + 293 '*RESULT a: a= [3,4,7,8] seconds\n' +
271 'Avg a: 5.500000seconds\n' + 294 'Avg a: 5.500000seconds\n' +
272 'Sd a: 2.380476seconds\n', 295 'Sd a: 2.380476seconds\n',
273 'RESULT telemetry_page_measurement_results: ' + 296 'RESULT telemetry_page_measurement_results: ' +
274 'num_failed= 0 count\n', 297 'num_failed= 0 count\n',
275 'RESULT telemetry_page_measurement_results: ' + 298 'RESULT telemetry_page_measurement_results: ' +
276 'num_errored= 0 count\n' 299 'num_errored= 0 count\n'
277 ] 300 ]
278 self.assertEquals(expected, self._test_output_stream.output_data) 301 self.assertEquals(expected, self._test_output_stream.output_data)
279 302
280 def test_overall_results_trace_tag(self): 303 def test_overall_results_trace_tag(self):
281 test_page_set = _MakePageSet() 304 test_page_set = _MakePageSet()
282 305
283 measurement_results = page_test_results.PageTestResults() 306 measurement_results = page_test_results.PageTestResults()
284 measurement_results.AddSummaryValue( 307 measurement_results.AddSummaryValue(
285 scalar.ScalarValue(None, 'a', 'seconds', 1)) 308 scalar.ScalarValue(None, 'a', 'seconds', 1,
309 improvement_direction=improvement_direction.DOWN))
286 310
287 measurement_results.WillRunPage(test_page_set.pages[0]) 311 measurement_results.WillRunPage(test_page_set.pages[0])
288 measurement_results.AddValue(scalar.ScalarValue( 312 measurement_results.AddValue(scalar.ScalarValue(
289 test_page_set.pages[0], 'b', 'seconds', 2)) 313 test_page_set.pages[0], 'b', 'seconds', 2,
314 improvement_direction=improvement_direction.DOWN))
290 measurement_results.DidRunPage(test_page_set.pages[0]) 315 measurement_results.DidRunPage(test_page_set.pages[0])
291 316
292 measurement_results.WillRunPage(test_page_set.pages[1]) 317 measurement_results.WillRunPage(test_page_set.pages[1])
293 measurement_results.AddValue(scalar.ScalarValue( 318 measurement_results.AddValue(scalar.ScalarValue(
294 test_page_set.pages[1], 'b', 'seconds', 3)) 319 test_page_set.pages[1], 'b', 'seconds', 3,
320 improvement_direction=improvement_direction.DOWN))
295 measurement_results.DidRunPage(test_page_set.pages[1]) 321 measurement_results.DidRunPage(test_page_set.pages[1])
296 322
297 measurement_results.AddSummaryValue( 323 measurement_results.AddSummaryValue(
298 scalar.ScalarValue(None, 'c', 'seconds', 4)) 324 scalar.ScalarValue(None, 'c', 'seconds', 4,
325 improvement_direction=improvement_direction.DOWN))
299 326
300 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 327 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
301 self._test_output_stream, trace_tag='_ref') 328 self._test_output_stream, trace_tag='_ref')
302 formatter.Format(measurement_results) 329 formatter.Format(measurement_results)
303 330
304 expected = [ 331 expected = [
305 '*RESULT b: b_ref= [2,3] seconds\n' + 332 '*RESULT b: b_ref= [2,3] seconds\n' +
306 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', 333 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n',
307 '*RESULT a: a_ref= 1 seconds\n', 334 '*RESULT a: a_ref= 1 seconds\n',
308 '*RESULT c: c_ref= 4 seconds\n', 335 '*RESULT c: c_ref= 4 seconds\n',
309 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', 336 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n',
310 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] 337 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']
311 self.assertEquals(expected, self._test_output_stream.output_data) 338 self.assertEquals(expected, self._test_output_stream.output_data)
312 339
313 340
314 def test_overall_results_page_runs_twice(self): 341 def test_overall_results_page_runs_twice(self):
315 test_page_set = _MakePageSet() 342 test_page_set = _MakePageSet()
316 343
317 measurement_results = page_test_results.PageTestResults() 344 measurement_results = page_test_results.PageTestResults()
318 345
319 measurement_results.AddSummaryValue( 346 measurement_results.AddSummaryValue(
320 scalar.ScalarValue(None, 'a', 'seconds', 1)) 347 scalar.ScalarValue(None, 'a', 'seconds', 1,
348 improvement_direction=improvement_direction.DOWN))
321 349
322 measurement_results.WillRunPage(test_page_set.pages[0]) 350 measurement_results.WillRunPage(test_page_set.pages[0])
323 measurement_results.AddValue(scalar.ScalarValue( 351 measurement_results.AddValue(scalar.ScalarValue(
324 test_page_set.pages[0], 'b', 'seconds', 2)) 352 test_page_set.pages[0], 'b', 'seconds', 2,
353 improvement_direction=improvement_direction.DOWN))
325 measurement_results.DidRunPage(test_page_set.pages[0]) 354 measurement_results.DidRunPage(test_page_set.pages[0])
326 355
327 measurement_results.WillRunPage(test_page_set.pages[0]) 356 measurement_results.WillRunPage(test_page_set.pages[0])
328 measurement_results.AddValue(scalar.ScalarValue( 357 measurement_results.AddValue(scalar.ScalarValue(
329 test_page_set.pages[0], 'b', 'seconds', 3)) 358 test_page_set.pages[0], 'b', 'seconds', 3,
359 improvement_direction=improvement_direction.DOWN))
330 measurement_results.DidRunPage(test_page_set.pages[0]) 360 measurement_results.DidRunPage(test_page_set.pages[0])
331 361
332 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 362 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
333 self._test_output_stream) 363 self._test_output_stream)
334 formatter.Format(measurement_results) 364 formatter.Format(measurement_results)
335 365
336 expected = [ 366 expected = [
337 'RESULT b: http___www.foo.com_= [2,3] seconds\n' + 367 'RESULT b: http___www.foo.com_= [2,3] seconds\n' +
338 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', 368 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n',
339 '*RESULT b: b= [2,3] seconds\n' + 369 '*RESULT b: b= [2,3] seconds\n' +
340 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', 370 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n',
341 '*RESULT a: a= 1 seconds\n', 371 '*RESULT a: a= 1 seconds\n',
342 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', 372 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n',
343 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] 373 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']
344 self.assertEquals(expected, self._test_output_stream.output_data) 374 self.assertEquals(expected, self._test_output_stream.output_data)
345 375
346 def test_unimportant_results(self): 376 def test_unimportant_results(self):
347 test_page_set = _MakePageSet() 377 test_page_set = _MakePageSet()
348 378
349 measurement_results = page_test_results.PageTestResults() 379 measurement_results = page_test_results.PageTestResults()
350 380
351 measurement_results.AddSummaryValue( 381 measurement_results.AddSummaryValue(
352 scalar.ScalarValue(None, 'a', 'seconds', 1, important=False)) 382 scalar.ScalarValue(None, 'a', 'seconds', 1, important=False,
383 improvement_direction=improvement_direction.DOWN))
353 384
354 measurement_results.WillRunPage(test_page_set.pages[0]) 385 measurement_results.WillRunPage(test_page_set.pages[0])
355 measurement_results.AddValue(scalar.ScalarValue( 386 measurement_results.AddValue(scalar.ScalarValue(
356 test_page_set.pages[0], 'b', 'seconds', 2, important=False)) 387 test_page_set.pages[0], 'b', 'seconds', 2, important=False,
388 improvement_direction=improvement_direction.DOWN))
357 measurement_results.DidRunPage(test_page_set.pages[0]) 389 measurement_results.DidRunPage(test_page_set.pages[0])
358 390
359 measurement_results.WillRunPage(test_page_set.pages[1]) 391 measurement_results.WillRunPage(test_page_set.pages[1])
360 measurement_results.AddValue(scalar.ScalarValue( 392 measurement_results.AddValue(scalar.ScalarValue(
361 test_page_set.pages[1], 'b', 'seconds', 3, important=False)) 393 test_page_set.pages[1], 'b', 'seconds', 3, important=False,
394 improvement_direction=improvement_direction.DOWN))
362 measurement_results.DidRunPage(test_page_set.pages[1]) 395 measurement_results.DidRunPage(test_page_set.pages[1])
363 396
364 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 397 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
365 self._test_output_stream) 398 self._test_output_stream)
366 formatter.Format(measurement_results) 399 formatter.Format(measurement_results)
367 400
368 self.assertEquals( 401 self.assertEquals(
369 self._test_output_stream.output_data, 402 self._test_output_stream.output_data,
370 ['RESULT b: http___www.bar.com_= 3 seconds\n', 403 ['RESULT b: http___www.bar.com_= 3 seconds\n',
371 'RESULT b: http___www.foo.com_= 2 seconds\n', 404 'RESULT b: http___www.foo.com_= 2 seconds\n',
372 'RESULT b: b= [2,3] seconds\n' + 405 'RESULT b: b= [2,3] seconds\n' +
373 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', 406 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n',
374 'RESULT a: a= 1 seconds\n', 407 'RESULT a: a= 1 seconds\n',
375 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', 408 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n',
376 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']) 409 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'])
377 410
378 def test_list_value(self): 411 def test_list_value(self):
379 test_page_set = _MakePageSet() 412 test_page_set = _MakePageSet()
380 413
381 measurement_results = page_test_results.PageTestResults() 414 measurement_results = page_test_results.PageTestResults()
382 415
383 measurement_results.AddSummaryValue( 416 measurement_results.AddSummaryValue(
384 list_of_scalar_values.ListOfScalarValues(None, 'a', 'seconds', [1, 1])) 417 list_of_scalar_values.ListOfScalarValues(
418 None, 'a', 'seconds', [1, 1],
419 improvement_direction=improvement_direction.DOWN))
385 420
386 measurement_results.WillRunPage(test_page_set.pages[0]) 421 measurement_results.WillRunPage(test_page_set.pages[0])
387 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues( 422 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues(
388 test_page_set.pages[0], 'b', 'seconds', [2, 2])) 423 test_page_set.pages[0], 'b', 'seconds', [2, 2],
424 improvement_direction=improvement_direction.DOWN))
389 measurement_results.DidRunPage(test_page_set.pages[0]) 425 measurement_results.DidRunPage(test_page_set.pages[0])
390 426
391 measurement_results.WillRunPage(test_page_set.pages[1]) 427 measurement_results.WillRunPage(test_page_set.pages[1])
392 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues( 428 measurement_results.AddValue(list_of_scalar_values.ListOfScalarValues(
393 test_page_set.pages[1], 'b', 'seconds', [3, 3])) 429 test_page_set.pages[1], 'b', 'seconds', [3, 3],
430 improvement_direction=improvement_direction.DOWN))
394 measurement_results.DidRunPage(test_page_set.pages[1]) 431 measurement_results.DidRunPage(test_page_set.pages[1])
395 432
396 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 433 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
397 self._test_output_stream) 434 self._test_output_stream)
398 formatter.Format(measurement_results) 435 formatter.Format(measurement_results)
399 436
400 expected = [ 437 expected = [
401 'RESULT b: http___www.bar.com_= [3,3] seconds\n' + 438 'RESULT b: http___www.bar.com_= [3,3] seconds\n' +
402 'Avg b: 3.000000seconds\n', 439 'Avg b: 3.000000seconds\n',
403 'RESULT b: http___www.foo.com_= [2,2] seconds\n' + 440 'RESULT b: http___www.foo.com_= [2,2] seconds\n' +
404 'Avg b: 2.000000seconds\n', 441 'Avg b: 2.000000seconds\n',
405 '*RESULT b: b= [2,2,3,3] seconds\nAvg b: 2.500000seconds\n' + 442 '*RESULT b: b= [2,2,3,3] seconds\nAvg b: 2.500000seconds\n' +
406 'Sd b: 0.577350seconds\n', 443 'Sd b: 0.577350seconds\n',
407 '*RESULT a: a= [1,1] seconds\nAvg a: 1.000000seconds\n', 444 '*RESULT a: a= [1,1] seconds\nAvg a: 1.000000seconds\n',
408 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', 445 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n',
409 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] 446 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']
410 self.assertEquals(expected, self._test_output_stream.output_data) 447 self.assertEquals(expected, self._test_output_stream.output_data)
411 448
412 def test_histogram(self): 449 def test_histogram(self):
413 test_page_set = _MakePageSet() 450 test_page_set = _MakePageSet()
414 451
415 measurement_results = page_test_results.PageTestResults() 452 measurement_results = page_test_results.PageTestResults()
416 measurement_results.WillRunPage(test_page_set.pages[0]) 453 measurement_results.WillRunPage(test_page_set.pages[0])
417 measurement_results.AddValue(histogram.HistogramValue( 454 measurement_results.AddValue(histogram.HistogramValue(
418 test_page_set.pages[0], 'a', 'units', 455 test_page_set.pages[0], 'a', 'units',
419 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', 456 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
420 important=False)) 457 important=False, improvement_direction=improvement_direction.UP))
421 measurement_results.DidRunPage(test_page_set.pages[0]) 458 measurement_results.DidRunPage(test_page_set.pages[0])
422 459
423 measurement_results.WillRunPage(test_page_set.pages[1]) 460 measurement_results.WillRunPage(test_page_set.pages[1])
424 measurement_results.AddValue(histogram.HistogramValue( 461 measurement_results.AddValue(histogram.HistogramValue(
425 test_page_set.pages[1], 'a', 'units', 462 test_page_set.pages[1], 'a', 'units',
426 raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}', 463 raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}',
427 important=False)) 464 important=False, improvement_direction=improvement_direction.DOWN))
428 measurement_results.DidRunPage(test_page_set.pages[1]) 465 measurement_results.DidRunPage(test_page_set.pages[1])
429 466
430 formatter = buildbot_output_formatter.BuildbotOutputFormatter( 467 formatter = buildbot_output_formatter.BuildbotOutputFormatter(
431 self._test_output_stream) 468 self._test_output_stream)
432 formatter.Format(measurement_results) 469 formatter.Format(measurement_results)
433 470
434 expected = [ 471 expected = [
435 'HISTOGRAM a: http___www.bar.com_= ' + 472 'HISTOGRAM a: http___www.bar.com_= ' +
436 '{"buckets": [{"low": 2, "high": 3, "count": 1}]} units\n' + 473 '{"buckets": [{"low": 2, "high": 3, "count": 1}]} units\n' +
437 'Avg a: 2.500000units\n', 474 'Avg a: 2.500000units\n',
438 'HISTOGRAM a: http___www.foo.com_= ' + 475 'HISTOGRAM a: http___www.foo.com_= ' +
439 '{"buckets": [{"low": 1, "high": 2, "count": 1}]} units\n' + 476 '{"buckets": [{"low": 1, "high": 2, "count": 1}]} units\n' +
440 'Avg a: 1.500000units\n', 477 'Avg a: 1.500000units\n',
441 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', 478 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n',
442 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'] 479 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n']
443 self.assertEquals(expected, self._test_output_stream.output_data) 480 self.assertEquals(expected, self._test_output_stream.output_data)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698