OLD | NEW |
---|---|
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 from collections import OrderedDict | 5 from collections import OrderedDict |
6 | 6 |
7 from webkitpy.common.host_mock import MockHost | 7 from webkitpy.common.host_mock import MockHost |
8 from webkitpy.common.system.filesystem_mock import MockFileSystem | 8 from webkitpy.common.system.filesystem_mock import MockFileSystem |
9 from webkitpy.common.system.logtesting import LoggingTestCase | 9 from webkitpy.common.system.logtesting import LoggingTestCase |
10 from webkitpy.layout_tests.builder_list import BuilderList | 10 from webkitpy.layout_tests.builder_list import BuilderList |
(...skipping 17 matching lines...) Expand all Loading... | |
28 | 28 |
29 | 29 |
30 class FakeBotTestExpectationsFactory(object): | 30 class FakeBotTestExpectationsFactory(object): |
31 | 31 |
32 def __init__(self): | 32 def __init__(self): |
33 """The distinct results seen in at least one run of the test. | 33 """The distinct results seen in at least one run of the test. |
34 E.g. if the bot results for mytest.html are: | 34 E.g. if the bot results for mytest.html are: |
35 PASS PASS FAIL PASS TIMEOUT | 35 PASS PASS FAIL PASS TIMEOUT |
36 then _all_results_by_builder would be: | 36 then _all_results_by_builder would be: |
37 { | 37 { |
38 'WebKit Linux' : { | 38 'WebKit Linux Precise' : { |
Dirk Pranke
2016/09/16 22:39:57
Ideally we'd change these test cases to refer to a
jeffcarp
2016/09/21 19:21:50
That sounds good but sounds like it should be in a
| |
39 'mytest.html': ['FAIL', 'PASS', 'TIMEOUT'] | 39 'mytest.html': ['FAIL', 'PASS', 'TIMEOUT'] |
40 } | 40 } |
41 } | 41 } |
42 """ | 42 """ |
43 self._all_results_by_builder = {} | 43 self._all_results_by_builder = {} |
44 | 44 |
45 def expectations_for_builder(self, builder): | 45 def expectations_for_builder(self, builder): |
46 if builder not in self._all_results_by_builder: | 46 if builder not in self._all_results_by_builder: |
47 return None | 47 return None |
48 | 48 |
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
147 # Even though the results show all passing, none of the | 147 # Even though the results show all passing, none of the |
148 # expectations are flaky so we shouldn't remove any. | 148 # expectations are flaky so we shouldn't remove any. |
149 Bug(test) test/a.html [ Pass ] | 149 Bug(test) test/a.html [ Pass ] |
150 Bug(test) test/b.html [ Timeout ] | 150 Bug(test) test/b.html [ Timeout ] |
151 Bug(test) test/c.html [ Failure Timeout ] | 151 Bug(test) test/c.html [ Failure Timeout ] |
152 Bug(test) test/d.html [ Rebaseline ] | 152 Bug(test) test/d.html [ Rebaseline ] |
153 Bug(test) test/e.html [ NeedsManualRebaseline ] | 153 Bug(test) test/e.html [ NeedsManualRebaseline ] |
154 Bug(test) test/f.html [ NeedsRebaseline ]""" | 154 Bug(test) test/f.html [ NeedsRebaseline ]""" |
155 | 155 |
156 self._define_builders({ | 156 self._define_builders({ |
157 "WebKit Linux": { | 157 "WebKit Linux Precise": { |
158 "port_name": "linux-precise", | 158 "port_name": "linux-precise", |
159 "specifiers": ['Precise', 'Release'] | 159 "specifiers": ['Precise', 'Release'] |
160 }, | 160 }, |
161 }) | 161 }) |
162 self._port.all_build_types = ('release',) | 162 self._port.all_build_types = ('release',) |
163 self._port.all_systems = (('precise', 'x86_64'),) | 163 self._port.all_systems = (('precise', 'x86_64'),) |
164 | 164 |
165 self._parse_expectations(test_expectations_before) | 165 self._parse_expectations(test_expectations_before) |
166 self._expectation_factory._all_results_by_builder = { | 166 self._expectation_factory._all_results_by_builder = { |
167 'WebKit Linux': { | 167 'WebKit Linux Precise': { |
168 "test/a.html": ["PASS", "PASS"], | 168 "test/a.html": ["PASS", "PASS"], |
169 "test/b.html": ["PASS", "PASS"], | 169 "test/b.html": ["PASS", "PASS"], |
170 "test/c.html": ["PASS", "PASS"], | 170 "test/c.html": ["PASS", "PASS"], |
171 "test/d.html": ["PASS", "PASS"], | 171 "test/d.html": ["PASS", "PASS"], |
172 "test/e.html": ["PASS", "PASS"], | 172 "test/e.html": ["PASS", "PASS"], |
173 "test/f.html": ["PASS", "PASS"], | 173 "test/f.html": ["PASS", "PASS"], |
174 } | 174 } |
175 } | 175 } |
176 updated_expectations = ( | 176 updated_expectations = ( |
177 self._flake_remover.get_updated_test_expectations()) | 177 self._flake_remover.get_updated_test_expectations()) |
178 self._assert_expectations_match( | 178 self._assert_expectations_match( |
179 updated_expectations, test_expectations_before) | 179 updated_expectations, test_expectations_before) |
180 | 180 |
181 def test_dont_remove_skip(self): | 181 def test_dont_remove_skip(self): |
182 """Tests that lines with Skip are untouched. | 182 """Tests that lines with Skip are untouched. |
183 | 183 |
184 If a line is marked as Skip, it will eventually contain no results, | 184 If a line is marked as Skip, it will eventually contain no results, |
185 which is indistinguishable from "All Passing" so don't remove since we | 185 which is indistinguishable from "All Passing" so don't remove since we |
186 don't know what the results actually are. | 186 don't know what the results actually are. |
187 """ | 187 """ |
188 test_expectations_before = """ | 188 test_expectations_before = """ |
189 # Skip expectations should never be removed. | 189 # Skip expectations should never be removed. |
190 Bug(test) test/a.html [ Skip ] | 190 Bug(test) test/a.html [ Skip ] |
191 Bug(test) test/b.html [ Skip ] | 191 Bug(test) test/b.html [ Skip ] |
192 Bug(test) test/c.html [ Skip ]""" | 192 Bug(test) test/c.html [ Skip ]""" |
193 | 193 |
194 self._define_builders({ | 194 self._define_builders({ |
195 "WebKit Linux": { | 195 "WebKit Linux Precise": { |
196 "port_name": "linux-precise", | 196 "port_name": "linux-precise", |
197 "specifiers": ['Precise', 'Release'] | 197 "specifiers": ['Precise', 'Release'] |
198 }, | 198 }, |
199 }) | 199 }) |
200 self._port.all_build_types = ('release',) | 200 self._port.all_build_types = ('release',) |
201 self._port.all_systems = (('precise', 'x86_64'),) | 201 self._port.all_systems = (('precise', 'x86_64'),) |
202 | 202 |
203 self._parse_expectations(test_expectations_before) | 203 self._parse_expectations(test_expectations_before) |
204 self._expectation_factory._all_results_by_builder = { | 204 self._expectation_factory._all_results_by_builder = { |
205 'WebKit Linux': { | 205 'WebKit Linux Precise': { |
206 "test/a.html": ["PASS", "PASS"], | 206 "test/a.html": ["PASS", "PASS"], |
207 "test/b.html": ["PASS", "IMAGE"], | 207 "test/b.html": ["PASS", "IMAGE"], |
208 } | 208 } |
209 } | 209 } |
210 updated_expectations = ( | 210 updated_expectations = ( |
211 self._flake_remover.get_updated_test_expectations()) | 211 self._flake_remover.get_updated_test_expectations()) |
212 self._assert_expectations_match( | 212 self._assert_expectations_match( |
213 updated_expectations, test_expectations_before) | 213 updated_expectations, test_expectations_before) |
214 | 214 |
215 def test_dont_remove_rebaselines(self): | 215 def test_dont_remove_rebaselines(self): |
216 """Tests that lines with rebaseline expectations are untouched.""" | 216 """Tests that lines with rebaseline expectations are untouched.""" |
217 test_expectations_before = """ | 217 test_expectations_before = """ |
218 # Even though the results show all passing, none of the | 218 # Even though the results show all passing, none of the |
219 # expectations are flaky so we shouldn't remove any. | 219 # expectations are flaky so we shouldn't remove any. |
220 Bug(test) test/a.html [ Failure NeedsRebaseline Pass ] | 220 Bug(test) test/a.html [ Failure NeedsRebaseline Pass ] |
221 Bug(test) test/b.html [ Failure Pass Rebaseline ] | 221 Bug(test) test/b.html [ Failure Pass Rebaseline ] |
222 Bug(test) test/c.html [ Failure NeedsManualRebaseline Pass ]""" | 222 Bug(test) test/c.html [ Failure NeedsManualRebaseline Pass ]""" |
223 | 223 |
224 self._define_builders({ | 224 self._define_builders({ |
225 "WebKit Linux": { | 225 "WebKit Linux Precise": { |
226 "port_name": "linux-precise", | 226 "port_name": "linux-precise", |
227 "specifiers": ['Precise', 'Release'] | 227 "specifiers": ['Precise', 'Release'] |
228 }, | 228 }, |
229 }) | 229 }) |
230 self._port.all_build_types = ('release',) | 230 self._port.all_build_types = ('release',) |
231 self._port.all_systems = (('precise', 'x86_64'),) | 231 self._port.all_systems = (('precise', 'x86_64'),) |
232 | 232 |
233 self._parse_expectations(test_expectations_before) | 233 self._parse_expectations(test_expectations_before) |
234 self._expectation_factory._all_results_by_builder = { | 234 self._expectation_factory._all_results_by_builder = { |
235 'WebKit Linux': { | 235 'WebKit Linux Precise': { |
236 "test/a.html": ["PASS", "PASS"], | 236 "test/a.html": ["PASS", "PASS"], |
237 "test/b.html": ["PASS", "PASS"], | 237 "test/b.html": ["PASS", "PASS"], |
238 "test/c.html": ["PASS", "PASS"] | 238 "test/c.html": ["PASS", "PASS"] |
239 } | 239 } |
240 } | 240 } |
241 updated_expectations = ( | 241 updated_expectations = ( |
242 self._flake_remover.get_updated_test_expectations()) | 242 self._flake_remover.get_updated_test_expectations()) |
243 self._assert_expectations_match( | 243 self._assert_expectations_match( |
244 updated_expectations, test_expectations_before) | 244 updated_expectations, test_expectations_before) |
245 | 245 |
246 def test_all_failure_types(self): | 246 def test_all_failure_types(self): |
247 """Tests that all failure types are treated as failure.""" | 247 """Tests that all failure types are treated as failure.""" |
248 test_expectations_before = ( | 248 test_expectations_before = ( |
249 """Bug(test) test/a.html [ Failure Pass ] | 249 """Bug(test) test/a.html [ Failure Pass ] |
250 Bug(test) test/b.html [ Failure Pass ] | 250 Bug(test) test/b.html [ Failure Pass ] |
251 Bug(test) test/c.html [ Failure Pass ] | 251 Bug(test) test/c.html [ Failure Pass ] |
252 Bug(test) test/d.html [ Failure Pass ] | 252 Bug(test) test/d.html [ Failure Pass ] |
253 # Remove these two since CRASH and TIMEOUT aren't considered | 253 # Remove these two since CRASH and TIMEOUT aren't considered |
254 # Failure. | 254 # Failure. |
255 Bug(test) test/e.html [ Failure Pass ] | 255 Bug(test) test/e.html [ Failure Pass ] |
256 Bug(test) test/f.html [ Failure Pass ]""") | 256 Bug(test) test/f.html [ Failure Pass ]""") |
257 | 257 |
258 self._define_builders({ | 258 self._define_builders({ |
259 "WebKit Linux": { | 259 "WebKit Linux Precise": { |
260 "port_name": "linux-precise", | 260 "port_name": "linux-precise", |
261 "specifiers": ['Precise', 'Release'] | 261 "specifiers": ['Precise', 'Release'] |
262 }, | 262 }, |
263 }) | 263 }) |
264 self._port.all_build_types = ('release',) | 264 self._port.all_build_types = ('release',) |
265 self._port.all_systems = (('precise', 'x86_64'),) | 265 self._port.all_systems = (('precise', 'x86_64'),) |
266 | 266 |
267 self._parse_expectations(test_expectations_before) | 267 self._parse_expectations(test_expectations_before) |
268 self._expectation_factory._all_results_by_builder = { | 268 self._expectation_factory._all_results_by_builder = { |
269 'WebKit Linux': { | 269 'WebKit Linux Precise': { |
270 "test/a.html": ["PASS", "IMAGE"], | 270 "test/a.html": ["PASS", "IMAGE"], |
271 "test/b.html": ["PASS", "TEXT"], | 271 "test/b.html": ["PASS", "TEXT"], |
272 "test/c.html": ["PASS", "IMAGE+TEXT"], | 272 "test/c.html": ["PASS", "IMAGE+TEXT"], |
273 "test/d.html": ["PASS", "AUDIO"], | 273 "test/d.html": ["PASS", "AUDIO"], |
274 "test/e.html": ["PASS", "CRASH"], | 274 "test/e.html": ["PASS", "CRASH"], |
275 "test/f.html": ["PASS", "TIMEOUT"], | 275 "test/f.html": ["PASS", "TIMEOUT"], |
276 } | 276 } |
277 } | 277 } |
278 updated_expectations = ( | 278 updated_expectations = ( |
279 self._flake_remover.get_updated_test_expectations()) | 279 self._flake_remover.get_updated_test_expectations()) |
(...skipping 13 matching lines...) Expand all Loading... | |
293 """ | 293 """ |
294 test_expectations_before = ( | 294 test_expectations_before = ( |
295 """# Remove this since it's passing all runs. | 295 """# Remove this since it's passing all runs. |
296 Bug(test) test/a.html [ Failure Pass ] | 296 Bug(test) test/a.html [ Failure Pass ] |
297 # Remove this since, although there's a failure, it's not a timeout. | 297 # Remove this since, although there's a failure, it's not a timeout. |
298 Bug(test) test/b.html [ Pass Timeout ] | 298 Bug(test) test/b.html [ Pass Timeout ] |
299 # Keep since we have both crashes and passes. | 299 # Keep since we have both crashes and passes. |
300 Bug(test) test/c.html [ Crash Pass ]""") | 300 Bug(test) test/c.html [ Crash Pass ]""") |
301 | 301 |
302 self._define_builders({ | 302 self._define_builders({ |
303 "WebKit Linux": { | 303 "WebKit Linux Precise": { |
304 "port_name": "linux-precise", | 304 "port_name": "linux-precise", |
305 "specifiers": ['Precise', 'Release'] | 305 "specifiers": ['Precise', 'Release'] |
306 }, | 306 }, |
307 }) | 307 }) |
308 self._port.all_build_types = ('release',) | 308 self._port.all_build_types = ('release',) |
309 self._port.all_systems = (('precise', 'x86_64'),) | 309 self._port.all_systems = (('precise', 'x86_64'),) |
310 | 310 |
311 self._parse_expectations(test_expectations_before) | 311 self._parse_expectations(test_expectations_before) |
312 self._expectation_factory._all_results_by_builder = { | 312 self._expectation_factory._all_results_by_builder = { |
313 'WebKit Linux': { | 313 'WebKit Linux Precise': { |
314 "test/a.html": ["PASS", "PASS", "PASS"], | 314 "test/a.html": ["PASS", "PASS", "PASS"], |
315 "test/b.html": ["PASS", "IMAGE", "PASS"], | 315 "test/b.html": ["PASS", "IMAGE", "PASS"], |
316 "test/c.html": ["PASS", "CRASH", "PASS"], | 316 "test/c.html": ["PASS", "CRASH", "PASS"], |
317 } | 317 } |
318 } | 318 } |
319 updated_expectations = ( | 319 updated_expectations = ( |
320 self._flake_remover.get_updated_test_expectations()) | 320 self._flake_remover.get_updated_test_expectations()) |
321 self._assert_expectations_match(updated_expectations, ( | 321 self._assert_expectations_match(updated_expectations, ( |
322 """# Keep since we have both crashes and passes. | 322 """# Keep since we have both crashes and passes. |
323 Bug(test) test/c.html [ Crash Pass ]""")) | 323 Bug(test) test/c.html [ Crash Pass ]""")) |
324 | 324 |
325 def test_all_failure_case(self): | 325 def test_all_failure_case(self): |
326 """Tests that results with all failures are not treated as non-flaky.""" | 326 """Tests that results with all failures are not treated as non-flaky.""" |
327 test_expectations_before = ( | 327 test_expectations_before = ( |
328 """# Keep since it's all failures. | 328 """# Keep since it's all failures. |
329 Bug(test) test/a.html [ Failure Pass ]""") | 329 Bug(test) test/a.html [ Failure Pass ]""") |
330 | 330 |
331 self._define_builders({ | 331 self._define_builders({ |
332 "WebKit Linux": { | 332 "WebKit Linux Precise": { |
333 "port_name": "linux-precise", | 333 "port_name": "linux-precise", |
334 "specifiers": ['Precise', 'Release'] | 334 "specifiers": ['Precise', 'Release'] |
335 }, | 335 }, |
336 }) | 336 }) |
337 self._port.all_build_types = ('release',) | 337 self._port.all_build_types = ('release',) |
338 self._port.all_systems = (('precise', 'x86_64'),) | 338 self._port.all_systems = (('precise', 'x86_64'),) |
339 | 339 |
340 self._parse_expectations(test_expectations_before) | 340 self._parse_expectations(test_expectations_before) |
341 self._expectation_factory._all_results_by_builder = { | 341 self._expectation_factory._all_results_by_builder = { |
342 'WebKit Linux': { | 342 'WebKit Linux Precise': { |
343 "test/a.html": ["IMAGE", "IMAGE", "IMAGE"], | 343 "test/a.html": ["IMAGE", "IMAGE", "IMAGE"], |
344 } | 344 } |
345 } | 345 } |
346 updated_expectations = ( | 346 updated_expectations = ( |
347 self._flake_remover.get_updated_test_expectations()) | 347 self._flake_remover.get_updated_test_expectations()) |
348 self._assert_expectations_match(updated_expectations, ( | 348 self._assert_expectations_match(updated_expectations, ( |
349 """# Keep since it's all failures. | 349 """# Keep since it's all failures. |
350 Bug(test) test/a.html [ Failure Pass ]""")) | 350 Bug(test) test/a.html [ Failure Pass ]""")) |
351 | 351 |
352 def test_empty_test_expectations(self): | 352 def test_empty_test_expectations(self): |
353 """Running on an empty TestExpectations file outputs an empty file.""" | 353 """Running on an empty TestExpectations file outputs an empty file.""" |
354 test_expectations_before = "" | 354 test_expectations_before = "" |
355 | 355 |
356 self._define_builders({ | 356 self._define_builders({ |
357 "WebKit Linux": { | 357 "WebKit Linux Precise": { |
358 "port_name": "linux-precise", | 358 "port_name": "linux-precise", |
359 "specifiers": ['Precise', 'Release'] | 359 "specifiers": ['Precise', 'Release'] |
360 }, | 360 }, |
361 }) | 361 }) |
362 self._port.all_build_types = ('release',) | 362 self._port.all_build_types = ('release',) |
363 self._port.all_systems = (('precise', 'x86_64'),) | 363 self._port.all_systems = (('precise', 'x86_64'),) |
364 | 364 |
365 self._parse_expectations(test_expectations_before) | 365 self._parse_expectations(test_expectations_before) |
366 self._expectation_factory._all_results_by_builder = { | 366 self._expectation_factory._all_results_by_builder = { |
367 'WebKit Linux': { | 367 'WebKit Linux Precise': { |
368 "test/a.html": ["PASS", "PASS", "PASS"], | 368 "test/a.html": ["PASS", "PASS", "PASS"], |
369 } | 369 } |
370 } | 370 } |
371 updated_expectations = ( | 371 updated_expectations = ( |
372 self._flake_remover.get_updated_test_expectations()) | 372 self._flake_remover.get_updated_test_expectations()) |
373 self._assert_expectations_match(updated_expectations, "") | 373 self._assert_expectations_match(updated_expectations, "") |
374 | 374 |
375 def test_basic_multiple_builders(self): | 375 def test_basic_multiple_builders(self): |
376 """Tests basic functionality with multiple builders.""" | 376 """Tests basic functionality with multiple builders.""" |
377 test_expectations_before = ( | 377 test_expectations_before = ( |
378 """# Remove since it's passing on both builders. | 378 """# Remove since it's passing on both builders. |
379 Bug(test) test/a.html [ Failure Pass ] | 379 Bug(test) test/a.html [ Failure Pass ] |
380 # Keep since it's failing on the Mac builder. | 380 # Keep since it's failing on the Mac builder. |
381 Bug(test) test/b.html [ Failure Pass ] | 381 Bug(test) test/b.html [ Failure Pass ] |
382 # Keep since it's failing on the Linux builder. | 382 # Keep since it's failing on the Linux builder. |
383 Bug(test) test/c.html [ Failure Pass ]""") | 383 Bug(test) test/c.html [ Failure Pass ]""") |
384 | 384 |
385 self._define_builders({ | 385 self._define_builders({ |
386 "WebKit Linux": { | 386 "WebKit Linux Precise": { |
387 "port_name": "linux-precise", | 387 "port_name": "linux-precise", |
388 "specifiers": ['Precise', 'Release'] | 388 "specifiers": ['Precise', 'Release'] |
389 }, | 389 }, |
390 "WebKit Mac10.10": { | 390 "WebKit Mac10.10": { |
391 "port_name": "mac-mac10.10", | 391 "port_name": "mac-mac10.10", |
392 "specifiers": ['Mac10.10', 'Release'] | 392 "specifiers": ['Mac10.10', 'Release'] |
393 }, | 393 }, |
394 }) | 394 }) |
395 | 395 |
396 self._port.all_build_types = ('release',) | 396 self._port.all_build_types = ('release',) |
397 self._port.all_systems = (('mac10.10', 'x86'), | 397 self._port.all_systems = (('mac10.10', 'x86'), |
398 ('precise', 'x86_64')) | 398 ('precise', 'x86_64')) |
399 | 399 |
400 self._parse_expectations(test_expectations_before) | 400 self._parse_expectations(test_expectations_before) |
401 self._expectation_factory._all_results_by_builder = { | 401 self._expectation_factory._all_results_by_builder = { |
402 'WebKit Linux': { | 402 'WebKit Linux Precise': { |
403 "test/a.html": ["PASS", "PASS", "PASS"], | 403 "test/a.html": ["PASS", "PASS", "PASS"], |
404 "test/b.html": ["PASS", "PASS", "PASS"], | 404 "test/b.html": ["PASS", "PASS", "PASS"], |
405 "test/c.html": ["AUDIO", "AUDIO", "AUDIO"], | 405 "test/c.html": ["AUDIO", "AUDIO", "AUDIO"], |
406 }, | 406 }, |
407 'WebKit Mac10.10': { | 407 'WebKit Mac10.10': { |
408 "test/a.html": ["PASS", "PASS", "PASS"], | 408 "test/a.html": ["PASS", "PASS", "PASS"], |
409 "test/b.html": ["PASS", "PASS", "IMAGE"], | 409 "test/b.html": ["PASS", "PASS", "IMAGE"], |
410 "test/c.html": ["PASS", "PASS", "PASS"], | 410 "test/c.html": ["PASS", "PASS", "PASS"], |
411 }, | 411 }, |
412 } | 412 } |
(...skipping 15 matching lines...) Expand all Loading... | |
428 # Remove since it's passing on both Linux and Windows builders. | 428 # Remove since it's passing on both Linux and Windows builders. |
429 Bug(test) [ Linux Win ] test/c.html [ Failure Pass ] | 429 Bug(test) [ Linux Win ] test/c.html [ Failure Pass ] |
430 # Remove since it's passing on Mac results | 430 # Remove since it's passing on Mac results |
431 Bug(test) [ Mac ] test/d.html [ Failure Pass ]""") | 431 Bug(test) [ Mac ] test/d.html [ Failure Pass ]""") |
432 | 432 |
433 self._define_builders({ | 433 self._define_builders({ |
434 "WebKit Win7": { | 434 "WebKit Win7": { |
435 "port_name": "win-win7", | 435 "port_name": "win-win7", |
436 "specifiers": ['Win7', 'Release'] | 436 "specifiers": ['Win7', 'Release'] |
437 }, | 437 }, |
438 "WebKit Linux": { | 438 "WebKit Linux Precise": { |
439 "port_name": "linux-precise", | 439 "port_name": "linux-precise", |
440 "specifiers": ['Precise', 'Release'] | 440 "specifiers": ['Precise', 'Release'] |
441 }, | 441 }, |
442 "WebKit Mac10.10": { | 442 "WebKit Mac10.10": { |
443 "port_name": "mac-mac10.10", | 443 "port_name": "mac-mac10.10", |
444 "specifiers": ['Mac10.10', 'Release'] | 444 "specifiers": ['Mac10.10', 'Release'] |
445 }, | 445 }, |
446 }) | 446 }) |
447 self._port.all_build_types = ('release',) | 447 self._port.all_build_types = ('release',) |
448 self._port.all_systems = (('mac10.10', 'x86'), | 448 self._port.all_systems = (('mac10.10', 'x86'), |
449 ('win7', 'x86'), | 449 ('win7', 'x86'), |
450 ('precise', 'x86_64')) | 450 ('precise', 'x86_64')) |
451 | 451 |
452 self._parse_expectations(test_expectations_before) | 452 self._parse_expectations(test_expectations_before) |
453 self._expectation_factory._all_results_by_builder = { | 453 self._expectation_factory._all_results_by_builder = { |
454 'WebKit Linux': { | 454 'WebKit Linux Precise': { |
455 "test/a.html": ["PASS", "PASS", "PASS"], | 455 "test/a.html": ["PASS", "PASS", "PASS"], |
456 "test/b.html": ["PASS", "PASS", "PASS"], | 456 "test/b.html": ["PASS", "PASS", "PASS"], |
457 "test/c.html": ["PASS", "PASS", "PASS"], | 457 "test/c.html": ["PASS", "PASS", "PASS"], |
458 "test/d.html": ["IMAGE", "PASS", "PASS"], | 458 "test/d.html": ["IMAGE", "PASS", "PASS"], |
459 }, | 459 }, |
460 'WebKit Mac10.10': { | 460 'WebKit Mac10.10': { |
461 "test/a.html": ["PASS", "PASS", "IMAGE"], | 461 "test/a.html": ["PASS", "PASS", "IMAGE"], |
462 "test/b.html": ["PASS", "IMAGE", "PASS"], | 462 "test/b.html": ["PASS", "IMAGE", "PASS"], |
463 "test/c.html": ["PASS", "IMAGE", "PASS"], | 463 "test/c.html": ["PASS", "IMAGE", "PASS"], |
464 "test/d.html": ["PASS", "PASS", "PASS"], | 464 "test/d.html": ["PASS", "PASS", "PASS"], |
(...skipping 29 matching lines...) Expand all Loading... | |
494 | 494 |
495 self._define_builders({ | 495 self._define_builders({ |
496 "WebKit Win7": { | 496 "WebKit Win7": { |
497 "port_name": "win-win7", | 497 "port_name": "win-win7", |
498 "specifiers": ['Win7', 'Release'] | 498 "specifiers": ['Win7', 'Release'] |
499 }, | 499 }, |
500 "WebKit Win7 (dbg)": { | 500 "WebKit Win7 (dbg)": { |
501 "port_name": "win-win7", | 501 "port_name": "win-win7", |
502 "specifiers": ['Win7', 'Debug'] | 502 "specifiers": ['Win7', 'Debug'] |
503 }, | 503 }, |
504 "WebKit Linux": { | 504 "WebKit Linux Precise": { |
505 "port_name": "linux-precise", | 505 "port_name": "linux-precise", |
506 "specifiers": ['Precise', 'Release'] | 506 "specifiers": ['Precise', 'Release'] |
507 }, | 507 }, |
508 "WebKit Linux (dbg)": { | 508 "WebKit Linux Precise (dbg)": { |
509 "port_name": "linux-precise", | 509 "port_name": "linux-precise", |
510 "specifiers": ['Precise', 'Debug'] | 510 "specifiers": ['Precise', 'Debug'] |
511 }, | 511 }, |
512 }) | 512 }) |
513 self._port.all_build_types = ('release', 'debug') | 513 self._port.all_build_types = ('release', 'debug') |
514 self._port.all_systems = (('win7', 'x86'), | 514 self._port.all_systems = (('win7', 'x86'), |
515 ('precise', 'x86_64')) | 515 ('precise', 'x86_64')) |
516 | 516 |
517 self._parse_expectations(test_expectations_before) | 517 self._parse_expectations(test_expectations_before) |
518 self._expectation_factory._all_results_by_builder = { | 518 self._expectation_factory._all_results_by_builder = { |
519 'WebKit Linux': { | 519 'WebKit Linux Precise': { |
520 "test/a.html": ["PASS", "PASS", "PASS"], | 520 "test/a.html": ["PASS", "PASS", "PASS"], |
521 "test/b.html": ["PASS", "IMAGE", "PASS"], | 521 "test/b.html": ["PASS", "IMAGE", "PASS"], |
522 "test/c.html": ["PASS", "IMAGE", "PASS"], | 522 "test/c.html": ["PASS", "IMAGE", "PASS"], |
523 "test/d.html": ["PASS", "PASS", "PASS"], | 523 "test/d.html": ["PASS", "PASS", "PASS"], |
524 "test/e.html": ["PASS", "PASS", "PASS"], | 524 "test/e.html": ["PASS", "PASS", "PASS"], |
525 }, | 525 }, |
526 'WebKit Linux (dbg)': { | 526 'WebKit Linux Precise (dbg)': { |
527 "test/a.html": ["PASS", "IMAGE", "PASS"], | 527 "test/a.html": ["PASS", "IMAGE", "PASS"], |
528 "test/b.html": ["PASS", "PASS", "PASS"], | 528 "test/b.html": ["PASS", "PASS", "PASS"], |
529 "test/c.html": ["PASS", "PASS", "PASS"], | 529 "test/c.html": ["PASS", "PASS", "PASS"], |
530 "test/d.html": ["IMAGE", "PASS", "PASS"], | 530 "test/d.html": ["IMAGE", "PASS", "PASS"], |
531 "test/e.html": ["PASS", "PASS", "PASS"], | 531 "test/e.html": ["PASS", "PASS", "PASS"], |
532 }, | 532 }, |
533 'WebKit Win7 (dbg)': { | 533 'WebKit Win7 (dbg)': { |
534 "test/a.html": ["PASS", "PASS", "PASS"], | 534 "test/a.html": ["PASS", "PASS", "PASS"], |
535 "test/b.html": ["PASS", "PASS", "PASS"], | 535 "test/b.html": ["PASS", "PASS", "PASS"], |
536 "test/c.html": ["PASS", "PASS", "PASS"], | 536 "test/c.html": ["PASS", "PASS", "PASS"], |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
588 | 588 |
589 # Comment F - Keep since only b is removed | 589 # Comment F - Keep since only b is removed |
590 Bug(test) test/b.html [ Failure Pass ] | 590 Bug(test) test/b.html [ Failure Pass ] |
591 Bug(test) test/c.html [ Failure Pass ] | 591 Bug(test) test/c.html [ Failure Pass ] |
592 | 592 |
593 # Comment G - Should be removed since both d and e will be removed. | 593 # Comment G - Should be removed since both d and e will be removed. |
594 Bug(test) test/d.html [ Failure Pass ] | 594 Bug(test) test/d.html [ Failure Pass ] |
595 Bug(test) test/e.html [ Failure Pass ]""" | 595 Bug(test) test/e.html [ Failure Pass ]""" |
596 | 596 |
597 self._define_builders({ | 597 self._define_builders({ |
598 "WebKit Linux": { | 598 "WebKit Linux Precise": { |
599 "port_name": "linux-precise", | 599 "port_name": "linux-precise", |
600 "specifiers": ['Precise', 'Release'] | 600 "specifiers": ['Precise', 'Release'] |
601 }, | 601 }, |
602 }) | 602 }) |
603 self._port.all_build_types = ('release',) | 603 self._port.all_build_types = ('release',) |
604 self._port.all_systems = (('precise', 'x86_64'),) | 604 self._port.all_systems = (('precise', 'x86_64'),) |
605 | 605 |
606 self._parse_expectations(test_expectations_before) | 606 self._parse_expectations(test_expectations_before) |
607 self._expectation_factory._all_results_by_builder = { | 607 self._expectation_factory._all_results_by_builder = { |
608 'WebKit Linux': { | 608 'WebKit Linux Precise': { |
609 "test/a.html": ["PASS", "PASS", "PASS"], | 609 "test/a.html": ["PASS", "PASS", "PASS"], |
610 "test/b.html": ["PASS", "PASS", "PASS"], | 610 "test/b.html": ["PASS", "PASS", "PASS"], |
611 "test/c.html": ["PASS", "IMAGE", "PASS"], | 611 "test/c.html": ["PASS", "IMAGE", "PASS"], |
612 "test/d.html": ["PASS", "PASS", "PASS"], | 612 "test/d.html": ["PASS", "PASS", "PASS"], |
613 "test/e.html": ["PASS", "PASS", "PASS"], | 613 "test/e.html": ["PASS", "PASS", "PASS"], |
614 } | 614 } |
615 } | 615 } |
616 updated_expectations = ( | 616 updated_expectations = ( |
617 self._flake_remover.get_updated_test_expectations()) | 617 self._flake_remover.get_updated_test_expectations()) |
618 self._assert_expectations_match(updated_expectations, ( | 618 self._assert_expectations_match(updated_expectations, ( |
(...skipping 19 matching lines...) Expand all Loading... | |
638 Bug(test) test/a.html [ Skip ] | 638 Bug(test) test/a.html [ Skip ] |
639 # This line shouldn't be removed either since it's not flaky. | 639 # This line shouldn't be removed either since it's not flaky. |
640 Bug(test) test/b.html [ Failure Timeout ] | 640 Bug(test) test/b.html [ Failure Timeout ] |
641 # The lines below should be removed since they're flaky but all runs | 641 # The lines below should be removed since they're flaky but all runs |
642 # are passing. | 642 # are passing. |
643 Bug(test) test/c.html [ Failure Pass ] | 643 Bug(test) test/c.html [ Failure Pass ] |
644 Bug(test) test/d.html [ Pass Timeout ] | 644 Bug(test) test/d.html [ Pass Timeout ] |
645 Bug(test) test/e.html [ Crash Pass ]""" | 645 Bug(test) test/e.html [ Crash Pass ]""" |
646 | 646 |
647 self._define_builders({ | 647 self._define_builders({ |
648 "WebKit Linux": { | 648 "WebKit Linux Precise": { |
649 "port_name": "linux-precise", | 649 "port_name": "linux-precise", |
650 "specifiers": ['Precise', 'Release'] | 650 "specifiers": ['Precise', 'Release'] |
651 }, | 651 }, |
652 }) | 652 }) |
653 self._port.all_build_types = ('release',) | 653 self._port.all_build_types = ('release',) |
654 self._port.all_systems = (('precise', 'x86_64'),) | 654 self._port.all_systems = (('precise', 'x86_64'),) |
655 | 655 |
656 self._parse_expectations(test_expectations_before) | 656 self._parse_expectations(test_expectations_before) |
657 self._expectation_factory._all_results_by_builder = { | 657 self._expectation_factory._all_results_by_builder = { |
658 'WebKit Linux': { | 658 'WebKit Linux Precise': { |
659 } | 659 } |
660 } | 660 } |
661 updated_expectations = ( | 661 updated_expectations = ( |
662 self._flake_remover.get_updated_test_expectations()) | 662 self._flake_remover.get_updated_test_expectations()) |
663 self._assert_expectations_match(updated_expectations, """ | 663 self._assert_expectations_match(updated_expectations, """ |
664 # A Skip expectation probably won't have any results but we | 664 # A Skip expectation probably won't have any results but we |
665 # shouldn't consider those passing so this line should remain. | 665 # shouldn't consider those passing so this line should remain. |
666 Bug(test) test/a.html [ Skip ] | 666 Bug(test) test/a.html [ Skip ] |
667 # This line shouldn't be removed either since it's not flaky. | 667 # This line shouldn't be removed either since it's not flaky. |
668 Bug(test) test/b.html [ Failure Timeout ]""") | 668 Bug(test) test/b.html [ Failure Timeout ]""") |
(...skipping 10 matching lines...) Expand all Loading... | |
679 Bug(test) [ Linux ] test/b.html [ Failure Pass ] | 679 Bug(test) [ Linux ] test/b.html [ Failure Pass ] |
680 # This one shouldn't emit an error since it's not flaky, we don't | 680 # This one shouldn't emit an error since it's not flaky, we don't |
681 # have to check the builder results. | 681 # have to check the builder results. |
682 Bug(test) test/c.html [ Failure ] | 682 Bug(test) test/c.html [ Failure ] |
683 Bug(test) test/d.html [ Failure Pass ] | 683 Bug(test) test/d.html [ Failure Pass ] |
684 # This one shouldn't emit an error since it will only match the | 684 # This one shouldn't emit an error since it will only match the |
685 # existing Linux Release configuration | 685 # existing Linux Release configuration |
686 Bug(test) [ Linux Release ] test/e.html [ Failure Pass ]""" | 686 Bug(test) [ Linux Release ] test/e.html [ Failure Pass ]""" |
687 | 687 |
688 self._define_builders({ | 688 self._define_builders({ |
689 "WebKit Linux": { | 689 "WebKit Linux Precise": { |
690 "port_name": "linux-precise", | 690 "port_name": "linux-precise", |
691 "specifiers": ['Precise', 'Release'] | 691 "specifiers": ['Precise', 'Release'] |
692 }, | 692 }, |
693 }) | 693 }) |
694 | 694 |
695 # Three errors should be emitted: | 695 # Three errors should be emitted: |
696 # (1) There's no Windows builders so a.html will emit an error on the | 696 # (1) There's no Windows builders so a.html will emit an error on the |
697 # first missing one. | 697 # first missing one. |
698 # (2) There's no Linux debug builder so b.html will emit an error. | 698 # (2) There's no Linux debug builder so b.html will emit an error. |
699 # (3) c.html is missing will match both the Windows and Linux dbg | 699 # (3) c.html is missing will match both the Windows and Linux dbg |
700 # builders which are missing so it'll emit an error on the first one. | 700 # builders which are missing so it'll emit an error on the first one. |
701 self._port.all_build_types = ('release', 'debug') | 701 self._port.all_build_types = ('release', 'debug') |
702 self._port.all_systems = (('win7', 'x86'), | 702 self._port.all_systems = (('win7', 'x86'), |
703 ('precise', 'x86_64')) | 703 ('precise', 'x86_64')) |
704 | 704 |
705 self._parse_expectations(test_expectations_before) | 705 self._parse_expectations(test_expectations_before) |
706 self._expectation_factory._all_results_by_builder = { | 706 self._expectation_factory._all_results_by_builder = { |
707 'WebKit Linux': { | 707 'WebKit Linux Precise': { |
708 "test/a.html": ["PASS", "PASS", "PASS"], | 708 "test/a.html": ["PASS", "PASS", "PASS"], |
709 "test/b.html": ["PASS", "PASS", "PASS"], | 709 "test/b.html": ["PASS", "PASS", "PASS"], |
710 "test/c.html": ["PASS", "IMAGE", "PASS"], | 710 "test/c.html": ["PASS", "IMAGE", "PASS"], |
711 "test/d.html": ["PASS", "PASS", "PASS"], | 711 "test/d.html": ["PASS", "PASS", "PASS"], |
712 "test/e.html": ["PASS", "IMAGE", "PASS"], | 712 "test/e.html": ["PASS", "IMAGE", "PASS"], |
713 } | 713 } |
714 } | 714 } |
715 | 715 |
716 updated_expectations = ( | 716 updated_expectations = ( |
717 self._flake_remover.get_updated_test_expectations()) | 717 self._flake_remover.get_updated_test_expectations()) |
(...skipping 18 matching lines...) Expand all Loading... | |
736 test_expectations_before = """ | 736 test_expectations_before = """ |
737 Bug(test) [ Linux ] test/a.html [ Failure Pass ] | 737 Bug(test) [ Linux ] test/a.html [ Failure Pass ] |
738 # This line won't emit an error since the Linux Release results | 738 # This line won't emit an error since the Linux Release results |
739 # exist. | 739 # exist. |
740 Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] | 740 Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] |
741 Bug(test) [ Release ] test/c.html [ Failure Pass ] | 741 Bug(test) [ Release ] test/c.html [ Failure Pass ] |
742 # This line is not flaky so we shouldn't even check the results. | 742 # This line is not flaky so we shouldn't even check the results. |
743 Bug(test) [ Linux ] test/d.html [ Failure ]""" | 743 Bug(test) [ Linux ] test/d.html [ Failure ]""" |
744 | 744 |
745 self._define_builders({ | 745 self._define_builders({ |
746 "WebKit Linux": { | 746 "WebKit Linux Precise": { |
747 "port_name": "linux-precise", | 747 "port_name": "linux-precise", |
748 "specifiers": ['Precise', 'Release'] | 748 "specifiers": ['Precise', 'Release'] |
749 }, | 749 }, |
750 "WebKit Linux (dbg)": { | 750 "WebKit Linux Precise (dbg)": { |
751 "port_name": "linux-precise", | 751 "port_name": "linux-precise", |
752 "specifiers": ['Precise', 'Debug'] | 752 "specifiers": ['Precise', 'Debug'] |
753 }, | 753 }, |
754 "WebKit Win7": { | 754 "WebKit Win7": { |
755 "port_name": "win-win7", | 755 "port_name": "win-win7", |
756 "specifiers": ['Win7', 'Release'] | 756 "specifiers": ['Win7', 'Release'] |
757 }, | 757 }, |
758 "WebKit Win7 (dbg)": { | 758 "WebKit Win7 (dbg)": { |
759 "port_name": "win-win7", | 759 "port_name": "win-win7", |
760 "specifiers": ['Win7', 'Debug'] | 760 "specifiers": ['Win7', 'Debug'] |
761 }, | 761 }, |
762 }) | 762 }) |
763 | 763 |
764 # Two warnings and two errors should be emitted: | 764 # Two warnings and two errors should be emitted: |
765 # (1) A warning since the results don't contain anything for the Linux | 765 # (1) A warning since the results don't contain anything for the Linux |
766 # (dbg) builder | 766 # (dbg) builder |
767 # (2) A warning since the results don't contain anything for the Win | 767 # (2) A warning since the results don't contain anything for the Win |
768 # release builder | 768 # release builder |
769 # (3) The first line needs and is missing results for Linux (dbg). | 769 # (3) The first line needs and is missing results for Linux (dbg). |
770 # (4) The third line needs and is missing results for Win Release. | 770 # (4) The third line needs and is missing results for Win Release. |
771 self._port.all_build_types = ('release', 'debug') | 771 self._port.all_build_types = ('release', 'debug') |
772 self._port.all_systems = (('win7', 'x86'), | 772 self._port.all_systems = (('win7', 'x86'), |
773 ('precise', 'x86_64')) | 773 ('precise', 'x86_64')) |
774 | 774 |
775 self._parse_expectations(test_expectations_before) | 775 self._parse_expectations(test_expectations_before) |
776 self._expectation_factory._all_results_by_builder = { | 776 self._expectation_factory._all_results_by_builder = { |
777 'WebKit Linux': { | 777 'WebKit Linux Precise': { |
778 "test/a.html": ["PASS", "PASS", "PASS"], | 778 "test/a.html": ["PASS", "PASS", "PASS"], |
779 "test/b.html": ["PASS", "IMAGE", "PASS"], | 779 "test/b.html": ["PASS", "IMAGE", "PASS"], |
780 "test/c.html": ["PASS", "PASS", "PASS"], | 780 "test/c.html": ["PASS", "PASS", "PASS"], |
781 "test/d.html": ["PASS", "PASS", "PASS"], | 781 "test/d.html": ["PASS", "PASS", "PASS"], |
782 }, | 782 }, |
783 'WebKit Win7 (dbg)': { | 783 'WebKit Win7 (dbg)': { |
784 "test/a.html": ["PASS", "PASS", "PASS"], | 784 "test/a.html": ["PASS", "PASS", "PASS"], |
785 "test/b.html": ["PASS", "PASS", "PASS"], | 785 "test/b.html": ["PASS", "PASS", "PASS"], |
786 "test/c.html": ["PASS", "PASS", "PASS"], | 786 "test/c.html": ["PASS", "PASS", "PASS"], |
787 "test/d.html": ["PASS", "PASS", "PASS"], | 787 "test/d.html": ["PASS", "PASS", "PASS"], |
788 }, | 788 }, |
789 } | 789 } |
790 | 790 |
791 updated_expectations = ( | 791 updated_expectations = ( |
792 self._flake_remover.get_updated_test_expectations()) | 792 self._flake_remover.get_updated_test_expectations()) |
793 self.assertLog([ | 793 self.assertLog([ |
794 'WARNING: Downloaded results are missing results for builder "WebKit Linux (dbg)"\n', | 794 'WARNING: Downloaded results are missing results for builder "WebKit Linux Precise (dbg)"\n', |
795 'WARNING: Downloaded results are missing results for builder "WebKit Win7"\n', | 795 'WARNING: Downloaded results are missing results for builder "WebKit Win7"\n', |
796 'ERROR: Failed to find results for builder "WebKit Linux (dbg)"\n', | 796 'ERROR: Failed to find results for builder "WebKit Linux Precise (db g)"\n', |
797 'ERROR: Failed to find results for builder "WebKit Win7"\n', | 797 'ERROR: Failed to find results for builder "WebKit Win7"\n', |
798 ]) | 798 ]) |
799 | 799 |
800 # Also make sure we didn't remove any lines if some builders were | 800 # Also make sure we didn't remove any lines if some builders were |
801 # missing. | 801 # missing. |
802 self._assert_expectations_match( | 802 self._assert_expectations_match( |
803 updated_expectations, test_expectations_before) | 803 updated_expectations, test_expectations_before) |
804 | 804 |
805 def test_harness_updates_file(self): | 805 def test_harness_updates_file(self): |
806 """Tests that the call harness updates the TestExpectations file.""" | 806 """Tests that the call harness updates the TestExpectations file.""" |
807 | 807 |
808 self._define_builders({ | 808 self._define_builders({ |
809 "WebKit Linux": { | 809 "WebKit Linux Precise": { |
810 "port_name": "linux-precise", | 810 "port_name": "linux-precise", |
811 "specifiers": ['Precise', 'Release'] | 811 "specifiers": ['Precise', 'Release'] |
812 }, | 812 }, |
813 "WebKit Linux (dbg)": { | 813 "WebKit Linux Precise (dbg)": { |
814 "port_name": "linux-precise", | 814 "port_name": "linux-precise", |
815 "specifiers": ['Precise', 'Debug'] | 815 "specifiers": ['Precise', 'Debug'] |
816 }, | 816 }, |
817 }) | 817 }) |
818 | 818 |
819 # Setup the mock host and port. | 819 # Setup the mock host and port. |
820 host = MockHost() | 820 host = MockHost() |
821 host.port_factory = FakePortFactory(host) | 821 host.port_factory = FakePortFactory(host) |
822 host.port_factory._all_build_types = ('release', 'debug') | 822 host.port_factory._all_build_types = ('release', 'debug') |
823 host.port_factory._all_systems = (('precise', 'x86_64'),) | 823 host.port_factory._all_systems = (('precise', 'x86_64'),) |
(...skipping 12 matching lines...) Expand all Loading... | |
836 Bug(test) [ Linux ] test/d.html [ Failure ]""" | 836 Bug(test) [ Linux ] test/d.html [ Failure ]""" |
837 files = { | 837 files = { |
838 test_expectation_path: test_expectations | 838 test_expectation_path: test_expectations |
839 } | 839 } |
840 host.filesystem = MockFileSystem(files) | 840 host.filesystem = MockFileSystem(files) |
841 self._write_tests_into_filesystem(host.filesystem) | 841 self._write_tests_into_filesystem(host.filesystem) |
842 | 842 |
843 # Write out the fake builder bot results. | 843 # Write out the fake builder bot results. |
844 expectation_factory = FakeBotTestExpectationsFactory() | 844 expectation_factory = FakeBotTestExpectationsFactory() |
845 expectation_factory._all_results_by_builder = { | 845 expectation_factory._all_results_by_builder = { |
846 'WebKit Linux': { | 846 'WebKit Linux Precise': { |
847 "test/a.html": ["PASS", "PASS", "PASS"], | 847 "test/a.html": ["PASS", "PASS", "PASS"], |
848 "test/b.html": ["PASS", "IMAGE", "PASS"], | 848 "test/b.html": ["PASS", "IMAGE", "PASS"], |
849 "test/c.html": ["PASS", "PASS", "PASS"], | 849 "test/c.html": ["PASS", "PASS", "PASS"], |
850 "test/d.html": ["PASS", "PASS", "PASS"], | 850 "test/d.html": ["PASS", "PASS", "PASS"], |
851 }, | 851 }, |
852 'WebKit Linux (dbg)': { | 852 'WebKit Linux Precise (dbg)': { |
853 "test/a.html": ["PASS", "PASS", "PASS"], | 853 "test/a.html": ["PASS", "PASS", "PASS"], |
854 "test/b.html": ["PASS", "PASS", "PASS"], | 854 "test/b.html": ["PASS", "PASS", "PASS"], |
855 "test/c.html": ["PASS", "PASS", "PASS"], | 855 "test/c.html": ["PASS", "PASS", "PASS"], |
856 "test/d.html": ["IMAGE", "PASS", "PASS"], | 856 "test/d.html": ["IMAGE", "PASS", "PASS"], |
857 }, | 857 }, |
858 } | 858 } |
859 | 859 |
860 main(host, expectation_factory, []) | 860 main(host, expectation_factory, []) |
861 | 861 |
862 self.assertEqual(host.filesystem.files[test_expectation_path], ( | 862 self.assertEqual(host.filesystem.files[test_expectation_path], ( |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
897 ]) | 897 ]) |
898 self.assertFalse(host.filesystem.isfile(test_expectation_path)) | 898 self.assertFalse(host.filesystem.isfile(test_expectation_path)) |
899 | 899 |
900 def test_harness_remove_all(self): | 900 def test_harness_remove_all(self): |
901 """Tests that removing all expectations doesn't delete the file. | 901 """Tests that removing all expectations doesn't delete the file. |
902 | 902 |
903 Make sure we're prepared for the day when we exterminated flakes. | 903 Make sure we're prepared for the day when we exterminated flakes. |
904 """ | 904 """ |
905 | 905 |
906 self._define_builders({ | 906 self._define_builders({ |
907 "WebKit Linux": { | 907 "WebKit Linux Precise": { |
908 "port_name": "linux-precise", | 908 "port_name": "linux-precise", |
909 "specifiers": ['Precise', 'Release'] | 909 "specifiers": ['Precise', 'Release'] |
910 }, | 910 }, |
911 "WebKit Linux (dbg)": { | 911 "WebKit Linux Precise (dbg)": { |
912 "port_name": "linux-precise", | 912 "port_name": "linux-precise", |
913 "specifiers": ['Precise', 'Debug'] | 913 "specifiers": ['Precise', 'Debug'] |
914 }, | 914 }, |
915 }) | 915 }) |
916 | 916 |
917 # Setup the mock host and port. | 917 # Setup the mock host and port. |
918 host = MockHost() | 918 host = MockHost() |
919 host.port_factory = FakePortFactory(host) | 919 host.port_factory = FakePortFactory(host) |
920 host.port_factory._all_build_types = ('release', 'debug') | 920 host.port_factory._all_build_types = ('release', 'debug') |
921 host.port_factory._all_systems = (('precise', 'x86_64'),) | 921 host.port_factory._all_systems = (('precise', 'x86_64'),) |
922 | 922 |
923 # Write out a fake TestExpectations file. | 923 # Write out a fake TestExpectations file. |
924 test_expectation_path = ( | 924 test_expectation_path = ( |
925 host.port_factory.get().path_to_generic_test_expectations_file()) | 925 host.port_factory.get().path_to_generic_test_expectations_file()) |
926 test_expectations = """ | 926 test_expectations = """ |
927 # Remove since passing on both bots. | 927 # Remove since passing on both bots. |
928 Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" | 928 Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" |
929 | 929 |
930 files = { | 930 files = { |
931 test_expectation_path: test_expectations | 931 test_expectation_path: test_expectations |
932 } | 932 } |
933 host.filesystem = MockFileSystem(files) | 933 host.filesystem = MockFileSystem(files) |
934 self._write_tests_into_filesystem(host.filesystem) | 934 self._write_tests_into_filesystem(host.filesystem) |
935 | 935 |
936 # Write out the fake builder bot results. | 936 # Write out the fake builder bot results. |
937 expectation_factory = FakeBotTestExpectationsFactory() | 937 expectation_factory = FakeBotTestExpectationsFactory() |
938 expectation_factory._all_results_by_builder = { | 938 expectation_factory._all_results_by_builder = { |
939 'WebKit Linux': { | 939 'WebKit Linux Precise': { |
940 "test/a.html": ["PASS", "PASS", "PASS"], | 940 "test/a.html": ["PASS", "PASS", "PASS"], |
941 }, | 941 }, |
942 'WebKit Linux (dbg)': { | 942 'WebKit Linux Precise (dbg)': { |
943 "test/a.html": ["PASS", "PASS", "PASS"], | 943 "test/a.html": ["PASS", "PASS", "PASS"], |
944 }, | 944 }, |
945 } | 945 } |
946 | 946 |
947 main(host, expectation_factory, []) | 947 main(host, expectation_factory, []) |
948 | 948 |
949 self.assertTrue(host.filesystem.isfile(test_expectation_path)) | 949 self.assertTrue(host.filesystem.isfile(test_expectation_path)) |
950 self.assertEqual(host.filesystem.files[test_expectation_path], '') | 950 self.assertEqual(host.filesystem.files[test_expectation_path], '') |
OLD | NEW |