OLD | NEW |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 from collections import OrderedDict | 5 from collections import OrderedDict |
6 import logging | 6 import logging |
7 | 7 |
8 from webkitpy.common.host_mock import MockHost | 8 from webkitpy.common.host_mock import MockHost |
9 from webkitpy.common.system.filesystem_mock import MockFileSystem | 9 from webkitpy.common.system.filesystem_mock import MockFileSystem |
10 from webkitpy.common.system.logtesting import LoggingTestCase, LogTesting | 10 from webkitpy.common.system.logtesting import LoggingTestCase, LogTesting |
(...skipping 18 matching lines...) Expand all Loading... |
29 | 29 |
30 | 30 |
31 class FakeBotTestExpectationsFactory(object): | 31 class FakeBotTestExpectationsFactory(object): |
32 | 32 |
33 def __init__(self): | 33 def __init__(self): |
34 """The distinct results seen in at least one run of the test. | 34 """The distinct results seen in at least one run of the test. |
35 E.g. if the bot results for mytest.html are: | 35 E.g. if the bot results for mytest.html are: |
36 PASS PASS FAIL PASS TIMEOUT | 36 PASS PASS FAIL PASS TIMEOUT |
37 then _all_results_by_builder would be: | 37 then _all_results_by_builder would be: |
38 { | 38 { |
39 'WebKit Linux' : { | 39 'WebKit Linux Precise' : { |
40 'mytest.html': ['FAIL', 'PASS', 'TIMEOUT'] | 40 'mytest.html': ['FAIL', 'PASS', 'TIMEOUT'] |
41 } | 41 } |
42 } | 42 } |
43 """ | 43 """ |
44 self._all_results_by_builder = {} | 44 self._all_results_by_builder = {} |
45 | 45 |
46 def expectations_for_builder(self, builder): | 46 def expectations_for_builder(self, builder): |
47 if builder not in self._all_results_by_builder: | 47 if builder not in self._all_results_by_builder: |
48 return None | 48 return None |
49 | 49 |
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
148 # Even though the results show all passing, none of the | 148 # Even though the results show all passing, none of the |
149 # expectations are flaky so we shouldn't remove any. | 149 # expectations are flaky so we shouldn't remove any. |
150 Bug(test) test/a.html [ Pass ] | 150 Bug(test) test/a.html [ Pass ] |
151 Bug(test) test/b.html [ Timeout ] | 151 Bug(test) test/b.html [ Timeout ] |
152 Bug(test) test/c.html [ Failure Timeout ] | 152 Bug(test) test/c.html [ Failure Timeout ] |
153 Bug(test) test/d.html [ Rebaseline ] | 153 Bug(test) test/d.html [ Rebaseline ] |
154 Bug(test) test/e.html [ NeedsManualRebaseline ] | 154 Bug(test) test/e.html [ NeedsManualRebaseline ] |
155 Bug(test) test/f.html [ NeedsRebaseline ]""" | 155 Bug(test) test/f.html [ NeedsRebaseline ]""" |
156 | 156 |
157 self._define_builders({ | 157 self._define_builders({ |
158 "WebKit Linux": { | 158 "WebKit Linux Precise": { |
159 "port_name": "linux-precise", | 159 "port_name": "linux-precise", |
160 "specifiers": ['Precise', 'Release'] | 160 "specifiers": ['Precise', 'Release'] |
161 }, | 161 }, |
162 }) | 162 }) |
163 self._port.all_build_types = ('release',) | 163 self._port.all_build_types = ('release',) |
164 self._port.all_systems = (('precise', 'x86_64'),) | 164 self._port.all_systems = (('precise', 'x86_64'),) |
165 | 165 |
166 self._parse_expectations(test_expectations_before) | 166 self._parse_expectations(test_expectations_before) |
167 self._expectation_factory._all_results_by_builder = { | 167 self._expectation_factory._all_results_by_builder = { |
168 'WebKit Linux': { | 168 'WebKit Linux Precise': { |
169 "test/a.html": ["PASS", "PASS"], | 169 "test/a.html": ["PASS", "PASS"], |
170 "test/b.html": ["PASS", "PASS"], | 170 "test/b.html": ["PASS", "PASS"], |
171 "test/c.html": ["PASS", "PASS"], | 171 "test/c.html": ["PASS", "PASS"], |
172 "test/d.html": ["PASS", "PASS"], | 172 "test/d.html": ["PASS", "PASS"], |
173 "test/e.html": ["PASS", "PASS"], | 173 "test/e.html": ["PASS", "PASS"], |
174 "test/f.html": ["PASS", "PASS"], | 174 "test/f.html": ["PASS", "PASS"], |
175 } | 175 } |
176 } | 176 } |
177 updated_expectations = ( | 177 updated_expectations = ( |
178 self._flake_remover.get_updated_test_expectations()) | 178 self._flake_remover.get_updated_test_expectations()) |
179 self._assert_expectations_match( | 179 self._assert_expectations_match( |
180 updated_expectations, test_expectations_before) | 180 updated_expectations, test_expectations_before) |
181 | 181 |
182 def test_dont_remove_skip(self): | 182 def test_dont_remove_skip(self): |
183 """Tests that lines with Skip are untouched. | 183 """Tests that lines with Skip are untouched. |
184 | 184 |
185 If a line is marked as Skip, it will eventually contain no results, | 185 If a line is marked as Skip, it will eventually contain no results, |
186 which is indistinguishable from "All Passing" so don't remove since we | 186 which is indistinguishable from "All Passing" so don't remove since we |
187 don't know what the results actually are. | 187 don't know what the results actually are. |
188 """ | 188 """ |
189 test_expectations_before = """ | 189 test_expectations_before = """ |
190 # Skip expectations should never be removed. | 190 # Skip expectations should never be removed. |
191 Bug(test) test/a.html [ Skip ] | 191 Bug(test) test/a.html [ Skip ] |
192 Bug(test) test/b.html [ Skip ] | 192 Bug(test) test/b.html [ Skip ] |
193 Bug(test) test/c.html [ Skip ]""" | 193 Bug(test) test/c.html [ Skip ]""" |
194 | 194 |
195 self._define_builders({ | 195 self._define_builders({ |
196 "WebKit Linux": { | 196 "WebKit Linux Precise": { |
197 "port_name": "linux-precise", | 197 "port_name": "linux-precise", |
198 "specifiers": ['Precise', 'Release'] | 198 "specifiers": ['Precise', 'Release'] |
199 }, | 199 }, |
200 }) | 200 }) |
201 self._port.all_build_types = ('release',) | 201 self._port.all_build_types = ('release',) |
202 self._port.all_systems = (('precise', 'x86_64'),) | 202 self._port.all_systems = (('precise', 'x86_64'),) |
203 | 203 |
204 self._parse_expectations(test_expectations_before) | 204 self._parse_expectations(test_expectations_before) |
205 self._expectation_factory._all_results_by_builder = { | 205 self._expectation_factory._all_results_by_builder = { |
206 'WebKit Linux': { | 206 'WebKit Linux Precise': { |
207 "test/a.html": ["PASS", "PASS"], | 207 "test/a.html": ["PASS", "PASS"], |
208 "test/b.html": ["PASS", "IMAGE"], | 208 "test/b.html": ["PASS", "IMAGE"], |
209 } | 209 } |
210 } | 210 } |
211 updated_expectations = ( | 211 updated_expectations = ( |
212 self._flake_remover.get_updated_test_expectations()) | 212 self._flake_remover.get_updated_test_expectations()) |
213 self._assert_expectations_match( | 213 self._assert_expectations_match( |
214 updated_expectations, test_expectations_before) | 214 updated_expectations, test_expectations_before) |
215 | 215 |
216 def test_dont_remove_rebaselines(self): | 216 def test_dont_remove_rebaselines(self): |
217 """Tests that lines with rebaseline expectations are untouched.""" | 217 """Tests that lines with rebaseline expectations are untouched.""" |
218 test_expectations_before = """ | 218 test_expectations_before = """ |
219 # Even though the results show all passing, none of the | 219 # Even though the results show all passing, none of the |
220 # expectations are flaky so we shouldn't remove any. | 220 # expectations are flaky so we shouldn't remove any. |
221 Bug(test) test/a.html [ Failure NeedsRebaseline Pass ] | 221 Bug(test) test/a.html [ Failure NeedsRebaseline Pass ] |
222 Bug(test) test/b.html [ Failure Pass Rebaseline ] | 222 Bug(test) test/b.html [ Failure Pass Rebaseline ] |
223 Bug(test) test/c.html [ Failure NeedsManualRebaseline Pass ]""" | 223 Bug(test) test/c.html [ Failure NeedsManualRebaseline Pass ]""" |
224 | 224 |
225 self._define_builders({ | 225 self._define_builders({ |
226 "WebKit Linux": { | 226 "WebKit Linux Precise": { |
227 "port_name": "linux-precise", | 227 "port_name": "linux-precise", |
228 "specifiers": ['Precise', 'Release'] | 228 "specifiers": ['Precise', 'Release'] |
229 }, | 229 }, |
230 }) | 230 }) |
231 self._port.all_build_types = ('release',) | 231 self._port.all_build_types = ('release',) |
232 self._port.all_systems = (('precise', 'x86_64'),) | 232 self._port.all_systems = (('precise', 'x86_64'),) |
233 | 233 |
234 self._parse_expectations(test_expectations_before) | 234 self._parse_expectations(test_expectations_before) |
235 self._expectation_factory._all_results_by_builder = { | 235 self._expectation_factory._all_results_by_builder = { |
236 'WebKit Linux': { | 236 'WebKit Linux Precise': { |
237 "test/a.html": ["PASS", "PASS"], | 237 "test/a.html": ["PASS", "PASS"], |
238 "test/b.html": ["PASS", "PASS"], | 238 "test/b.html": ["PASS", "PASS"], |
239 "test/c.html": ["PASS", "PASS"] | 239 "test/c.html": ["PASS", "PASS"] |
240 } | 240 } |
241 } | 241 } |
242 updated_expectations = ( | 242 updated_expectations = ( |
243 self._flake_remover.get_updated_test_expectations()) | 243 self._flake_remover.get_updated_test_expectations()) |
244 self._assert_expectations_match( | 244 self._assert_expectations_match( |
245 updated_expectations, test_expectations_before) | 245 updated_expectations, test_expectations_before) |
246 | 246 |
247 def test_all_failure_types(self): | 247 def test_all_failure_types(self): |
248 """Tests that all failure types are treated as failure.""" | 248 """Tests that all failure types are treated as failure.""" |
249 test_expectations_before = ( | 249 test_expectations_before = ( |
250 """Bug(test) test/a.html [ Failure Pass ] | 250 """Bug(test) test/a.html [ Failure Pass ] |
251 Bug(test) test/b.html [ Failure Pass ] | 251 Bug(test) test/b.html [ Failure Pass ] |
252 Bug(test) test/c.html [ Failure Pass ] | 252 Bug(test) test/c.html [ Failure Pass ] |
253 Bug(test) test/d.html [ Failure Pass ] | 253 Bug(test) test/d.html [ Failure Pass ] |
254 # Remove these two since CRASH and TIMEOUT aren't considered | 254 # Remove these two since CRASH and TIMEOUT aren't considered |
255 # Failure. | 255 # Failure. |
256 Bug(test) test/e.html [ Failure Pass ] | 256 Bug(test) test/e.html [ Failure Pass ] |
257 Bug(test) test/f.html [ Failure Pass ]""") | 257 Bug(test) test/f.html [ Failure Pass ]""") |
258 | 258 |
259 self._define_builders({ | 259 self._define_builders({ |
260 "WebKit Linux": { | 260 "WebKit Linux Precise": { |
261 "port_name": "linux-precise", | 261 "port_name": "linux-precise", |
262 "specifiers": ['Precise', 'Release'] | 262 "specifiers": ['Precise', 'Release'] |
263 }, | 263 }, |
264 }) | 264 }) |
265 self._port.all_build_types = ('release',) | 265 self._port.all_build_types = ('release',) |
266 self._port.all_systems = (('precise', 'x86_64'),) | 266 self._port.all_systems = (('precise', 'x86_64'),) |
267 | 267 |
268 self._parse_expectations(test_expectations_before) | 268 self._parse_expectations(test_expectations_before) |
269 self._expectation_factory._all_results_by_builder = { | 269 self._expectation_factory._all_results_by_builder = { |
270 'WebKit Linux': { | 270 'WebKit Linux Precise': { |
271 "test/a.html": ["PASS", "IMAGE"], | 271 "test/a.html": ["PASS", "IMAGE"], |
272 "test/b.html": ["PASS", "TEXT"], | 272 "test/b.html": ["PASS", "TEXT"], |
273 "test/c.html": ["PASS", "IMAGE+TEXT"], | 273 "test/c.html": ["PASS", "IMAGE+TEXT"], |
274 "test/d.html": ["PASS", "AUDIO"], | 274 "test/d.html": ["PASS", "AUDIO"], |
275 "test/e.html": ["PASS", "CRASH"], | 275 "test/e.html": ["PASS", "CRASH"], |
276 "test/f.html": ["PASS", "TIMEOUT"], | 276 "test/f.html": ["PASS", "TIMEOUT"], |
277 } | 277 } |
278 } | 278 } |
279 updated_expectations = ( | 279 updated_expectations = ( |
280 self._flake_remover.get_updated_test_expectations()) | 280 self._flake_remover.get_updated_test_expectations()) |
(...skipping 13 matching lines...) Expand all Loading... |
294 """ | 294 """ |
295 test_expectations_before = ( | 295 test_expectations_before = ( |
296 """# Remove this since it's passing all runs. | 296 """# Remove this since it's passing all runs. |
297 Bug(test) test/a.html [ Failure Pass ] | 297 Bug(test) test/a.html [ Failure Pass ] |
298 # Remove this since, although there's a failure, it's not a timeout. | 298 # Remove this since, although there's a failure, it's not a timeout. |
299 Bug(test) test/b.html [ Pass Timeout ] | 299 Bug(test) test/b.html [ Pass Timeout ] |
300 # Keep since we have both crashes and passes. | 300 # Keep since we have both crashes and passes. |
301 Bug(test) test/c.html [ Crash Pass ]""") | 301 Bug(test) test/c.html [ Crash Pass ]""") |
302 | 302 |
303 self._define_builders({ | 303 self._define_builders({ |
304 "WebKit Linux": { | 304 "WebKit Linux Precise": { |
305 "port_name": "linux-precise", | 305 "port_name": "linux-precise", |
306 "specifiers": ['Precise', 'Release'] | 306 "specifiers": ['Precise', 'Release'] |
307 }, | 307 }, |
308 }) | 308 }) |
309 self._port.all_build_types = ('release',) | 309 self._port.all_build_types = ('release',) |
310 self._port.all_systems = (('precise', 'x86_64'),) | 310 self._port.all_systems = (('precise', 'x86_64'),) |
311 | 311 |
312 self._parse_expectations(test_expectations_before) | 312 self._parse_expectations(test_expectations_before) |
313 self._expectation_factory._all_results_by_builder = { | 313 self._expectation_factory._all_results_by_builder = { |
314 'WebKit Linux': { | 314 'WebKit Linux Precise': { |
315 "test/a.html": ["PASS", "PASS", "PASS"], | 315 "test/a.html": ["PASS", "PASS", "PASS"], |
316 "test/b.html": ["PASS", "IMAGE", "PASS"], | 316 "test/b.html": ["PASS", "IMAGE", "PASS"], |
317 "test/c.html": ["PASS", "CRASH", "PASS"], | 317 "test/c.html": ["PASS", "CRASH", "PASS"], |
318 } | 318 } |
319 } | 319 } |
320 updated_expectations = ( | 320 updated_expectations = ( |
321 self._flake_remover.get_updated_test_expectations()) | 321 self._flake_remover.get_updated_test_expectations()) |
322 self._assert_expectations_match(updated_expectations, ( | 322 self._assert_expectations_match(updated_expectations, ( |
323 """# Keep since we have both crashes and passes. | 323 """# Keep since we have both crashes and passes. |
324 Bug(test) test/c.html [ Crash Pass ]""")) | 324 Bug(test) test/c.html [ Crash Pass ]""")) |
325 | 325 |
326 def test_all_failure_case(self): | 326 def test_all_failure_case(self): |
327 """Tests that results with all failures are not treated as non-flaky.""" | 327 """Tests that results with all failures are not treated as non-flaky.""" |
328 test_expectations_before = ( | 328 test_expectations_before = ( |
329 """# Keep since it's all failures. | 329 """# Keep since it's all failures. |
330 Bug(test) test/a.html [ Failure Pass ]""") | 330 Bug(test) test/a.html [ Failure Pass ]""") |
331 | 331 |
332 self._define_builders({ | 332 self._define_builders({ |
333 "WebKit Linux": { | 333 "WebKit Linux Precise": { |
334 "port_name": "linux-precise", | 334 "port_name": "linux-precise", |
335 "specifiers": ['Precise', 'Release'] | 335 "specifiers": ['Precise', 'Release'] |
336 }, | 336 }, |
337 }) | 337 }) |
338 self._port.all_build_types = ('release',) | 338 self._port.all_build_types = ('release',) |
339 self._port.all_systems = (('precise', 'x86_64'),) | 339 self._port.all_systems = (('precise', 'x86_64'),) |
340 | 340 |
341 self._parse_expectations(test_expectations_before) | 341 self._parse_expectations(test_expectations_before) |
342 self._expectation_factory._all_results_by_builder = { | 342 self._expectation_factory._all_results_by_builder = { |
343 'WebKit Linux': { | 343 'WebKit Linux Precise': { |
344 "test/a.html": ["IMAGE", "IMAGE", "IMAGE"], | 344 "test/a.html": ["IMAGE", "IMAGE", "IMAGE"], |
345 } | 345 } |
346 } | 346 } |
347 updated_expectations = ( | 347 updated_expectations = ( |
348 self._flake_remover.get_updated_test_expectations()) | 348 self._flake_remover.get_updated_test_expectations()) |
349 self._assert_expectations_match(updated_expectations, ( | 349 self._assert_expectations_match(updated_expectations, ( |
350 """# Keep since it's all failures. | 350 """# Keep since it's all failures. |
351 Bug(test) test/a.html [ Failure Pass ]""")) | 351 Bug(test) test/a.html [ Failure Pass ]""")) |
352 | 352 |
353 def test_empty_test_expectations(self): | 353 def test_empty_test_expectations(self): |
354 """Running on an empty TestExpectations file outputs an empty file.""" | 354 """Running on an empty TestExpectations file outputs an empty file.""" |
355 test_expectations_before = "" | 355 test_expectations_before = "" |
356 | 356 |
357 self._define_builders({ | 357 self._define_builders({ |
358 "WebKit Linux": { | 358 "WebKit Linux Precise": { |
359 "port_name": "linux-precise", | 359 "port_name": "linux-precise", |
360 "specifiers": ['Precise', 'Release'] | 360 "specifiers": ['Precise', 'Release'] |
361 }, | 361 }, |
362 }) | 362 }) |
363 self._port.all_build_types = ('release',) | 363 self._port.all_build_types = ('release',) |
364 self._port.all_systems = (('precise', 'x86_64'),) | 364 self._port.all_systems = (('precise', 'x86_64'),) |
365 | 365 |
366 self._parse_expectations(test_expectations_before) | 366 self._parse_expectations(test_expectations_before) |
367 self._expectation_factory._all_results_by_builder = { | 367 self._expectation_factory._all_results_by_builder = { |
368 'WebKit Linux': { | 368 'WebKit Linux Precise': { |
369 "test/a.html": ["PASS", "PASS", "PASS"], | 369 "test/a.html": ["PASS", "PASS", "PASS"], |
370 } | 370 } |
371 } | 371 } |
372 updated_expectations = ( | 372 updated_expectations = ( |
373 self._flake_remover.get_updated_test_expectations()) | 373 self._flake_remover.get_updated_test_expectations()) |
374 self._assert_expectations_match(updated_expectations, "") | 374 self._assert_expectations_match(updated_expectations, "") |
375 | 375 |
376 def test_basic_multiple_builders(self): | 376 def test_basic_multiple_builders(self): |
377 """Tests basic functionality with multiple builders.""" | 377 """Tests basic functionality with multiple builders.""" |
378 test_expectations_before = ( | 378 test_expectations_before = ( |
379 """# Remove since it's passing on both builders. | 379 """# Remove since it's passing on both builders. |
380 Bug(test) test/a.html [ Failure Pass ] | 380 Bug(test) test/a.html [ Failure Pass ] |
381 # Keep since it's failing on the Mac builder. | 381 # Keep since it's failing on the Mac builder. |
382 Bug(test) test/b.html [ Failure Pass ] | 382 Bug(test) test/b.html [ Failure Pass ] |
383 # Keep since it's failing on the Linux builder. | 383 # Keep since it's failing on the Linux builder. |
384 Bug(test) test/c.html [ Failure Pass ]""") | 384 Bug(test) test/c.html [ Failure Pass ]""") |
385 | 385 |
386 self._define_builders({ | 386 self._define_builders({ |
387 "WebKit Linux": { | 387 "WebKit Linux Precise": { |
388 "port_name": "linux-precise", | 388 "port_name": "linux-precise", |
389 "specifiers": ['Precise', 'Release'] | 389 "specifiers": ['Precise', 'Release'] |
390 }, | 390 }, |
391 "WebKit Mac10.10": { | 391 "WebKit Mac10.10": { |
392 "port_name": "mac-mac10.10", | 392 "port_name": "mac-mac10.10", |
393 "specifiers": ['Mac10.10', 'Release'] | 393 "specifiers": ['Mac10.10', 'Release'] |
394 }, | 394 }, |
395 }) | 395 }) |
396 | 396 |
397 self._port.all_build_types = ('release',) | 397 self._port.all_build_types = ('release',) |
398 self._port.all_systems = (('mac10.10', 'x86'), | 398 self._port.all_systems = (('mac10.10', 'x86'), |
399 ('precise', 'x86_64')) | 399 ('precise', 'x86_64')) |
400 | 400 |
401 self._parse_expectations(test_expectations_before) | 401 self._parse_expectations(test_expectations_before) |
402 self._expectation_factory._all_results_by_builder = { | 402 self._expectation_factory._all_results_by_builder = { |
403 'WebKit Linux': { | 403 'WebKit Linux Precise': { |
404 "test/a.html": ["PASS", "PASS", "PASS"], | 404 "test/a.html": ["PASS", "PASS", "PASS"], |
405 "test/b.html": ["PASS", "PASS", "PASS"], | 405 "test/b.html": ["PASS", "PASS", "PASS"], |
406 "test/c.html": ["AUDIO", "AUDIO", "AUDIO"], | 406 "test/c.html": ["AUDIO", "AUDIO", "AUDIO"], |
407 }, | 407 }, |
408 'WebKit Mac10.10': { | 408 'WebKit Mac10.10': { |
409 "test/a.html": ["PASS", "PASS", "PASS"], | 409 "test/a.html": ["PASS", "PASS", "PASS"], |
410 "test/b.html": ["PASS", "PASS", "IMAGE"], | 410 "test/b.html": ["PASS", "PASS", "IMAGE"], |
411 "test/c.html": ["PASS", "PASS", "PASS"], | 411 "test/c.html": ["PASS", "PASS", "PASS"], |
412 }, | 412 }, |
413 } | 413 } |
(...skipping 15 matching lines...) Expand all Loading... |
429 # Remove since it's passing on both Linux and Windows builders. | 429 # Remove since it's passing on both Linux and Windows builders. |
430 Bug(test) [ Linux Win ] test/c.html [ Failure Pass ] | 430 Bug(test) [ Linux Win ] test/c.html [ Failure Pass ] |
431 # Remove since it's passing on Mac results | 431 # Remove since it's passing on Mac results |
432 Bug(test) [ Mac ] test/d.html [ Failure Pass ]""") | 432 Bug(test) [ Mac ] test/d.html [ Failure Pass ]""") |
433 | 433 |
434 self._define_builders({ | 434 self._define_builders({ |
435 "WebKit Win7": { | 435 "WebKit Win7": { |
436 "port_name": "win-win7", | 436 "port_name": "win-win7", |
437 "specifiers": ['Win7', 'Release'] | 437 "specifiers": ['Win7', 'Release'] |
438 }, | 438 }, |
439 "WebKit Linux": { | 439 "WebKit Linux Precise": { |
440 "port_name": "linux-precise", | 440 "port_name": "linux-precise", |
441 "specifiers": ['Precise', 'Release'] | 441 "specifiers": ['Precise', 'Release'] |
442 }, | 442 }, |
443 "WebKit Mac10.10": { | 443 "WebKit Mac10.10": { |
444 "port_name": "mac-mac10.10", | 444 "port_name": "mac-mac10.10", |
445 "specifiers": ['Mac10.10', 'Release'] | 445 "specifiers": ['Mac10.10', 'Release'] |
446 }, | 446 }, |
447 }) | 447 }) |
448 self._port.all_build_types = ('release',) | 448 self._port.all_build_types = ('release',) |
449 self._port.all_systems = (('mac10.10', 'x86'), | 449 self._port.all_systems = (('mac10.10', 'x86'), |
450 ('win7', 'x86'), | 450 ('win7', 'x86'), |
451 ('precise', 'x86_64')) | 451 ('precise', 'x86_64')) |
452 | 452 |
453 self._parse_expectations(test_expectations_before) | 453 self._parse_expectations(test_expectations_before) |
454 self._expectation_factory._all_results_by_builder = { | 454 self._expectation_factory._all_results_by_builder = { |
455 'WebKit Linux': { | 455 'WebKit Linux Precise': { |
456 "test/a.html": ["PASS", "PASS", "PASS"], | 456 "test/a.html": ["PASS", "PASS", "PASS"], |
457 "test/b.html": ["PASS", "PASS", "PASS"], | 457 "test/b.html": ["PASS", "PASS", "PASS"], |
458 "test/c.html": ["PASS", "PASS", "PASS"], | 458 "test/c.html": ["PASS", "PASS", "PASS"], |
459 "test/d.html": ["IMAGE", "PASS", "PASS"], | 459 "test/d.html": ["IMAGE", "PASS", "PASS"], |
460 }, | 460 }, |
461 'WebKit Mac10.10': { | 461 'WebKit Mac10.10': { |
462 "test/a.html": ["PASS", "PASS", "IMAGE"], | 462 "test/a.html": ["PASS", "PASS", "IMAGE"], |
463 "test/b.html": ["PASS", "IMAGE", "PASS"], | 463 "test/b.html": ["PASS", "IMAGE", "PASS"], |
464 "test/c.html": ["PASS", "IMAGE", "PASS"], | 464 "test/c.html": ["PASS", "IMAGE", "PASS"], |
465 "test/d.html": ["PASS", "PASS", "PASS"], | 465 "test/d.html": ["PASS", "PASS", "PASS"], |
(...skipping 29 matching lines...) Expand all Loading... |
495 | 495 |
496 self._define_builders({ | 496 self._define_builders({ |
497 "WebKit Win7": { | 497 "WebKit Win7": { |
498 "port_name": "win-win7", | 498 "port_name": "win-win7", |
499 "specifiers": ['Win7', 'Release'] | 499 "specifiers": ['Win7', 'Release'] |
500 }, | 500 }, |
501 "WebKit Win7 (dbg)": { | 501 "WebKit Win7 (dbg)": { |
502 "port_name": "win-win7", | 502 "port_name": "win-win7", |
503 "specifiers": ['Win7', 'Debug'] | 503 "specifiers": ['Win7', 'Debug'] |
504 }, | 504 }, |
505 "WebKit Linux": { | 505 "WebKit Linux Precise": { |
506 "port_name": "linux-precise", | 506 "port_name": "linux-precise", |
507 "specifiers": ['Precise', 'Release'] | 507 "specifiers": ['Precise', 'Release'] |
508 }, | 508 }, |
509 "WebKit Linux (dbg)": { | 509 "WebKit Linux Precise (dbg)": { |
510 "port_name": "linux-precise", | 510 "port_name": "linux-precise", |
511 "specifiers": ['Precise', 'Debug'] | 511 "specifiers": ['Precise', 'Debug'] |
512 }, | 512 }, |
513 }) | 513 }) |
514 self._port.all_build_types = ('release', 'debug') | 514 self._port.all_build_types = ('release', 'debug') |
515 self._port.all_systems = (('win7', 'x86'), | 515 self._port.all_systems = (('win7', 'x86'), |
516 ('precise', 'x86_64')) | 516 ('precise', 'x86_64')) |
517 | 517 |
518 self._parse_expectations(test_expectations_before) | 518 self._parse_expectations(test_expectations_before) |
519 self._expectation_factory._all_results_by_builder = { | 519 self._expectation_factory._all_results_by_builder = { |
520 'WebKit Linux': { | 520 'WebKit Linux Precise': { |
521 "test/a.html": ["PASS", "PASS", "PASS"], | 521 "test/a.html": ["PASS", "PASS", "PASS"], |
522 "test/b.html": ["PASS", "IMAGE", "PASS"], | 522 "test/b.html": ["PASS", "IMAGE", "PASS"], |
523 "test/c.html": ["PASS", "IMAGE", "PASS"], | 523 "test/c.html": ["PASS", "IMAGE", "PASS"], |
524 "test/d.html": ["PASS", "PASS", "PASS"], | 524 "test/d.html": ["PASS", "PASS", "PASS"], |
525 "test/e.html": ["PASS", "PASS", "PASS"], | 525 "test/e.html": ["PASS", "PASS", "PASS"], |
526 }, | 526 }, |
527 'WebKit Linux (dbg)': { | 527 'WebKit Linux Precise (dbg)': { |
528 "test/a.html": ["PASS", "IMAGE", "PASS"], | 528 "test/a.html": ["PASS", "IMAGE", "PASS"], |
529 "test/b.html": ["PASS", "PASS", "PASS"], | 529 "test/b.html": ["PASS", "PASS", "PASS"], |
530 "test/c.html": ["PASS", "PASS", "PASS"], | 530 "test/c.html": ["PASS", "PASS", "PASS"], |
531 "test/d.html": ["IMAGE", "PASS", "PASS"], | 531 "test/d.html": ["IMAGE", "PASS", "PASS"], |
532 "test/e.html": ["PASS", "PASS", "PASS"], | 532 "test/e.html": ["PASS", "PASS", "PASS"], |
533 }, | 533 }, |
534 'WebKit Win7 (dbg)': { | 534 'WebKit Win7 (dbg)': { |
535 "test/a.html": ["PASS", "PASS", "PASS"], | 535 "test/a.html": ["PASS", "PASS", "PASS"], |
536 "test/b.html": ["PASS", "PASS", "PASS"], | 536 "test/b.html": ["PASS", "PASS", "PASS"], |
537 "test/c.html": ["PASS", "PASS", "PASS"], | 537 "test/c.html": ["PASS", "PASS", "PASS"], |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
589 | 589 |
590 # Comment F - Keep since only b is removed | 590 # Comment F - Keep since only b is removed |
591 Bug(test) test/b.html [ Failure Pass ] | 591 Bug(test) test/b.html [ Failure Pass ] |
592 Bug(test) test/c.html [ Failure Pass ] | 592 Bug(test) test/c.html [ Failure Pass ] |
593 | 593 |
594 # Comment G - Should be removed since both d and e will be removed. | 594 # Comment G - Should be removed since both d and e will be removed. |
595 Bug(test) test/d.html [ Failure Pass ] | 595 Bug(test) test/d.html [ Failure Pass ] |
596 Bug(test) test/e.html [ Failure Pass ]""" | 596 Bug(test) test/e.html [ Failure Pass ]""" |
597 | 597 |
598 self._define_builders({ | 598 self._define_builders({ |
599 "WebKit Linux": { | 599 "WebKit Linux Precise": { |
600 "port_name": "linux-precise", | 600 "port_name": "linux-precise", |
601 "specifiers": ['Precise', 'Release'] | 601 "specifiers": ['Precise', 'Release'] |
602 }, | 602 }, |
603 }) | 603 }) |
604 self._port.all_build_types = ('release',) | 604 self._port.all_build_types = ('release',) |
605 self._port.all_systems = (('precise', 'x86_64'),) | 605 self._port.all_systems = (('precise', 'x86_64'),) |
606 | 606 |
607 self._parse_expectations(test_expectations_before) | 607 self._parse_expectations(test_expectations_before) |
608 self._expectation_factory._all_results_by_builder = { | 608 self._expectation_factory._all_results_by_builder = { |
609 'WebKit Linux': { | 609 'WebKit Linux Precise': { |
610 "test/a.html": ["PASS", "PASS", "PASS"], | 610 "test/a.html": ["PASS", "PASS", "PASS"], |
611 "test/b.html": ["PASS", "PASS", "PASS"], | 611 "test/b.html": ["PASS", "PASS", "PASS"], |
612 "test/c.html": ["PASS", "IMAGE", "PASS"], | 612 "test/c.html": ["PASS", "IMAGE", "PASS"], |
613 "test/d.html": ["PASS", "PASS", "PASS"], | 613 "test/d.html": ["PASS", "PASS", "PASS"], |
614 "test/e.html": ["PASS", "PASS", "PASS"], | 614 "test/e.html": ["PASS", "PASS", "PASS"], |
615 } | 615 } |
616 } | 616 } |
617 updated_expectations = ( | 617 updated_expectations = ( |
618 self._flake_remover.get_updated_test_expectations()) | 618 self._flake_remover.get_updated_test_expectations()) |
619 self._assert_expectations_match(updated_expectations, ( | 619 self._assert_expectations_match(updated_expectations, ( |
(...skipping 19 matching lines...) Expand all Loading... |
639 Bug(test) test/a.html [ Skip ] | 639 Bug(test) test/a.html [ Skip ] |
640 # This line shouldn't be removed either since it's not flaky. | 640 # This line shouldn't be removed either since it's not flaky. |
641 Bug(test) test/b.html [ Failure Timeout ] | 641 Bug(test) test/b.html [ Failure Timeout ] |
642 # The lines below should be removed since they're flaky but all runs | 642 # The lines below should be removed since they're flaky but all runs |
643 # are passing. | 643 # are passing. |
644 Bug(test) test/c.html [ Failure Pass ] | 644 Bug(test) test/c.html [ Failure Pass ] |
645 Bug(test) test/d.html [ Pass Timeout ] | 645 Bug(test) test/d.html [ Pass Timeout ] |
646 Bug(test) test/e.html [ Crash Pass ]""" | 646 Bug(test) test/e.html [ Crash Pass ]""" |
647 | 647 |
648 self._define_builders({ | 648 self._define_builders({ |
649 "WebKit Linux": { | 649 "WebKit Linux Precise": { |
650 "port_name": "linux-precise", | 650 "port_name": "linux-precise", |
651 "specifiers": ['Precise', 'Release'] | 651 "specifiers": ['Precise', 'Release'] |
652 }, | 652 }, |
653 }) | 653 }) |
654 self._port.all_build_types = ('release',) | 654 self._port.all_build_types = ('release',) |
655 self._port.all_systems = (('precise', 'x86_64'),) | 655 self._port.all_systems = (('precise', 'x86_64'),) |
656 | 656 |
657 self._parse_expectations(test_expectations_before) | 657 self._parse_expectations(test_expectations_before) |
658 self._expectation_factory._all_results_by_builder = { | 658 self._expectation_factory._all_results_by_builder = { |
659 'WebKit Linux': { | 659 'WebKit Linux Precise': { |
660 } | 660 } |
661 } | 661 } |
662 updated_expectations = ( | 662 updated_expectations = ( |
663 self._flake_remover.get_updated_test_expectations()) | 663 self._flake_remover.get_updated_test_expectations()) |
664 self._assert_expectations_match(updated_expectations, """ | 664 self._assert_expectations_match(updated_expectations, """ |
665 # A Skip expectation probably won't have any results but we | 665 # A Skip expectation probably won't have any results but we |
666 # shouldn't consider those passing so this line should remain. | 666 # shouldn't consider those passing so this line should remain. |
667 Bug(test) test/a.html [ Skip ] | 667 Bug(test) test/a.html [ Skip ] |
668 # This line shouldn't be removed either since it's not flaky. | 668 # This line shouldn't be removed either since it's not flaky. |
669 Bug(test) test/b.html [ Failure Timeout ]""") | 669 Bug(test) test/b.html [ Failure Timeout ]""") |
(...skipping 25 matching lines...) Expand all Loading... |
695 # This one is marked as flaky and there are some matching | 695 # This one is marked as flaky and there are some matching |
696 # configurations with no builders, but for all configurations | 696 # configurations with no builders, but for all configurations |
697 # with existing builders, it is non-flaky. | 697 # with existing builders, it is non-flaky. |
698 Bug(test) test/d.html [ Failure Pass ] | 698 Bug(test) test/d.html [ Failure Pass ] |
699 | 699 |
700 # This one only matches the existing linux release builder, | 700 # This one only matches the existing linux release builder, |
701 # and it's still flaky, so it shouldn't be removed. | 701 # and it's still flaky, so it shouldn't be removed. |
702 Bug(test) [ Linux Release ] test/e.html [ Failure Pass ]""" | 702 Bug(test) [ Linux Release ] test/e.html [ Failure Pass ]""" |
703 | 703 |
704 self._define_builders({ | 704 self._define_builders({ |
705 "WebKit Linux": { | 705 "WebKit Linux Precise": { |
706 "port_name": "linux-precise", | 706 "port_name": "linux-precise", |
707 "specifiers": ['Precise', 'Release'] | 707 "specifiers": ['Precise', 'Release'] |
708 }, | 708 }, |
709 }) | 709 }) |
710 | 710 |
711 self._port.all_build_types = ('release', 'debug') | 711 self._port.all_build_types = ('release', 'debug') |
712 self._port.all_systems = ( | 712 self._port.all_systems = ( |
713 ('win7', 'x86'), | 713 ('win7', 'x86'), |
714 ('precise', 'x86_64'), | 714 ('precise', 'x86_64'), |
715 ) | 715 ) |
716 | 716 |
717 self._parse_expectations(test_expectations_before) | 717 self._parse_expectations(test_expectations_before) |
718 self._expectation_factory._all_results_by_builder = { | 718 self._expectation_factory._all_results_by_builder = { |
719 'WebKit Linux': { | 719 'WebKit Linux Precise': { |
720 "test/a.html": ["PASS", "PASS", "PASS"], | 720 "test/a.html": ["PASS", "PASS", "PASS"], |
721 "test/b.html": ["PASS", "PASS", "PASS"], | 721 "test/b.html": ["PASS", "PASS", "PASS"], |
722 "test/c.html": ["PASS", "IMAGE", "PASS"], | 722 "test/c.html": ["PASS", "IMAGE", "PASS"], |
723 "test/d.html": ["PASS", "PASS", "PASS"], | 723 "test/d.html": ["PASS", "PASS", "PASS"], |
724 "test/e.html": ["PASS", "IMAGE", "PASS"], | 724 "test/e.html": ["PASS", "IMAGE", "PASS"], |
725 } | 725 } |
726 } | 726 } |
727 | 727 |
728 updated_expectations = ( | 728 updated_expectations = ( |
729 self._flake_remover.get_updated_test_expectations()) | 729 self._flake_remover.get_updated_test_expectations()) |
730 | 730 |
731 self.assertLog([ | 731 self.assertLog([ |
732 'DEBUG: No builder with config <win7, x86, release>\n', | 732 'DEBUG: No builder with config <win7, x86, release>\n', |
733 'DEBUG: No builder with config <win7, x86, debug>\n', | 733 'DEBUG: No builder with config <win7, x86, debug>\n', |
734 'WARNING: No matching builders for line, deleting line.\n', | 734 'WARNING: No matching builders for line, deleting line.\n', |
735 'INFO: Deleting line "Bug(test) [ Win ] test/a.html [ Failure Pass ]
"\n', | 735 'INFO: Deleting line "Bug(test) [ Win ] test/a.html [ Failure Pass ]
"\n', |
736 'DEBUG: No builder with config <precise, x86_64, debug>\n', | 736 'DEBUG: No builder with config <precise, x86_64, debug>\n', |
737 'DEBUG: Checked builders:\n WebKit Linux\n', | 737 'DEBUG: Checked builders:\n WebKit Linux Precise\n', |
738 'INFO: Deleting line "Bug(test) [ Linux ] test/b.html [ Failure Pass
]"\n', | 738 'INFO: Deleting line "Bug(test) [ Linux ] test/b.html [ Failure Pass
]"\n', |
739 'DEBUG: No builder with config <win7, x86, release>\n', | 739 'DEBUG: No builder with config <win7, x86, release>\n', |
740 'DEBUG: No builder with config <win7, x86, debug>\n', | 740 'DEBUG: No builder with config <win7, x86, debug>\n', |
741 'DEBUG: No builder with config <precise, x86_64, debug>\n', | 741 'DEBUG: No builder with config <precise, x86_64, debug>\n', |
742 'DEBUG: Checked builders:\n WebKit Linux\n', | 742 'DEBUG: Checked builders:\n WebKit Linux Precise\n', |
743 'INFO: Deleting line "Bug(test) test/d.html [ Failure Pass ]"\n', | 743 'INFO: Deleting line "Bug(test) test/d.html [ Failure Pass ]"\n', |
744 ]) | 744 ]) |
745 self._assert_expectations_match( | 745 self._assert_expectations_match( |
746 updated_expectations, | 746 updated_expectations, |
747 """ | 747 """ |
748 # No message should be emitted for this one because it's not | 748 # No message should be emitted for this one because it's not |
749 # marked as flaky, so we don't need to check builder results. | 749 # marked as flaky, so we don't need to check builder results. |
750 Bug(test) test/c.html [ Failure ] | 750 Bug(test) test/c.html [ Failure ] |
751 | 751 |
752 # This one only matches the existing linux release builder, | 752 # This one only matches the existing linux release builder, |
(...skipping 10 matching lines...) Expand all Loading... |
763 test_expectations_before = """ | 763 test_expectations_before = """ |
764 Bug(test) [ Linux ] test/a.html [ Failure Pass ] | 764 Bug(test) [ Linux ] test/a.html [ Failure Pass ] |
765 # This line won't emit an error since the Linux Release results | 765 # This line won't emit an error since the Linux Release results |
766 # exist. | 766 # exist. |
767 Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] | 767 Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] |
768 Bug(test) [ Release ] test/c.html [ Failure Pass ] | 768 Bug(test) [ Release ] test/c.html [ Failure Pass ] |
769 # This line is not flaky so we shouldn't even check the results. | 769 # This line is not flaky so we shouldn't even check the results. |
770 Bug(test) [ Linux ] test/d.html [ Failure ]""" | 770 Bug(test) [ Linux ] test/d.html [ Failure ]""" |
771 | 771 |
772 self._define_builders({ | 772 self._define_builders({ |
773 "WebKit Linux": { | 773 "WebKit Linux Precise": { |
774 "port_name": "linux-precise", | 774 "port_name": "linux-precise", |
775 "specifiers": ['Precise', 'Release'] | 775 "specifiers": ['Precise', 'Release'] |
776 }, | 776 }, |
777 "WebKit Linux (dbg)": { | 777 "WebKit Linux Precise (dbg)": { |
778 "port_name": "linux-precise", | 778 "port_name": "linux-precise", |
779 "specifiers": ['Precise', 'Debug'] | 779 "specifiers": ['Precise', 'Debug'] |
780 }, | 780 }, |
781 "WebKit Win7": { | 781 "WebKit Win7": { |
782 "port_name": "win-win7", | 782 "port_name": "win-win7", |
783 "specifiers": ['Win7', 'Release'] | 783 "specifiers": ['Win7', 'Release'] |
784 }, | 784 }, |
785 "WebKit Win7 (dbg)": { | 785 "WebKit Win7 (dbg)": { |
786 "port_name": "win-win7", | 786 "port_name": "win-win7", |
787 "specifiers": ['Win7', 'Debug'] | 787 "specifiers": ['Win7', 'Debug'] |
788 }, | 788 }, |
789 }) | 789 }) |
790 | 790 |
791 # Two warnings and two errors should be emitted: | 791 # Two warnings and two errors should be emitted: |
792 # (1) A warning since the results don't contain anything for the Linux | 792 # (1) A warning since the results don't contain anything for the Linux |
793 # (dbg) builder | 793 # (dbg) builder |
794 # (2) A warning since the results don't contain anything for the Win | 794 # (2) A warning since the results don't contain anything for the Win |
795 # release builder | 795 # release builder |
796 # (3) The first line needs and is missing results for Linux (dbg). | 796 # (3) The first line needs and is missing results for Linux (dbg). |
797 # (4) The third line needs and is missing results for Win Release. | 797 # (4) The third line needs and is missing results for Win Release. |
798 self._port.all_build_types = ('release', 'debug') | 798 self._port.all_build_types = ('release', 'debug') |
799 self._port.all_systems = (('win7', 'x86'), | 799 self._port.all_systems = (('win7', 'x86'), |
800 ('precise', 'x86_64')) | 800 ('precise', 'x86_64')) |
801 | 801 |
802 self._parse_expectations(test_expectations_before) | 802 self._parse_expectations(test_expectations_before) |
803 self._expectation_factory._all_results_by_builder = { | 803 self._expectation_factory._all_results_by_builder = { |
804 'WebKit Linux': { | 804 'WebKit Linux Precise': { |
805 "test/a.html": ["PASS", "PASS", "PASS"], | 805 "test/a.html": ["PASS", "PASS", "PASS"], |
806 "test/b.html": ["PASS", "IMAGE", "PASS"], | 806 "test/b.html": ["PASS", "IMAGE", "PASS"], |
807 "test/c.html": ["PASS", "PASS", "PASS"], | 807 "test/c.html": ["PASS", "PASS", "PASS"], |
808 "test/d.html": ["PASS", "PASS", "PASS"], | 808 "test/d.html": ["PASS", "PASS", "PASS"], |
809 }, | 809 }, |
810 'WebKit Win7 (dbg)': { | 810 'WebKit Win7 (dbg)': { |
811 "test/a.html": ["PASS", "PASS", "PASS"], | 811 "test/a.html": ["PASS", "PASS", "PASS"], |
812 "test/b.html": ["PASS", "PASS", "PASS"], | 812 "test/b.html": ["PASS", "PASS", "PASS"], |
813 "test/c.html": ["PASS", "PASS", "PASS"], | 813 "test/c.html": ["PASS", "PASS", "PASS"], |
814 "test/d.html": ["PASS", "PASS", "PASS"], | 814 "test/d.html": ["PASS", "PASS", "PASS"], |
815 }, | 815 }, |
816 } | 816 } |
817 | 817 |
818 updated_expectations = ( | 818 updated_expectations = ( |
819 self._flake_remover.get_updated_test_expectations()) | 819 self._flake_remover.get_updated_test_expectations()) |
820 self.assertLog([ | 820 self.assertLog([ |
821 'WARNING: Downloaded results are missing results for builder "WebKit
Linux (dbg)"\n', | 821 'WARNING: Downloaded results are missing results for builder "WebKit
Linux Precise (dbg)"\n', |
822 'WARNING: Downloaded results are missing results for builder "WebKit
Win7"\n', | 822 'WARNING: Downloaded results are missing results for builder "WebKit
Win7"\n', |
823 'ERROR: Failed to find results for builder "WebKit Linux (dbg)"\n', | 823 'ERROR: Failed to find results for builder "WebKit Linux Precise (db
g)"\n', |
824 'ERROR: Failed to find results for builder "WebKit Win7"\n', | 824 'ERROR: Failed to find results for builder "WebKit Win7"\n', |
825 ]) | 825 ]) |
826 | 826 |
827 # Also make sure we didn't remove any lines if some builders were | 827 # Also make sure we didn't remove any lines if some builders were |
828 # missing. | 828 # missing. |
829 self._assert_expectations_match( | 829 self._assert_expectations_match( |
830 updated_expectations, test_expectations_before) | 830 updated_expectations, test_expectations_before) |
831 | 831 |
832 def test_harness_updates_file(self): | 832 def test_harness_updates_file(self): |
833 """Tests that the call harness updates the TestExpectations file.""" | 833 """Tests that the call harness updates the TestExpectations file.""" |
834 | 834 |
835 self._define_builders({ | 835 self._define_builders({ |
836 "WebKit Linux": { | 836 "WebKit Linux Precise": { |
837 "port_name": "linux-precise", | 837 "port_name": "linux-precise", |
838 "specifiers": ['Precise', 'Release'] | 838 "specifiers": ['Precise', 'Release'] |
839 }, | 839 }, |
840 "WebKit Linux (dbg)": { | 840 "WebKit Linux Precise (dbg)": { |
841 "port_name": "linux-precise", | 841 "port_name": "linux-precise", |
842 "specifiers": ['Precise', 'Debug'] | 842 "specifiers": ['Precise', 'Debug'] |
843 }, | 843 }, |
844 }) | 844 }) |
845 | 845 |
846 # Setup the mock host and port. | 846 # Setup the mock host and port. |
847 host = MockHost() | 847 host = MockHost() |
848 host.port_factory = FakePortFactory(host) | 848 host.port_factory = FakePortFactory(host) |
849 host.port_factory._all_build_types = ('release', 'debug') | 849 host.port_factory._all_build_types = ('release', 'debug') |
850 host.port_factory._all_systems = (('precise', 'x86_64'),) | 850 host.port_factory._all_systems = (('precise', 'x86_64'),) |
(...skipping 12 matching lines...) Expand all Loading... |
863 Bug(test) [ Linux ] test/d.html [ Failure ]""" | 863 Bug(test) [ Linux ] test/d.html [ Failure ]""" |
864 files = { | 864 files = { |
865 test_expectation_path: test_expectations | 865 test_expectation_path: test_expectations |
866 } | 866 } |
867 host.filesystem = MockFileSystem(files) | 867 host.filesystem = MockFileSystem(files) |
868 self._write_tests_into_filesystem(host.filesystem) | 868 self._write_tests_into_filesystem(host.filesystem) |
869 | 869 |
870 # Write out the fake builder bot results. | 870 # Write out the fake builder bot results. |
871 expectation_factory = FakeBotTestExpectationsFactory() | 871 expectation_factory = FakeBotTestExpectationsFactory() |
872 expectation_factory._all_results_by_builder = { | 872 expectation_factory._all_results_by_builder = { |
873 'WebKit Linux': { | 873 'WebKit Linux Precise': { |
874 "test/a.html": ["PASS", "PASS", "PASS"], | 874 "test/a.html": ["PASS", "PASS", "PASS"], |
875 "test/b.html": ["PASS", "IMAGE", "PASS"], | 875 "test/b.html": ["PASS", "IMAGE", "PASS"], |
876 "test/c.html": ["PASS", "PASS", "PASS"], | 876 "test/c.html": ["PASS", "PASS", "PASS"], |
877 "test/d.html": ["PASS", "PASS", "PASS"], | 877 "test/d.html": ["PASS", "PASS", "PASS"], |
878 }, | 878 }, |
879 'WebKit Linux (dbg)': { | 879 'WebKit Linux Precise (dbg)': { |
880 "test/a.html": ["PASS", "PASS", "PASS"], | 880 "test/a.html": ["PASS", "PASS", "PASS"], |
881 "test/b.html": ["PASS", "PASS", "PASS"], | 881 "test/b.html": ["PASS", "PASS", "PASS"], |
882 "test/c.html": ["PASS", "PASS", "PASS"], | 882 "test/c.html": ["PASS", "PASS", "PASS"], |
883 "test/d.html": ["IMAGE", "PASS", "PASS"], | 883 "test/d.html": ["IMAGE", "PASS", "PASS"], |
884 }, | 884 }, |
885 } | 885 } |
886 | 886 |
887 main(host, expectation_factory, []) | 887 main(host, expectation_factory, []) |
888 | 888 |
889 self.assertEqual(host.filesystem.files[test_expectation_path], ( | 889 self.assertEqual(host.filesystem.files[test_expectation_path], ( |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
924 ]) | 924 ]) |
925 self.assertFalse(host.filesystem.isfile(test_expectation_path)) | 925 self.assertFalse(host.filesystem.isfile(test_expectation_path)) |
926 | 926 |
927 def test_harness_remove_all(self): | 927 def test_harness_remove_all(self): |
928 """Tests that removing all expectations doesn't delete the file. | 928 """Tests that removing all expectations doesn't delete the file. |
929 | 929 |
930 Make sure we're prepared for the day when we exterminated flakes. | 930 Make sure we're prepared for the day when we exterminated flakes. |
931 """ | 931 """ |
932 | 932 |
933 self._define_builders({ | 933 self._define_builders({ |
934 "WebKit Linux": { | 934 "WebKit Linux Precise": { |
935 "port_name": "linux-precise", | 935 "port_name": "linux-precise", |
936 "specifiers": ['Precise', 'Release'] | 936 "specifiers": ['Precise', 'Release'] |
937 }, | 937 }, |
938 "WebKit Linux (dbg)": { | 938 "WebKit Linux Precise (dbg)": { |
939 "port_name": "linux-precise", | 939 "port_name": "linux-precise", |
940 "specifiers": ['Precise', 'Debug'] | 940 "specifiers": ['Precise', 'Debug'] |
941 }, | 941 }, |
942 }) | 942 }) |
943 | 943 |
944 # Setup the mock host and port. | 944 # Setup the mock host and port. |
945 host = MockHost() | 945 host = MockHost() |
946 host.port_factory = FakePortFactory(host) | 946 host.port_factory = FakePortFactory(host) |
947 host.port_factory._all_build_types = ('release', 'debug') | 947 host.port_factory._all_build_types = ('release', 'debug') |
948 host.port_factory._all_systems = (('precise', 'x86_64'),) | 948 host.port_factory._all_systems = (('precise', 'x86_64'),) |
949 | 949 |
950 # Write out a fake TestExpectations file. | 950 # Write out a fake TestExpectations file. |
951 test_expectation_path = ( | 951 test_expectation_path = ( |
952 host.port_factory.get().path_to_generic_test_expectations_file()) | 952 host.port_factory.get().path_to_generic_test_expectations_file()) |
953 test_expectations = """ | 953 test_expectations = """ |
954 # Remove since passing on both bots. | 954 # Remove since passing on both bots. |
955 Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" | 955 Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" |
956 | 956 |
957 files = { | 957 files = { |
958 test_expectation_path: test_expectations | 958 test_expectation_path: test_expectations |
959 } | 959 } |
960 host.filesystem = MockFileSystem(files) | 960 host.filesystem = MockFileSystem(files) |
961 self._write_tests_into_filesystem(host.filesystem) | 961 self._write_tests_into_filesystem(host.filesystem) |
962 | 962 |
963 # Write out the fake builder bot results. | 963 # Write out the fake builder bot results. |
964 expectation_factory = FakeBotTestExpectationsFactory() | 964 expectation_factory = FakeBotTestExpectationsFactory() |
965 expectation_factory._all_results_by_builder = { | 965 expectation_factory._all_results_by_builder = { |
966 'WebKit Linux': { | 966 'WebKit Linux Precise': { |
967 "test/a.html": ["PASS", "PASS", "PASS"], | 967 "test/a.html": ["PASS", "PASS", "PASS"], |
968 }, | 968 }, |
969 'WebKit Linux (dbg)': { | 969 'WebKit Linux Precise (dbg)': { |
970 "test/a.html": ["PASS", "PASS", "PASS"], | 970 "test/a.html": ["PASS", "PASS", "PASS"], |
971 }, | 971 }, |
972 } | 972 } |
973 | 973 |
974 main(host, expectation_factory, []) | 974 main(host, expectation_factory, []) |
975 | 975 |
976 self.assertTrue(host.filesystem.isfile(test_expectation_path)) | 976 self.assertTrue(host.filesystem.isfile(test_expectation_path)) |
977 self.assertEqual(host.filesystem.files[test_expectation_path], '') | 977 self.assertEqual(host.filesystem.files[test_expectation_path], '') |
OLD | NEW |