OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
92 "actual": "FAIL" | 92 "actual": "FAIL" |
93 }, | 93 }, |
94 "flaky-failed.html": { | 94 "flaky-failed.html": { |
95 "expected": "PASS FAIL", | 95 "expected": "PASS FAIL", |
96 "actual": "FAIL" | 96 "actual": "FAIL" |
97 }, | 97 }, |
98 "media-document-audio-repaint.html": { | 98 "media-document-audio-repaint.html": { |
99 "expected": "IMAGE", | 99 "expected": "IMAGE", |
100 "actual": "IMAGE", | 100 "actual": "IMAGE", |
101 "time": 0.1 | 101 "time": 0.1 |
| 102 }, |
| 103 "unexpected-leak.html": { |
| 104 "expected": "PASS", |
| 105 "actual": "LEAK" |
102 } | 106 } |
103 } | 107 } |
104 }, | 108 }, |
105 "skipped": 2, | 109 "skipped": 2, |
106 "num_regressions": 0, | 110 "num_regressions": 0, |
107 "build_number": "3", | 111 "build_number": "3", |
108 "interrupted": false, | 112 "interrupted": false, |
109 "layout_tests_dir": "\/tmp\/cr\/src\/third_party\/WebKit\/LayoutTests", | 113 "layout_tests_dir": "\/tmp\/cr\/src\/third_party\/WebKit\/LayoutTests", |
110 "version": 3, | 114 "version": 3, |
111 "builder_name": "Webkit", | 115 "builder_name": "Webkit", |
112 "num_passes": 10, | 116 "num_passes": 10, |
113 "pixel_tests_enabled": true, | 117 "pixel_tests_enabled": true, |
114 "blink_revision": "1234", | 118 "blink_revision": "1234", |
115 "has_pretty_patch": true, | 119 "has_pretty_patch": true, |
116 "fixable": 25, | 120 "fixable": 25, |
117 "num_flaky": 0, | 121 "num_flaky": 0, |
118 "num_failures_by_type": { | 122 "num_failures_by_type": { |
119 "CRASH": 3, | 123 "CRASH": 3, |
120 "MISSING": 0, | 124 "MISSING": 0, |
121 "TEXT": 3, | 125 "TEXT": 3, |
122 "IMAGE": 1, | 126 "IMAGE": 1, |
123 "PASS": 10, | 127 "PASS": 10, |
124 "SKIP": 2, | 128 "SKIP": 2, |
125 "TIMEOUT": 16, | 129 "TIMEOUT": 16, |
126 "IMAGE+TEXT": 0, | 130 "IMAGE+TEXT": 0, |
127 "FAIL": 2, | 131 "FAIL": 2, |
128 "AUDIO": 0 | 132 "AUDIO": 0, |
| 133 "LEAK": 1 |
129 }, | 134 }, |
130 "has_wdiff": true, | 135 "has_wdiff": true, |
131 "chromium_revision": "5678" | 136 "chromium_revision": "5678" |
132 });""" | 137 });""" |
133 | 138 |
134 JSON_RESULTS_OLD_TEMPLATE = ( | 139 JSON_RESULTS_OLD_TEMPLATE = ( |
135 '{"[BUILDER_NAME]":{' | 140 '{"[BUILDER_NAME]":{' |
136 '"allFixableCount":[[TESTDATA_COUNT]],' | 141 '"allFixableCount":[[TESTDATA_COUNT]],' |
137 '"blinkRevision":[[TESTDATA_WEBKITREVISION]],' | 142 '"blinkRevision":[[TESTDATA_WEBKITREVISION]],' |
138 '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],' | 143 '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],' |
(...skipping 230 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
369 } | 374 } |
370 })) | 375 })) |
371 | 376 |
372 def test_merge_full_results_format(self): | 377 def test_merge_full_results_format(self): |
373 expected_incremental_results = { | 378 expected_incremental_results = { |
374 "Webkit": { | 379 "Webkit": { |
375 "blinkRevision": ["1234"], | 380 "blinkRevision": ["1234"], |
376 "buildNumbers": ["3"], | 381 "buildNumbers": ["3"], |
377 "chromeRevision": ["5678"], | 382 "chromeRevision": ["5678"], |
378 "failure_map": CHAR_TO_FAILURE, | 383 "failure_map": CHAR_TO_FAILURE, |
379 "num_failures_by_type": {"AUDIO": [0], "CRASH": [3], "FAIL": [2]
, "IMAGE": [1], "IMAGE+TEXT": [0], "MISSING": [0], "PASS": [10], "SKIP": [2], "T
EXT": [3], "TIMEOUT": [16]}, | 384 "num_failures_by_type": {"AUDIO": [0], "CRASH": [3], "FAIL": [2]
, "IMAGE": [1], "IMAGE+TEXT": [0], "MISSING": [0], "PASS": [10], "SKIP": [2], "T
EXT": [3], "TIMEOUT": [16], "LEAK": [1]}, |
380 "secondsSinceEpoch": [1368146629], | 385 "secondsSinceEpoch": [1368146629], |
381 "tests": { | 386 "tests": { |
382 "media": { | 387 "media": { |
383 "W3C": { | 388 "W3C": { |
384 "audio": { | 389 "audio": { |
385 "src": { | 390 "src": { |
386 "src_removal_does_not_trigger_loadstart.html
": { | 391 "src_removal_does_not_trigger_loadstart.html
": { |
387 "results": [[1, PASS]], | 392 "results": [[1, PASS]], |
388 "times": [[1, 4]], | 393 "times": [[1, 4]], |
389 } | 394 } |
(...skipping 25 matching lines...) Expand all Loading... |
415 }, | 420 }, |
416 "flaky-failed.html": { | 421 "flaky-failed.html": { |
417 "expected": "PASS FAIL", | 422 "expected": "PASS FAIL", |
418 "results": [[1, FAIL]], | 423 "results": [[1, FAIL]], |
419 "times": [[1, 0]], | 424 "times": [[1, 0]], |
420 }, | 425 }, |
421 "unexpected-fail.html": { | 426 "unexpected-fail.html": { |
422 "results": [[1, FAIL]], | 427 "results": [[1, FAIL]], |
423 "times": [[1, 0]], | 428 "times": [[1, 0]], |
424 }, | 429 }, |
| 430 "unexpected-leak.html": { |
| 431 "results": [[1, LEAK]], |
| 432 "times": [[1, 0]], |
| 433 }, |
425 } | 434 } |
426 } | 435 } |
427 }, | 436 }, |
428 "version": 4 | 437 "version": 4 |
429 } | 438 } |
430 | 439 |
431 aggregated_results = "" | 440 aggregated_results = "" |
432 incremental_json, _ = JsonResults._get_incremental_json(self._builder, F
ULL_RESULT_EXAMPLE, is_full_results_format=True) | 441 incremental_json, _ = JsonResults._get_incremental_json(self._builder, F
ULL_RESULT_EXAMPLE, is_full_results_format=True) |
433 merged_results, _ = JsonResults.merge("Webkit", aggregated_results, incr
emental_json, num_runs=jsonresults.JSON_RESULTS_MAX_BUILDS, sort_keys=True) | 442 merged_results, _ = JsonResults.merge("Webkit", aggregated_results, incr
emental_json, num_runs=jsonresults.JSON_RESULTS_MAX_BUILDS, sort_keys=True) |
434 self.assert_json_equal(merged_results, expected_incremental_results) | 443 self.assert_json_equal(merged_results, expected_incremental_results) |
(...skipping 630 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1065 "results": [[1, TEXT]], | 1074 "results": [[1, TEXT]], |
1066 "times": [[1, 0]]}, | 1075 "times": [[1, 0]]}, |
1067 "test.failed": { | 1076 "test.failed": { |
1068 "results": [[10, FAIL]], | 1077 "results": [[10, FAIL]], |
1069 "times": [[10, 0]]}, | 1078 "times": [[10, 0]]}, |
1070 }, | 1079 }, |
1071 "version": 4}) | 1080 "version": 4}) |
1072 | 1081 |
1073 if __name__ == '__main__': | 1082 if __name__ == '__main__': |
1074 unittest.main() | 1083 unittest.main() |
OLD | NEW |