Chromium Code Reviews Chromium Code Reviews
Issue 372053:
Added test runner for sputnik tests. (Closed)
| OLD | NEW |
|---|---|
| (Empty) | |
| 1 # Copyright 2008 the V8 project authors. All rights reserved. | |
|
Mads Ager (chromium)
2009/11/09 09:26:51
2009
| |
| 2 # Redistribution and use in source and binary forms, with or without | |
| 3 # modification, are permitted provided that the following conditions are | |
| 4 # met: | |
| 5 # | |
| 6 # * Redistributions of source code must retain the above copyright | |
| 7 # notice, this list of conditions and the following disclaimer. | |
| 8 # * Redistributions in binary form must reproduce the above | |
| 9 # copyright notice, this list of conditions and the following | |
| 10 # disclaimer in the documentation and/or other materials provided | |
| 11 # with the distribution. | |
| 12 # * Neither the name of Google Inc. nor the names of its | |
| 13 # contributors may be used to endorse or promote products derived | |
| 14 # from this software without specific prior written permission. | |
| 15 # | |
| 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 27 | |
| 28 prefix sputnik | |
| 29 def FAIL_OK = FAIL, OKAY | |
| 30 | |
| 31 ##################### DELIBERATE INCOMPATIBILITIES ##################### | |
| 32 | |
| 33 # 900066: Deleting elements in .arguments should disconnect the | |
| 34 # element from the actual arguments. Implementing this is nontrivial | |
| 35 # and we have no indication that anything on the web depends on this | |
| 36 # feature. | |
| 37 S13_A13_T1: FAIL_OK | |
| 38 S13_A13_T2: FAIL_OK | |
| 39 S13_A13_T3: FAIL_OK | |
| 40 | |
| 41 # This tests precision of trignometric functions. We're slightly off | |
| 42 # from the implementation in libc (~ 1e-17) but it's not clear if we | |
| 43 # or they are closer to the right answer, or if it even matters. | |
| 44 S15.8.2.16_A7: PASS || FAIL_OK | |
| 45 S15.8.2.18_A7: PASS || FAIL_OK | |
| 46 S15.8.2.13_A23: PASS || FAIL_OK | |
| 47 | |
| 48 # We allow calls to regexp exec() with no arguments to fail for | |
| 49 # compatibility reasons. | |
| 50 S15.10.6.2_A1_T16: FAIL_OK | |
| 51 S15.10.6.3_A1_T16: FAIL_OK | |
| 52 | |
| 53 # We allow regexps to be called as functions for compatibility reasons. | |
| 54 S15.10.7_A1_T1: FAIL_OK | |
| 55 S15.10.7_A1_T2: FAIL_OK | |
| 56 | |
| 57 # We allow construct calls to built-in functions, and we allow built-in | |
| 58 # functions to have prototypes. | |
| 59 S15.1.2.1_A4.6: FAIL_OK | |
| 60 S15.1.2.1_A4.7: FAIL_OK | |
| 61 S15.1.2.2_A9.6: FAIL_OK | |
| 62 S15.1.2.2_A9.7: FAIL_OK | |
| 63 S15.1.2.3_A7.6: FAIL_OK | |
| 64 S15.1.2.3_A7.7: FAIL_OK | |
| 65 S15.1.2.4_A2.6: FAIL_OK | |
| 66 S15.1.2.4_A2.7: FAIL_OK | |
| 67 S15.1.2.5_A2.6: FAIL_OK | |
| 68 S15.1.2.5_A2.7: FAIL_OK | |
| 69 S15.1.3.1_A5.6: FAIL_OK | |
| 70 S15.1.3.1_A5.7: FAIL_OK | |
| 71 S15.1.3.2_A5.6: FAIL_OK | |
| 72 S15.1.3.2_A5.7: FAIL_OK | |
| 73 S15.1.3.3_A5.6: FAIL_OK | |
| 74 S15.1.3.3_A5.7: FAIL_OK | |
| 75 S15.1.3.4_A5.6: FAIL_OK | |
| 76 S15.1.3.4_A5.7: FAIL_OK | |
| 77 S15.10.6.2_A6: FAIL_OK | |
| 78 S15.10.6.3_A6: FAIL_OK | |
| 79 S15.10.6.4_A6: FAIL_OK | |
| 80 S15.10.6.4_A7: FAIL_OK | |
| 81 S15.2.4.2_A6: FAIL_OK | |
| 82 S15.2.4.3_A6: FAIL_OK | |
| 83 S15.2.4.4_A6: FAIL_OK | |
| 84 S15.2.4.5_A6: FAIL_OK | |
| 85 S15.2.4.6_A6: FAIL_OK | |
| 86 S15.2.4.7_A6: FAIL_OK | |
| 87 S15.3.4.2_A6: FAIL_OK | |
| 88 S15.4.4.10_A5.6: FAIL_OK | |
| 89 S15.4.4.10_A5.7: FAIL_OK | |
| 90 S15.4.4.11_A7.6: FAIL_OK | |
| 91 S15.4.4.11_A7.7: FAIL_OK | |
| 92 S15.4.4.12_A5.6: FAIL_OK | |
| 93 S15.4.4.12_A5.7: FAIL_OK | |
| 94 S15.4.4.13_A5.6: FAIL_OK | |
| 95 S15.4.4.13_A5.7: FAIL_OK | |
| 96 S15.4.4.2_A4.6: FAIL_OK | |
| 97 S15.4.4.3_A4.6: FAIL_OK | |
| 98 S15.4.4.3_A4.6: FAIL_OK | |
| 99 S15.4.4.4_A4.6: FAIL_OK | |
| 100 S15.4.4.4_A4.7: FAIL_OK | |
| 101 S15.4.4.5_A6.6: FAIL_OK | |
| 102 S15.4.4.5_A6.7: FAIL_OK | |
| 103 S15.4.4.6_A5.6: FAIL_OK | |
| 104 S15.4.4.6_A5.7: FAIL_OK | |
| 105 S15.4.4.7_A6.6: FAIL_OK | |
| 106 S15.4.4.7_A6.7: FAIL_OK | |
| 107 S15.4.4.8_A5.6: FAIL_OK | |
| 108 S15.4.4.8_A5.7: FAIL_OK | |
| 109 S15.4.4.9_A5.6: FAIL_OK | |
| 110 S15.4.4.9_A5.7: FAIL_OK | |
| 111 S15.5.4.10_A6: FAIL_OK | |
| 112 S15.5.4.11_A6: FAIL_OK | |
| 113 S15.5.4.12_A6: FAIL_OK | |
| 114 S15.5.4.13_A6: FAIL_OK | |
| 115 S15.5.4.14_A6: FAIL_OK | |
| 116 S15.5.4.15_A6: FAIL_OK | |
| 117 S15.5.4.16_A6: FAIL_OK | |
| 118 S15.5.4.17_A6: FAIL_OK | |
| 119 S15.5.4.18_A6: FAIL_OK | |
| 120 S15.5.4.19_A6: FAIL_OK | |
| 121 S15.5.4.4_A6: FAIL_OK | |
| 122 S15.5.4.5_A6: FAIL_OK | |
| 123 S15.5.4.6_A6: FAIL_OK | |
| 124 S15.5.4.7_A6: FAIL_OK | |
| 125 S15.5.4.9_A6: FAIL_OK | |
| 126 S15.3.4.3_A12: FAIL_OK | |
| 127 S15.3.4.4_A12: FAIL_OK | |
| 128 S15.5.4.8_A6: FAIL_OK | |
| 129 | |
| 130 # We are silent in some regexp cases where the spec wants us to give | |
| 131 # errors, for compatibility. | |
| 132 S15.10.2.11_A1_T2: FAIL | |
| 133 S15.10.2.11_A1_T3: FAIL | |
| 134 S15.10.4.1_A5_T1: FAIL | |
| 135 S15.10.4.1_A5_T2: FAIL | |
| 136 S15.10.4.1_A5_T3: FAIL | |
| 137 S15.10.4.1_A5_T4: FAIL | |
| 138 S15.10.4.1_A5_T5: FAIL | |
| 139 S15.10.4.1_A5_T6: FAIL | |
| 140 S15.10.4.1_A5_T7: FAIL | |
| 141 S15.10.4.1_A5_T8: FAIL | |
| 142 S15.10.4.1_A5_T9: FAIL | |
| 143 | |
| 144 # We are more lenient in which string character escapes we allow than | |
| 145 # the spec (7.8.4 p. 19) wants us to be. This is for compatibility. | |
| 146 S7.8.4_A4.3_T2: FAIL_OK | |
| 147 S7.8.4_A4.3_T2: FAIL_OK | |
| 148 S7.8.4_A6.2_T2: FAIL_OK | |
| 149 S7.8.4_A6.1_T4: FAIL_OK | |
| 150 S7.8.4_A4.3_T4: FAIL_OK | |
| 151 S7.8.4_A7.2_T2: FAIL_OK | |
| 152 S7.8.4_A7.1_T4: FAIL_OK | |
| 153 S7.8.4_A6.4_T2: FAIL_OK | |
| 154 S7.8.4_A7.4_T2: FAIL_OK | |
| 155 S7.8.4_A7.2_T4: FAIL_OK | |
| 156 S7.8.4_A4.3_T6: FAIL_OK | |
| 157 S7.8.4_A7.2_T6: FAIL_OK | |
| 158 S7.8.4_A4.3_T1: FAIL_OK | |
| 159 S7.8.4_A6.2_T1: FAIL_OK | |
| 160 S7.8.4_A4.3_T3: FAIL_OK | |
| 161 S7.8.4_A7.2_T1: FAIL_OK | |
| 162 S7.8.4_A6.4_T1: FAIL_OK | |
| 163 S7.8.4_A7.2_T3: FAIL_OK | |
| 164 S7.8.4_A7.4_T1: FAIL_OK | |
| 165 S7.8.4_A4.3_T5: FAIL_OK | |
| 166 S7.8.4_A7.2_T5: FAIL_OK | |
| 167 S7.8.4_A4.3_T1: FAIL_OK | |
| 168 S7.8.4_A6.2_T1: FAIL_OK | |
| 169 S7.8.4_A4.3_T3: FAIL_OK | |
| 170 S7.8.4_A7.2_T1: FAIL_OK | |
| 171 S7.8.4_A6.4_T1: FAIL_OK | |
| 172 S7.8.4_A7.2_T3: FAIL_OK | |
| 173 S7.8.4_A7.4_T1: FAIL_OK | |
| 174 S7.8.4_A4.3_T5: FAIL_OK | |
| 175 S7.8.4_A7.2_T5: FAIL_OK | |
| 176 | |
| 177 # We allow some keywords to be used as identifiers | |
| 178 S7.5.3_A1.17: FAIL_OK | |
| 179 S7.5.3_A1.26: FAIL_OK | |
| 180 S7.5.3_A1.18: FAIL_OK | |
| 181 S7.5.3_A1.27: FAIL_OK | |
| 182 S7.5.3_A1.28: FAIL_OK | |
| 183 S7.5.3_A1.19: FAIL_OK | |
| 184 S7.5.3_A1.29: FAIL_OK | |
| 185 S7.5.3_A1.1: FAIL_OK | |
| 186 S7.5.3_A1.2: FAIL_OK | |
| 187 S7.5.3_A1.3: FAIL_OK | |
| 188 S7.5.3_A1.4: FAIL_OK | |
| 189 S7.5.3_A1.5: FAIL_OK | |
| 190 S7.5.3_A1.8: FAIL_OK | |
| 191 S7.5.3_A1.9: FAIL_OK | |
| 192 S7.5.3_A1.10: FAIL_OK | |
| 193 S7.5.3_A1.11: FAIL_OK | |
| 194 S7.5.3_A1.21: FAIL_OK | |
| 195 S7.5.3_A1.12: FAIL_OK | |
| 196 S7.5.3_A1.30: FAIL_OK | |
| 197 S7.5.3_A1.31: FAIL_OK | |
| 198 S7.5.3_A1.13: FAIL_OK | |
| 199 S7.5.3_A1.22: FAIL_OK | |
| 200 S7.5.3_A1.23: FAIL_OK | |
| 201 S7.5.3_A1.14: FAIL_OK | |
| 202 S7.5.3_A1.15: FAIL_OK | |
| 203 S7.5.3_A1.24: FAIL_OK | |
| 204 S7.5.3_A1.25: FAIL_OK | |
| 205 S7.5.3_A1.16: FAIL_OK | |
| 206 | |
| 207 # This checks for non-262 behavior | |
| 208 S12.6.4_A14_T1: PASS || FAIL_OK | |
| 209 S12.6.4_R1: PASS || FAIL_OK | |
| 210 S12.6.4_R2: PASS || FAIL_OK | |
| 211 S8.4_D2.1: PASS || FAIL_OK | |
| 212 S8.4_D2.2: PASS || FAIL_OK | |
| 213 S8.4_D2.3: PASS || FAIL_OK | |
| 214 S8.4_D2.4: PASS || FAIL_OK | |
| 215 S8.4_D2.5: PASS || FAIL_OK | |
| 216 S8.4_D2.6: PASS || FAIL_OK | |
| 217 S8.4_D2.7: PASS || FAIL_OK | |
| 218 S8.4_D1.1: PASS || FAIL_OK | |
| 219 S13.2_D1.2: PASS || FAIL_OK | |
| 220 S11.4.3_D1.2: PASS || FAIL_OK | |
| 221 S7.6_D1: PASS || FAIL_OK | |
| 222 S7.6_D2: PASS || FAIL_OK | |
| 223 S15.1.2.2_D1.2: PASS || FAIL_OK | |
| 224 S13_D1_T1: PASS || FAIL_OK | |
| 225 S14_D4_T3: PASS || FAIL_OK | |
| 226 S14_D7: PASS || FAIL_OK | |
| 227 S15.5.4.11_D1.1_T2: PASS || FAIL_OK | |
| 228 S15.5.4.11_D1.1_T4: PASS || FAIL_OK | |
| 229 S15.5.2_D2: PASS || FAIL_OK | |
| 230 S15.5.4.11_D1.1_T1: PASS || FAIL_OK | |
| 231 S15.5.4.11_D1.1_T3: PASS || FAIL_OK | |
| 232 S12.6.4_D1: PASS || FAIL_OK | |
| 233 | |
| 234 # We deliberately don't throw type errors when iterating through the | |
| 235 # undefined object | |
| 236 S9.9_A1: FAIL_OK | |
| 237 S9.9_A2: FAIL_OK | |
| 238 | |
| 239 # We allow function declarations within statements | |
| 240 S12.5_A9_T1: FAIL_OK | |
| 241 S12.5_A9_T2: FAIL_OK | |
| 242 # S12.6.2_A13_T3: FAIL_OK | |
| 243 # S12.5_A9_T3: FAIL_OK | |
| 244 # S12.6.1_A13_T3: FAIL_OK | |
| 245 S12.1_A1: FAIL_OK | |
| 246 S12.6.2_A13_T1: FAIL_OK | |
| 247 S12.6.2_A13_T2: FAIL_OK | |
| 248 S12.6.1_A13_T1: FAIL_OK | |
| 249 S12.6.1_A13_T2: FAIL_OK | |
| 250 S12.6.4_A13_T1: FAIL_OK | |
| 251 S12.6.4_A13_T2: FAIL_OK | |
| 252 #S12.6.4_A13_T3: FAIL_OK | |
| 253 S15.3.4.2_A1_T1: FAIL_OK | |
| 254 | |
| 255 # Linux and Mac defaults to extended 80 bit floating point format in the FPU. | |
| 256 # We follow the other major JS engines by keeping this default. | |
| 257 S8.5_A2.2: PASS, FAIL if $system == linux, FAIL if $system == macos | |
| 258 S8.5_A2.1: PASS, FAIL if $system == linux, FAIL if $system == macos | |
| 259 | |
| 260 ##################### SKIPPED TESTS ##################### | |
| 261 | |
| 262 # These tests take a looong time to run in debug mode. | |
| 263 S15.1.3.2_A2.5_T1: PASS, SKIP if $mode == debug | |
| 264 S15.1.3.1_A2.5_T1: PASS, SKIP if $mode == debug | |
| 265 | |
| 266 | |
| 267 # These tests fail because we had to add bugs to be compatible with JSC. See | |
| 268 # http://code.google.com/p/chromium/issues/detail?id=1717 | |
| 269 S15.4.4_A1.1_T2: FAIL_OK | |
| 270 S15.5.4.1_A1_T2: FAIL_OK | |
| 271 S15.5.4_A1: FAIL_OK | |
| 272 S15.5.4_A3: FAIL_OK | |
| 273 S15.9.3.1_A3_T1.1: FAIL_OK | |
| 274 S15.9.3.1_A3_T2.1: FAIL_OK | |
| 275 S15.9.3.1_A3_T3.1: FAIL_OK | |
| 276 S15.9.3.1_A3_T4.1: FAIL_OK | |
| 277 S15.9.3.1_A3_T5.1: FAIL_OK | |
| 278 S15.9.3.1_A3_T6.1: FAIL_OK | |
| 279 S15.9.3.2_A3_T1.1: FAIL_OK | |
| 280 S15.9.5.10_A1_T2: FAIL_OK | |
| 281 S15.9.5.11_A1_T2: FAIL_OK | |
| 282 S15.9.5.12_A1_T2: FAIL_OK | |
| 283 S15.9.5.13_A1_T2: FAIL_OK | |
| 284 S15.9.5.14_A1_T2: FAIL_OK | |
| 285 S15.9.5.15_A1_T2: FAIL_OK | |
| 286 S15.9.5.16_A1_T2: FAIL_OK | |
| 287 S15.9.5.17_A1_T2: FAIL_OK | |
| 288 S15.9.5.18_A1_T2: FAIL_OK | |
| 289 S15.9.5.19_A1_T2: FAIL_OK | |
| 290 S15.9.5.20_A1_T2: FAIL_OK | |
| 291 S15.9.5.21_A1_T2: FAIL_OK | |
| 292 S15.9.5.22_A1_T2: FAIL_OK | |
| 293 S15.9.5.23_A1_T2: FAIL_OK | |
| 294 S15.9.5.24_A1_T2: FAIL_OK | |
| 295 S15.9.5.25_A1_T2: FAIL_OK | |
| 296 S15.9.5.26_A1_T2: FAIL_OK | |
| 297 S15.9.5.27_A1_T2: FAIL_OK | |
| 298 S15.9.5.28_A1_T2: FAIL_OK | |
| 299 S15.9.5.29_A1_T2: FAIL_OK | |
| 300 S15.9.5.2_A1_T2: FAIL_OK | |
| 301 S15.9.5.30_A1_T2: FAIL_OK | |
| 302 S15.9.5.31_A1_T2: FAIL_OK | |
| 303 S15.9.5.32_A1_T2: FAIL_OK | |
| 304 S15.9.5.33_A1_T2: FAIL_OK | |
| 305 S15.9.5.34_A1_T2: FAIL_OK | |
| 306 S15.9.5.35_A1_T2: FAIL_OK | |
| 307 S15.9.5.36_A1_T2: FAIL_OK | |
| 308 S15.9.5.37_A1_T2: FAIL_OK | |
| 309 S15.9.5.38_A1_T2: FAIL_OK | |
| 310 S15.9.5.39_A1_T2: FAIL_OK | |
| 311 S15.9.5.3_A1_T2: FAIL_OK | |
| 312 S15.9.5.40_A1_T2: FAIL_OK | |
| 313 S15.9.5.41_A1_T2: FAIL_OK | |
| 314 S15.9.5.42_A1_T2: FAIL_OK | |
| 315 S15.9.5.4_A1_T2: FAIL_OK | |
| 316 S15.9.5.5_A1_T2: FAIL_OK | |
| 317 S15.9.5.6_A1_T2: FAIL_OK | |
| 318 S15.9.5.7_A1_T2: FAIL_OK | |
| 319 S15.9.5.8_A1_T2: FAIL_OK | |
| 320 S15.9.5.9_A1_T2: FAIL_OK | |
| 321 | |
| 322 # Regexps have type "function", not "object". | |
| 323 S11.4.3_A3.6: FAIL_OK | |
| 324 S15.10.7_A3_T2: FAIL_OK | |
| 325 S15.10.7_A3_T1: FAIL_OK | |
| OLD | NEW |
