| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 library testrunner_test; | |
| 6 | |
| 7 import 'dart:async'; | |
| 8 import 'dart:io'; | |
| 9 import 'package:unittest/unittest.dart'; | |
| 10 | |
| 11 var dart; | |
| 12 var debug = false; | |
| 13 | |
| 14 Future runTestrunner( | |
| 15 command, List<String> args, List<String> stdout, List<String> stderr) { | |
| 16 if (debug) { | |
| 17 print("Running $command ${args.join(' ')}"); | |
| 18 } | |
| 19 return Process.run(command, args).then((ProcessResult result) { | |
| 20 var lineEndings = new RegExp("\r\n|\n"); | |
| 21 stdout.addAll(result.stdout.trim().split(lineEndings)); | |
| 22 stderr.addAll(result.stderr.trim().split(lineEndings)); | |
| 23 }).catchError((e) { | |
| 24 stderr.add("Error starting process:"); | |
| 25 stderr.add(" Command: $command"); | |
| 26 stderr.add(" Error: ${e}"); | |
| 27 completer.complete(-1); | |
| 28 }); | |
| 29 } | |
| 30 | |
| 31 // Useful utility for debugging test failures. | |
| 32 void dump(label, list) { | |
| 33 if (!debug) return; | |
| 34 print('\n@=[ $label ]=============================\n'); | |
| 35 for (var i = 0; i < list.length; i++) { | |
| 36 print('@ ${list[i]}\n'); | |
| 37 } | |
| 38 print('------------------------------------------\n'); | |
| 39 } | |
| 40 | |
| 41 int stringCompare(String s1, String s2) => s1.compareTo(s2); | |
| 42 | |
| 43 Future runTest(List<String> args, List<String> expected_stdout, | |
| 44 {List<String> expected_stderr, sort: false}) { | |
| 45 var stdout = new List<String>(); | |
| 46 var stderr = new List<String>(); | |
| 47 for (var i = 0; i < expected_stdout.length; i++) { | |
| 48 expected_stdout[i] = | |
| 49 expected_stdout[i].replaceAll('/', Platform.pathSeparator); | |
| 50 } | |
| 51 if (debug) { | |
| 52 args.insert(1, "--log=stderr"); | |
| 53 } | |
| 54 var rtn = runTestrunner(dart, args, stdout, stderr); | |
| 55 rtn.then((_) { | |
| 56 dump('stderr', stderr); | |
| 57 dump('stdout', stdout); | |
| 58 | |
| 59 if (expected_stderr != null) { | |
| 60 expect(stderr.length, orderedEquals(expected_stderr)); | |
| 61 } | |
| 62 var i, l = 0, matched = 0; | |
| 63 if (sort) { | |
| 64 stdout.sort(stringCompare); | |
| 65 expected_stdout.sort(stringCompare); | |
| 66 } | |
| 67 for (i = 0; i < stdout.length; i++) { | |
| 68 if (!stdout[i].startsWith('@')) { | |
| 69 if (expected_stdout.length <= l) { | |
| 70 fail("Extra text in output: ${stdout[i]}"); | |
| 71 return; | |
| 72 } | |
| 73 var actual = stdout[i].trim(); | |
| 74 if (debug) { | |
| 75 print("Compare <$actual> and <${expected_stdout[l]}>"); | |
| 76 } | |
| 77 if (expected_stdout[l].startsWith('*')) { | |
| 78 expect(actual, endsWith(expected_stdout[l].substring(1))); | |
| 79 } else if (expected_stdout[l].startsWith('?')) { | |
| 80 var pat = expected_stdout[l].substring(1); | |
| 81 if (Platform.operatingSystem == 'windows') { | |
| 82 // The joys of Windows... | |
| 83 pat = pat.replaceAll('\\', '\\\\'); | |
| 84 } | |
| 85 expect(actual, matches(pat)); | |
| 86 } else { | |
| 87 expect(actual, expected_stdout[l]); | |
| 88 } | |
| 89 ++l; | |
| 90 } | |
| 91 } | |
| 92 if (l < expected_stdout.length) { | |
| 93 fail("Only matched $l of ${expected_stdout.length} lines"); | |
| 94 } | |
| 95 }); | |
| 96 return rtn; | |
| 97 } | |
| 98 | |
| 99 // A useful function to quickly disable a group of tests; just | |
| 100 // replace group() with skip_group(). | |
| 101 skip_group(_1, _2) {} | |
| 102 | |
| 103 main() { | |
| 104 dart = Platform.executable; | |
| 105 var idx = dart.indexOf('dart-sdk'); | |
| 106 if (idx < 0) { | |
| 107 print("Please run using the dart executable from the Dart SDK"); | |
| 108 exit(-1); | |
| 109 } | |
| 110 var _ = Platform.pathSeparator; | |
| 111 var testrunner = '../../testrunner/testrunner.dart' | |
| 112 .replaceAll('/', Platform.pathSeparator); | |
| 113 | |
| 114 group("list tests", () { | |
| 115 test('list file', () { | |
| 116 return runTest([testrunner, '--list-files', 'non_browser_tests'], | |
| 117 ['?.*/non_browser_tests/non_browser_test.dart']); | |
| 118 }); | |
| 119 test('list files', () { | |
| 120 return runTest([ | |
| 121 testrunner, | |
| 122 '--recurse', | |
| 123 '--sort', | |
| 124 '--list-files', | |
| 125 '--test-file-pattern=.dart\$' | |
| 126 ], [ | |
| 127 '*browser_tests/web/browser_test.dart', | |
| 128 '*http_client_tests/http_client_test.dart', | |
| 129 '*layout_tests/web/layout_test.dart', | |
| 130 '*non_browser_tests/non_browser_test.dart', | |
| 131 '*non_browser_tests/non_browser_toast.dart', | |
| 132 '*/testrunner_test.dart' | |
| 133 ]); | |
| 134 }); | |
| 135 test('list files', () { | |
| 136 return runTest([ | |
| 137 testrunner, | |
| 138 '--list-files', | |
| 139 '--test-file-pattern=.dart\$', | |
| 140 'non_browser_tests' | |
| 141 ], [ | |
| 142 '*non_browser_tests/non_browser_test.dart', | |
| 143 '*non_browser_tests/non_browser_toast.dart' | |
| 144 ], sort: true); | |
| 145 }); | |
| 146 test('list groups', () { | |
| 147 return runTest([ | |
| 148 testrunner, | |
| 149 '--list-groups', | |
| 150 'non_browser_tests' | |
| 151 ], [ | |
| 152 '*non_browser_tests/non_browser_test.dart group1', | |
| 153 '*non_browser_tests/non_browser_test.dart group2' | |
| 154 ]); | |
| 155 }); | |
| 156 test('list tests', () { | |
| 157 return runTest([ | |
| 158 testrunner, | |
| 159 '--list-tests', | |
| 160 'non_browser_tests' | |
| 161 ], [ | |
| 162 '*non_browser_tests/non_browser_test.dart group1 test1', | |
| 163 '*non_browser_tests/non_browser_test.dart group2 test2' | |
| 164 ]); | |
| 165 }); | |
| 166 }); | |
| 167 | |
| 168 group("vm", () { | |
| 169 test("vm without timing info", () { | |
| 170 return runTest([ | |
| 171 testrunner, | |
| 172 '--recurse', | |
| 173 'non_browser_tests' | |
| 174 ], [ | |
| 175 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1' | |
| 176 ' Expected: false', | |
| 177 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2' | |
| 178 ]); | |
| 179 }); | |
| 180 | |
| 181 test("vm with timing info", () { | |
| 182 return runTest([ | |
| 183 testrunner, | |
| 184 '--recurse', | |
| 185 '--time', | |
| 186 'non_browser_tests' | |
| 187 ], [ | |
| 188 '?FAIL [0-9.]+s .*/non_browser_tests/non_browser_test.dart group1' | |
| 189 ' test1 Expected: false', | |
| 190 '?PASS [0-9.]+s .*/non_browser_tests/non_browser_test.dart group2' | |
| 191 ' test2' | |
| 192 ]); | |
| 193 }); | |
| 194 }); | |
| 195 | |
| 196 group("selection", () { | |
| 197 test("--include", () { | |
| 198 return runTest([ | |
| 199 testrunner, | |
| 200 '--recurse', | |
| 201 '--include=group1', | |
| 202 'non_browser_tests' | |
| 203 ], [ | |
| 204 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1 ' | |
| 205 'Expected: false' | |
| 206 ]); | |
| 207 }); | |
| 208 | |
| 209 test("--exclude", () { | |
| 210 return runTest( | |
| 211 [testrunner, '--recurse', '--exclude=group1', 'non_browser_tests'], | |
| 212 ['?PASS .*/non_browser_tests/non_browser_test.dart group2 test2']); | |
| 213 }); | |
| 214 | |
| 215 test("test file pattern", () { | |
| 216 return runTest([ | |
| 217 testrunner, | |
| 218 '--recurse', | |
| 219 '--test-file-pattern=toast', | |
| 220 'non_browser_tests' | |
| 221 ], [ | |
| 222 '?PASS .*/non_browser_tests/non_browser_toast.dart foo bar' | |
| 223 ]); | |
| 224 }); | |
| 225 }); | |
| 226 | |
| 227 group("stop on failure tests", () { | |
| 228 test("without stop", () { | |
| 229 return runTest([ | |
| 230 testrunner, | |
| 231 '--recurse', | |
| 232 '--sort', | |
| 233 '--tasks=1', | |
| 234 '--test-file-pattern=.dart\$', | |
| 235 'non_browser_tests' | |
| 236 ], [ | |
| 237 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1 ' | |
| 238 'Expected: false', | |
| 239 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2', | |
| 240 '?PASS .*/non_browser_tests/non_browser_toast.dart foo bar' | |
| 241 ]); | |
| 242 }); | |
| 243 test("with stop", () { | |
| 244 return runTest([ | |
| 245 testrunner, | |
| 246 '--recurse', | |
| 247 '--sort', | |
| 248 '--tasks=1', | |
| 249 '--test-file-pattern=.dart\$', | |
| 250 '--stop-on-failure', | |
| 251 'non_browser_tests' | |
| 252 ], [ | |
| 253 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1 ' | |
| 254 'Expected: false', | |
| 255 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2' | |
| 256 ]); | |
| 257 }); | |
| 258 }); | |
| 259 | |
| 260 group("output control", () { | |
| 261 test("summary test", () { | |
| 262 return runTest([ | |
| 263 testrunner, | |
| 264 '--recurse', | |
| 265 '--summary', | |
| 266 'non_browser_tests' | |
| 267 ], [ | |
| 268 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1 ' | |
| 269 'Expected: false', | |
| 270 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2', | |
| 271 '', | |
| 272 '?.*/non_browser_tests/non_browser_test.dart: ' | |
| 273 '1 PASSED, 1 FAILED, 0 ERRORS' | |
| 274 ]); | |
| 275 }); | |
| 276 | |
| 277 test('list tests with custom format', () { | |
| 278 return runTest([ | |
| 279 testrunner, | |
| 280 '--list-tests', | |
| 281 '--list-format="<FILENAME><TESTNAME>"', | |
| 282 'non_browser_tests' | |
| 283 ], [ | |
| 284 '?.*/non_browser_tests/non_browser_test.dart test1', | |
| 285 '?.*/non_browser_tests/non_browser_test.dart test2' | |
| 286 ]); | |
| 287 }); | |
| 288 | |
| 289 test("custom message formatting", () { | |
| 290 return runTest([ | |
| 291 testrunner, | |
| 292 '--recurse', | |
| 293 '--pass-format=YIPPEE! <GROUPNAME><TESTNAME>', | |
| 294 '--fail-format=EPIC FAIL! <GROUPNAME><TESTNAME>', | |
| 295 'non_browser_tests' | |
| 296 ], [ | |
| 297 'EPIC FAIL! group1 test1', | |
| 298 'YIPPEE! group2 test2' | |
| 299 ]); | |
| 300 }); | |
| 301 }); | |
| 302 | |
| 303 test("checked mode test", () { | |
| 304 return runTest([ | |
| 305 testrunner, | |
| 306 '--recurse', | |
| 307 '--checked', | |
| 308 'non_browser_tests' | |
| 309 ], [ | |
| 310 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1 ' | |
| 311 'Expected: false', | |
| 312 "?FAIL .*/non_browser_tests/non_browser_test.dart group2 test2 " | |
| 313 "Caught type 'int' is not a subtype of type 'bool' of 'x'." | |
| 314 ]); | |
| 315 }); | |
| 316 | |
| 317 group("browser", () { | |
| 318 test("native test", () { | |
| 319 return runTest([ | |
| 320 testrunner, | |
| 321 '--recurse', | |
| 322 '--runtime=drt-dart', | |
| 323 'browser_tests' | |
| 324 ], [ | |
| 325 '?FAIL .*/browser_tests/web/browser_test.dart group1 test1 ' | |
| 326 'Expected: false', | |
| 327 '?PASS .*/browser_tests/web/browser_test.dart group2 test2' | |
| 328 ]); | |
| 329 }); | |
| 330 | |
| 331 test("compiled test", () { | |
| 332 return runTest([ | |
| 333 testrunner, | |
| 334 '--recurse', | |
| 335 '--runtime=drt-js', | |
| 336 'browser_tests' | |
| 337 ], [ | |
| 338 '?FAIL .*/browser_tests/web/browser_test.dart group1 test1 ' | |
| 339 'Expected: false', | |
| 340 '?PASS .*/browser_tests/web/browser_test.dart group2 test2' | |
| 341 ]); | |
| 342 }); | |
| 343 }); | |
| 344 | |
| 345 group("textual layout tests", () { | |
| 346 group("drt-dart", () { | |
| 347 test("no baseline", () { | |
| 348 var f = new File("layout_tests/web/layout_test/layout.txt"); | |
| 349 if (f.existsSync()) { | |
| 350 f.deleteSync(); | |
| 351 } | |
| 352 return runTest([ | |
| 353 testrunner, | |
| 354 '--runtime=drt-dart', | |
| 355 '--recurse', | |
| 356 '--layout-text', | |
| 357 'layout_tests' | |
| 358 ], [ | |
| 359 '?FAIL .*/layout_tests/web/layout_test.dart layout ' | |
| 360 'No expectation file' | |
| 361 ]); | |
| 362 }); | |
| 363 test("create baseline", () { | |
| 364 return runTest([ | |
| 365 testrunner, | |
| 366 '--runtime=drt-dart', | |
| 367 '--recurse', | |
| 368 '--layout-text', | |
| 369 '--regenerate', | |
| 370 'layout_tests' | |
| 371 ], [ | |
| 372 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 373 ]); | |
| 374 }); | |
| 375 test("test baseline", () { | |
| 376 return runTest([ | |
| 377 testrunner, | |
| 378 '--runtime=drt-dart', | |
| 379 '--recurse', | |
| 380 '--layout-text', | |
| 381 'layout_tests' | |
| 382 ], [ | |
| 383 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 384 ]); | |
| 385 }); | |
| 386 }); | |
| 387 group("drt-js", () { | |
| 388 test("no baseline", () { | |
| 389 var f = new File("layout_tests/web/layout_test/layout.txt"); | |
| 390 if (f.existsSync()) { | |
| 391 f.deleteSync(); | |
| 392 } | |
| 393 return runTest([ | |
| 394 testrunner, | |
| 395 '--runtime=drt-js', | |
| 396 '--recurse', | |
| 397 '--layout-text', | |
| 398 'layout_tests' | |
| 399 ], [ | |
| 400 '?FAIL .*/layout_tests/web/layout_test.dart layout ' | |
| 401 'No expectation file' | |
| 402 ]); | |
| 403 }); | |
| 404 test("create baseline", () { | |
| 405 return runTest([ | |
| 406 testrunner, | |
| 407 '--runtime=drt-js', | |
| 408 '--recurse', | |
| 409 '--layout-text', | |
| 410 '--regenerate', | |
| 411 'layout_tests' | |
| 412 ], [ | |
| 413 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 414 ]); | |
| 415 }); | |
| 416 test("test baseline", () { | |
| 417 return runTest([ | |
| 418 testrunner, | |
| 419 '--runtime=drt-js', | |
| 420 '--recurse', | |
| 421 '--layout-text', | |
| 422 'layout_tests' | |
| 423 ], [ | |
| 424 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 425 ]); | |
| 426 }); | |
| 427 }); | |
| 428 }); | |
| 429 | |
| 430 group("pixel layout tests", () { | |
| 431 group("drt-dart", () { | |
| 432 test("no baseline", () { | |
| 433 var f = new File("layout_tests/web/layout_test/layout.png"); | |
| 434 if (f.existsSync()) { | |
| 435 f.deleteSync(); | |
| 436 } | |
| 437 return runTest([ | |
| 438 testrunner, | |
| 439 '--runtime=drt-dart', | |
| 440 '--recurse', | |
| 441 '--layout-pixel', | |
| 442 'layout_tests' | |
| 443 ], [ | |
| 444 '?FAIL .*/layout_tests/web/layout_test.dart layout ' | |
| 445 'No expectation file' | |
| 446 ]); | |
| 447 }); | |
| 448 test("create baseline", () { | |
| 449 return runTest([ | |
| 450 testrunner, | |
| 451 '--runtime=drt-dart', | |
| 452 '--recurse', | |
| 453 '--layout-pixel', | |
| 454 '--regenerate', | |
| 455 'layout_tests' | |
| 456 ], [ | |
| 457 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 458 ]); | |
| 459 }); | |
| 460 test("test baseline", () { | |
| 461 return runTest([ | |
| 462 testrunner, | |
| 463 '--runtime=drt-dart', | |
| 464 '--recurse', | |
| 465 '--layout-pixel', | |
| 466 'layout_tests' | |
| 467 ], [ | |
| 468 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 469 ]); | |
| 470 }); | |
| 471 // TODO(gram): Should add a test that changes a byte of the | |
| 472 // expectation .png. | |
| 473 }); | |
| 474 group("drt-js", () { | |
| 475 test("no baseline", () { | |
| 476 var f = new File("layout_tests/web/layout_test/layout.png"); | |
| 477 if (f.existsSync()) { | |
| 478 f.deleteSync(); | |
| 479 } | |
| 480 return runTest([ | |
| 481 testrunner, | |
| 482 '--runtime=drt-js', | |
| 483 '--recurse', | |
| 484 '--layout-pixel', | |
| 485 'layout_tests' | |
| 486 ], [ | |
| 487 '?FAIL .*/layout_tests/web/layout_test.dart layout ' | |
| 488 'No expectation file' | |
| 489 ]); | |
| 490 }); | |
| 491 test("create baseline", () { | |
| 492 return runTest([ | |
| 493 testrunner, | |
| 494 '--runtime=drt-js', | |
| 495 '--recurse', | |
| 496 '--layout-pixel', | |
| 497 '--regenerate', | |
| 498 'layout_tests' | |
| 499 ], [ | |
| 500 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 501 ]); | |
| 502 }); | |
| 503 test("test baseline", () { | |
| 504 return runTest([ | |
| 505 testrunner, | |
| 506 '--runtime=drt-js', | |
| 507 '--recurse', | |
| 508 '--layout-pixel', | |
| 509 'layout_tests' | |
| 510 ], [ | |
| 511 '?PASS .*/layout_tests/web/layout_test.dart layout' | |
| 512 ]); | |
| 513 }); | |
| 514 }); | |
| 515 }); | |
| 516 | |
| 517 group("run in isolate", () { | |
| 518 test("vm", () { | |
| 519 return runTest([ | |
| 520 testrunner, | |
| 521 '--runtime=vm', | |
| 522 '--recurse', | |
| 523 '--isolate', | |
| 524 'non_browser_tests' | |
| 525 ], [ | |
| 526 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1' | |
| 527 ' Expected: false', | |
| 528 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2' | |
| 529 ]); | |
| 530 }); | |
| 531 test("drt-dart", () { | |
| 532 return runTest([ | |
| 533 testrunner, | |
| 534 '--runtime=drt-dart', | |
| 535 '--recurse', | |
| 536 '--isolate', | |
| 537 'non_browser_tests' | |
| 538 ], [ | |
| 539 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1' | |
| 540 ' Expected: false', | |
| 541 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2' | |
| 542 ]); | |
| 543 }); | |
| 544 test("drt-js", () { | |
| 545 return runTest([ | |
| 546 testrunner, | |
| 547 '--runtime=drt-js', | |
| 548 '--recurse', | |
| 549 '--isolate', | |
| 550 'non_browser_tests' | |
| 551 ], [ | |
| 552 '?FAIL .*/non_browser_tests/non_browser_test.dart group1 test1 ' | |
| 553 'Expected: false', | |
| 554 '?PASS .*/non_browser_tests/non_browser_test.dart group2 test2' | |
| 555 ]); | |
| 556 }); | |
| 557 }); | |
| 558 | |
| 559 group("embedded server", () { | |
| 560 test("get test", () { | |
| 561 return runTest([ | |
| 562 testrunner, | |
| 563 '--recurse', | |
| 564 '--server', | |
| 565 '--port=3456', | |
| 566 '--root=${Directory.current.path}', | |
| 567 'http_client_tests' | |
| 568 ], [ | |
| 569 '?PASS .*/http_client_tests/http_client_test.dart test1', | |
| 570 '?PASS .*/http_client_tests/http_client_test.dart test2' | |
| 571 ]); | |
| 572 }); | |
| 573 }); | |
| 574 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2976313002:
Delete old tests that aren't used and/or useful any more. (Closed)
