| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| 11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
| 12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
| 13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
| 14 */ | 14 */ |
| 15 #library("test_suite"); | 15 #library("test_suite"); |
| 16 | 16 |
| 17 #import("dart:io"); | 17 #import("dart:io"); |
| 18 #import("dart:isolate"); | 18 #import("dart:isolate"); |
| 19 #import("status_file_parser.dart"); | 19 #import("status_file_parser.dart"); |
| 20 #import("test_runner.dart"); | 20 #import("test_runner.dart"); |
| 21 #import("multitest.dart"); | 21 #import("multitest.dart"); |
| 22 #import("drt_updater.dart"); |
| 22 | 23 |
| 23 #source("browser_test.dart"); | 24 #source("browser_test.dart"); |
| 24 | 25 |
| 25 | 26 |
| 26 /** | 27 /** |
| 27 * A TestSuite represents a collection of tests. It creates a [TestCase] | 28 * A TestSuite represents a collection of tests. It creates a [TestCase] |
| 28 * object for each test to be run, and passes the test cases to a callback. | 29 * object for each test to be run, and passes the test cases to a callback. |
| 29 * | 30 * |
| 30 * Most TestSuites represent a directory or directory tree containing tests, | 31 * Most TestSuites represent a directory or directory tree containing tests, |
| 31 * and a status file containing the expected results when these tests are run. | 32 * and a status file containing the expected results when these tests are run. |
| (...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 215 bool isTestFile(String filename) => filename.endsWith("Test.dart"); | 216 bool isTestFile(String filename) => filename.endsWith("Test.dart"); |
| 216 | 217 |
| 217 bool listRecursively() => false; | 218 bool listRecursively() => false; |
| 218 | 219 |
| 219 String shellPath() => TestUtils.dartShellFileName(configuration); | 220 String shellPath() => TestUtils.dartShellFileName(configuration); |
| 220 | 221 |
| 221 List<String> additionalOptions(String filename) => []; | 222 List<String> additionalOptions(String filename) => []; |
| 222 | 223 |
| 223 void forEachTest(Function onTest, Map testCache, String globalTempDir(), | 224 void forEachTest(Function onTest, Map testCache, String globalTempDir(), |
| 224 [Function onDone = null]) { | 225 [Function onDone = null]) { |
| 226 // If DumpRenderTree is required, and not yet updated, wait for update. |
| 227 if (DumpRenderTreeUpdater.componentRequiresDRT(configuration['component']) |
| 228 && !DumpRenderTreeUpdater.updated) { |
| 229 Expect.isTrue(DumpRenderTreeUpdater.isActive); |
| 230 DumpRenderTreeUpdater.onUpdated.add(() { |
| 231 forEachTest(onTest, testCache, globalTempDir, onDone); |
| 232 }); |
| 233 return; |
| 234 } |
| 235 |
| 225 doTest = onTest; | 236 doTest = onTest; |
| 226 doDone = (onDone != null) ? onDone : (() => null); | 237 doDone = (onDone != null) ? onDone : (() => null); |
| 227 globalTemporaryDirectory = globalTempDir; | 238 globalTemporaryDirectory = globalTempDir; |
| 228 | 239 |
| 229 var filesRead = 0; | 240 var filesRead = 0; |
| 230 void statusFileRead() { | 241 void statusFileRead() { |
| 231 filesRead++; | 242 filesRead++; |
| 232 if (filesRead == statusFilePaths.length) { | 243 if (filesRead == statusFilePaths.length) { |
| 233 // Checked if we have already found and generated the tests for | 244 // Checked if we have already found and generated the tests for |
| 234 // this suite. | 245 // this suite. |
| (...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 318 } | 329 } |
| 319 Set<String> expectations = testExpectations.expectations(testName); | 330 Set<String> expectations = testExpectations.expectations(testName); |
| 320 if (configuration['report']) { | 331 if (configuration['report']) { |
| 321 // Tests with multiple VMOptions are counted more than once. | 332 // Tests with multiple VMOptions are counted more than once. |
| 322 for (var dummy in optionsFromFile["vmOptions"]) { | 333 for (var dummy in optionsFromFile["vmOptions"]) { |
| 323 SummaryReport.add(expectations); | 334 SummaryReport.add(expectations); |
| 324 } | 335 } |
| 325 } | 336 } |
| 326 if (expectations.contains(SKIP)) return; | 337 if (expectations.contains(SKIP)) return; |
| 327 | 338 |
| 328 switch (configuration['component']) { | 339 if (TestUtils.isBrowserComponent(configuration['component'])) { |
| 329 case 'dartium': | 340 enqueueBrowserTest(info, testName, expectations); |
| 330 case 'chromium': | 341 } else { |
| 331 case 'frogium': | 342 enqueueStandardTest(info, testName, expectations); |
| 332 case 'legium': | |
| 333 case 'webdriver': | |
| 334 enqueueBrowserTest(filename, testName, optionsFromFile, | |
| 335 expectations, isNegative); | |
| 336 break; | |
| 337 default: | |
| 338 isNegative = isNegative || | |
| 339 (configuration['checked'] && info.isNegativeIfChecked); | |
| 340 | |
| 341 if (configuration['component'] == 'dartc') { | |
| 342 // dartc can detect static type warnings by the | |
| 343 // format of the error line | |
| 344 if (info.hasFatalTypeErrors) { | |
| 345 isNegative = true; | |
| 346 } else if (info.hasRuntimeErrors) { | |
| 347 isNegative = false; | |
| 348 } | |
| 349 } | |
| 350 | |
| 351 var argumentLists = argumentListsFromFile(filename, | |
| 352 optionsFromFile); | |
| 353 | |
| 354 for (var args in argumentLists) { | |
| 355 doTest(new TestCase('$suiteName/$testName', | |
| 356 [new Command(shellPath(), args)], | |
| 357 configuration, | |
| 358 completeHandler, | |
| 359 expectations, | |
| 360 isNegative, | |
| 361 info)); | |
| 362 } | |
| 363 } | 343 } |
| 364 } | 344 } |
| 365 | 345 |
| 346 void enqueueStandardTest(TestInfo info, |
| 347 String testName, |
| 348 Set<String> expectations) { |
| 349 bool isNegative = info.isNegative || |
| 350 (configuration['checked'] && info.isNegativeIfChecked); |
| 351 |
| 352 if (configuration['component'] == 'dartc') { |
| 353 // dartc can detect static type warnings by the |
| 354 // format of the error line |
| 355 if (info.hasFatalTypeErrors) { |
| 356 isNegative = true; |
| 357 } else if (info.hasRuntimeErrors) { |
| 358 isNegative = false; |
| 359 } |
| 360 } |
| 361 |
| 362 var argumentLists = argumentListsFromFile(info.filename, |
| 363 info.optionsFromFile); |
| 364 |
| 365 for (var args in argumentLists) { |
| 366 doTest(new TestCase('$suiteName/$testName', |
| 367 [new Command(shellPath(), args)], |
| 368 configuration, |
| 369 completeHandler, |
| 370 expectations, |
| 371 isNegative, |
| 372 info)); |
| 373 } |
| 374 } |
| 375 |
| 366 Function makeTestCaseCreator(Map optionsFromFile) { | 376 Function makeTestCaseCreator(Map optionsFromFile) { |
| 367 return (String filename, | 377 return (String filename, |
| 368 bool isNegative, | 378 bool isNegative, |
| 369 [bool isNegativeIfChecked = false, | 379 [bool isNegativeIfChecked = false, |
| 370 bool hasFatalTypeErrors = false, | 380 bool hasFatalTypeErrors = false, |
| 371 bool hasRuntimeErrors = false, | 381 bool hasRuntimeErrors = false, |
| 372 Set<String> multitestOutcome = null]) { | 382 Set<String> multitestOutcome = null]) { |
| 373 // Cache the test information for each test case. | 383 // Cache the test information for each test case. |
| 374 var info = new TestInformation(filename, | 384 var info = new TestInformation(filename, |
| 375 optionsFromFile, | 385 optionsFromFile, |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 410 * The [StandardTestSuite] has support for testing components that | 420 * The [StandardTestSuite] has support for testing components that |
| 411 * compile a test from Dart to Javascript, and then run the resulting | 421 * compile a test from Dart to Javascript, and then run the resulting |
| 412 * Javascript. This function creates a working directory to hold the | 422 * Javascript. This function creates a working directory to hold the |
| 413 * Javascript version of the test, and copies the appropriate framework | 423 * Javascript version of the test, and copies the appropriate framework |
| 414 * files to that directory. It creates a [BrowserTestCase], which has | 424 * files to that directory. It creates a [BrowserTestCase], which has |
| 415 * two sequential steps to be run by the [ProcessQueue when] the test is | 425 * two sequential steps to be run by the [ProcessQueue when] the test is |
| 416 * executed: a compilation | 426 * executed: a compilation |
| 417 * step and an execution step, both with the appropriate executable and | 427 * step and an execution step, both with the appropriate executable and |
| 418 * arguments. | 428 * arguments. |
| 419 */ | 429 */ |
| 420 void enqueueBrowserTest(String filename, | 430 void enqueueBrowserTest(TestInformation info, |
| 421 String testName, | 431 String testName, |
| 422 Map optionsFromFile, | 432 Set<String> expectations) { |
| 423 Set<String> expectations, | 433 Map optionsFromFile = info.optionsFromFile; |
| 424 bool isNegative) { | 434 String filename = info.filename; |
| 425 if (optionsFromFile['isMultitest']) return; | 435 if (optionsFromFile['isMultitest']) return; |
| 426 bool isWebTest = optionsFromFile['containsDomImport']; | 436 bool isWebTest = optionsFromFile['containsDomImport']; |
| 427 bool isLibraryDefinition = optionsFromFile['isLibraryDefinition']; | 437 bool isLibraryDefinition = optionsFromFile['isLibraryDefinition']; |
| 428 if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) { | 438 if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) { |
| 429 print('Warning for $filename: Browser tests require #library ' + | 439 print('Warning for $filename: Browser tests require #library ' + |
| 430 'in any file that uses #import, #source, or #resource'); | 440 'in any file that uses #import, #source, or #resource'); |
| 431 } | 441 } |
| 432 | 442 |
| 433 final String component = configuration['component']; | 443 final String component = configuration['component']; |
| 444 Expect.isTrue(DumpRenderTreeUpdater.componentRequiresDRT(component)); |
| 434 final String testPath = | 445 final String testPath = |
| 435 new File(filename).fullPathSync().replaceAll('\\', '/'); | 446 new File(filename).fullPathSync().replaceAll('\\', '/'); |
| 436 | 447 |
| 437 for (var vmOptions in optionsFromFile['vmOptions']) { | 448 for (var vmOptions in optionsFromFile['vmOptions']) { |
| 438 // Create a unique temporary directory for each set of vmOptions. | 449 // Create a unique temporary directory for each set of vmOptions. |
| 439 // TODO(dart:429): Replace separate replaceAlls with a RegExp when | 450 // TODO(dart:429): Replace separate replaceAlls with a RegExp when |
| 440 // replaceAll(RegExp, String) is implemented. | 451 // replaceAll(RegExp, String) is implemented. |
| 441 String optionsName = ''; | 452 String optionsName = ''; |
| 442 if (optionsFromFile['vmOptions'].length > 1) { | 453 if (optionsFromFile['vmOptions'].length > 1) { |
| 443 optionsName = Strings.join(vmOptions, '-').replaceAll('-','') | 454 optionsName = Strings.join(vmOptions, '-').replaceAll('-','') |
| (...skipping 719 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1163 args.add('--enable_asserts'); | 1174 args.add('--enable_asserts'); |
| 1164 args.add("--enable_type_checks"); | 1175 args.add("--enable_type_checks"); |
| 1165 } | 1176 } |
| 1166 if (configuration["component"] == "leg" | 1177 if (configuration["component"] == "leg" |
| 1167 || configuration["component"] == "legium") { | 1178 || configuration["component"] == "legium") { |
| 1168 args.add("--verbose"); | 1179 args.add("--verbose"); |
| 1169 args.add("--leg"); | 1180 args.add("--leg"); |
| 1170 } | 1181 } |
| 1171 return args; | 1182 return args; |
| 1172 } | 1183 } |
| 1184 |
| 1185 static bool isBrowserComponent(String component) => |
| 1186 const <String>['dartium', |
| 1187 'chromium', |
| 1188 'frogium', |
| 1189 'legium', |
| 1190 'webdriver'].some((x) => x == component); |
| 1173 } | 1191 } |
| 1174 | 1192 |
| 1175 class SummaryReport { | 1193 class SummaryReport { |
| 1176 static int total = 0; | 1194 static int total = 0; |
| 1177 static int skipped = 0; | 1195 static int skipped = 0; |
| 1178 static int noCrash = 0; | 1196 static int noCrash = 0; |
| 1179 static int pass = 0; | 1197 static int pass = 0; |
| 1180 static int failOk = 0; | 1198 static int failOk = 0; |
| 1181 static int fail = 0; | 1199 static int fail = 0; |
| 1182 static int crash = 0; | 1200 static int crash = 0; |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1216 * $noCrash tests are expected to be flaky but not crash | 1234 * $noCrash tests are expected to be flaky but not crash |
| 1217 * $pass tests are expected to pass | 1235 * $pass tests are expected to pass |
| 1218 * $failOk tests are expected to fail that we won't fix | 1236 * $failOk tests are expected to fail that we won't fix |
| 1219 * $fail tests are expected to fail that we should fix | 1237 * $fail tests are expected to fail that we should fix |
| 1220 * $crash tests are expected to crash that we should fix | 1238 * $crash tests are expected to crash that we should fix |
| 1221 * $timeout tests are allowed to timeout | 1239 * $timeout tests are allowed to timeout |
| 1222 """; | 1240 """; |
| 1223 print(report); | 1241 print(report); |
| 1224 } | 1242 } |
| 1225 } | 1243 } |
| OLD | NEW |