Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(76)

Side by Side Diff: tools/testing/dart/test_suite.dart

Issue 9666053: Add get_drt.py to test.dart, so that testing browser components updates DumpRenderTree. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: address comments Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/testing/dart/test_options.dart ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /** 5 /**
6 * Classes and methods for enumerating and preparing tests. 6 * Classes and methods for enumerating and preparing tests.
7 * 7 *
8 * This library includes: 8 * This library includes:
9 * 9 *
10 * - Creating tests by listing all the Dart files in certain directories, 10 * - Creating tests by listing all the Dart files in certain directories,
11 * and creating [TestCase]s for those files that meet the relevant criteria. 11 * and creating [TestCase]s for those files that meet the relevant criteria.
12 * - Preparing tests, including copying files and frameworks to temporary 12 * - Preparing tests, including copying files and frameworks to temporary
13 * directories, and computing the command line and arguments to be run. 13 * directories, and computing the command line and arguments to be run.
14 */ 14 */
15 #library("test_suite"); 15 #library("test_suite");
16 16
17 #import("dart:io"); 17 #import("dart:io");
18 #import("dart:builtin"); 18 #import("dart:builtin");
19 #import("dart:isolate"); 19 #import("dart:isolate");
20 #import("status_file_parser.dart"); 20 #import("status_file_parser.dart");
21 #import("test_runner.dart"); 21 #import("test_runner.dart");
22 #import("multitest.dart"); 22 #import("multitest.dart");
23 #import("drt_updater.dart");
23 24
24 #source("browser_test.dart"); 25 #source("browser_test.dart");
25 26
26 27
27 /** 28 /**
28 * A TestSuite represents a collection of tests. It creates a [TestCase] 29 * A TestSuite represents a collection of tests. It creates a [TestCase]
29 * object for each test to be run, and passes the test cases to a callback. 30 * object for each test to be run, and passes the test cases to a callback.
30 * 31 *
31 * Most TestSuites represent a directory or directory tree containing tests, 32 * Most TestSuites represent a directory or directory tree containing tests,
32 * and a status file containing the expected results when these tests are run. 33 * and a status file containing the expected results when these tests are run.
(...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after
216 bool isTestFile(String filename) => filename.endsWith("Test.dart"); 217 bool isTestFile(String filename) => filename.endsWith("Test.dart");
217 218
218 bool listRecursively() => false; 219 bool listRecursively() => false;
219 220
220 String shellPath() => TestUtils.dartShellFileName(configuration); 221 String shellPath() => TestUtils.dartShellFileName(configuration);
221 222
222 List<String> additionalOptions(String filename) => []; 223 List<String> additionalOptions(String filename) => [];
223 224
224 void forEachTest(Function onTest, Map testCache, String globalTempDir(), 225 void forEachTest(Function onTest, Map testCache, String globalTempDir(),
225 [Function onDone = null]) { 226 [Function onDone = null]) {
227 // If DumpRenderTree is required, and not yet updated, wait for update.
228 if (DumpRenderTreeUpdater.componentRequiresDRT(configuration['component'])
229 && !DumpRenderTreeUpdater.updated) {
230 Expect.isTrue(DumpRenderTreeUpdater.isActive);
231 DumpRenderTreeUpdater.onUpdated.add(() {
232 forEachTest(onTest, testCache, globalTempDir, onDone);
233 });
234 return;
235 }
236
226 doTest = onTest; 237 doTest = onTest;
227 doDone = (onDone != null) ? onDone : (() => null); 238 doDone = (onDone != null) ? onDone : (() => null);
228 globalTemporaryDirectory = globalTempDir; 239 globalTemporaryDirectory = globalTempDir;
229 240
230 var filesRead = 0; 241 var filesRead = 0;
231 void statusFileRead() { 242 void statusFileRead() {
232 filesRead++; 243 filesRead++;
233 if (filesRead == statusFilePaths.length) { 244 if (filesRead == statusFilePaths.length) {
234 // Checked if we have already found and generated the tests for 245 // Checked if we have already found and generated the tests for
235 // this suite. 246 // this suite.
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after
319 } 330 }
320 Set<String> expectations = testExpectations.expectations(testName); 331 Set<String> expectations = testExpectations.expectations(testName);
321 if (configuration['report']) { 332 if (configuration['report']) {
322 // Tests with multiple VMOptions are counted more than once. 333 // Tests with multiple VMOptions are counted more than once.
323 for (var dummy in optionsFromFile["vmOptions"]) { 334 for (var dummy in optionsFromFile["vmOptions"]) {
324 SummaryReport.add(expectations); 335 SummaryReport.add(expectations);
325 } 336 }
326 } 337 }
327 if (expectations.contains(SKIP)) return; 338 if (expectations.contains(SKIP)) return;
328 339
329 switch (configuration['component']) { 340 if (TestUtils.isBrowserComponent(configuration['component'])) {
330 case 'dartium': 341 enqueueBrowserTest(info, testName, expectations);
331 case 'chromium': 342 } else {
332 case 'frogium': 343 enqueueStandardTest(info, testName, expectations);
333 case 'legium':
334 case 'webdriver':
335 enqueueBrowserTest(filename, testName, optionsFromFile,
336 expectations, isNegative);
337 break;
338 default:
339 isNegative = isNegative ||
340 (configuration['checked'] && info.isNegativeIfChecked);
341
342 if (configuration['component'] == 'dartc') {
343 // dartc can detect static type warnings by the
344 // format of the error line
345 if (info.hasFatalTypeErrors) {
346 isNegative = true;
347 } else if (info.hasRuntimeErrors) {
348 isNegative = false;
349 }
350 }
351
352 var argumentLists = argumentListsFromFile(filename,
353 optionsFromFile);
354
355 for (var args in argumentLists) {
356 doTest(new TestCase('$suiteName/$testName',
357 [new Command(shellPath(), args)],
358 configuration,
359 completeHandler,
360 expectations,
361 isNegative,
362 info));
363 }
364 } 344 }
365 } 345 }
366 346
347 void enqueueStandardTest(TestInformation info,
348 String testName,
349 Set<String> expectations) {
350 bool isNegative = info.isNegative ||
351 (configuration['checked'] && info.isNegativeIfChecked);
352
353 if (configuration['component'] == 'dartc') {
354 // dartc can detect static type warnings by the
355 // format of the error line
356 if (info.hasFatalTypeErrors) {
357 isNegative = true;
358 } else if (info.hasRuntimeErrors) {
359 isNegative = false;
360 }
361 }
362
363 var argumentLists = argumentListsFromFile(info.filename,
364 info.optionsFromFile);
365
366 for (var args in argumentLists) {
367 doTest(new TestCase('$suiteName/$testName',
368 [new Command(shellPath(), args)],
369 configuration,
370 completeHandler,
371 expectations,
372 isNegative,
373 info));
374 }
375 }
376
367 Function makeTestCaseCreator(Map optionsFromFile) { 377 Function makeTestCaseCreator(Map optionsFromFile) {
368 return (String filename, 378 return (String filename,
369 bool isNegative, 379 bool isNegative,
370 [bool isNegativeIfChecked = false, 380 [bool isNegativeIfChecked = false,
371 bool hasFatalTypeErrors = false, 381 bool hasFatalTypeErrors = false,
372 bool hasRuntimeErrors = false, 382 bool hasRuntimeErrors = false,
373 Set<String> multitestOutcome = null]) { 383 Set<String> multitestOutcome = null]) {
374 // Cache the test information for each test case. 384 // Cache the test information for each test case.
375 var info = new TestInformation(filename, 385 var info = new TestInformation(filename,
376 optionsFromFile, 386 optionsFromFile,
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after
411 * The [StandardTestSuite] has support for testing components that 421 * The [StandardTestSuite] has support for testing components that
412 * compile a test from Dart to Javascript, and then run the resulting 422 * compile a test from Dart to Javascript, and then run the resulting
413 * Javascript. This function creates a working directory to hold the 423 * Javascript. This function creates a working directory to hold the
414 * Javascript version of the test, and copies the appropriate framework 424 * Javascript version of the test, and copies the appropriate framework
415 * files to that directory. It creates a [BrowserTestCase], which has 425 * files to that directory. It creates a [BrowserTestCase], which has
416 * two sequential steps to be run by the [ProcessQueue when] the test is 426 * two sequential steps to be run by the [ProcessQueue when] the test is
417 * executed: a compilation 427 * executed: a compilation
418 * step and an execution step, both with the appropriate executable and 428 * step and an execution step, both with the appropriate executable and
419 * arguments. 429 * arguments.
420 */ 430 */
421 void enqueueBrowserTest(String filename, 431 void enqueueBrowserTest(TestInformation info,
422 String testName, 432 String testName,
423 Map optionsFromFile, 433 Set<String> expectations) {
424 Set<String> expectations, 434 Map optionsFromFile = info.optionsFromFile;
425 bool isNegative) { 435 String filename = info.filename;
426 if (optionsFromFile['isMultitest']) return; 436 if (optionsFromFile['isMultitest']) return;
427 bool isWebTest = optionsFromFile['containsDomImport']; 437 bool isWebTest = optionsFromFile['containsDomImport'];
428 bool isLibraryDefinition = optionsFromFile['isLibraryDefinition']; 438 bool isLibraryDefinition = optionsFromFile['isLibraryDefinition'];
429 if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) { 439 if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) {
430 print('Warning for $filename: Browser tests require #library ' + 440 print('Warning for $filename: Browser tests require #library ' +
431 'in any file that uses #import, #source, or #resource'); 441 'in any file that uses #import, #source, or #resource');
432 } 442 }
433 443
434 final String component = configuration['component']; 444 final String component = configuration['component'];
445 Expect.isTrue(DumpRenderTreeUpdater.componentRequiresDRT(component));
435 final String testPath = 446 final String testPath =
436 new File(filename).fullPathSync().replaceAll('\\', '/'); 447 new File(filename).fullPathSync().replaceAll('\\', '/');
437 448
438 for (var vmOptions in optionsFromFile['vmOptions']) { 449 for (var vmOptions in optionsFromFile['vmOptions']) {
439 // Create a unique temporary directory for each set of vmOptions. 450 // Create a unique temporary directory for each set of vmOptions.
440 // TODO(dart:429): Replace separate replaceAlls with a RegExp when 451 // TODO(dart:429): Replace separate replaceAlls with a RegExp when
441 // replaceAll(RegExp, String) is implemented. 452 // replaceAll(RegExp, String) is implemented.
442 String optionsName = ''; 453 String optionsName = '';
443 if (optionsFromFile['vmOptions'].length > 1) { 454 if (optionsFromFile['vmOptions'].length > 1) {
444 optionsName = Strings.join(vmOptions, '-').replaceAll('-','') 455 optionsName = Strings.join(vmOptions, '-').replaceAll('-','')
(...skipping 191 matching lines...) Expand 10 before | Expand all | Expand 10 after
636 // including any intermediate directories that don't exist. 647 // including any intermediate directories that don't exist.
637 var generatedTestPath = ['generated_tests', 648 var generatedTestPath = ['generated_tests',
638 configuration['component'], 649 configuration['component'],
639 testUniqueName]; 650 testUniqueName];
640 651
641 String tempDirPath = TestUtils.buildDir(configuration); 652 String tempDirPath = TestUtils.buildDir(configuration);
642 if (requiresCleanTemporaryDirectory) { 653 if (requiresCleanTemporaryDirectory) {
643 tempDirPath = globalTemporaryDirectory(); 654 tempDirPath = globalTemporaryDirectory();
644 String debugMode = 655 String debugMode =
645 (configuration['mode'] == 'debug') ? 'Debug_' : 'Release_'; 656 (configuration['mode'] == 'debug') ? 'Debug_' : 'Release_';
646 generatedTestPath = ['${debugMode}_${configuration["arch"]}'] 657 var temp = ['${debugMode}_${configuration["arch"]}'];
647 .addAll(generatedTestPath); 658 temp.addAll(generatedTestPath);
659 generatedTestPath = temp;
648 } 660 }
649 Directory tempDir = new Directory(tempDirPath); 661 Directory tempDir = new Directory(tempDirPath);
650 if (!tempDir.existsSync()) { 662 if (!tempDir.existsSync()) {
651 // Dartium tests can be run with no build step, with no output directory. 663 // Dartium tests can be run with no build step, with no output directory.
652 // This special case builds the build directory that should be there. 664 // This special case builds the build directory that should be there.
653 var buildPath = tempDirPath.split('/'); 665 var buildPath = tempDirPath.split('/');
654 tempDirPath = buildPath[0]; 666 tempDirPath = buildPath[0];
655 if (tempDirPath == '') { 667 if (tempDirPath == '') {
656 throw new Exception( 668 throw new Exception(
657 'Non-relative path to build directory in test_suite.dart'); 669 'Non-relative path to build directory in test_suite.dart');
(...skipping 506 matching lines...) Expand 10 before | Expand all | Expand 10 after
1164 args.add('--enable_asserts'); 1176 args.add('--enable_asserts');
1165 args.add("--enable_type_checks"); 1177 args.add("--enable_type_checks");
1166 } 1178 }
1167 if (configuration["component"] == "leg" 1179 if (configuration["component"] == "leg"
1168 || configuration["component"] == "legium") { 1180 || configuration["component"] == "legium") {
1169 args.add("--verbose"); 1181 args.add("--verbose");
1170 args.add("--leg"); 1182 args.add("--leg");
1171 } 1183 }
1172 return args; 1184 return args;
1173 } 1185 }
1186
1187 static bool isBrowserComponent(String component) =>
1188 const <String>['dartium',
1189 'frogium',
1190 'legium',
1191 'webdriver'].some((x) => x == component);
1174 } 1192 }
1175 1193
1176 class SummaryReport { 1194 class SummaryReport {
1177 static int total = 0; 1195 static int total = 0;
1178 static int skipped = 0; 1196 static int skipped = 0;
1179 static int noCrash = 0; 1197 static int noCrash = 0;
1180 static int pass = 0; 1198 static int pass = 0;
1181 static int failOk = 0; 1199 static int failOk = 0;
1182 static int fail = 0; 1200 static int fail = 0;
1183 static int crash = 0; 1201 static int crash = 0;
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
1217 * $noCrash tests are expected to be flaky but not crash 1235 * $noCrash tests are expected to be flaky but not crash
1218 * $pass tests are expected to pass 1236 * $pass tests are expected to pass
1219 * $failOk tests are expected to fail that we won't fix 1237 * $failOk tests are expected to fail that we won't fix
1220 * $fail tests are expected to fail that we should fix 1238 * $fail tests are expected to fail that we should fix
1221 * $crash tests are expected to crash that we should fix 1239 * $crash tests are expected to crash that we should fix
1222 * $timeout tests are allowed to timeout 1240 * $timeout tests are allowed to timeout
1223 """; 1241 """;
1224 print(report); 1242 print(report);
1225 } 1243 }
1226 } 1244 }
OLDNEW
« no previous file with comments | « tools/testing/dart/test_options.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698