Use status files for DDC's presubmit script/travis bot

pkg/dev_compiler/test/codegen_test.dart

Index: pkg/dev_compiler/test/codegen_test.dart
diff --git a/pkg/dev_compiler/test/codegen_test.dart b/pkg/dev_compiler/test/codegen_test.dart
index 0dcaa243ba8af1477d3a825529fb062ef7dd29c0..1d5e2ff8e251b1bb377a1f8f4903463c285f3474 100644
--- a/pkg/dev_compiler/test/codegen_test.dart
+++ b/pkg/dev_compiler/test/codegen_test.dart
@@ -12,6 +12,7 @@ library dev_compiler.test.codegen_test;
 // compiles stuff. This should be changed to not use unittest and just be a
 // regular program that outputs files.
 
+import 'dart:convert';
 import 'dart:io' show Directory, File, Platform;
 import 'package:analyzer/analyzer.dart'
     show
@@ -32,9 +33,13 @@ import 'package:dev_compiler/src/compiler/module_builder.dart'
     show ModuleFormat, addModuleFormatOptions, parseModuleFormatOption;
 import 'package:path/path.dart' as path;
 import 'package:test/test.dart' show expect, isFalse, isTrue, test;
+import 'package:status_file/expectation.dart';
+import 'package:test_dart/path.dart' as test_dart;
+import 'package:test_dart/test_suite.dart' show StandardTestSuite;
+import 'package:test_dart/utils.dart';
+import 'package:test_dart/options.dart';
 
 import '../tool/build_sdk.dart' as build_sdk;
-import 'compile_error_tests.dart';
 import 'multitest.dart' show extractTestsFromMultitest, isMultiTest;
 import 'testing.dart' show repoDirectory, testDirectory;
 
@@ -82,21 +87,14 @@ main(List<String> arguments) {
   var sharedCompiler = new ModuleCompiler(new AnalyzerOptions.basic(
       dartSdkSummaryPath: sdkSummaryFile, summaryPaths: summaryPaths));
 
-  var testDirs = [
-    'language',
-    'corelib',
-    path.join('lib', 'async'),
-    path.join('lib', 'collection'),
-    path.join('lib', 'convert'),
-    path.join('lib', 'html'),
-    path.join('lib', 'math'),
-    path.join('lib', 'mirrors'),
-    path.join('lib', 'typed_data'),
-  ];
+  var testDirs = ['language', 'corelib', 'lib'];
 
   // Copy all of the test files and expanded multitest files to
   // gen/codegen_tests. We'll compile from there.
+  TestUtils.setDartDirUri(Platform.script.resolve('../../..'));
   var testFiles = _setUpTests(testDirs);
+  _writeRuntimeStatus(testFiles);
+  return; //!!!

[vsm, 2017/07/31 21:02:20]

Did you mean to leave that return here?

[Jennifer Messerly, 2017/07/31 21:24:55]

Oops! after I got it passing with the compile error change, I started working on
`_writeRuntimeStatus` and debugging that a lot. I'll upload with that removed :)

 
   // Our default compiler options. Individual tests can override these.
   var defaultOptions = ['--no-source-map', '--no-summarize'];
@@ -118,12 +116,12 @@ main(List<String> arguments) {
   ];
 
   // Compile each test file to JS and put the result in gen/codegen_output.
-  for (var testFile in testFiles) {
+  testFiles.forEach((testFile, expectations) {
     var relativePath = path.relative(testFile, from: codegenTestDir);
 
     // Only compile the top-level files for generating coverage.
     bool isTopLevelTest = path.dirname(relativePath) == ".";
-    if (codeCoverage && !isTopLevelTest) continue;
+    if (codeCoverage && !isTopLevelTest) return;
 
     var name = path.withoutExtension(relativePath);
     test('dartdevc $name', () {
@@ -169,13 +167,7 @@ main(List<String> arguments) {
       var intentionalCompileError = contents.contains(': compile-time error') ||
           contents.contains('/*@compile-error=');
 
-      // This covers tests that should not produce a static error but that
-      // currently do due to issues in our implementation.
-      var knownCompileError = compileErrorTests.contains(name);
-
-      var crashing = _crashingTests.contains(name);
-      var inconsistent = _inconsistentTests.contains(name);
-
+      var crashing = expectations.contains(Expectation.crash);
       if (module == null) {
         expect(crashing, isTrue,
             reason: "test $name crashes during compilation.\n"
@@ -192,13 +184,9 @@ main(List<String> arguments) {
 
       expect(crashing, isFalse, reason: "test $name no longer crashes.");
 
-      if (inconsistent) {
-        // An inconsistent test will only compile on some platforms (see
-        // comment below). It should not crash however.
-      } else if (module.isValid) {
-        // TODO(vsm): We don't seem to trip on non-strong errors?
-        // expect(expectedCompileTimeError, isFalse,
-        //    reason: "test $name expected compilation errors, but compiled.");
+      var knownCompileError =
+          expectations.contains(Expectation.compileTimeError);
+      if (module.isValid) {
         expect(knownCompileError, isFalse,
             reason: "test $name expected static errors, but compiled.");
       } else {
@@ -208,7 +196,7 @@ main(List<String> arguments) {
                 "\n\n${module.errors.join('\n')}.");
       }
     });
-  }
+  });
 
   if (filePattern.hasMatch('sunflower')) {
     test('sunflower', () {
@@ -276,9 +264,40 @@ String _moduleForLibrary(Source source) {
   throw new Exception('Module not found for library "${source.fullName}"');
 }
 
-List<String> _setUpTests(List<String> testDirs) {
-  var testFiles = <String>[];
+void _writeRuntimeStatus(Map<String, Set<Expectation>> testFiles) {
+  var runtimeStatus = <String, String>{};
+  testFiles.forEach((name, status) {
+    name = path.withoutExtension(path.relative(name, from: codegenTestDir));
+    // Skip tests that we don't expect to compile.
+    if (status.contains(Expectation.compileTimeError) ||
+        status.contains(Expectation.crash)) {
+      return;
+    }
+    // Normalize the expectations for the Karma language_test.js runner.
+    if (status.remove(Expectation.ok)) assert(status.isNotEmpty);
+    if (status.remove(Expectation.missingCompileTimeError) ||
+        status.remove(Expectation.missingRuntimeError)) {
+      status.add(Expectation.pass);
+    }
+
+    // Skip passing tests.
+    // TODO(jmesserly): we could record these for extra sanity checks.
+    if (status.length == 1 && status.contains(Expectation.pass)) {
+      return;
+    }
+    
+    runtimeStatus[name] = status.map((s) => '$s').join(',');
+  });
+  new File(path.join(codegenOutputDir, 'test_status.js')).writeAsStringSync('''
+define([], function() {
+  'use strict';
+  return ${new JsonEncoder.withIndent(' ').convert(runtimeStatus)};
+});
+''');
+}
 
+Map<String, Set<Expectation>> _setUpTests(List<String> testDirs) {
+  var testFiles = <String, Set<Expectation>>{};
   for (var testDir in testDirs) {
     // TODO(rnystrom): Simplify this when the Dart 2.0 test migration is
     // complete (#30183).
@@ -287,13 +306,23 @@ List<String> _setUpTests(List<String> testDirs) {
     var dirParts = path.split(testDir);
 
     for (var suffix in const ["_2", "_strong"]) {
-      var sdkTestDir =
-          path.join(dirParts[0] + suffix, path.joinAll(dirParts.skip(1)));
-      var inputPath =
-          path.join(testDirectory, '..', '..', '..', 'tests', sdkTestDir);
+      var sdkTestDir = path.join(
+          'tests', dirParts[0] + suffix, path.joinAll(dirParts.skip(1)));
+      var inputPath = path.join(testDirectory, '..', '..', '..', sdkTestDir);
 
       if (!new Directory(inputPath).existsSync()) continue;
 
+      var browsers = Platform.environment['DDC_BROWSERS'];
+      var runtime = browsers == 'Firefox' ? 'firefox' : 'chrome';
+      var config = new OptionsParser()
+          .parse('-m release -c dartdevc --use-sdk --strong'.split(' ')
+            ..addAll(['-r', runtime, '--suite_dir', sdkTestDir]))
+          .single;
+
+      var testSuite = new StandardTestSuite.forDirectory(
+          config, new test_dart.Path(sdkTestDir));
+      var expectations = testSuite.readExpectations();
+
       for (var file in _listFiles(inputPath, recursive: true)) {
         var relativePath = path.relative(file, from: inputPath);
         var outputPath = path.join(codegenTestDir, testDir, relativePath);
@@ -301,6 +330,7 @@ List<String> _setUpTests(List<String> testDirs) {
         _ensureDirectory(path.dirname(outputPath));
 
         if (file.endsWith("_test.dart")) {
+          var statusPath = path.withoutExtension(relativePath);
 
           void _writeTest(String outputPath, String contents) {
             if (contents.contains('package:unittest/')) {
@@ -318,21 +348,21 @@ List<String> _setUpTests(List<String> testDirs) {
           if (isMultiTest(contents)) {
             // It's a multitest, so expand it and add all of the variants.
             var tests = <String, String>{};
-            var outcomes = <String, Set<String>>{};
-            extractTestsFromMultitest(file, contents, tests, outcomes);
+            extractTestsFromMultitest(file, contents, tests, {});
 
             var fileName = path.basenameWithoutExtension(file);
             var outputDir = path.dirname(outputPath);
             tests.forEach((name, contents) {
               var multiFile =
                   path.join(outputDir, '${fileName}_${name}_multi.dart');
-              testFiles.add(multiFile);
+              testFiles[multiFile] =
+                  expectations.expectations("$statusPath/$name");
 
               _writeTest(multiFile, contents);
             });
           } else {
             // It's a single test suite.
-            testFiles.add(outputPath);
+            testFiles[outputPath] = expectations.expectations(statusPath);
           }
 
           // Write the test file.
@@ -356,7 +386,7 @@ List<String> _setUpTests(List<String> testDirs) {
 
     new File(file).copySync(outputPath);
     if (file.endsWith(".dart")) {
-      testFiles.add(outputPath);
+      testFiles[outputPath] = new Set()..add(Expectation.pass);
     }
   }
 
@@ -372,14 +402,8 @@ void _ensureDirectory(String dir) {
 Iterable<String> _listFiles(String dir, {bool recursive: false}) {
   return new Directory(dir)
       .listSync(recursive: recursive, followLinks: false)
-      .where((entry) {
-    if (entry is! File) return false;
-
-    var filePath = entry.path;
-    if (!filePattern.hasMatch(filePath)) return false;
-
-    return true;
-  }).map((file) => file.path);
+      .where((e) => e is File && filePattern.hasMatch(e.path))
+      .map((f) => f.path);
 }
 
 /// Parse directives from [contents] and find the complete set of transitive
@@ -425,21 +449,3 @@ String _resolveDirective(UriBasedDirective directive) {
       ? uriContent
       : null;
 }
-
-/// Tests that, due to bugs, are strong-mode clean only on some platforms.
-final _inconsistentTests = new Set<String>.from([
-  // This test is clean on windows, but not linux/mac due to newline encoding.
-  // See: https://github.com/dart-lang/sdk/issues/27224
-  'language/multiline_newline_test_02_multi',
-].map((p) => p.replaceAll('/', path.separator)));
-
-final _crashingTests = new Set<String>.from([
-  // TODO(vsm): Fix these - they import files from a different directory
-  // - this triggers an invalid library root build error.
-  'lib/html/custom/attribute_changed_callback_test',
-  'lib/html/custom/constructor_calls_created_synchronously_test',
-  'lib/html/custom/entered_left_view_test',
-  'lib/html/custom/js_custom_test',
-  'lib/html/custom/mirrors_test',
-  'lib/html/custom/regress_194523002_test',
-].map((p) => p.replaceAll('/', path.separator)));