Added PkgBuildTestSuite (aka 'pkgbuild')

dart/tools/testing/dart/test_runner.dart

Index: dart/tools/testing/dart/test_runner.dart
diff --git a/dart/tools/testing/dart/test_runner.dart b/dart/tools/testing/dart/test_runner.dart
index d39598c0dd9b9e753136b3cea95bbab2bc337db1..0f3f548b4e57d80e7bbc6d66a638d72ebf6bab6e 100644
--- a/dart/tools/testing/dart/test_runner.dart
+++ b/dart/tools/testing/dart/test_runner.dart
@@ -156,8 +156,12 @@ class ProcessCommand extends Command {
   }
 
   String get reproductionCommand {
-    return ([executable]..addAll(arguments))
+    var command = ([executable]..addAll(arguments))
         .map(escapeCommandLineArgument).join(' ');
+    if (workingDirectory != null) {
+      command = "$command (working directory: $workingDirectory)";
+    }
+    return command;
   }
 
   Future<bool> get outputIsUpToDate => new Future.value(false);
@@ -362,6 +366,32 @@ class JSCommandlineCommand extends ProcessCommand {
                 environmentOverrides);
 }
 
+class PubCommand extends ProcessCommand {
+  final String command;
+
+  PubCommand._(String pubCommand,
+               String pubExecutable,
+               String pubspecYamlDirectory,
+               String pubCacheDirectory)
+      : super._('pub_$pubCommand',
+                new io.File(pubExecutable).absolute.path,
+                [pubCommand],
+                {'PUB_CACHE' : pubCacheDirectory},
+                pubspecYamlDirectory), command = pubCommand;
+
+  void _buildHashCode(HashCodeBuilder builder) {
+    super._buildHashCode(builder);
+    builder.add(command);
+  }
+
+  bool _equal(Command other) {
+    return
+        other is PubCommand &&
+        super._equal(other) &&
+        command == other.command;
+  }
+}
+
 /* [ScriptCommand]s are executed by dart code. */
 abstract class ScriptCommand extends Command {
   ScriptCommand._(String displayName) : super._(displayName);
@@ -369,6 +399,118 @@ abstract class ScriptCommand extends Command {
   Future<ScriptCommandOutputImpl> run();
 }
 
+class CleanDirectoryCopyCommand extends ScriptCommand {
+  final String _sourceDirectory;
+  final String _destinationDirectory;
+
+  CleanDirectoryCopyCommand._(this._sourceDirectory, this._destinationDirectory)
+    : super._('dir_copy');
+
+  String get reproductionCommand =>
+      "Copying '$_sourceDirectory' to '$_destinationDirectory'.";
+
+  Future<ScriptCommandOutputImpl> run() {
+    var watch = new Stopwatch()..start();
+
+    var source = new io.Directory(_sourceDirectory);
+    var destination = new io.Directory(_destinationDirectory);
+
+    return destination.exists().then((bool exists) {
+      var cleanDirectoryFuture;
+      if (exists) {
+        cleanDirectoryFuture = destination.delete(recursive: true);
+      } else {
+        cleanDirectoryFuture = new Future.value(null);
+      }
+      return cleanDirectoryFuture.then((_) {
+        return TestUtils.copyDirectory(_sourceDirectory, _destinationDirectory);
+      });
+    }).then((_) {
+      return new ScriptCommandOutputImpl(
+          this, Expectation.PASS, "", watch.elapsed);
+    }).catchError((error) {
+      return new ScriptCommandOutputImpl(
+          this, Expectation.FAIL, "An error occured: $error.", watch.elapsed);
+    });
+  }
+
+  void _buildHashCode(HashCodeBuilder builder) {
+    super._buildHashCode(builder);
+    builder.add(_sourceDirectory);
+    builder.add(_destinationDirectory);
+  }
+
+  bool _equal(Command other) {
+    return
+        other is CleanDirectoryCopyCommand &&
+        super._equal(other) &&
+        _sourceDirectory == other._sourceDirectory &&
+        _destinationDirectory == other._destinationDirectory;
+  }
+}
+
+class ModifyPubspecYamlCommand extends ScriptCommand {
+  String _pubspecYamlFile;
+  String _destinationFile;
+  Map<String, Map> _dependencyOverrides;
+
+  ModifyPubspecYamlCommand._(this._pubspecYamlFile,
+                             this._destinationFile,
+                             this._dependencyOverrides)
+    : super._("modify_pubspec");
+
+  String get reproductionCommand =>
+      "Adding necessary dependency overrides to '$_pubspecYamlFile' "
+      "(destination = $_destinationFile).";
+
+  Future<ScriptCommandOutputImpl> run() {
+    var watch = new Stopwatch()..start();
+
+    var file = new io.File(_pubspecYamlFile);
+    var destinationFile = new io.File(_destinationFile);
+    return file.readAsString().then((String yamlString) {
+      var dependencyOverrideSection = new StringBuffer();
+      if (_dependencyOverrides.isNotEmpty) {
+        dependencyOverrideSection.write(
+            "\n"
+            "# This section was autogenerated by test.py!\n"
+            "dependency_overrides:\n");

[Bob Nystrom, 2014/01/07 17:13:43]

This will do bad things if the pubspec already contains dependency_overrides.

One option is to parse the entire pubspec using the yaml package (which is in
the Dart repo). That will give you back a regular map. You can then find/add the
"dependency_overrides" key, and add them to that. Then you can just resave that
pubspec by serializing it to JSON. YAML is a superset of JSON, so pub will be
able to read that in fine.

[kustermann, 2014/01/10 15:48:05]

I was aware that this could cause issues. But so far it doesn't.

At the beginning I actually started out with using a copy of pkg/yaml (we need a
copy, we can't just import this one, because the testing scripts have to be
independent of the packages we're testing).

Then I ran into two issues:
- pkg/yaml didn't provide a way to save it again
- manipulating json is not what I want:
  * what I would like to have is a 'class PubspecYaml' represents exactly what
is allowed in pubspec.yaml files

So I started looking into sdk/lib/_internal/pub/lib/src/pubspec.dart. But this
file doesn't just represent pubspec.yaml, it's importing tons of other dart
files (and I don't want to make a copy these).

I've tried going the simple hackisch way, and it works, so I stayed with that.

I don't understand why yaml should be a subset/superset of json. Looking at
syntax it's just different. But I guess you're right and I just have to look
into the yaml/json specs to figure it out.

[Bob Nystrom, 2014/01/10 17:54:58]

Yeah, that's an open issue, but just serializing to JSON is a valid way to save
back to YAML. It's just not the same YAML you originally parsed.
Yeah, that would be nice, I agree.
YAML has a pretty flexible syntax. Maps and lists can be created using
whitespace for indentation, but it also allows {} and []. When you do that, you
effectively have JSON. From the YAML spec:

1.3. Relation to JSON

Both JSON and YAML aim to be human readable data interchange formats. However,
JSON and YAML have different priorities. JSON’s foremost design goal is
simplicity and universality. Thus, JSON is trivial to generate and parse, at the
cost of reduced human readability. It also uses a lowest common denominator
information model, ensuring any JSON data can be easily processed by every
modern programming environment.

In contrast, YAML’s foremost design goals are human readability and support for
serializing arbitrary native data structures. Thus, YAML allows for extremely
readable files, but is more complex to generate and parse. In addition, YAML
ventures beyond the lowest common denominator data types, requiring more complex
processing when crossing between different programming environments.

YAML can therefore be viewed as a natural superset of JSON, offering improved
human readability and a more complete information model. This is also the case
in practice; every JSON file is also a valid YAML file. This makes it easy to
migrate from JSON to YAML if/when the additional features are required.

+        _dependencyOverrides.forEach((String packageName, Map override) {
+          dependencyOverrideSection.write("  $packageName:\n");
+          override.forEach((overrideKey, overrideValue) {
+            dependencyOverrideSection.write(
+                "    $overrideKey: $overrideValue\n");
+          });
+        });
+      }
+      var modifiedYamlString = "$yamlString\n$dependencyOverrideSection";
+      return destinationFile.writeAsString(modifiedYamlString);
+    }).then((_) {
+      return new ScriptCommandOutputImpl(
+          this, Expectation.PASS, "", watch.elapsed);
+    }).catchError((error) {
+      return new ScriptCommandOutputImpl(
+          this, Expectation.FAIL, "An error occured: $error.", watch.elapsed);
+    });
+  }
+
+  void _buildHashCode(HashCodeBuilder builder) {
+    super._buildHashCode(builder);
+    builder.add(_pubspecYamlFile);
+    builder.add(_destinationFile);
+    builder.addJson(_dependencyOverrides);
+  }
+
+  bool _equal(Command other) {
+    return
+        other is ModifyPubspecYamlCommand &&
+        super._equal(other) &&
+        _pubspecYamlFile == other._pubspecYamlFile &&
+        _destinationFile == other._destinationFile &&
+        deepJsonCompare(_dependencyOverrides, other._dependencyOverrides);
+  }
+}
+
 class CommandBuilder {
   static final CommandBuilder instance = new CommandBuilder._();
 
@@ -437,6 +579,30 @@ class CommandBuilder {
     return _getUniqueCommand(command);
   }
 
+  Command getCopyCommand(String sourceDirectory, String destinationDirectory) {
+    var command = new CleanDirectoryCopyCommand._(sourceDirectory,
+                                                  destinationDirectory);
+    return _getUniqueCommand(command);
+  }
+
+  Command getPubCommand(String pubCommand,
+                        String pubExecutable,
+                        String pubspecYamlDirectory,
+                        String pubCacheDirectory) {
+    var command = new PubCommand._(pubCommand,
+                                   pubExecutable,
+                                   pubspecYamlDirectory,
+                                   pubCacheDirectory);
+    return _getUniqueCommand(command);
+  }
+
+  Command getModifyPubspecCommand(String pubspecYamlFile, Map depsOverrides,
+                                  {String destinationFile: null}) {
+    if (destinationFile == null) destinationFile = pubspecYamlFile;
+    return _getUniqueCommand(new ModifyPubspecYamlCommand._(
+        pubspecYamlFile, destinationFile, depsOverrides));
+  }
+
   Command _getUniqueCommand(Command command) {
     // All Command classes have hashCode/operator==, so we check if this command
     // has already been build, if so we return the cached one, otherwise we
@@ -1337,6 +1503,26 @@ class JsCommandlineOutputImpl extends CommandOutputImpl
   }
 }
 
+class PubCommandOutputImpl extends CommandOutputImpl {
+  PubCommandOutputImpl(PubCommand command, int exitCode, bool timedOut,
+      List<int> stdout, List<int> stderr, Duration time)
+      : super(command, exitCode, timedOut, stdout, stderr, time, false);
+
+  Expectation result(TestCase testCase) {
+    // Handle crashes and timeouts first
+    if (hasCrashed) return Expectation.CRASH;
+    if (hasTimedOut) return Expectation.TIMEOUT;
+
+    if (exitCode == 0) {
+      return Expectation.PASS;
+    } else if ((command as PubCommand).command == 'get') {
+      return Expectation.PUB_GET_ERROR;
+    } else {
+      return Expectation.FAIL;
+    }
+  }
+}
+
 class ScriptCommandOutputImpl extends CommandOutputImpl {
   final Expectation _result;
 
@@ -1383,6 +1569,9 @@ CommandOutput createCommandOutput(Command command,
   } else if (command is JSCommandlineCommand) {
     return new JsCommandlineOutputImpl(
         command, exitCode, timedOut, stdout, stderr, time);
+  } else if (command is PubCommand) {
+    return new PubCommandOutputImpl(
+        command, exitCode, timedOut, stdout, stderr, time);
   }
 
   return new CommandOutputImpl(