Switch to copy files synchronously (address the too-many-open-files bug)

pkg/polymer/lib/src/build/runner.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Definitions used to run the polymer linter and deploy tools without using
  * pub serve or pub deploy.
  */
 library polymer.src.build.runner;
 
@@ skipping 179 matching lines @@
 
 /**
  * Emits all outputs of [barback] and copies files that we didn't process (like
  * polymer's libraries).
  */
 Future _emitAllFiles(Barback barback, BarbackOptions options) {
   return barback.getAllAssets().then((assets) {
     // Delete existing output folder before we generate anything
     var dir = new Directory(options.outDir);
     if (dir.existsSync()) dir.deleteSync(recursive: true);
-    return _emitPackagesDir(options)
-      .then((_) => _emitTransformedFiles(assets, options))
+    _emitPackagesDir(options);
+    return _emitTransformedFiles(assets, options)
       .then((_) => _addPackagesSymlinks(assets, options))
       .then((_) => assets);
   });
 }
 
 Future _emitTransformedFiles(AssetSet assets, BarbackOptions options) {
   // Copy all the assets we transformed
   var futures = [];
   var currentPackage = options.currentPackage;
   var transformTests = options.transformTests;
@@ skipping 46 matching lines @@
         link.createSync(targetPath);
       }
     }
   }
 }
 
 /**
  * Emits a 'packages' directory directly under `out/packages` with the contents
  * of every file that was not transformed by barback.
  */
-Future _emitPackagesDir(BarbackOptions options) {
-  if (options.transformPolymerDependencies) return new Future.value(null);
+void _emitPackagesDir(BarbackOptions options) {
+  if (options.transformPolymerDependencies) return;
   var outPackages = path.join(options.outDir, 'packages');
   _ensureDir(outPackages);
 
   // Copy all the files we didn't process
-  var futures = [];
   var dirs = options.packageDirs;
   for (var package in _polymerPackageDependencies) {
     for (var relpath in _listPackageDir(package, 'lib', options)) {
       var inpath = path.join(dirs[package], relpath);
       var outpath = path.join(outPackages, package, relpath.substring(4));
-      futures.add(_copyFile(inpath, outpath));

[kevmoo-old, 2013/10/05 04:23:12]

Could this be the problem? We're adding a bunch of running futures to a list.

Try using Future.forEach here. I bet this would solve the problem without
getting rid of the async code.

[Jennifer Messerly, 2013/10/05 05:23:26]

Sure, but then you're running them all serially. How is that better than just
doing synchronous code?

The point of async is to take advantage of parallelism at the OS level, by
creating essentially a tree of operations that can resume as soon as their data
is ready. If your futures are one big chain, you might as well have done it
sync.

I suppose you might have other code running in other isolates. Or similarity we
could try to parallelize our Futures to keep them inside 256 files. Either way
tho, it still risks going over the limit.

I do think we should add a TODO & have a bug filed. This is a big trap in
dart:io on Mac. By trying to do the "right thing" and parallelize your futures
you end up shooting yourself in the foot :|

[Søren Gjesse, 2013/10/07 06:40:27]

With async primitives you have the ability to easily do things in parallel.
However that does not mean that you don't have to take care of resource
management. In principle the code above is fine, but somewhere there should be
resource control. That could be in the _copyFile function which could do
throttling so that it is only actively copying a limited number of files
simultaneously. Of cause that is only local resource management of the file
copying, but is't important.

+      _copyFileSync(inpath, outpath);
     }
   }
-  return Future.wait(futures);
 }
 
 /** Ensure [dirpath] exists. */
 void _ensureDir(String dirpath) {
   new Directory(dirpath).createSync(recursive: true);
 }
 
 /**
  * Returns the first directory name on a url-style path, or null if there are no
  * slashes.
  */
 String _firstDir(String url) {
   var firstSlash = url.indexOf('/');
   if (firstSlash == -1) return null;
   return url.substring(0, firstSlash);
 }
 
 /** Copy a file from [inpath] to [outpath]. */
-Future _copyFile(String inpath, String outpath) {
+void _copyFileSync(String inpath, String outpath) {
   _ensureDir(path.dirname(outpath));
-  var writer = new File(outpath).openWrite();
-  return writer.addStream(new File(inpath).openRead())
-      .then((_) => writer.close());
+  new File(outpath).writeAsBytesSync(new File(inpath).readAsBytesSync());
 }
 
 /** Write contents of an [asset] into a file at [filepath]. */
 Future _writeAsset(String filepath, Asset asset) {
   _ensureDir(path.dirname(filepath));
   var writer = new File(filepath).openWrite();
   return writer.addStream(asset.read()).then((_) => writer.close());
 }