| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 /// Helper functionality to make working with IO easier. | |
| 6 library pub.io; | |
| 7 | |
| 8 import 'dart:async'; | |
| 9 import 'dart:collection'; | |
| 10 import 'dart:convert'; | |
| 11 import 'dart:io'; | |
| 12 | |
| 13 import 'package:path/path.dart' as path; | |
| 14 import 'package:pool/pool.dart'; | |
| 15 import 'package:http/http.dart' show ByteStream; | |
| 16 import 'package:http_multi_server/http_multi_server.dart'; | |
| 17 import 'package:stack_trace/stack_trace.dart'; | |
| 18 | |
| 19 import 'exit_codes.dart' as exit_codes; | |
| 20 import 'exceptions.dart'; | |
| 21 import 'error_group.dart'; | |
| 22 import 'log.dart' as log; | |
| 23 import 'sdk.dart' as sdk; | |
| 24 import 'utils.dart'; | |
| 25 | |
| 26 export 'package:http/http.dart' show ByteStream; | |
| 27 | |
| 28 /// The pool used for restricting access to asynchronous operations that consume | |
| 29 /// file descriptors. | |
| 30 /// | |
| 31 /// The maximum number of allocated descriptors is based on empirical tests that | |
| 32 /// indicate that beyond 32, additional file reads don't provide substantial | |
| 33 /// additional throughput. | |
| 34 final _descriptorPool = new Pool(32); | |
| 35 | |
| 36 /// Determines if a file or directory exists at [path]. | |
| 37 bool entryExists(String path) => | |
| 38 dirExists(path) || fileExists(path) || linkExists(path); | |
| 39 | |
| 40 /// Returns whether [link] exists on the file system. | |
| 41 /// | |
| 42 /// This returns `true` for any symlink, regardless of what it points at or | |
| 43 /// whether it's broken. | |
| 44 bool linkExists(String link) => new Link(link).existsSync(); | |
| 45 | |
| 46 /// Returns whether [file] exists on the file system. | |
| 47 /// | |
| 48 /// This returns `true` for a symlink only if that symlink is unbroken and | |
| 49 /// points to a file. | |
| 50 bool fileExists(String file) => new File(file).existsSync(); | |
| 51 | |
| 52 /// Returns the canonical path for [pathString]. | |
| 53 /// | |
| 54 /// This is the normalized, absolute path, with symlinks resolved. As in | |
| 55 /// [transitiveTarget], broken or recursive symlinks will not be fully resolved. | |
| 56 /// | |
| 57 /// This doesn't require [pathString] to point to a path that exists on the | |
| 58 /// filesystem; nonexistent or unreadable path entries are treated as normal | |
| 59 /// directories. | |
| 60 String canonicalize(String pathString) { | |
| 61 var seen = new Set<String>(); | |
| 62 var components = new Queue<String>.from( | |
| 63 path.split(path.normalize(path.absolute(pathString)))); | |
| 64 | |
| 65 // The canonical path, built incrementally as we iterate through [components]. | |
| 66 var newPath = components.removeFirst(); | |
| 67 | |
| 68 // Move through the components of the path, resolving each one's symlinks as | |
| 69 // necessary. A resolved component may also add new components that need to be | |
| 70 // resolved in turn. | |
| 71 while (!components.isEmpty) { | |
| 72 seen.add(path.join(newPath, path.joinAll(components))); | |
| 73 var resolvedPath = resolveLink( | |
| 74 path.join(newPath, components.removeFirst())); | |
| 75 var relative = path.relative(resolvedPath, from: newPath); | |
| 76 | |
| 77 // If the resolved path of the component relative to `newPath` is just ".", | |
| 78 // that means component was a symlink pointing to its parent directory. We | |
| 79 // can safely ignore such components. | |
| 80 if (relative == '.') continue; | |
| 81 | |
| 82 var relativeComponents = new Queue<String>.from(path.split(relative)); | |
| 83 | |
| 84 // If the resolved path is absolute relative to `newPath`, that means it's | |
| 85 // on a different drive. We need to canonicalize the entire target of that | |
| 86 // symlink again. | |
| 87 if (path.isAbsolute(relative)) { | |
| 88 // If we've already tried to canonicalize the new path, we've encountered | |
| 89 // a symlink loop. Avoid going infinite by treating the recursive symlink | |
| 90 // as the canonical path. | |
| 91 if (seen.contains(relative)) { | |
| 92 newPath = relative; | |
| 93 } else { | |
| 94 newPath = relativeComponents.removeFirst(); | |
| 95 relativeComponents.addAll(components); | |
| 96 components = relativeComponents; | |
| 97 } | |
| 98 continue; | |
| 99 } | |
| 100 | |
| 101 // Pop directories off `newPath` if the component links upwards in the | |
| 102 // directory hierarchy. | |
| 103 while (relativeComponents.first == '..') { | |
| 104 newPath = path.dirname(newPath); | |
| 105 relativeComponents.removeFirst(); | |
| 106 } | |
| 107 | |
| 108 // If there's only one component left, [resolveLink] guarantees that it's | |
| 109 // not a link (or is a broken link). We can just add it to `newPath` and | |
| 110 // continue resolving the remaining components. | |
| 111 if (relativeComponents.length == 1) { | |
| 112 newPath = path.join(newPath, relativeComponents.single); | |
| 113 continue; | |
| 114 } | |
| 115 | |
| 116 // If we've already tried to canonicalize the new path, we've encountered a | |
| 117 // symlink loop. Avoid going infinite by treating the recursive symlink as | |
| 118 // the canonical path. | |
| 119 var newSubPath = path.join(newPath, path.joinAll(relativeComponents)); | |
| 120 if (seen.contains(newSubPath)) { | |
| 121 newPath = newSubPath; | |
| 122 continue; | |
| 123 } | |
| 124 | |
| 125 // If there are multiple new components to resolve, add them to the | |
| 126 // beginning of the queue. | |
| 127 relativeComponents.addAll(components); | |
| 128 components = relativeComponents; | |
| 129 } | |
| 130 return newPath; | |
| 131 } | |
| 132 | |
| 133 /// Returns the transitive target of [link] (if A links to B which links to C, | |
| 134 /// this will return C). | |
| 135 /// | |
| 136 /// If [link] is part of a symlink loop (e.g. A links to B which links back to | |
| 137 /// A), this returns the path to the first repeated link (so | |
| 138 /// `transitiveTarget("A")` would return `"A"` and `transitiveTarget("A")` would | |
| 139 /// return `"B"`). | |
| 140 /// | |
| 141 /// This accepts paths to non-links or broken links, and returns them as-is. | |
| 142 String resolveLink(String link) { | |
| 143 var seen = new Set<String>(); | |
| 144 while (linkExists(link) && !seen.contains(link)) { | |
| 145 seen.add(link); | |
| 146 link = path.normalize(path.join( | |
| 147 path.dirname(link), new Link(link).targetSync())); | |
| 148 } | |
| 149 return link; | |
| 150 } | |
| 151 | |
| 152 /// Reads the contents of the text file [file]. | |
| 153 String readTextFile(String file) => | |
| 154 new File(file).readAsStringSync(encoding: UTF8); | |
| 155 | |
| 156 /// Reads the contents of the binary file [file]. | |
| 157 List<int> readBinaryFile(String file) { | |
| 158 log.io("Reading binary file $file."); | |
| 159 var contents = new File(file).readAsBytesSync(); | |
| 160 log.io("Read ${contents.length} bytes from $file."); | |
| 161 return contents; | |
| 162 } | |
| 163 | |
| 164 /// Creates [file] and writes [contents] to it. | |
| 165 /// | |
| 166 /// If [dontLogContents] is true, the contents of the file will never be logged. | |
| 167 String writeTextFile(String file, String contents, | |
| 168 {bool dontLogContents: false}) { | |
| 169 // Sanity check: don't spew a huge file. | |
| 170 log.io("Writing ${contents.length} characters to text file $file."); | |
| 171 if (!dontLogContents && contents.length < 1024 * 1024) { | |
| 172 log.fine("Contents:\n$contents"); | |
| 173 } | |
| 174 | |
| 175 new File(file).writeAsStringSync(contents); | |
| 176 return file; | |
| 177 } | |
| 178 | |
| 179 /// Creates [file] and writes [contents] to it. | |
| 180 String writeBinaryFile(String file, List<int> contents) { | |
| 181 log.io("Writing ${contents.length} bytes to binary file $file."); | |
| 182 new File(file).openSync(mode: FileMode.WRITE) | |
| 183 ..writeFromSync(contents) | |
| 184 ..closeSync(); | |
| 185 log.fine("Wrote text file $file."); | |
| 186 return file; | |
| 187 } | |
| 188 | |
| 189 /// Writes [stream] to a new file at path [file]. | |
| 190 /// | |
| 191 /// Replaces any file already at that path. Completes when the file is done | |
| 192 /// being written. | |
| 193 Future<String> createFileFromStream(Stream<List<int>> stream, String file) { | |
| 194 // TODO(nweiz): remove extra logging when we figure out the windows bot issue. | |
| 195 log.io("Creating $file from stream."); | |
| 196 | |
| 197 return _descriptorPool.withResource(() { | |
| 198 return stream.pipe(new File(file).openWrite()).then((_) { | |
| 199 log.fine("Created $file from stream."); | |
| 200 return file; | |
| 201 }); | |
| 202 }); | |
| 203 } | |
| 204 | |
| 205 /// Copies all files in [files] to the directory [destination]. | |
| 206 /// | |
| 207 /// Their locations in [destination] will be determined by their relative | |
| 208 /// location to [baseDir]. Any existing files at those paths will be | |
| 209 /// overwritten. | |
| 210 void copyFiles(Iterable<String> files, String baseDir, String destination) { | |
| 211 for (var file in files) { | |
| 212 var newPath = path.join(destination, path.relative(file, from: baseDir)); | |
| 213 ensureDir(path.dirname(newPath)); | |
| 214 copyFile(file, newPath); | |
| 215 } | |
| 216 } | |
| 217 | |
| 218 /// Copies a file from [source] to [destination]. | |
| 219 void copyFile(String source, String destination) { | |
| 220 writeBinaryFile(destination, readBinaryFile(source)); | |
| 221 } | |
| 222 | |
| 223 /// Creates a directory [dir]. | |
| 224 String createDir(String dir) { | |
| 225 new Directory(dir).createSync(); | |
| 226 return dir; | |
| 227 } | |
| 228 | |
| 229 /// Ensures that [dir] and all its parent directories exist. | |
| 230 /// | |
| 231 /// If they don't exist, creates them. | |
| 232 String ensureDir(String dir) { | |
| 233 new Directory(dir).createSync(recursive: true); | |
| 234 return dir; | |
| 235 } | |
| 236 | |
| 237 /// Creates a temp directory in [dir], whose name will be [prefix] with | |
| 238 /// characters appended to it to make a unique name. | |
| 239 /// | |
| 240 /// Returns the path of the created directory. | |
| 241 String createTempDir(String base, String prefix) { | |
| 242 var tempDir = new Directory(base).createTempSync(prefix); | |
| 243 log.io("Created temp directory ${tempDir.path}"); | |
| 244 return tempDir.path; | |
| 245 } | |
| 246 | |
| 247 /// Creates a temp directory in the system temp directory, whose name will be | |
| 248 /// 'pub_' with characters appended to it to make a unique name. | |
| 249 /// | |
| 250 /// Returns the path of the created directory. | |
| 251 String createSystemTempDir() { | |
| 252 var tempDir = Directory.systemTemp.createTempSync('pub_'); | |
| 253 log.io("Created temp directory ${tempDir.path}"); | |
| 254 return tempDir.path; | |
| 255 } | |
| 256 | |
| 257 /// Lists the contents of [dir]. | |
| 258 /// | |
| 259 /// If [recursive] is `true`, lists subdirectory contents (defaults to `false`). | |
| 260 /// If [includeHidden] is `true`, includes files and directories beginning with | |
| 261 /// `.` (defaults to `false`). If [includeDirs] is `true`, includes directories | |
| 262 /// as well as files (defaults to `true`). | |
| 263 /// | |
| 264 /// [whiteList] is a list of hidden filenames to include even when | |
| 265 /// [includeHidden] is `false`. | |
| 266 /// | |
| 267 /// Note that dart:io handles recursive symlinks in an unfortunate way. You | |
| 268 /// end up with two copies of every entity that is within the recursive loop. | |
| 269 /// We originally had our own directory list code that addressed that, but it | |
| 270 /// had a noticeable performance impact. In the interest of speed, we'll just | |
| 271 /// live with that annoying behavior. | |
| 272 /// | |
| 273 /// The returned paths are guaranteed to begin with [dir]. Broken symlinks won't | |
| 274 /// be returned. | |
| 275 List<String> listDir(String dir, {bool recursive: false, | |
| 276 bool includeHidden: false, bool includeDirs: true, | |
| 277 Iterable<String> whitelist}) { | |
| 278 if (whitelist == null) whitelist = []; | |
| 279 var whitelistFilter = createFileFilter(whitelist); | |
| 280 | |
| 281 // This is used in some performance-sensitive paths and can list many, many | |
| 282 // files. As such, it leans more havily towards optimization as opposed to | |
| 283 // readability than most code in pub. In particular, it avoids using the path | |
| 284 // package, since re-parsing a path is very expensive relative to string | |
| 285 // operations. | |
| 286 return new Directory(dir).listSync( | |
| 287 recursive: recursive, followLinks: true).where((entity) { | |
| 288 if (!includeDirs && entity is Directory) return false; | |
| 289 if (entity is Link) return false; | |
| 290 if (includeHidden) return true; | |
| 291 | |
| 292 // Using substring here is generally problematic in cases where dir has one | |
| 293 // or more trailing slashes. If you do listDir("foo"), you'll get back | |
| 294 // paths like "foo/bar". If you do listDir("foo/"), you'll get "foo/bar" | |
| 295 // (note the trailing slash was dropped. If you do listDir("foo//"), you'll | |
| 296 // get "foo//bar". | |
| 297 // | |
| 298 // This means if you strip off the prefix, the resulting string may have a | |
| 299 // leading separator (if the prefix did not have a trailing one) or it may | |
| 300 // not. However, since we are only using the results of that to call | |
| 301 // contains() on, the leading separator is harmless. | |
| 302 assert(entity.path.startsWith(dir)); | |
| 303 var pathInDir = entity.path.substring(dir.length); | |
| 304 | |
| 305 // If the basename is whitelisted, don't count its "/." as making the file | |
| 306 // hidden. | |
| 307 var whitelistedBasename = whitelistFilter.firstWhere(pathInDir.contains, | |
| 308 orElse: () => null); | |
| 309 if (whitelistedBasename != null) { | |
| 310 pathInDir = pathInDir.substring( | |
| 311 0, pathInDir.length - whitelistedBasename.length); | |
| 312 } | |
| 313 | |
| 314 if (pathInDir.contains("/.")) return false; | |
| 315 if (Platform.operatingSystem != "windows") return true; | |
| 316 return !pathInDir.contains("\\."); | |
| 317 }).map((entity) => entity.path).toList(); | |
| 318 } | |
| 319 | |
| 320 /// Returns whether [dir] exists on the file system. | |
| 321 /// | |
| 322 /// This returns `true` for a symlink only if that symlink is unbroken and | |
| 323 /// points to a directory. | |
| 324 bool dirExists(String dir) => new Directory(dir).existsSync(); | |
| 325 | |
| 326 /// Tries to resiliently perform [operation]. | |
| 327 /// | |
| 328 /// Some file system operations can intermittently fail on Windows because | |
| 329 /// other processes are locking a file. We've seen this with virus scanners | |
| 330 /// when we try to delete or move something while it's being scanned. To | |
| 331 /// mitigate that, on Windows, this will retry the operation a few times if it | |
| 332 /// fails. | |
| 333 void _attempt(String description, void operation()) { | |
| 334 if (Platform.operatingSystem != 'windows') { | |
| 335 operation(); | |
| 336 return; | |
| 337 } | |
| 338 | |
| 339 getErrorReason(error) { | |
| 340 if (error.osError.errorCode == 5) { | |
| 341 return "access was denied"; | |
| 342 } | |
| 343 | |
| 344 if (error.osError.errorCode == 32) { | |
| 345 return "it was in use by another process"; | |
| 346 } | |
| 347 | |
| 348 return null; | |
| 349 } | |
| 350 | |
| 351 for (var i = 0; i < 2; i++) { | |
| 352 try { | |
| 353 operation(); | |
| 354 return; | |
| 355 } on FileSystemException catch (error) { | |
| 356 var reason = getErrorReason(error); | |
| 357 if (reason == null) rethrow; | |
| 358 | |
| 359 log.io("Failed to $description because $reason. " | |
| 360 "Retrying in 50ms."); | |
| 361 sleep(new Duration(milliseconds: 50)); | |
| 362 } | |
| 363 } | |
| 364 | |
| 365 try { | |
| 366 operation(); | |
| 367 } on FileSystemException catch (error) { | |
| 368 var reason = getErrorReason(error); | |
| 369 if (reason == null) rethrow; | |
| 370 | |
| 371 fail("Failed to $description because $reason.\n" | |
| 372 "This may be caused by a virus scanner or having a file\n" | |
| 373 "in the directory open in another application."); | |
| 374 } | |
| 375 } | |
| 376 | |
| 377 /// Deletes whatever's at [path], whether it's a file, directory, or symlink. | |
| 378 /// | |
| 379 /// If it's a directory, it will be deleted recursively. | |
| 380 void deleteEntry(String path) { | |
| 381 _attempt("delete entry", () { | |
| 382 if (linkExists(path)) { | |
| 383 log.io("Deleting link $path."); | |
| 384 new Link(path).deleteSync(); | |
| 385 } else if (dirExists(path)) { | |
| 386 log.io("Deleting directory $path."); | |
| 387 new Directory(path).deleteSync(recursive: true); | |
| 388 } else if (fileExists(path)) { | |
| 389 log.io("Deleting file $path."); | |
| 390 new File(path).deleteSync(); | |
| 391 } | |
| 392 }); | |
| 393 } | |
| 394 | |
| 395 /// Attempts to delete whatever's at [path], but doesn't throw an exception if | |
| 396 /// the deletion fails. | |
| 397 void tryDeleteEntry(String path) { | |
| 398 try { | |
| 399 deleteEntry(path); | |
| 400 } catch (error, stackTrace) { | |
| 401 log.fine("Failed to delete $path: $error\n" | |
| 402 "${new Chain.forTrace(stackTrace)}"); | |
| 403 } | |
| 404 } | |
| 405 | |
| 406 /// "Cleans" [dir]. | |
| 407 /// | |
| 408 /// If that directory already exists, it is deleted. Then a new empty directory | |
| 409 /// is created. | |
| 410 void cleanDir(String dir) { | |
| 411 if (entryExists(dir)) deleteEntry(dir); | |
| 412 ensureDir(dir); | |
| 413 } | |
| 414 | |
| 415 /// Renames (i.e. moves) the directory [from] to [to]. | |
| 416 void renameDir(String from, String to) { | |
| 417 _attempt("rename directory", () { | |
| 418 log.io("Renaming directory $from to $to."); | |
| 419 try { | |
| 420 new Directory(from).renameSync(to); | |
| 421 } on IOException { | |
| 422 // Ensure that [to] isn't left in an inconsistent state. See issue 12436. | |
| 423 if (entryExists(to)) deleteEntry(to); | |
| 424 rethrow; | |
| 425 } | |
| 426 }); | |
| 427 } | |
| 428 | |
| 429 /// Creates a new symlink at path [symlink] that points to [target]. | |
| 430 /// | |
| 431 /// Returns a [Future] which completes to the path to the symlink file. | |
| 432 /// | |
| 433 /// If [relative] is true, creates a symlink with a relative path from the | |
| 434 /// symlink to the target. Otherwise, uses the [target] path unmodified. | |
| 435 /// | |
| 436 /// Note that on Windows, only directories may be symlinked to. | |
| 437 void createSymlink(String target, String symlink, | |
| 438 {bool relative: false}) { | |
| 439 if (relative) { | |
| 440 // Relative junction points are not supported on Windows. Instead, just | |
| 441 // make sure we have a clean absolute path because it will interpret a | |
| 442 // relative path to be relative to the cwd, not the symlink, and will be | |
| 443 // confused by forward slashes. | |
| 444 if (Platform.operatingSystem == 'windows') { | |
| 445 target = path.normalize(path.absolute(target)); | |
| 446 } else { | |
| 447 // If the directory where we're creating the symlink was itself reached | |
| 448 // by traversing a symlink, we want the relative path to be relative to | |
| 449 // it's actual location, not the one we went through to get to it. | |
| 450 var symlinkDir = canonicalize(path.dirname(symlink)); | |
| 451 target = path.normalize(path.relative(target, from: symlinkDir)); | |
| 452 } | |
| 453 } | |
| 454 | |
| 455 log.fine("Creating $symlink pointing to $target"); | |
| 456 new Link(symlink).createSync(target); | |
| 457 } | |
| 458 | |
| 459 /// Creates a new symlink that creates an alias at [symlink] that points to the | |
| 460 /// `lib` directory of package [target]. | |
| 461 /// | |
| 462 /// If [target] does not have a `lib` directory, this shows a warning if | |
| 463 /// appropriate and then does nothing. | |
| 464 /// | |
| 465 /// If [relative] is true, creates a symlink with a relative path from the | |
| 466 /// symlink to the target. Otherwise, uses the [target] path unmodified. | |
| 467 void createPackageSymlink(String name, String target, String symlink, | |
| 468 {bool isSelfLink: false, bool relative: false}) { | |
| 469 // See if the package has a "lib" directory. If not, there's nothing to | |
| 470 // symlink to. | |
| 471 target = path.join(target, 'lib'); | |
| 472 if (!dirExists(target)) return; | |
| 473 | |
| 474 log.fine("Creating ${isSelfLink ? "self" : ""}link for package '$name'."); | |
| 475 createSymlink(target, symlink, relative: relative); | |
| 476 } | |
| 477 | |
| 478 /// Whether pub is running from within the Dart SDK, as opposed to from the Dart | |
| 479 /// source repository. | |
| 480 final bool runningFromSdk = Platform.script.path.endsWith('.snapshot'); | |
| 481 | |
| 482 /// Resolves [target] relative to the path to pub's `asset` directory. | |
| 483 String assetPath(String target) { | |
| 484 if (runningFromSdk) { | |
| 485 return path.join( | |
| 486 sdk.rootDirectory, 'lib', '_internal', 'pub', 'asset', target); | |
| 487 } else { | |
| 488 return path.join(pubRoot, 'asset', target); | |
| 489 } | |
| 490 } | |
| 491 | |
| 492 /// Returns the path to the root of pub's sources in the Dart repo. | |
| 493 String get pubRoot => path.join(repoRoot, 'sdk', 'lib', '_internal', 'pub'); | |
| 494 | |
| 495 /// Returns the path to the root of the Dart repository. | |
| 496 /// | |
| 497 /// This throws a [StateError] if it's called when running pub from the SDK. | |
| 498 String get repoRoot { | |
| 499 if (runningFromSdk) { | |
| 500 throw new StateError("Can't get the repo root from the SDK."); | |
| 501 } | |
| 502 | |
| 503 // Get the path to the directory containing this very file. | |
| 504 var libDir = path.dirname(libraryPath('pub.io')); | |
| 505 | |
| 506 // Assume we're running directly from the source location in the repo: | |
| 507 // | |
| 508 // <repo>/sdk/lib/_internal/pub/lib/src | |
| 509 return path.normalize(path.join(libDir, '..', '..', '..', '..', '..', '..')); | |
| 510 } | |
| 511 | |
| 512 /// A line-by-line stream of standard input. | |
| 513 final Stream<String> stdinLines = streamToLines( | |
| 514 new ByteStream(stdin).toStringStream()); | |
| 515 | |
| 516 /// Displays a message and reads a yes/no confirmation from the user. | |
| 517 /// | |
| 518 /// Returns a [Future] that completes to `true` if the user confirms or `false` | |
| 519 /// if they do not. | |
| 520 /// | |
| 521 /// This will automatically append " (y/n)?" to the message, so [message] | |
| 522 /// should just be a fragment like, "Are you sure you want to proceed". | |
| 523 Future<bool> confirm(String message) { | |
| 524 log.fine('Showing confirm message: $message'); | |
| 525 if (runningAsTest) { | |
| 526 log.message("$message (y/n)?"); | |
| 527 } else { | |
| 528 stdout.write(log.format("$message (y/n)? ")); | |
| 529 } | |
| 530 return streamFirst(stdinLines) | |
| 531 .then((line) => new RegExp(r"^[yY]").hasMatch(line)); | |
| 532 } | |
| 533 | |
| 534 /// Reads and discards all output from [stream]. | |
| 535 /// | |
| 536 /// Returns a [Future] that completes when the stream is closed. | |
| 537 Future drainStream(Stream stream) { | |
| 538 return stream.fold(null, (x, y) {}); | |
| 539 } | |
| 540 | |
| 541 /// Flushes the stdout and stderr streams, then exits the program with the given | |
| 542 /// status code. | |
| 543 /// | |
| 544 /// This returns a Future that will never complete, since the program will have | |
| 545 /// exited already. This is useful to prevent Future chains from proceeding | |
| 546 /// after you've decided to exit. | |
| 547 Future flushThenExit(int status) { | |
| 548 return Future.wait([ | |
| 549 stdout.close(), | |
| 550 stderr.close() | |
| 551 ]).then((_) => exit(status)); | |
| 552 } | |
| 553 | |
| 554 /// Returns a [EventSink] that pipes all data to [consumer] and a [Future] that | |
| 555 /// will succeed when [EventSink] is closed or fail with any errors that occur | |
| 556 /// while writing. | |
| 557 Pair<EventSink, Future> consumerToSink(StreamConsumer consumer) { | |
| 558 var controller = new StreamController(sync: true); | |
| 559 var done = controller.stream.pipe(consumer); | |
| 560 return new Pair<EventSink, Future>(controller.sink, done); | |
| 561 } | |
| 562 | |
| 563 // TODO(nweiz): remove this when issue 7786 is fixed. | |
| 564 /// Pipes all data and errors from [stream] into [sink]. | |
| 565 /// | |
| 566 /// When [stream] is done, the returned [Future] is completed and [sink] is | |
| 567 /// closed if [closeSink] is true. | |
| 568 /// | |
| 569 /// When an error occurs on [stream], that error is passed to [sink]. If | |
| 570 /// [cancelOnError] is true, [Future] will be completed successfully and no | |
| 571 /// more data or errors will be piped from [stream] to [sink]. If | |
| 572 /// [cancelOnError] and [closeSink] are both true, [sink] will then be | |
| 573 /// closed. | |
| 574 Future store(Stream stream, EventSink sink, | |
| 575 {bool cancelOnError: true, bool closeSink: true}) { | |
| 576 var completer = new Completer(); | |
| 577 stream.listen(sink.add, onError: (e, stackTrace) { | |
| 578 sink.addError(e, stackTrace); | |
| 579 if (cancelOnError) { | |
| 580 completer.complete(); | |
| 581 if (closeSink) sink.close(); | |
| 582 } | |
| 583 }, onDone: () { | |
| 584 if (closeSink) sink.close(); | |
| 585 completer.complete(); | |
| 586 }, cancelOnError: cancelOnError); | |
| 587 return completer.future; | |
| 588 } | |
| 589 | |
| 590 /// Spawns and runs the process located at [executable], passing in [args]. | |
| 591 /// | |
| 592 /// Returns a [Future] that will complete with the results of the process after | |
| 593 /// it has ended. | |
| 594 /// | |
| 595 /// The spawned process will inherit its parent's environment variables. If | |
| 596 /// [environment] is provided, that will be used to augment (not replace) the | |
| 597 /// the inherited variables. | |
| 598 Future<PubProcessResult> runProcess(String executable, List<String> args, | |
| 599 {workingDir, Map<String, String> environment}) { | |
| 600 return _descriptorPool.withResource(() { | |
| 601 return _doProcess(Process.run, executable, args, workingDir, environment) | |
| 602 .then((result) { | |
| 603 var pubResult = new PubProcessResult( | |
| 604 result.stdout, result.stderr, result.exitCode); | |
| 605 log.processResult(executable, pubResult); | |
| 606 return pubResult; | |
| 607 }); | |
| 608 }); | |
| 609 } | |
| 610 | |
| 611 /// Spawns the process located at [executable], passing in [args]. | |
| 612 /// | |
| 613 /// Returns a [Future] that will complete with the [Process] once it's been | |
| 614 /// started. | |
| 615 /// | |
| 616 /// The spawned process will inherit its parent's environment variables. If | |
| 617 /// [environment] is provided, that will be used to augment (not replace) the | |
| 618 /// the inherited variables. | |
| 619 Future<PubProcess> startProcess(String executable, List<String> args, | |
| 620 {workingDir, Map<String, String> environment}) { | |
| 621 return _descriptorPool.request().then((resource) { | |
| 622 return _doProcess(Process.start, executable, args, workingDir, environment) | |
| 623 .then((ioProcess) { | |
| 624 var process = new PubProcess(ioProcess); | |
| 625 process.exitCode.whenComplete(resource.release); | |
| 626 return process; | |
| 627 }); | |
| 628 }); | |
| 629 } | |
| 630 | |
| 631 /// Like [runProcess], but synchronous. | |
| 632 PubProcessResult runProcessSync(String executable, List<String> args, | |
| 633 {String workingDir, Map<String, String> environment}) { | |
| 634 var result = _doProcess( | |
| 635 Process.runSync, executable, args, workingDir, environment); | |
| 636 var pubResult = new PubProcessResult( | |
| 637 result.stdout, result.stderr, result.exitCode); | |
| 638 log.processResult(executable, pubResult); | |
| 639 return pubResult; | |
| 640 } | |
| 641 | |
| 642 /// A wrapper around [Process] that exposes `dart:async`-style APIs. | |
| 643 class PubProcess { | |
| 644 /// The underlying `dart:io` [Process]. | |
| 645 final Process _process; | |
| 646 | |
| 647 /// The mutable field for [stdin]. | |
| 648 EventSink<List<int>> _stdin; | |
| 649 | |
| 650 /// The mutable field for [stdinClosed]. | |
| 651 Future _stdinClosed; | |
| 652 | |
| 653 /// The mutable field for [stdout]. | |
| 654 ByteStream _stdout; | |
| 655 | |
| 656 /// The mutable field for [stderr]. | |
| 657 ByteStream _stderr; | |
| 658 | |
| 659 /// The mutable field for [exitCode]. | |
| 660 Future<int> _exitCode; | |
| 661 | |
| 662 /// The sink used for passing data to the process's standard input stream. | |
| 663 /// | |
| 664 /// Errors on this stream are surfaced through [stdinClosed], [stdout], | |
| 665 /// [stderr], and [exitCode], which are all members of an [ErrorGroup]. | |
| 666 EventSink<List<int>> get stdin => _stdin; | |
| 667 | |
| 668 // TODO(nweiz): write some more sophisticated Future machinery so that this | |
| 669 // doesn't surface errors from the other streams/futures, but still passes its | |
| 670 // unhandled errors to them. Right now it's impossible to recover from a stdin | |
| 671 // error and continue interacting with the process. | |
| 672 /// A [Future] that completes when [stdin] is closed, either by the user or by | |
| 673 /// the process itself. | |
| 674 /// | |
| 675 /// This is in an [ErrorGroup] with [stdout], [stderr], and [exitCode], so any | |
| 676 /// error in process will be passed to it, but won't reach the top-level error | |
| 677 /// handler unless nothing has handled it. | |
| 678 Future get stdinClosed => _stdinClosed; | |
| 679 | |
| 680 /// The process's standard output stream. | |
| 681 /// | |
| 682 /// This is in an [ErrorGroup] with [stdinClosed], [stderr], and [exitCode], | |
| 683 /// so any error in process will be passed to it, but won't reach the | |
| 684 /// top-level error handler unless nothing has handled it. | |
| 685 ByteStream get stdout => _stdout; | |
| 686 | |
| 687 /// The process's standard error stream. | |
| 688 /// | |
| 689 /// This is in an [ErrorGroup] with [stdinClosed], [stdout], and [exitCode], | |
| 690 /// so any error in process will be passed to it, but won't reach the | |
| 691 /// top-level error handler unless nothing has handled it. | |
| 692 ByteStream get stderr => _stderr; | |
| 693 | |
| 694 /// A [Future] that will complete to the process's exit code once the process | |
| 695 /// has finished running. | |
| 696 /// | |
| 697 /// This is in an [ErrorGroup] with [stdinClosed], [stdout], and [stderr], so | |
| 698 /// any error in process will be passed to it, but won't reach the top-level | |
| 699 /// error handler unless nothing has handled it. | |
| 700 Future<int> get exitCode => _exitCode; | |
| 701 | |
| 702 /// Creates a new [PubProcess] wrapping [process]. | |
| 703 PubProcess(Process process) | |
| 704 : _process = process { | |
| 705 var errorGroup = new ErrorGroup(); | |
| 706 | |
| 707 var pair = consumerToSink(process.stdin); | |
| 708 _stdin = pair.first; | |
| 709 _stdinClosed = errorGroup.registerFuture(pair.last); | |
| 710 | |
| 711 _stdout = new ByteStream( | |
| 712 errorGroup.registerStream(process.stdout)); | |
| 713 _stderr = new ByteStream( | |
| 714 errorGroup.registerStream(process.stderr)); | |
| 715 | |
| 716 var exitCodeCompleter = new Completer(); | |
| 717 _exitCode = errorGroup.registerFuture(exitCodeCompleter.future); | |
| 718 _process.exitCode.then((code) => exitCodeCompleter.complete(code)); | |
| 719 } | |
| 720 | |
| 721 /// Sends [signal] to the underlying process. | |
| 722 bool kill([ProcessSignal signal = ProcessSignal.SIGTERM]) => | |
| 723 _process.kill(signal); | |
| 724 } | |
| 725 | |
| 726 /// Calls [fn] with appropriately modified arguments. | |
| 727 /// | |
| 728 /// [fn] should have the same signature as [Process.start], except that the | |
| 729 /// returned value may have any return type. | |
| 730 _doProcess(Function fn, String executable, List<String> args, | |
| 731 String workingDir, Map<String, String> environment) { | |
| 732 // TODO(rnystrom): Should dart:io just handle this? | |
| 733 // Spawning a process on Windows will not look for the executable in the | |
| 734 // system path. So, if executable looks like it needs that (i.e. it doesn't | |
| 735 // have any path separators in it), then spawn it through a shell. | |
| 736 if ((Platform.operatingSystem == "windows") && | |
| 737 (executable.indexOf('\\') == -1)) { | |
| 738 args = flatten(["/c", executable, args]); | |
| 739 executable = "cmd"; | |
| 740 } | |
| 741 | |
| 742 log.process(executable, args, workingDir == null ? '.' : workingDir); | |
| 743 | |
| 744 return fn(executable, args, | |
| 745 workingDirectory: workingDir, | |
| 746 environment: environment); | |
| 747 } | |
| 748 | |
| 749 /// Wraps [input], an asynchronous network operation to provide a timeout. | |
| 750 /// | |
| 751 /// If [input] completes before [milliseconds] have passed, then the return | |
| 752 /// value completes in the same way. However, if [milliseconds] pass before | |
| 753 /// [input] has completed, it completes with a [TimeoutException] with | |
| 754 /// [description] (which should be a fragment describing the action that timed | |
| 755 /// out). | |
| 756 /// | |
| 757 /// [url] is the URL being accessed asynchronously. | |
| 758 /// | |
| 759 /// Note that timing out will not cancel the asynchronous operation behind | |
| 760 /// [input]. | |
| 761 Future timeout(Future input, int milliseconds, Uri url, String description) { | |
| 762 // TODO(nwiez): Replace this with [Future.timeout]. | |
| 763 var completer = new Completer(); | |
| 764 var duration = new Duration(milliseconds: milliseconds); | |
| 765 var timer = new Timer(duration, () { | |
| 766 // Include the duration ourselves in the message instead of passing it to | |
| 767 // TimeoutException since we show nicer output. | |
| 768 var message = 'Timed out after ${niceDuration(duration)} while ' | |
| 769 '$description.'; | |
| 770 | |
| 771 if (url.host == "pub.dartlang.org" || | |
| 772 url.host == "storage.googleapis.com") { | |
| 773 message += "\nThis is likely a transient error. Please try again later."; | |
| 774 } | |
| 775 | |
| 776 completer.completeError(new TimeoutException(message), new Chain.current()); | |
| 777 }); | |
| 778 input.then((value) { | |
| 779 if (completer.isCompleted) return; | |
| 780 timer.cancel(); | |
| 781 completer.complete(value); | |
| 782 }).catchError((e, stackTrace) { | |
| 783 if (completer.isCompleted) return; | |
| 784 timer.cancel(); | |
| 785 completer.completeError(e, stackTrace); | |
| 786 }); | |
| 787 return completer.future; | |
| 788 } | |
| 789 | |
| 790 /// Creates a temporary directory and passes its path to [fn]. | |
| 791 /// | |
| 792 /// Once the [Future] returned by [fn] completes, the temporary directory and | |
| 793 /// all its contents are deleted. [fn] can also return `null`, in which case | |
| 794 /// the temporary directory is deleted immediately afterwards. | |
| 795 /// | |
| 796 /// Returns a future that completes to the value that the future returned from | |
| 797 /// [fn] completes to. | |
| 798 Future withTempDir(Future fn(String path)) { | |
| 799 return new Future.sync(() { | |
| 800 var tempDir = createSystemTempDir(); | |
| 801 return new Future.sync(() => fn(tempDir)) | |
| 802 .whenComplete(() => deleteEntry(tempDir)); | |
| 803 }); | |
| 804 } | |
| 805 | |
| 806 /// Binds an [HttpServer] to [host] and [port]. | |
| 807 /// | |
| 808 /// If [host] is "localhost", this will automatically listen on both the IPv4 | |
| 809 /// and IPv6 loopback addresses. | |
| 810 Future<HttpServer> bindServer(String host, int port) { | |
| 811 if (host == 'localhost') return HttpMultiServer.loopback(port); | |
| 812 return HttpServer.bind(host, port); | |
| 813 } | |
| 814 | |
| 815 /// Extracts a `.tar.gz` file from [stream] to [destination]. | |
| 816 /// | |
| 817 /// Returns whether or not the extraction was successful. | |
| 818 Future<bool> extractTarGz(Stream<List<int>> stream, String destination) { | |
| 819 log.fine("Extracting .tar.gz stream to $destination."); | |
| 820 | |
| 821 if (Platform.operatingSystem == "windows") { | |
| 822 return _extractTarGzWindows(stream, destination); | |
| 823 } | |
| 824 | |
| 825 var args = ["--extract", "--gunzip", "--directory", destination]; | |
| 826 if (_noUnknownKeyword) { | |
| 827 // BSD tar (the default on OS X) can insert strange headers to a tarfile | |
| 828 // that GNU tar (the default on Linux) is unable to understand. This will | |
| 829 // cause GNU tar to emit a number of harmless but scary-looking warnings | |
| 830 // which are silenced by this flag. | |
| 831 args.insert(0, "--warning=no-unknown-keyword"); | |
| 832 } | |
| 833 | |
| 834 return startProcess("tar", args).then((process) { | |
| 835 // Ignore errors on process.std{out,err}. They'll be passed to | |
| 836 // process.exitCode, and we don't want them being top-levelled by | |
| 837 // std{out,err}Sink. | |
| 838 store(process.stdout.handleError((_) {}), stdout, closeSink: false); | |
| 839 store(process.stderr.handleError((_) {}), stderr, closeSink: false); | |
| 840 return Future.wait([ | |
| 841 store(stream, process.stdin), | |
| 842 process.exitCode | |
| 843 ]); | |
| 844 }).then((results) { | |
| 845 var exitCode = results[1]; | |
| 846 if (exitCode != exit_codes.SUCCESS) { | |
| 847 throw new Exception("Failed to extract .tar.gz stream to $destination " | |
| 848 "(exit code $exitCode)."); | |
| 849 } | |
| 850 log.fine("Extracted .tar.gz stream to $destination. Exit code $exitCode."); | |
| 851 }); | |
| 852 } | |
| 853 | |
| 854 /// Whether to include "--warning=no-unknown-keyword" when invoking tar. | |
| 855 /// | |
| 856 /// This flag quiets warnings that come from opening OS X-generated tarballs on | |
| 857 /// Linux, but only GNU tar >= 1.26 supports it. | |
| 858 final bool _noUnknownKeyword = _computeNoUnknownKeyword(); | |
| 859 bool _computeNoUnknownKeyword() { | |
| 860 if (!Platform.isLinux) return false; | |
| 861 var result = Process.runSync("tar", ["--version"]); | |
| 862 if (result.exitCode != 0) { | |
| 863 throw new ApplicationException( | |
| 864 "Failed to run tar (exit code ${result.exitCode}):\n${result.stderr}"); | |
| 865 } | |
| 866 | |
| 867 var match = new RegExp(r"^tar \(GNU tar\) (\d+).(\d+)\n") | |
| 868 .firstMatch(result.stdout); | |
| 869 if (match == null) return false; | |
| 870 | |
| 871 var major = int.parse(match[1]); | |
| 872 var minor = int.parse(match[2]); | |
| 873 return major >= 2 || (major == 1 && minor >= 23); | |
| 874 } | |
| 875 | |
| 876 String get pathTo7zip { | |
| 877 if (runningFromSdk) return assetPath(path.join('7zip', '7za.exe')); | |
| 878 return path.join(repoRoot, 'third_party', '7zip', '7za.exe'); | |
| 879 } | |
| 880 | |
| 881 Future<bool> _extractTarGzWindows(Stream<List<int>> stream, | |
| 882 String destination) { | |
| 883 // TODO(rnystrom): In the repo's history, there is an older implementation of | |
| 884 // this that does everything in memory by piping streams directly together | |
| 885 // instead of writing out temp files. The code is simpler, but unfortunately, | |
| 886 // 7zip seems to periodically fail when we invoke it from Dart and tell it to | |
| 887 // read from stdin instead of a file. Consider resurrecting that version if | |
| 888 // we can figure out why it fails. | |
| 889 | |
| 890 return withTempDir((tempDir) { | |
| 891 // Write the archive to a temp file. | |
| 892 var dataFile = path.join(tempDir, 'data.tar.gz'); | |
| 893 return createFileFromStream(stream, dataFile).then((_) { | |
| 894 // 7zip can't unarchive from gzip -> tar -> destination all in one step | |
| 895 // first we un-gzip it to a tar file. | |
| 896 // Note: Setting the working directory instead of passing in a full file | |
| 897 // path because 7zip says "A full path is not allowed here." | |
| 898 return runProcess(pathTo7zip, ['e', 'data.tar.gz'], workingDir: tempDir); | |
| 899 }).then((result) { | |
| 900 if (result.exitCode != exit_codes.SUCCESS) { | |
| 901 throw new Exception('Could not un-gzip (exit code ${result.exitCode}). ' | |
| 902 'Error:\n' | |
| 903 '${result.stdout.join("\n")}\n' | |
| 904 '${result.stderr.join("\n")}'); | |
| 905 } | |
| 906 | |
| 907 // Find the tar file we just created since we don't know its name. | |
| 908 var tarFile = listDir(tempDir).firstWhere( | |
| 909 (file) => path.extension(file) == '.tar', | |
| 910 orElse: () { | |
| 911 throw new FormatException('The gzip file did not contain a tar file.'); | |
| 912 }); | |
| 913 | |
| 914 // Untar the archive into the destination directory. | |
| 915 return runProcess(pathTo7zip, ['x', tarFile], workingDir: destination); | |
| 916 }).then((result) { | |
| 917 if (result.exitCode != exit_codes.SUCCESS) { | |
| 918 throw new Exception('Could not un-tar (exit code ${result.exitCode}). ' | |
| 919 'Error:\n' | |
| 920 '${result.stdout.join("\n")}\n' | |
| 921 '${result.stderr.join("\n")}'); | |
| 922 } | |
| 923 return true; | |
| 924 }); | |
| 925 }); | |
| 926 } | |
| 927 | |
| 928 /// Create a .tar.gz archive from a list of entries. | |
| 929 /// | |
| 930 /// Each entry can be a [String], [Directory], or [File] object. The root of | |
| 931 /// the archive is considered to be [baseDir], which defaults to the current | |
| 932 /// working directory. | |
| 933 /// | |
| 934 /// Returns a [ByteStream] that emits the contents of the archive. | |
| 935 ByteStream createTarGz(List contents, {baseDir}) { | |
| 936 return new ByteStream(futureStream(new Future.sync(() async { | |
| 937 var buffer = new StringBuffer(); | |
| 938 buffer.write('Creating .tag.gz stream containing:\n'); | |
| 939 contents.forEach((file) => buffer.write('$file\n')); | |
| 940 log.fine(buffer.toString()); | |
| 941 | |
| 942 if (baseDir == null) baseDir = path.current; | |
| 943 baseDir = path.absolute(baseDir); | |
| 944 contents = contents.map((entry) { | |
| 945 entry = path.absolute(entry); | |
| 946 if (!path.isWithin(baseDir, entry)) { | |
| 947 throw new ArgumentError('Entry $entry is not inside $baseDir.'); | |
| 948 } | |
| 949 return path.relative(entry, from: baseDir); | |
| 950 }).toList(); | |
| 951 | |
| 952 if (Platform.operatingSystem != "windows") { | |
| 953 var args = [ | |
| 954 "--create", | |
| 955 "--gzip", | |
| 956 "--directory", | |
| 957 baseDir, | |
| 958 "--files-from", | |
| 959 "/dev/stdin" | |
| 960 ]; | |
| 961 | |
| 962 var process = await startProcess("tar", args); | |
| 963 process.stdin.add(UTF8.encode(contents.join("\n"))); | |
| 964 process.stdin.close(); | |
| 965 return process.stdout; | |
| 966 } | |
| 967 | |
| 968 // Don't use [withTempDir] here because we don't want to delete the temp | |
| 969 // directory until the returned stream has closed. | |
| 970 var tempDir = createSystemTempDir(); | |
| 971 | |
| 972 try { | |
| 973 // Create the file containing the list of files to compress. | |
| 974 var contentsPath = path.join(tempDir, "files.txt"); | |
| 975 writeTextFile(contentsPath, contents.join("\n")); | |
| 976 | |
| 977 // Create the tar file. | |
| 978 var tarFile = path.join(tempDir, "intermediate.tar"); | |
| 979 var args = ["a", "-w$baseDir", tarFile, "@$contentsPath"]; | |
| 980 | |
| 981 // We're passing 'baseDir' both as '-w' and setting it as the working | |
| 982 // directory explicitly here intentionally. The former ensures that the | |
| 983 // files added to the archive have the correct relative path in the | |
| 984 // archive. The latter enables relative paths in the "-i" args to be | |
| 985 // resolved. | |
| 986 await runProcess(pathTo7zip, args, workingDir: baseDir); | |
| 987 | |
| 988 // GZIP it. 7zip doesn't support doing both as a single operation. | |
| 989 // Send the output to stdout. | |
| 990 args = ["a", "unused", "-tgzip", "-so", tarFile]; | |
| 991 return (await startProcess(pathTo7zip, args)) | |
| 992 .stdout | |
| 993 .transform(onDoneTransformer(() => deleteEntry(tempDir))); | |
| 994 } catch (_) { | |
| 995 deleteEntry(tempDir); | |
| 996 rethrow; | |
| 997 } | |
| 998 }))); | |
| 999 } | |
| 1000 | |
| 1001 /// Contains the results of invoking a [Process] and waiting for it to complete. | |
| 1002 class PubProcessResult { | |
| 1003 final List<String> stdout; | |
| 1004 final List<String> stderr; | |
| 1005 final int exitCode; | |
| 1006 | |
| 1007 PubProcessResult(String stdout, String stderr, this.exitCode) | |
| 1008 : this.stdout = _toLines(stdout), | |
| 1009 this.stderr = _toLines(stderr); | |
| 1010 | |
| 1011 // TODO(rnystrom): Remove this and change to returning one string. | |
| 1012 static List<String> _toLines(String output) { | |
| 1013 var lines = splitLines(output); | |
| 1014 if (!lines.isEmpty && lines.last == "") lines.removeLast(); | |
| 1015 return lines; | |
| 1016 } | |
| 1017 | |
| 1018 bool get success => exitCode == exit_codes.SUCCESS; | |
| 1019 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1165473002:
Start pulling pub from its own repo. (Closed)
Base URL: git@github.com:dart-lang/sdk.git@master