| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 library pub.source.hosted; | |
| 6 | |
| 7 import 'dart:async'; | |
| 8 import 'dart:io' as io; | |
| 9 import "dart:convert"; | |
| 10 | |
| 11 import 'package:http/http.dart' as http; | |
| 12 import 'package:path/path.dart' as path; | |
| 13 import 'package:pub_semver/pub_semver.dart'; | |
| 14 | |
| 15 import '../exceptions.dart'; | |
| 16 import '../http.dart'; | |
| 17 import '../io.dart'; | |
| 18 import '../log.dart' as log; | |
| 19 import '../package.dart'; | |
| 20 import '../pubspec.dart'; | |
| 21 import '../utils.dart'; | |
| 22 import 'cached.dart'; | |
| 23 | |
| 24 /// A package source that gets packages from a package hosting site that uses | |
| 25 /// the same API as pub.dartlang.org. | |
| 26 class HostedSource extends CachedSource { | |
| 27 final name = "hosted"; | |
| 28 final hasMultipleVersions = true; | |
| 29 | |
| 30 /// Gets the default URL for the package server for hosted dependencies. | |
| 31 static String get defaultUrl { | |
| 32 var url = io.Platform.environment["PUB_HOSTED_URL"]; | |
| 33 if (url != null) return url; | |
| 34 | |
| 35 return "https://pub.dartlang.org"; | |
| 36 } | |
| 37 | |
| 38 /// Downloads a list of all versions of a package that are available from the | |
| 39 /// site. | |
| 40 Future<List<Version>> getVersions(String name, description) { | |
| 41 var url = | |
| 42 _makeUrl(description, (server, package) => "$server/api/packages/$packag
e"); | |
| 43 | |
| 44 log.io("Get versions from $url."); | |
| 45 return httpClient.read(url, headers: PUB_API_HEADERS).then((body) { | |
| 46 var doc = JSON.decode(body); | |
| 47 return doc['versions'].map( | |
| 48 (version) => new Version.parse(version['version'])).toList(); | |
| 49 }).catchError((ex, stackTrace) { | |
| 50 var parsed = _parseDescription(description); | |
| 51 _throwFriendlyError(ex, stackTrace, parsed.first, parsed.last); | |
| 52 }); | |
| 53 } | |
| 54 | |
| 55 /// Downloads and parses the pubspec for a specific version of a package that | |
| 56 /// is available from the site. | |
| 57 Future<Pubspec> describeUncached(PackageId id) { | |
| 58 // Request it from the server. | |
| 59 var url = _makeVersionUrl( | |
| 60 id, | |
| 61 (server, package, version) => | |
| 62 "$server/api/packages/$package/versions/$version"); | |
| 63 | |
| 64 log.io("Describe package at $url."); | |
| 65 return httpClient.read(url, headers: PUB_API_HEADERS).then((version) { | |
| 66 version = JSON.decode(version); | |
| 67 | |
| 68 // TODO(rnystrom): After this is pulled down, we could place it in | |
| 69 // a secondary cache of just pubspecs. This would let us have a | |
| 70 // persistent cache for pubspecs for packages that haven't actually | |
| 71 // been downloaded. | |
| 72 return new Pubspec.fromMap( | |
| 73 version['pubspec'], | |
| 74 systemCache.sources, | |
| 75 expectedName: id.name, | |
| 76 location: url); | |
| 77 }).catchError((ex, stackTrace) { | |
| 78 var parsed = _parseDescription(id.description); | |
| 79 _throwFriendlyError(ex, stackTrace, id.name, parsed.last); | |
| 80 }); | |
| 81 } | |
| 82 | |
| 83 /// Downloads the package identified by [id] to the system cache. | |
| 84 Future<Package> downloadToSystemCache(PackageId id) { | |
| 85 return isInSystemCache(id).then((inCache) { | |
| 86 // Already cached so don't download it. | |
| 87 if (inCache) return true; | |
| 88 | |
| 89 var packageDir = _getDirectory(id); | |
| 90 ensureDir(path.dirname(packageDir)); | |
| 91 var parsed = _parseDescription(id.description); | |
| 92 return _download(parsed.last, parsed.first, id.version, packageDir); | |
| 93 }).then((found) { | |
| 94 if (!found) fail('Package $id not found.'); | |
| 95 return new Package.load(id.name, _getDirectory(id), systemCache.sources); | |
| 96 }); | |
| 97 } | |
| 98 | |
| 99 /// The system cache directory for the hosted source contains subdirectories | |
| 100 /// for each separate repository URL that's used on the system. | |
| 101 /// | |
| 102 /// Each of these subdirectories then contains a subdirectory for each | |
| 103 /// package downloaded from that site. | |
| 104 Future<String> getDirectory(PackageId id) => | |
| 105 new Future.value(_getDirectory(id)); | |
| 106 | |
| 107 String _getDirectory(PackageId id) { | |
| 108 var parsed = _parseDescription(id.description); | |
| 109 var dir = _urlToDirectory(parsed.last); | |
| 110 return path.join(systemCacheRoot, dir, "${parsed.first}-${id.version}"); | |
| 111 } | |
| 112 | |
| 113 String packageName(description) => _parseDescription(description).first; | |
| 114 | |
| 115 bool descriptionsEqual(description1, description2) => | |
| 116 _parseDescription(description1) == _parseDescription(description2); | |
| 117 | |
| 118 /// Ensures that [description] is a valid hosted package description. | |
| 119 /// | |
| 120 /// There are two valid formats. A plain string refers to a package with the | |
| 121 /// given name from the default host, while a map with keys "name" and "url" | |
| 122 /// refers to a package with the given name from the host at the given URL. | |
| 123 dynamic parseDescription(String containingPath, description, | |
| 124 {bool fromLockFile: false}) { | |
| 125 _parseDescription(description); | |
| 126 return description; | |
| 127 } | |
| 128 | |
| 129 /// Re-downloads all packages that have been previously downloaded into the | |
| 130 /// system cache from any server. | |
| 131 Future<Pair<int, int>> repairCachedPackages() { | |
| 132 final completer0 = new Completer(); | |
| 133 scheduleMicrotask(() { | |
| 134 try { | |
| 135 join0() { | |
| 136 var successes = 0; | |
| 137 var failures = 0; | |
| 138 var it0 = listDir(systemCacheRoot).iterator; | |
| 139 break0() { | |
| 140 completer0.complete(new Pair(successes, failures)); | |
| 141 } | |
| 142 var trampoline0; | |
| 143 continue0() { | |
| 144 trampoline0 = null; | |
| 145 if (it0.moveNext()) { | |
| 146 var serverDir = it0.current; | |
| 147 var url = _directoryToUrl(path.basename(serverDir)); | |
| 148 var packages = | |
| 149 _getCachedPackagesInDirectory(path.basename(serverDir)); | |
| 150 packages.sort(Package.orderByNameAndVersion); | |
| 151 var it1 = packages.iterator; | |
| 152 break1() { | |
| 153 trampoline0 = continue0; | |
| 154 do trampoline0(); while (trampoline0 != null); | |
| 155 } | |
| 156 var trampoline1; | |
| 157 continue1() { | |
| 158 trampoline1 = null; | |
| 159 if (it1.moveNext()) { | |
| 160 var package = it1.current; | |
| 161 join1() { | |
| 162 trampoline1 = continue1; | |
| 163 do trampoline1(); while (trampoline1 != null); | |
| 164 } | |
| 165 catch0(error, stackTrace) { | |
| 166 try { | |
| 167 failures++; | |
| 168 var message = | |
| 169 "Failed to repair ${log.bold(package.name)} " "${packa
ge.version}"; | |
| 170 join2() { | |
| 171 log.error("${message}. Error:\n${error}"); | |
| 172 log.fine(stackTrace); | |
| 173 tryDeleteEntry(package.dir); | |
| 174 join1(); | |
| 175 } | |
| 176 if (url != defaultUrl) { | |
| 177 message += " from ${url}"; | |
| 178 join2(); | |
| 179 } else { | |
| 180 join2(); | |
| 181 } | |
| 182 } catch (error, stackTrace) { | |
| 183 completer0.completeError(error, stackTrace); | |
| 184 } | |
| 185 } | |
| 186 try { | |
| 187 new Future.value( | |
| 188 _download(url, package.name, package.version, package.di
r)).then((x0) { | |
| 189 trampoline1 = () { | |
| 190 trampoline1 = null; | |
| 191 try { | |
| 192 x0; | |
| 193 successes++; | |
| 194 join1(); | |
| 195 } catch (e0, s0) { | |
| 196 catch0(e0, s0); | |
| 197 } | |
| 198 }; | |
| 199 do trampoline1(); while (trampoline1 != null); | |
| 200 }, onError: catch0); | |
| 201 } catch (e1, s1) { | |
| 202 catch0(e1, s1); | |
| 203 } | |
| 204 } else { | |
| 205 break1(); | |
| 206 } | |
| 207 } | |
| 208 trampoline1 = continue1; | |
| 209 do trampoline1(); while (trampoline1 != null); | |
| 210 } else { | |
| 211 break0(); | |
| 212 } | |
| 213 } | |
| 214 trampoline0 = continue0; | |
| 215 do trampoline0(); while (trampoline0 != null); | |
| 216 } | |
| 217 if (!dirExists(systemCacheRoot)) { | |
| 218 completer0.complete(new Pair(0, 0)); | |
| 219 } else { | |
| 220 join0(); | |
| 221 } | |
| 222 } catch (e, s) { | |
| 223 completer0.completeError(e, s); | |
| 224 } | |
| 225 }); | |
| 226 return completer0.future; | |
| 227 } | |
| 228 | |
| 229 /// Gets all of the packages that have been downloaded into the system cache | |
| 230 /// from the default server. | |
| 231 List<Package> getCachedPackages() { | |
| 232 return _getCachedPackagesInDirectory(_urlToDirectory(defaultUrl)); | |
| 233 } | |
| 234 | |
| 235 /// Gets all of the packages that have been downloaded into the system cache | |
| 236 /// into [dir]. | |
| 237 List<Package> _getCachedPackagesInDirectory(String dir) { | |
| 238 var cacheDir = path.join(systemCacheRoot, dir); | |
| 239 if (!dirExists(cacheDir)) return []; | |
| 240 | |
| 241 return listDir( | |
| 242 cacheDir).map( | |
| 243 (entry) => new Package.load(null, entry, systemCache.sources)).toLis
t(); | |
| 244 } | |
| 245 | |
| 246 /// Downloads package [package] at [version] from [server], and unpacks it | |
| 247 /// into [destPath]. | |
| 248 Future<bool> _download(String server, String package, Version version, | |
| 249 String destPath) { | |
| 250 return new Future.sync(() { | |
| 251 var url = Uri.parse("$server/packages/$package/versions/$version.tar.gz"); | |
| 252 log.io("Get package from $url."); | |
| 253 log.message('Downloading ${log.bold(package)} ${version}...'); | |
| 254 | |
| 255 // Download and extract the archive to a temp directory. | |
| 256 var tempDir = systemCache.createTempDir(); | |
| 257 return httpClient.send( | |
| 258 new http.Request( | |
| 259 "GET", | |
| 260 url)).then((response) => response.stream).then((stream) { | |
| 261 return timeout( | |
| 262 extractTarGz(stream, tempDir), | |
| 263 HTTP_TIMEOUT, | |
| 264 url, | |
| 265 'downloading $url'); | |
| 266 }).then((_) { | |
| 267 // Remove the existing directory if it exists. This will happen if | |
| 268 // we're forcing a download to repair the cache. | |
| 269 if (dirExists(destPath)) deleteEntry(destPath); | |
| 270 | |
| 271 // Now that the get has succeeded, move it to the real location in the | |
| 272 // cache. This ensures that we don't leave half-busted ghost | |
| 273 // directories in the user's pub cache if a get fails. | |
| 274 renameDir(tempDir, destPath); | |
| 275 return true; | |
| 276 }); | |
| 277 }); | |
| 278 } | |
| 279 | |
| 280 /// When an error occurs trying to read something about [package] from [url], | |
| 281 /// this tries to translate into a more user friendly error message. | |
| 282 /// | |
| 283 /// Always throws an error, either the original one or a better one. | |
| 284 void _throwFriendlyError(error, StackTrace stackTrace, String package, | |
| 285 String url) { | |
| 286 if (error is PubHttpException && error.response.statusCode == 404) { | |
| 287 throw new PackageNotFoundException( | |
| 288 "Could not find package $package at $url.", | |
| 289 error, | |
| 290 stackTrace); | |
| 291 } | |
| 292 | |
| 293 if (error is TimeoutException) { | |
| 294 fail( | |
| 295 "Timed out trying to find package $package at $url.", | |
| 296 error, | |
| 297 stackTrace); | |
| 298 } | |
| 299 | |
| 300 if (error is io.SocketException) { | |
| 301 fail( | |
| 302 "Got socket error trying to find package $package at $url.", | |
| 303 error, | |
| 304 stackTrace); | |
| 305 } | |
| 306 | |
| 307 // Otherwise re-throw the original exception. | |
| 308 throw error; | |
| 309 } | |
| 310 } | |
| 311 | |
| 312 /// This is the modified hosted source used when pub get or upgrade are run | |
| 313 /// with "--offline". | |
| 314 /// | |
| 315 /// This uses the system cache to get the list of available packages and does | |
| 316 /// no network access. | |
| 317 class OfflineHostedSource extends HostedSource { | |
| 318 /// Gets the list of all versions of [name] that are in the system cache. | |
| 319 Future<List<Version>> getVersions(String name, description) { | |
| 320 return newFuture(() { | |
| 321 var parsed = _parseDescription(description); | |
| 322 var server = parsed.last; | |
| 323 log.io( | |
| 324 "Finding versions of $name in " "$systemCacheRoot/${_urlToDirectory(se
rver)}"); | |
| 325 return _getCachedPackagesInDirectory( | |
| 326 _urlToDirectory( | |
| 327 server)).where( | |
| 328 (package) => package.name == name).map((package) => package.ve
rsion).toList(); | |
| 329 }).then((versions) { | |
| 330 // If there are no versions in the cache, report a clearer error. | |
| 331 if (versions.isEmpty) fail("Could not find package $name in cache."); | |
| 332 | |
| 333 return versions; | |
| 334 }); | |
| 335 } | |
| 336 | |
| 337 Future<bool> _download(String server, String package, Version version, | |
| 338 String destPath) { | |
| 339 // Since HostedSource is cached, this will only be called for uncached | |
| 340 // packages. | |
| 341 throw new UnsupportedError("Cannot download packages when offline."); | |
| 342 } | |
| 343 | |
| 344 Future<Pubspec> doDescribeUncached(PackageId id) { | |
| 345 // [getVersions()] will only return packages that are already cached. | |
| 346 // [CachedSource] will only call [doDescribeUncached()] on a package after | |
| 347 // it has failed to find it in the cache, so this code should not be | |
| 348 // reached. | |
| 349 throw new UnsupportedError("Cannot describe packages when offline."); | |
| 350 } | |
| 351 } | |
| 352 | |
| 353 /// Given a URL, returns a "normalized" string to be used as a directory name | |
| 354 /// for packages downloaded from the server at that URL. | |
| 355 /// | |
| 356 /// This normalization strips off the scheme (which is presumed to be HTTP or | |
| 357 /// HTTPS) and *sort of* URL-encodes it. I say "sort of" because it does it | |
| 358 /// incorrectly: it uses the character's *decimal* ASCII value instead of hex. | |
| 359 /// | |
| 360 /// This could cause an ambiguity since some characters get encoded as three | |
| 361 /// digits and others two. It's possible for one to be a prefix of the other. | |
| 362 /// In practice, the set of characters that are encoded don't happen to have | |
| 363 /// any collisions, so the encoding is reversible. | |
| 364 /// | |
| 365 /// This behavior is a bug, but is being preserved for compatibility. | |
| 366 String _urlToDirectory(String url) { | |
| 367 // Normalize all loopback URLs to "localhost". | |
| 368 url = url.replaceAllMapped( | |
| 369 new RegExp(r"^https?://(127\.0\.0\.1|\[::1\])?"), | |
| 370 (match) => match[1] == null ? '' : 'localhost'); | |
| 371 return replace( | |
| 372 url, | |
| 373 new RegExp(r'[<>:"\\/|?*%]'), | |
| 374 (match) => '%${match[0].codeUnitAt(0)}'); | |
| 375 } | |
| 376 | |
| 377 /// Given a directory name in the system cache, returns the URL of the server | |
| 378 /// whose packages it contains. | |
| 379 /// | |
| 380 /// See [_urlToDirectory] for details on the mapping. Note that because the | |
| 381 /// directory name does not preserve the scheme, this has to guess at it. It | |
| 382 /// chooses "http" for loopback URLs (mainly to support the pub tests) and | |
| 383 /// "https" for all others. | |
| 384 String _directoryToUrl(String url) { | |
| 385 // Decode the pseudo-URL-encoded characters. | |
| 386 var chars = '<>:"\\/|?*%'; | |
| 387 for (var i = 0; i < chars.length; i++) { | |
| 388 var c = chars.substring(i, i + 1); | |
| 389 url = url.replaceAll("%${c.codeUnitAt(0)}", c); | |
| 390 } | |
| 391 | |
| 392 // Figure out the scheme. | |
| 393 var scheme = "https"; | |
| 394 | |
| 395 // See if it's a loopback IP address. | |
| 396 if (isLoopback(url.replaceAll(new RegExp(":.*"), ""))) scheme = "http"; | |
| 397 return "$scheme://$url"; | |
| 398 } | |
| 399 | |
| 400 /// Parses [description] into its server and package name components, then | |
| 401 /// converts that to a Uri given [pattern]. | |
| 402 /// | |
| 403 /// Ensures the package name is properly URL encoded. | |
| 404 Uri _makeUrl(description, String pattern(String server, String package)) { | |
| 405 var parsed = _parseDescription(description); | |
| 406 var server = parsed.last; | |
| 407 var package = Uri.encodeComponent(parsed.first); | |
| 408 return Uri.parse(pattern(server, package)); | |
| 409 } | |
| 410 | |
| 411 /// Parses [id] into its server, package name, and version components, then | |
| 412 /// converts that to a Uri given [pattern]. | |
| 413 /// | |
| 414 /// Ensures the package name is properly URL encoded. | |
| 415 Uri _makeVersionUrl(PackageId id, String pattern(String server, String package, | |
| 416 String version)) { | |
| 417 var parsed = _parseDescription(id.description); | |
| 418 var server = parsed.last; | |
| 419 var package = Uri.encodeComponent(parsed.first); | |
| 420 var version = Uri.encodeComponent(id.version.toString()); | |
| 421 return Uri.parse(pattern(server, package, version)); | |
| 422 } | |
| 423 | |
| 424 /// Parses the description for a package. | |
| 425 /// | |
| 426 /// If the package parses correctly, this returns a (name, url) pair. If not, | |
| 427 /// this throws a descriptive FormatException. | |
| 428 Pair<String, String> _parseDescription(description) { | |
| 429 if (description is String) { | |
| 430 return new Pair<String, String>(description, HostedSource.defaultUrl); | |
| 431 } | |
| 432 | |
| 433 if (description is! Map) { | |
| 434 throw new FormatException("The description must be a package name or map."); | |
| 435 } | |
| 436 | |
| 437 if (!description.containsKey("name")) { | |
| 438 throw new FormatException("The description map must contain a 'name' key."); | |
| 439 } | |
| 440 | |
| 441 var name = description["name"]; | |
| 442 if (name is! String) { | |
| 443 throw new FormatException("The 'name' key must have a string value."); | |
| 444 } | |
| 445 | |
| 446 var url = description["url"]; | |
| 447 if (url == null) url = HostedSource.defaultUrl; | |
| 448 | |
| 449 return new Pair<String, String>(name, url); | |
| 450 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 937243002:
Revert "Revert "Use native async/await support in pub."" (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart