| OLD | NEW |
| (Empty) |
| 1 #!/usr/bin/env dart | |
| 2 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file | |
| 3 // for details. All rights reserved. Use of this source code is governed by a | |
| 4 // BSD-style license that can be found in the LICENSE file. | |
| 5 | |
| 6 /// Command line tool to merge the SDK libraries and our patch files. | |
| 7 /// This is currently designed as an offline tool, but we could automate it. | |
| 8 | |
| 9 import 'dart:io'; | |
| 10 import 'dart:math' as math; | |
| 11 | |
| 12 import 'package:analyzer/analyzer.dart'; | |
| 13 import 'package:analyzer/src/generated/sdk.dart'; | |
| 14 import 'package:path/path.dart' as path; | |
| 15 | |
| 16 void main(List<String> argv) { | |
| 17 var base = path.fromUri(Platform.script); | |
| 18 var dartDir = path.dirname(path.dirname(path.absolute(base))); | |
| 19 | |
| 20 if (argv.length != 4 || | |
| 21 !argv.isEmpty && argv.first != 'vm' && argv.first != 'ddc') { | |
| 22 var self = path.relative(base); | |
| 23 print('Usage: $self MODE SDK_DIR PATCH_DIR OUTPUT_DIR'); | |
| 24 print('MODE must be one of ddc or vm.'); | |
| 25 | |
| 26 var toolDir = path.relative(path.dirname(base)); | |
| 27 var sdkExample = path.join(toolDir, 'input_sdk'); | |
| 28 var patchExample = path.join(sdkExample, 'patch'); | |
| 29 var outExample = | |
| 30 path.relative(path.normalize(path.join('gen', 'patched_sdk'))); | |
| 31 print('For example:'); | |
| 32 print('\$ $self ddc $sdkExample $patchExample $outExample'); | |
| 33 | |
| 34 var repositoryDir = path.relative(path.dirname(path.dirname(base))); | |
| 35 sdkExample = path.relative(path.join(repositoryDir, 'sdk')); | |
| 36 patchExample = path.relative(path.join(repositoryDir, 'out', 'DebugX64', | |
| 37 'obj', 'gen', 'patch')); | |
| 38 outExample = path.relative(path.join(repositoryDir, 'out', 'DebugX64', | |
| 39 'obj', 'gen', 'patched_sdk')); | |
| 40 print('or:'); | |
| 41 print('\$ $self vm $sdkExample $patchExample $outExample'); | |
| 42 | |
| 43 exit(1); | |
| 44 } | |
| 45 | |
| 46 var mode = argv[0]; | |
| 47 var input = argv[1]; | |
| 48 var sdkLibIn = path.join(input, 'lib'); | |
| 49 var patchIn = argv[2]; | |
| 50 var sdkOut = path.join(argv[3], 'lib'); | |
| 51 | |
| 52 var privateIn = path.join(input, 'private'); | |
| 53 var INTERNAL_PATH = '_internal/compiler/js_lib/'; | |
| 54 | |
| 55 // Copy and patch libraries.dart and version | |
| 56 var libContents = new File(path.join(sdkLibIn, '_internal', | |
| 57 'sdk_library_metadata', 'lib', 'libraries.dart')).readAsStringSync(); | |
| 58 var patchedLibContents = libContents; | |
| 59 if (mode == 'vm') { | |
| 60 libContents = libContents.replaceAll( | |
| 61 ' libraries = const {', | |
| 62 ''' libraries = const { | |
| 63 | |
| 64 "_builtin": const LibraryInfo( | |
| 65 "_builtin/_builtin.dart", | |
| 66 categories: "Client,Server", | |
| 67 implementation: true, | |
| 68 documented: false, | |
| 69 platforms: VM_PLATFORM), | |
| 70 | |
| 71 "profiler": const LibraryInfo( | |
| 72 "profiler/profiler.dart", | |
| 73 maturity: Maturity.DEPRECATED, | |
| 74 documented: false), | |
| 75 | |
| 76 "_vmservice": const LibraryInfo( | |
| 77 "vmservice/vmservice.dart", | |
| 78 implementation: true, | |
| 79 documented: false, | |
| 80 platforms: VM_PLATFORM), | |
| 81 | |
| 82 "vmservice_io": const LibraryInfo( | |
| 83 "vmservice_io/vmservice_io.dart", | |
| 84 implementation: true, | |
| 85 documented: false, | |
| 86 platforms: VM_PLATFORM), | |
| 87 | |
| 88 '''); | |
| 89 } | |
| 90 _writeSync( | |
| 91 path.join( | |
| 92 sdkOut, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'), | |
| 93 libContents); | |
| 94 if (mode == 'ddc') { | |
| 95 _writeSync(path.join(sdkOut, '..', 'version'), | |
| 96 new File(path.join(sdkLibIn, '..', 'version')).readAsStringSync()); | |
| 97 } | |
| 98 | |
| 99 // Parse libraries.dart | |
| 100 var sdkLibraries = _getSdkLibraries(libContents); | |
| 101 | |
| 102 // Enumerate core libraries and apply patches | |
| 103 for (SdkLibrary library in sdkLibraries) { | |
| 104 // TODO(jmesserly): analyzer does not handle the default case of | |
| 105 // "both platforms" correctly, and treats it as being supported on neither. | |
| 106 // So instead we skip explicitly marked as either VM or dart2js libs. | |
| 107 if (mode == 'ddc' ? libary.isVmLibrary : library.isDart2JsLibrary) { | |
| 108 continue; | |
| 109 } | |
| 110 | |
| 111 var libraryOut = path.join(sdkLibIn, library.path); | |
| 112 var libraryIn; | |
| 113 if (mode == 'vm' && library.path.contains('typed_data.dart')) { | |
| 114 // dart:typed_data is unlike the other libraries in the SDK. The VM does | |
| 115 // not apply a patch to the base SDK implementation of the library. | |
| 116 // Instead, the VM provides a replacement implementation and ignores the | |
| 117 // sources in the SDK. | |
| 118 libraryIn = | |
| 119 path.join(dartDir, 'runtime', 'lib', 'typed_data.dart'); | |
| 120 } else if (mode == 'ddc' && library.path.contains(INTERNAL_PATH)) { | |
| 121 libraryIn = | |
| 122 path.join(privateIn, library.path.replaceAll(INTERNAL_PATH, '')); | |
| 123 } else { | |
| 124 libraryIn = libraryOut; | |
| 125 } | |
| 126 | |
| 127 var libraryFile = new File(libraryIn); | |
| 128 if (libraryFile.existsSync()) { | |
| 129 var outPaths = <String>[libraryOut]; | |
| 130 var libraryContents = libraryFile.readAsStringSync(); | |
| 131 | |
| 132 int inputModifyTime = | |
| 133 libraryFile.lastModifiedSync().millisecondsSinceEpoch; | |
| 134 var partFiles = <File>[]; | |
| 135 for (var part in parseDirectives(libraryContents).directives) { | |
| 136 if (part is PartDirective) { | |
| 137 var partPath = part.uri.stringValue; | |
| 138 outPaths.add(path.join(path.dirname(libraryOut), partPath)); | |
| 139 | |
| 140 var partFile = new File(path.join(path.dirname(libraryIn), partPath)); | |
| 141 partFiles.add(partFile); | |
| 142 inputModifyTime = math.max(inputModifyTime, | |
| 143 partFile.lastModifiedSync().millisecondsSinceEpoch); | |
| 144 } | |
| 145 } | |
| 146 | |
| 147 // See if we can find a patch file. | |
| 148 var patchPath = path.join( | |
| 149 patchIn, path.basenameWithoutExtension(libraryIn) + '_patch.dart'); | |
| 150 | |
| 151 var patchFile = new File(patchPath); | |
| 152 bool patchExists = patchFile.existsSync(); | |
| 153 if (patchExists) { | |
| 154 inputModifyTime = math.max(inputModifyTime, | |
| 155 patchFile.lastModifiedSync().millisecondsSinceEpoch); | |
| 156 } | |
| 157 | |
| 158 // Compute output paths | |
| 159 outPaths = outPaths | |
| 160 .map((p) => path.join(sdkOut, path.relative(p, from: sdkLibIn))) | |
| 161 .toList(); | |
| 162 | |
| 163 // Compare output modify time with input modify time. | |
| 164 bool needsUpdate = false; | |
| 165 for (var outPath in outPaths) { | |
| 166 var outFile = new File(outPath); | |
| 167 if (!outFile.existsSync() || | |
| 168 outFile.lastModifiedSync().millisecondsSinceEpoch < | |
| 169 inputModifyTime) { | |
| 170 needsUpdate = true; | |
| 171 break; | |
| 172 } | |
| 173 } | |
| 174 | |
| 175 if (needsUpdate) { | |
| 176 var contents = <String>[libraryContents]; | |
| 177 contents.addAll(partFiles.map((f) => f.readAsStringSync())); | |
| 178 if (patchExists) { | |
| 179 var patchContents = patchFile.readAsStringSync(); | |
| 180 contents = _patchLibrary( | |
| 181 patchFile.toString(), contents, patchContents); | |
| 182 } | |
| 183 | |
| 184 for (var i = 0; i < outPaths.length; i++) { | |
| 185 if (path.basename(outPaths[i]) == 'internal.dart') { | |
| 186 contents[i] += ''' | |
| 187 | |
| 188 /// Marks a function as an external implementation ("native" in the Dart VM). | |
| 189 /// | |
| 190 /// Provides a backend-specific String that can be used to identify the | |
| 191 /// function's implementation | |
| 192 class ExternalName { | |
| 193 final String name; | |
| 194 const ExternalName(this.name); | |
| 195 } | |
| 196 '''; | |
| 197 } | |
| 198 | |
| 199 _writeSync(outPaths[i], contents[i]); | |
| 200 } | |
| 201 } | |
| 202 } | |
| 203 } | |
| 204 if (mode == 'vm') { | |
| 205 | |
| 206 for (var tuple in [['_builtin', 'builtin.dart']]) { | |
| 207 var vmLibrary = tuple[0]; | |
| 208 var dartFile = tuple[1]; | |
| 209 | |
| 210 // The "dart:_builtin" library is only available for the DartVM. | |
| 211 var builtinLibraryIn = path.join(dartDir, 'runtime', 'bin', dartFile); | |
| 212 var builtinLibraryOut = path.join(sdkOut, vmLibrary, '${vmLibrary}.dart'); | |
| 213 _writeSync(builtinLibraryOut, new File(builtinLibraryIn).readAsStringSync(
)); | |
| 214 } | |
| 215 | |
| 216 for (var file in ['loader.dart', 'server.dart', 'vmservice_io.dart']) { | |
| 217 var libraryIn = path.join(dartDir, 'runtime', 'bin', 'vmservice', file); | |
| 218 var libraryOut = path.join(sdkOut, 'vmservice_io', file); | |
| 219 _writeSync(libraryOut, new File(libraryIn).readAsStringSync()); | |
| 220 } | |
| 221 } | |
| 222 } | |
| 223 | |
| 224 /// Writes a file, creating the directory if needed. | |
| 225 void _writeSync(String filePath, String contents) { | |
| 226 var outDir = new Directory(path.dirname(filePath)); | |
| 227 if (!outDir.existsSync()) outDir.createSync(recursive: true); | |
| 228 | |
| 229 new File(filePath).writeAsStringSync(contents); | |
| 230 } | |
| 231 | |
| 232 /// Merges dart:* library code with code from *_patch.dart file. | |
| 233 /// | |
| 234 /// Takes a list of the library's parts contents, with the main library contents | |
| 235 /// first in the list, and the contents of the patch file. | |
| 236 /// | |
| 237 /// The result will have `@patch` implementations merged into the correct place | |
| 238 /// (e.g. the class or top-level function declaration) and all other | |
| 239 /// declarations introduced by the patch will be placed into the main library | |
| 240 /// file. | |
| 241 /// | |
| 242 /// This is purely a syntactic transformation. Unlike dart2js patch files, there | |
| 243 /// is no semantic meaning given to the *_patch files, and they do not magically | |
| 244 /// get their own library scope, etc. | |
| 245 /// | |
| 246 /// Editorializing: the dart2js approach requires a Dart front end such as | |
| 247 /// package:analyzer to semantically model a feature beyond what is specified | |
| 248 /// in the Dart language. Since this feature is only for the convenience of | |
| 249 /// writing the dart:* libraries, and not a tool given to Dart developers, it | |
| 250 /// seems like a non-ideal situation. Instead we keep the preprocessing simple. | |
| 251 List<String> _patchLibrary(String name, | |
| 252 List<String> partsContents, | |
| 253 String patchContents) { | |
| 254 var results = <StringEditBuffer>[]; | |
| 255 | |
| 256 // Parse the patch first. We'll need to extract bits of this as we go through | |
| 257 // the other files. | |
| 258 final patchFinder = new PatchFinder.parseAndVisit(name, patchContents); | |
| 259 | |
| 260 // Merge `external` declarations with the corresponding `@patch` code. | |
| 261 for (var partContent in partsContents) { | |
| 262 var partEdits = new StringEditBuffer(partContent); | |
| 263 var partUnit = parseCompilationUnit(partContent); | |
| 264 partUnit.accept(new PatchApplier(partEdits, patchFinder)); | |
| 265 results.add(partEdits); | |
| 266 } | |
| 267 return new List<String>.from(results.map((e) => e.toString())); | |
| 268 } | |
| 269 | |
| 270 /// Merge `@patch` declarations into `external` declarations. | |
| 271 class PatchApplier extends GeneralizingAstVisitor { | |
| 272 final StringEditBuffer edits; | |
| 273 final PatchFinder patch; | |
| 274 | |
| 275 bool _isLibrary = true; // until proven otherwise. | |
| 276 | |
| 277 PatchApplier(this.edits, this.patch); | |
| 278 | |
| 279 @override | |
| 280 visitCompilationUnit(CompilationUnit node) { | |
| 281 super.visitCompilationUnit(node); | |
| 282 if (_isLibrary) _mergeUnpatched(node); | |
| 283 } | |
| 284 | |
| 285 void _merge(AstNode node, int pos) { | |
| 286 var code = patch.contents.substring(node.offset, node.end); | |
| 287 edits.insert(pos, '\n' + code); | |
| 288 } | |
| 289 | |
| 290 /// Merges directives and declarations that are not `@patch` into the library. | |
| 291 void _mergeUnpatched(CompilationUnit unit) { | |
| 292 // Merge imports from the patch | |
| 293 // TODO(jmesserly): remove duplicate imports | |
| 294 | |
| 295 // To patch a library, we must have a library directive | |
| 296 var libDir = unit.directives.first as LibraryDirective; | |
| 297 int importPos = unit.directives | |
| 298 .lastWhere((d) => d is ImportDirective, orElse: () => libDir) | |
| 299 .end; | |
| 300 for (var d in patch.unit.directives.where((d) => d is ImportDirective)) { | |
| 301 _merge(d, importPos); | |
| 302 } | |
| 303 | |
| 304 int partPos = unit.directives.last.end; | |
| 305 for (var d in patch.unit.directives.where((d) => d is PartDirective)) { | |
| 306 _merge(d, partPos); | |
| 307 } | |
| 308 | |
| 309 // Merge declarations from the patch | |
| 310 int declPos = edits.original.length; | |
| 311 for (var d in patch.mergeDeclarations) { | |
| 312 _merge(d, declPos); | |
| 313 } | |
| 314 } | |
| 315 | |
| 316 @override | |
| 317 visitPartOfDirective(PartOfDirective node) { | |
| 318 _isLibrary = false; | |
| 319 } | |
| 320 | |
| 321 @override | |
| 322 visitFunctionDeclaration(FunctionDeclaration node) { | |
| 323 _maybePatch(node); | |
| 324 } | |
| 325 | |
| 326 /// Merge patches and extensions into the class | |
| 327 @override | |
| 328 visitClassDeclaration(ClassDeclaration node) { | |
| 329 node.members.forEach(_maybePatch); | |
| 330 | |
| 331 var mergeMembers = patch.mergeMembers[_qualifiedName(node)]; | |
| 332 if (mergeMembers == null) return; | |
| 333 | |
| 334 // Merge members from the patch | |
| 335 var pos = node.members.last.end; | |
| 336 for (var member in mergeMembers) { | |
| 337 var code = patch.contents.substring(member.offset, member.end); | |
| 338 edits.insert(pos, '\n\n ' + code); | |
| 339 } | |
| 340 } | |
| 341 | |
| 342 void _maybePatch(AstNode node) { | |
| 343 if (node is FieldDeclaration) return; | |
| 344 | |
| 345 var externalKeyword = (node as dynamic).externalKeyword; | |
| 346 if (externalKeyword == null) return; | |
| 347 | |
| 348 var name = _qualifiedName(node); | |
| 349 var patchNode = patch.patches[name]; | |
| 350 if (patchNode == null) { | |
| 351 print('warning: patch not found for $name: $node'); | |
| 352 return; | |
| 353 } | |
| 354 | |
| 355 Annotation patchMeta = patchNode.metadata.lastWhere(_isPatchAnnotation); | |
| 356 int start = patchMeta.endToken.next.offset; | |
| 357 var code = patch.contents.substring(start, patchNode.end); | |
| 358 | |
| 359 // For some node like static fields, the node's offset doesn't include | |
| 360 // the external keyword. Also starting from the keyword lets us preserve | |
| 361 // documentation comments. | |
| 362 edits.replace(externalKeyword.offset, node.end, code); | |
| 363 } | |
| 364 } | |
| 365 | |
| 366 class PatchFinder extends GeneralizingAstVisitor { | |
| 367 final String contents; | |
| 368 final CompilationUnit unit; | |
| 369 | |
| 370 final Map patches = <String, Declaration>{}; | |
| 371 final Map mergeMembers = <String, List<ClassMember>>{}; | |
| 372 final List mergeDeclarations = <CompilationUnitMember>[]; | |
| 373 | |
| 374 PatchFinder.parseAndVisit(String name, String contents) | |
| 375 : contents = contents, | |
| 376 unit = parseCompilationUnit(contents, name: name) { | |
| 377 visitCompilationUnit(unit); | |
| 378 } | |
| 379 | |
| 380 @override | |
| 381 visitCompilationUnitMember(CompilationUnitMember node) { | |
| 382 mergeDeclarations.add(node); | |
| 383 } | |
| 384 | |
| 385 @override | |
| 386 visitClassDeclaration(ClassDeclaration node) { | |
| 387 if (_isPatch(node)) { | |
| 388 var members = <ClassMember>[]; | |
| 389 for (var member in node.members) { | |
| 390 if (_isPatch(member)) { | |
| 391 patches[_qualifiedName(member)] = member; | |
| 392 } else { | |
| 393 members.add(member); | |
| 394 } | |
| 395 } | |
| 396 if (members.isNotEmpty) { | |
| 397 mergeMembers[_qualifiedName(node)] = members; | |
| 398 } | |
| 399 } else { | |
| 400 mergeDeclarations.add(node); | |
| 401 } | |
| 402 } | |
| 403 | |
| 404 @override | |
| 405 visitFunctionDeclaration(FunctionDeclaration node) { | |
| 406 if (_isPatch(node)) { | |
| 407 patches[_qualifiedName(node)] = node; | |
| 408 } else { | |
| 409 mergeDeclarations.add(node); | |
| 410 } | |
| 411 } | |
| 412 | |
| 413 @override | |
| 414 visitFunctionBody(node) {} // skip method bodies | |
| 415 } | |
| 416 | |
| 417 String _qualifiedName(Declaration node) { | |
| 418 var parent = node.parent; | |
| 419 var className = ''; | |
| 420 if (parent is ClassDeclaration) { | |
| 421 className = parent.name.name + '.'; | |
| 422 } | |
| 423 var name = (node as dynamic).name; | |
| 424 name = (name != null ? name.name : ''); | |
| 425 | |
| 426 var accessor = ''; | |
| 427 if (node is MethodDeclaration) { | |
| 428 if (node.isGetter) accessor = 'get:'; | |
| 429 else if (node.isSetter) accessor = 'set:'; | |
| 430 } | |
| 431 return className + accessor + name; | |
| 432 } | |
| 433 | |
| 434 bool _isPatch(AnnotatedNode node) => node.metadata.any(_isPatchAnnotation); | |
| 435 | |
| 436 bool _isPatchAnnotation(Annotation m) => | |
| 437 m.name.name == 'patch' && m.constructorName == null && m.arguments == null; | |
| 438 | |
| 439 /// Editable string buffer. | |
| 440 /// | |
| 441 /// Applies a series of edits (insertions, removals, replacements) using | |
| 442 /// original location information, and composes them into the edited string. | |
| 443 /// | |
| 444 /// For example, starting with a parsed AST with original source locations, | |
| 445 /// this type allows edits to be made without regards to other edits. | |
| 446 class StringEditBuffer { | |
| 447 final String original; | |
| 448 final _edits = <_StringEdit>[]; | |
| 449 | |
| 450 /// Creates a new transaction. | |
| 451 StringEditBuffer(this.original); | |
| 452 | |
| 453 bool get hasEdits => _edits.length > 0; | |
| 454 | |
| 455 /// Edit the original text, replacing text on the range [begin] and | |
| 456 /// exclusive [end] with the [replacement] string. | |
| 457 void replace(int begin, int end, String replacement) { | |
| 458 _edits.add(new _StringEdit(begin, end, replacement)); | |
| 459 } | |
| 460 | |
| 461 /// Insert [string] at [offset]. | |
| 462 /// Equivalent to `replace(offset, offset, string)`. | |
| 463 void insert(int offset, String string) => replace(offset, offset, string); | |
| 464 | |
| 465 /// Remove text from the range [begin] to exclusive [end]. | |
| 466 /// Equivalent to `replace(begin, end, '')`. | |
| 467 void remove(int begin, int end) => replace(begin, end, ''); | |
| 468 | |
| 469 /// Applies all pending [edit]s and returns a new string. | |
| 470 /// | |
| 471 /// This method is non-destructive: it does not discard existing edits or | |
| 472 /// change the [original] string. Further edits can be added and this method | |
| 473 /// can be called again. | |
| 474 /// | |
| 475 /// Throws [UnsupportedError] if the edits were overlapping. If no edits were | |
| 476 /// made, the original string will be returned. | |
| 477 String toString() { | |
| 478 var sb = new StringBuffer(); | |
| 479 if (_edits.length == 0) return original; | |
| 480 | |
| 481 // Sort edits by start location. | |
| 482 _edits.sort(); | |
| 483 | |
| 484 int consumed = 0; | |
| 485 for (var edit in _edits) { | |
| 486 if (consumed > edit.begin) { | |
| 487 sb = new StringBuffer(); | |
| 488 sb.write('overlapping edits. Insert at offset '); | |
| 489 sb.write(edit.begin); | |
| 490 sb.write(' but have consumed '); | |
| 491 sb.write(consumed); | |
| 492 sb.write(' input characters. List of edits:'); | |
| 493 for (var e in _edits) { | |
| 494 sb.write('\n '); | |
| 495 sb.write(e); | |
| 496 } | |
| 497 throw new UnsupportedError(sb.toString()); | |
| 498 } | |
| 499 | |
| 500 // Add characters from the original string between this edit and the last | |
| 501 // one, if any. | |
| 502 var betweenEdits = original.substring(consumed, edit.begin); | |
| 503 sb.write(betweenEdits); | |
| 504 sb.write(edit.replace); | |
| 505 consumed = edit.end; | |
| 506 } | |
| 507 | |
| 508 // Add any text from the end of the original string that was not replaced. | |
| 509 sb.write(original.substring(consumed)); | |
| 510 return sb.toString(); | |
| 511 } | |
| 512 } | |
| 513 | |
| 514 class _StringEdit implements Comparable<_StringEdit> { | |
| 515 final int begin; | |
| 516 final int end; | |
| 517 final String replace; | |
| 518 | |
| 519 _StringEdit(this.begin, this.end, this.replace); | |
| 520 | |
| 521 int get length => end - begin; | |
| 522 | |
| 523 String toString() => '(Edit @ $begin,$end: "$replace")'; | |
| 524 | |
| 525 int compareTo(_StringEdit other) { | |
| 526 int diff = begin - other.begin; | |
| 527 if (diff != 0) return diff; | |
| 528 return end - other.end; | |
| 529 } | |
| 530 } | |
| 531 | |
| 532 List<SdkLibrary> _getSdkLibraries(String contents) { | |
| 533 var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(true); | |
| 534 parseCompilationUnit(contents).accept(libraryBuilder); | |
| 535 return libraryBuilder.librariesMap.sdkLibraries; | |
| 536 } | |
| OLD | NEW |