OLD | NEW |
| (Empty) |
1 #!/usr/bin/env dart | |
2 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file | |
3 // for details. All rights reserved. Use of this source code is governed by a | |
4 // BSD-style license that can be found in the LICENSE file. | |
5 | |
6 /// Command line tool to merge the SDK libraries and our patch files. | |
7 /// This is currently designed as an offline tool, but we could automate it. | |
8 | |
9 import 'dart:io'; | |
10 import 'dart:math' as math; | |
11 | |
12 import 'package:analyzer/analyzer.dart'; | |
13 import 'package:analyzer/src/generated/sdk.dart'; | |
14 import 'package:path/path.dart' as path; | |
15 | |
16 void main(List<String> argv) { | |
17 var base = path.fromUri(Platform.script); | |
18 var dartDir = path.dirname(path.dirname(path.absolute(base))); | |
19 | |
20 if (argv.length != 4 || | |
21 !argv.isEmpty && argv.first != 'vm' && argv.first != 'ddc') { | |
22 var self = path.relative(base); | |
23 print('Usage: $self MODE SDK_DIR PATCH_DIR OUTPUT_DIR'); | |
24 print('MODE must be one of ddc or vm.'); | |
25 | |
26 var toolDir = path.relative(path.dirname(base)); | |
27 var sdkExample = path.join(toolDir, 'input_sdk'); | |
28 var patchExample = path.join(sdkExample, 'patch'); | |
29 var outExample = | |
30 path.relative(path.normalize(path.join('gen', 'patched_sdk'))); | |
31 print('For example:'); | |
32 print('\$ $self ddc $sdkExample $patchExample $outExample'); | |
33 | |
34 var repositoryDir = path.relative(path.dirname(path.dirname(base))); | |
35 sdkExample = path.relative(path.join(repositoryDir, 'sdk')); | |
36 patchExample = path.relative(path.join(repositoryDir, 'out', 'DebugX64', | |
37 'obj', 'gen', 'patch')); | |
38 outExample = path.relative(path.join(repositoryDir, 'out', 'DebugX64', | |
39 'obj', 'gen', 'patched_sdk')); | |
40 print('or:'); | |
41 print('\$ $self vm $sdkExample $patchExample $outExample'); | |
42 | |
43 exit(1); | |
44 } | |
45 | |
46 var mode = argv[0]; | |
47 var input = argv[1]; | |
48 var sdkLibIn = path.join(input, 'lib'); | |
49 var patchIn = argv[2]; | |
50 var sdkOut = path.join(argv[3], 'lib'); | |
51 | |
52 var privateIn = path.join(input, 'private'); | |
53 var INTERNAL_PATH = '_internal/compiler/js_lib/'; | |
54 | |
55 // Copy and patch libraries.dart and version | |
56 var libContents = new File(path.join(sdkLibIn, '_internal', | |
57 'sdk_library_metadata', 'lib', 'libraries.dart')).readAsStringSync(); | |
58 var patchedLibContents = libContents; | |
59 if (mode == 'vm') { | |
60 libContents = libContents.replaceAll( | |
61 ' libraries = const {', | |
62 ''' libraries = const { | |
63 | |
64 "_builtin": const LibraryInfo( | |
65 "_builtin/_builtin.dart", | |
66 categories: "Client,Server", | |
67 implementation: true, | |
68 documented: false, | |
69 platforms: VM_PLATFORM), | |
70 | |
71 "profiler": const LibraryInfo( | |
72 "profiler/profiler.dart", | |
73 maturity: Maturity.DEPRECATED, | |
74 documented: false), | |
75 | |
76 "_vmservice": const LibraryInfo( | |
77 "vmservice/vmservice.dart", | |
78 implementation: true, | |
79 documented: false, | |
80 platforms: VM_PLATFORM), | |
81 | |
82 "vmservice_io": const LibraryInfo( | |
83 "vmservice_io/vmservice_io.dart", | |
84 implementation: true, | |
85 documented: false, | |
86 platforms: VM_PLATFORM), | |
87 | |
88 '''); | |
89 } | |
90 _writeSync( | |
91 path.join( | |
92 sdkOut, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'), | |
93 libContents); | |
94 if (mode == 'ddc') { | |
95 _writeSync(path.join(sdkOut, '..', 'version'), | |
96 new File(path.join(sdkLibIn, '..', 'version')).readAsStringSync()); | |
97 } | |
98 | |
99 // Parse libraries.dart | |
100 var sdkLibraries = _getSdkLibraries(libContents); | |
101 | |
102 // Enumerate core libraries and apply patches | |
103 for (SdkLibrary library in sdkLibraries) { | |
104 // TODO(jmesserly): analyzer does not handle the default case of | |
105 // "both platforms" correctly, and treats it as being supported on neither. | |
106 // So instead we skip explicitly marked as either VM or dart2js libs. | |
107 if (mode == 'ddc' ? libary.isVmLibrary : library.isDart2JsLibrary) { | |
108 continue; | |
109 } | |
110 | |
111 var libraryOut = path.join(sdkLibIn, library.path); | |
112 var libraryIn; | |
113 if (mode == 'vm' && library.path.contains('typed_data.dart')) { | |
114 // dart:typed_data is unlike the other libraries in the SDK. The VM does | |
115 // not apply a patch to the base SDK implementation of the library. | |
116 // Instead, the VM provides a replacement implementation and ignores the | |
117 // sources in the SDK. | |
118 libraryIn = | |
119 path.join(dartDir, 'runtime', 'lib', 'typed_data.dart'); | |
120 } else if (mode == 'ddc' && library.path.contains(INTERNAL_PATH)) { | |
121 libraryIn = | |
122 path.join(privateIn, library.path.replaceAll(INTERNAL_PATH, '')); | |
123 } else { | |
124 libraryIn = libraryOut; | |
125 } | |
126 | |
127 var libraryFile = new File(libraryIn); | |
128 if (libraryFile.existsSync()) { | |
129 var outPaths = <String>[libraryOut]; | |
130 var libraryContents = libraryFile.readAsStringSync(); | |
131 | |
132 int inputModifyTime = | |
133 libraryFile.lastModifiedSync().millisecondsSinceEpoch; | |
134 var partFiles = <File>[]; | |
135 for (var part in parseDirectives(libraryContents).directives) { | |
136 if (part is PartDirective) { | |
137 var partPath = part.uri.stringValue; | |
138 outPaths.add(path.join(path.dirname(libraryOut), partPath)); | |
139 | |
140 var partFile = new File(path.join(path.dirname(libraryIn), partPath)); | |
141 partFiles.add(partFile); | |
142 inputModifyTime = math.max(inputModifyTime, | |
143 partFile.lastModifiedSync().millisecondsSinceEpoch); | |
144 } | |
145 } | |
146 | |
147 // See if we can find a patch file. | |
148 var patchPath = path.join( | |
149 patchIn, path.basenameWithoutExtension(libraryIn) + '_patch.dart'); | |
150 | |
151 var patchFile = new File(patchPath); | |
152 bool patchExists = patchFile.existsSync(); | |
153 if (patchExists) { | |
154 inputModifyTime = math.max(inputModifyTime, | |
155 patchFile.lastModifiedSync().millisecondsSinceEpoch); | |
156 } | |
157 | |
158 // Compute output paths | |
159 outPaths = outPaths | |
160 .map((p) => path.join(sdkOut, path.relative(p, from: sdkLibIn))) | |
161 .toList(); | |
162 | |
163 // Compare output modify time with input modify time. | |
164 bool needsUpdate = false; | |
165 for (var outPath in outPaths) { | |
166 var outFile = new File(outPath); | |
167 if (!outFile.existsSync() || | |
168 outFile.lastModifiedSync().millisecondsSinceEpoch < | |
169 inputModifyTime) { | |
170 needsUpdate = true; | |
171 break; | |
172 } | |
173 } | |
174 | |
175 if (needsUpdate) { | |
176 var contents = <String>[libraryContents]; | |
177 contents.addAll(partFiles.map((f) => f.readAsStringSync())); | |
178 if (patchExists) { | |
179 var patchContents = patchFile.readAsStringSync(); | |
180 contents = _patchLibrary( | |
181 patchFile.toString(), contents, patchContents); | |
182 } | |
183 | |
184 for (var i = 0; i < outPaths.length; i++) { | |
185 if (path.basename(outPaths[i]) == 'internal.dart') { | |
186 contents[i] += ''' | |
187 | |
188 /// Marks a function as an external implementation ("native" in the Dart VM). | |
189 /// | |
190 /// Provides a backend-specific String that can be used to identify the | |
191 /// function's implementation | |
192 class ExternalName { | |
193 final String name; | |
194 const ExternalName(this.name); | |
195 } | |
196 '''; | |
197 } | |
198 | |
199 _writeSync(outPaths[i], contents[i]); | |
200 } | |
201 } | |
202 } | |
203 } | |
204 if (mode == 'vm') { | |
205 | |
206 for (var tuple in [['_builtin', 'builtin.dart']]) { | |
207 var vmLibrary = tuple[0]; | |
208 var dartFile = tuple[1]; | |
209 | |
210 // The "dart:_builtin" library is only available for the DartVM. | |
211 var builtinLibraryIn = path.join(dartDir, 'runtime', 'bin', dartFile); | |
212 var builtinLibraryOut = path.join(sdkOut, vmLibrary, '${vmLibrary}.dart'); | |
213 _writeSync(builtinLibraryOut, new File(builtinLibraryIn).readAsStringSync(
)); | |
214 } | |
215 | |
216 for (var file in ['loader.dart', 'server.dart', 'vmservice_io.dart']) { | |
217 var libraryIn = path.join(dartDir, 'runtime', 'bin', 'vmservice', file); | |
218 var libraryOut = path.join(sdkOut, 'vmservice_io', file); | |
219 _writeSync(libraryOut, new File(libraryIn).readAsStringSync()); | |
220 } | |
221 } | |
222 } | |
223 | |
224 /// Writes a file, creating the directory if needed. | |
225 void _writeSync(String filePath, String contents) { | |
226 var outDir = new Directory(path.dirname(filePath)); | |
227 if (!outDir.existsSync()) outDir.createSync(recursive: true); | |
228 | |
229 new File(filePath).writeAsStringSync(contents); | |
230 } | |
231 | |
232 /// Merges dart:* library code with code from *_patch.dart file. | |
233 /// | |
234 /// Takes a list of the library's parts contents, with the main library contents | |
235 /// first in the list, and the contents of the patch file. | |
236 /// | |
237 /// The result will have `@patch` implementations merged into the correct place | |
238 /// (e.g. the class or top-level function declaration) and all other | |
239 /// declarations introduced by the patch will be placed into the main library | |
240 /// file. | |
241 /// | |
242 /// This is purely a syntactic transformation. Unlike dart2js patch files, there | |
243 /// is no semantic meaning given to the *_patch files, and they do not magically | |
244 /// get their own library scope, etc. | |
245 /// | |
246 /// Editorializing: the dart2js approach requires a Dart front end such as | |
247 /// package:analyzer to semantically model a feature beyond what is specified | |
248 /// in the Dart language. Since this feature is only for the convenience of | |
249 /// writing the dart:* libraries, and not a tool given to Dart developers, it | |
250 /// seems like a non-ideal situation. Instead we keep the preprocessing simple. | |
251 List<String> _patchLibrary(String name, | |
252 List<String> partsContents, | |
253 String patchContents) { | |
254 var results = <StringEditBuffer>[]; | |
255 | |
256 // Parse the patch first. We'll need to extract bits of this as we go through | |
257 // the other files. | |
258 final patchFinder = new PatchFinder.parseAndVisit(name, patchContents); | |
259 | |
260 // Merge `external` declarations with the corresponding `@patch` code. | |
261 for (var partContent in partsContents) { | |
262 var partEdits = new StringEditBuffer(partContent); | |
263 var partUnit = parseCompilationUnit(partContent); | |
264 partUnit.accept(new PatchApplier(partEdits, patchFinder)); | |
265 results.add(partEdits); | |
266 } | |
267 return new List<String>.from(results.map((e) => e.toString())); | |
268 } | |
269 | |
270 /// Merge `@patch` declarations into `external` declarations. | |
271 class PatchApplier extends GeneralizingAstVisitor { | |
272 final StringEditBuffer edits; | |
273 final PatchFinder patch; | |
274 | |
275 bool _isLibrary = true; // until proven otherwise. | |
276 | |
277 PatchApplier(this.edits, this.patch); | |
278 | |
279 @override | |
280 visitCompilationUnit(CompilationUnit node) { | |
281 super.visitCompilationUnit(node); | |
282 if (_isLibrary) _mergeUnpatched(node); | |
283 } | |
284 | |
285 void _merge(AstNode node, int pos) { | |
286 var code = patch.contents.substring(node.offset, node.end); | |
287 edits.insert(pos, '\n' + code); | |
288 } | |
289 | |
290 /// Merges directives and declarations that are not `@patch` into the library. | |
291 void _mergeUnpatched(CompilationUnit unit) { | |
292 // Merge imports from the patch | |
293 // TODO(jmesserly): remove duplicate imports | |
294 | |
295 // To patch a library, we must have a library directive | |
296 var libDir = unit.directives.first as LibraryDirective; | |
297 int importPos = unit.directives | |
298 .lastWhere((d) => d is ImportDirective, orElse: () => libDir) | |
299 .end; | |
300 for (var d in patch.unit.directives.where((d) => d is ImportDirective)) { | |
301 _merge(d, importPos); | |
302 } | |
303 | |
304 int partPos = unit.directives.last.end; | |
305 for (var d in patch.unit.directives.where((d) => d is PartDirective)) { | |
306 _merge(d, partPos); | |
307 } | |
308 | |
309 // Merge declarations from the patch | |
310 int declPos = edits.original.length; | |
311 for (var d in patch.mergeDeclarations) { | |
312 _merge(d, declPos); | |
313 } | |
314 } | |
315 | |
316 @override | |
317 visitPartOfDirective(PartOfDirective node) { | |
318 _isLibrary = false; | |
319 } | |
320 | |
321 @override | |
322 visitFunctionDeclaration(FunctionDeclaration node) { | |
323 _maybePatch(node); | |
324 } | |
325 | |
326 /// Merge patches and extensions into the class | |
327 @override | |
328 visitClassDeclaration(ClassDeclaration node) { | |
329 node.members.forEach(_maybePatch); | |
330 | |
331 var mergeMembers = patch.mergeMembers[_qualifiedName(node)]; | |
332 if (mergeMembers == null) return; | |
333 | |
334 // Merge members from the patch | |
335 var pos = node.members.last.end; | |
336 for (var member in mergeMembers) { | |
337 var code = patch.contents.substring(member.offset, member.end); | |
338 edits.insert(pos, '\n\n ' + code); | |
339 } | |
340 } | |
341 | |
342 void _maybePatch(AstNode node) { | |
343 if (node is FieldDeclaration) return; | |
344 | |
345 var externalKeyword = (node as dynamic).externalKeyword; | |
346 if (externalKeyword == null) return; | |
347 | |
348 var name = _qualifiedName(node); | |
349 var patchNode = patch.patches[name]; | |
350 if (patchNode == null) { | |
351 print('warning: patch not found for $name: $node'); | |
352 return; | |
353 } | |
354 | |
355 Annotation patchMeta = patchNode.metadata.lastWhere(_isPatchAnnotation); | |
356 int start = patchMeta.endToken.next.offset; | |
357 var code = patch.contents.substring(start, patchNode.end); | |
358 | |
359 // For some node like static fields, the node's offset doesn't include | |
360 // the external keyword. Also starting from the keyword lets us preserve | |
361 // documentation comments. | |
362 edits.replace(externalKeyword.offset, node.end, code); | |
363 } | |
364 } | |
365 | |
366 class PatchFinder extends GeneralizingAstVisitor { | |
367 final String contents; | |
368 final CompilationUnit unit; | |
369 | |
370 final Map patches = <String, Declaration>{}; | |
371 final Map mergeMembers = <String, List<ClassMember>>{}; | |
372 final List mergeDeclarations = <CompilationUnitMember>[]; | |
373 | |
374 PatchFinder.parseAndVisit(String name, String contents) | |
375 : contents = contents, | |
376 unit = parseCompilationUnit(contents, name: name) { | |
377 visitCompilationUnit(unit); | |
378 } | |
379 | |
380 @override | |
381 visitCompilationUnitMember(CompilationUnitMember node) { | |
382 mergeDeclarations.add(node); | |
383 } | |
384 | |
385 @override | |
386 visitClassDeclaration(ClassDeclaration node) { | |
387 if (_isPatch(node)) { | |
388 var members = <ClassMember>[]; | |
389 for (var member in node.members) { | |
390 if (_isPatch(member)) { | |
391 patches[_qualifiedName(member)] = member; | |
392 } else { | |
393 members.add(member); | |
394 } | |
395 } | |
396 if (members.isNotEmpty) { | |
397 mergeMembers[_qualifiedName(node)] = members; | |
398 } | |
399 } else { | |
400 mergeDeclarations.add(node); | |
401 } | |
402 } | |
403 | |
404 @override | |
405 visitFunctionDeclaration(FunctionDeclaration node) { | |
406 if (_isPatch(node)) { | |
407 patches[_qualifiedName(node)] = node; | |
408 } else { | |
409 mergeDeclarations.add(node); | |
410 } | |
411 } | |
412 | |
413 @override | |
414 visitFunctionBody(node) {} // skip method bodies | |
415 } | |
416 | |
417 String _qualifiedName(Declaration node) { | |
418 var parent = node.parent; | |
419 var className = ''; | |
420 if (parent is ClassDeclaration) { | |
421 className = parent.name.name + '.'; | |
422 } | |
423 var name = (node as dynamic).name; | |
424 name = (name != null ? name.name : ''); | |
425 | |
426 var accessor = ''; | |
427 if (node is MethodDeclaration) { | |
428 if (node.isGetter) accessor = 'get:'; | |
429 else if (node.isSetter) accessor = 'set:'; | |
430 } | |
431 return className + accessor + name; | |
432 } | |
433 | |
434 bool _isPatch(AnnotatedNode node) => node.metadata.any(_isPatchAnnotation); | |
435 | |
436 bool _isPatchAnnotation(Annotation m) => | |
437 m.name.name == 'patch' && m.constructorName == null && m.arguments == null; | |
438 | |
439 /// Editable string buffer. | |
440 /// | |
441 /// Applies a series of edits (insertions, removals, replacements) using | |
442 /// original location information, and composes them into the edited string. | |
443 /// | |
444 /// For example, starting with a parsed AST with original source locations, | |
445 /// this type allows edits to be made without regards to other edits. | |
446 class StringEditBuffer { | |
447 final String original; | |
448 final _edits = <_StringEdit>[]; | |
449 | |
450 /// Creates a new transaction. | |
451 StringEditBuffer(this.original); | |
452 | |
453 bool get hasEdits => _edits.length > 0; | |
454 | |
455 /// Edit the original text, replacing text on the range [begin] and | |
456 /// exclusive [end] with the [replacement] string. | |
457 void replace(int begin, int end, String replacement) { | |
458 _edits.add(new _StringEdit(begin, end, replacement)); | |
459 } | |
460 | |
461 /// Insert [string] at [offset]. | |
462 /// Equivalent to `replace(offset, offset, string)`. | |
463 void insert(int offset, String string) => replace(offset, offset, string); | |
464 | |
465 /// Remove text from the range [begin] to exclusive [end]. | |
466 /// Equivalent to `replace(begin, end, '')`. | |
467 void remove(int begin, int end) => replace(begin, end, ''); | |
468 | |
469 /// Applies all pending [edit]s and returns a new string. | |
470 /// | |
471 /// This method is non-destructive: it does not discard existing edits or | |
472 /// change the [original] string. Further edits can be added and this method | |
473 /// can be called again. | |
474 /// | |
475 /// Throws [UnsupportedError] if the edits were overlapping. If no edits were | |
476 /// made, the original string will be returned. | |
477 String toString() { | |
478 var sb = new StringBuffer(); | |
479 if (_edits.length == 0) return original; | |
480 | |
481 // Sort edits by start location. | |
482 _edits.sort(); | |
483 | |
484 int consumed = 0; | |
485 for (var edit in _edits) { | |
486 if (consumed > edit.begin) { | |
487 sb = new StringBuffer(); | |
488 sb.write('overlapping edits. Insert at offset '); | |
489 sb.write(edit.begin); | |
490 sb.write(' but have consumed '); | |
491 sb.write(consumed); | |
492 sb.write(' input characters. List of edits:'); | |
493 for (var e in _edits) { | |
494 sb.write('\n '); | |
495 sb.write(e); | |
496 } | |
497 throw new UnsupportedError(sb.toString()); | |
498 } | |
499 | |
500 // Add characters from the original string between this edit and the last | |
501 // one, if any. | |
502 var betweenEdits = original.substring(consumed, edit.begin); | |
503 sb.write(betweenEdits); | |
504 sb.write(edit.replace); | |
505 consumed = edit.end; | |
506 } | |
507 | |
508 // Add any text from the end of the original string that was not replaced. | |
509 sb.write(original.substring(consumed)); | |
510 return sb.toString(); | |
511 } | |
512 } | |
513 | |
514 class _StringEdit implements Comparable<_StringEdit> { | |
515 final int begin; | |
516 final int end; | |
517 final String replace; | |
518 | |
519 _StringEdit(this.begin, this.end, this.replace); | |
520 | |
521 int get length => end - begin; | |
522 | |
523 String toString() => '(Edit @ $begin,$end: "$replace")'; | |
524 | |
525 int compareTo(_StringEdit other) { | |
526 int diff = begin - other.begin; | |
527 if (diff != 0) return diff; | |
528 return end - other.end; | |
529 } | |
530 } | |
531 | |
532 List<SdkLibrary> _getSdkLibraries(String contents) { | |
533 var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(true); | |
534 parseCompilationUnit(contents).accept(libraryBuilder); | |
535 return libraryBuilder.librariesMap.sdkLibraries; | |
536 } | |
OLD | NEW |