Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(225)

Side by Side Diff: tools/patch_sdk.dart

Issue 2451893004: Revert "Reland "Merge more Kernel infrastructure from kernel_sdk SDK fork."" (Closed)
Patch Set: Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tests/language/language_kernel.status ('k') | tools/patch_sdk.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env dart
2 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
3 // for details. All rights reserved. Use of this source code is governed by a
4 // BSD-style license that can be found in the LICENSE file.
5
6 /// Command line tool to merge the SDK libraries and our patch files.
7 /// This is currently designed as an offline tool, but we could automate it.
8
9 import 'dart:io';
10 import 'dart:math' as math;
11
12 import 'package:analyzer/analyzer.dart';
13 import 'package:analyzer/src/generated/sdk.dart';
14 import 'package:path/path.dart' as path;
15
16 void main(List<String> argv) {
17 var base = path.fromUri(Platform.script);
18 var dartDir = path.dirname(path.dirname(path.absolute(base)));
19
20 if (argv.length != 4 ||
21 !argv.isEmpty && argv.first != 'vm' && argv.first != 'ddc') {
22 var self = path.relative(base);
23 print('Usage: $self MODE SDK_DIR PATCH_DIR OUTPUT_DIR');
24 print('MODE must be one of ddc or vm.');
25
26 var toolDir = path.relative(path.dirname(base));
27 var sdkExample = path.join(toolDir, 'input_sdk');
28 var patchExample = path.join(sdkExample, 'patch');
29 var outExample =
30 path.relative(path.normalize(path.join('gen', 'patched_sdk')));
31 print('For example:');
32 print('\$ $self ddc $sdkExample $patchExample $outExample');
33
34 var repositoryDir = path.relative(path.dirname(path.dirname(base)));
35 sdkExample = path.relative(path.join(repositoryDir, 'sdk'));
36 patchExample = path.relative(path.join(repositoryDir, 'out', 'DebugX64',
37 'obj', 'gen', 'patch'));
38 outExample = path.relative(path.join(repositoryDir, 'out', 'DebugX64',
39 'obj', 'gen', 'patched_sdk'));
40 print('or:');
41 print('\$ $self vm $sdkExample $patchExample $outExample');
42
43 exit(1);
44 }
45
46 var mode = argv[0];
47 var input = argv[1];
48 var sdkLibIn = path.join(input, 'lib');
49 var patchIn = argv[2];
50 var sdkOut = path.join(argv[3], 'lib');
51
52 var privateIn = path.join(input, 'private');
53 var INTERNAL_PATH = '_internal/compiler/js_lib/';
54
55 // Copy and patch libraries.dart and version
56 var libContents = new File(path.join(sdkLibIn, '_internal',
57 'sdk_library_metadata', 'lib', 'libraries.dart')).readAsStringSync();
58 var patchedLibContents = libContents;
59 if (mode == 'vm') {
60 libContents = libContents.replaceAll(
61 ' libraries = const {',
62 ''' libraries = const {
63
64 "_builtin": const LibraryInfo(
65 "_builtin/_builtin.dart",
66 categories: "Client,Server",
67 implementation: true,
68 documented: false,
69 platforms: VM_PLATFORM),
70
71 "profiler": const LibraryInfo(
72 "profiler/profiler.dart",
73 maturity: Maturity.DEPRECATED,
74 documented: false),
75
76 "_vmservice": const LibraryInfo(
77 "vmservice/vmservice.dart",
78 implementation: true,
79 documented: false,
80 platforms: VM_PLATFORM),
81
82 "vmservice_io": const LibraryInfo(
83 "vmservice_io/vmservice_io.dart",
84 implementation: true,
85 documented: false,
86 platforms: VM_PLATFORM),
87
88 ''');
89 }
90 _writeSync(
91 path.join(
92 sdkOut, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'),
93 libContents);
94 if (mode == 'ddc') {
95 _writeSync(path.join(sdkOut, '..', 'version'),
96 new File(path.join(sdkLibIn, '..', 'version')).readAsStringSync());
97 }
98
99 // Parse libraries.dart
100 var sdkLibraries = _getSdkLibraries(libContents);
101
102 // Enumerate core libraries and apply patches
103 for (SdkLibrary library in sdkLibraries) {
104 // TODO(jmesserly): analyzer does not handle the default case of
105 // "both platforms" correctly, and treats it as being supported on neither.
106 // So instead we skip explicitly marked as either VM or dart2js libs.
107 if (mode == 'ddc' ? libary.isVmLibrary : library.isDart2JsLibrary) {
108 continue;
109 }
110
111 var libraryOut = path.join(sdkLibIn, library.path);
112 var libraryIn;
113 if (mode == 'vm' && library.path.contains('typed_data.dart')) {
114 // dart:typed_data is unlike the other libraries in the SDK. The VM does
115 // not apply a patch to the base SDK implementation of the library.
116 // Instead, the VM provides a replacement implementation and ignores the
117 // sources in the SDK.
118 libraryIn =
119 path.join(dartDir, 'runtime', 'lib', 'typed_data.dart');
120 } else if (mode == 'ddc' && library.path.contains(INTERNAL_PATH)) {
121 libraryIn =
122 path.join(privateIn, library.path.replaceAll(INTERNAL_PATH, ''));
123 } else {
124 libraryIn = libraryOut;
125 }
126
127 var libraryFile = new File(libraryIn);
128 if (libraryFile.existsSync()) {
129 var outPaths = <String>[libraryOut];
130 var libraryContents = libraryFile.readAsStringSync();
131
132 int inputModifyTime =
133 libraryFile.lastModifiedSync().millisecondsSinceEpoch;
134 var partFiles = <File>[];
135 for (var part in parseDirectives(libraryContents).directives) {
136 if (part is PartDirective) {
137 var partPath = part.uri.stringValue;
138 outPaths.add(path.join(path.dirname(libraryOut), partPath));
139
140 var partFile = new File(path.join(path.dirname(libraryIn), partPath));
141 partFiles.add(partFile);
142 inputModifyTime = math.max(inputModifyTime,
143 partFile.lastModifiedSync().millisecondsSinceEpoch);
144 }
145 }
146
147 // See if we can find a patch file.
148 var patchPath = path.join(
149 patchIn, path.basenameWithoutExtension(libraryIn) + '_patch.dart');
150
151 var patchFile = new File(patchPath);
152 bool patchExists = patchFile.existsSync();
153 if (patchExists) {
154 inputModifyTime = math.max(inputModifyTime,
155 patchFile.lastModifiedSync().millisecondsSinceEpoch);
156 }
157
158 // Compute output paths
159 outPaths = outPaths
160 .map((p) => path.join(sdkOut, path.relative(p, from: sdkLibIn)))
161 .toList();
162
163 // Compare output modify time with input modify time.
164 bool needsUpdate = false;
165 for (var outPath in outPaths) {
166 var outFile = new File(outPath);
167 if (!outFile.existsSync() ||
168 outFile.lastModifiedSync().millisecondsSinceEpoch <
169 inputModifyTime) {
170 needsUpdate = true;
171 break;
172 }
173 }
174
175 if (needsUpdate) {
176 var contents = <String>[libraryContents];
177 contents.addAll(partFiles.map((f) => f.readAsStringSync()));
178 if (patchExists) {
179 var patchContents = patchFile.readAsStringSync();
180 contents = _patchLibrary(
181 patchFile.toString(), contents, patchContents);
182 }
183
184 for (var i = 0; i < outPaths.length; i++) {
185 if (path.basename(outPaths[i]) == 'internal.dart') {
186 contents[i] += '''
187
188 /// Marks a function as an external implementation ("native" in the Dart VM).
189 ///
190 /// Provides a backend-specific String that can be used to identify the
191 /// function's implementation
192 class ExternalName {
193 final String name;
194 const ExternalName(this.name);
195 }
196 ''';
197 }
198
199 _writeSync(outPaths[i], contents[i]);
200 }
201 }
202 }
203 }
204 if (mode == 'vm') {
205
206 for (var tuple in [['_builtin', 'builtin.dart']]) {
207 var vmLibrary = tuple[0];
208 var dartFile = tuple[1];
209
210 // The "dart:_builtin" library is only available for the DartVM.
211 var builtinLibraryIn = path.join(dartDir, 'runtime', 'bin', dartFile);
212 var builtinLibraryOut = path.join(sdkOut, vmLibrary, '${vmLibrary}.dart');
213 _writeSync(builtinLibraryOut, new File(builtinLibraryIn).readAsStringSync( ));
214 }
215
216 for (var file in ['loader.dart', 'server.dart', 'vmservice_io.dart']) {
217 var libraryIn = path.join(dartDir, 'runtime', 'bin', 'vmservice', file);
218 var libraryOut = path.join(sdkOut, 'vmservice_io', file);
219 _writeSync(libraryOut, new File(libraryIn).readAsStringSync());
220 }
221 }
222 }
223
224 /// Writes a file, creating the directory if needed.
225 void _writeSync(String filePath, String contents) {
226 var outDir = new Directory(path.dirname(filePath));
227 if (!outDir.existsSync()) outDir.createSync(recursive: true);
228
229 new File(filePath).writeAsStringSync(contents);
230 }
231
232 /// Merges dart:* library code with code from *_patch.dart file.
233 ///
234 /// Takes a list of the library's parts contents, with the main library contents
235 /// first in the list, and the contents of the patch file.
236 ///
237 /// The result will have `@patch` implementations merged into the correct place
238 /// (e.g. the class or top-level function declaration) and all other
239 /// declarations introduced by the patch will be placed into the main library
240 /// file.
241 ///
242 /// This is purely a syntactic transformation. Unlike dart2js patch files, there
243 /// is no semantic meaning given to the *_patch files, and they do not magically
244 /// get their own library scope, etc.
245 ///
246 /// Editorializing: the dart2js approach requires a Dart front end such as
247 /// package:analyzer to semantically model a feature beyond what is specified
248 /// in the Dart language. Since this feature is only for the convenience of
249 /// writing the dart:* libraries, and not a tool given to Dart developers, it
250 /// seems like a non-ideal situation. Instead we keep the preprocessing simple.
251 List<String> _patchLibrary(String name,
252 List<String> partsContents,
253 String patchContents) {
254 var results = <StringEditBuffer>[];
255
256 // Parse the patch first. We'll need to extract bits of this as we go through
257 // the other files.
258 final patchFinder = new PatchFinder.parseAndVisit(name, patchContents);
259
260 // Merge `external` declarations with the corresponding `@patch` code.
261 for (var partContent in partsContents) {
262 var partEdits = new StringEditBuffer(partContent);
263 var partUnit = parseCompilationUnit(partContent);
264 partUnit.accept(new PatchApplier(partEdits, patchFinder));
265 results.add(partEdits);
266 }
267 return new List<String>.from(results.map((e) => e.toString()));
268 }
269
270 /// Merge `@patch` declarations into `external` declarations.
271 class PatchApplier extends GeneralizingAstVisitor {
272 final StringEditBuffer edits;
273 final PatchFinder patch;
274
275 bool _isLibrary = true; // until proven otherwise.
276
277 PatchApplier(this.edits, this.patch);
278
279 @override
280 visitCompilationUnit(CompilationUnit node) {
281 super.visitCompilationUnit(node);
282 if (_isLibrary) _mergeUnpatched(node);
283 }
284
285 void _merge(AstNode node, int pos) {
286 var code = patch.contents.substring(node.offset, node.end);
287 edits.insert(pos, '\n' + code);
288 }
289
290 /// Merges directives and declarations that are not `@patch` into the library.
291 void _mergeUnpatched(CompilationUnit unit) {
292 // Merge imports from the patch
293 // TODO(jmesserly): remove duplicate imports
294
295 // To patch a library, we must have a library directive
296 var libDir = unit.directives.first as LibraryDirective;
297 int importPos = unit.directives
298 .lastWhere((d) => d is ImportDirective, orElse: () => libDir)
299 .end;
300 for (var d in patch.unit.directives.where((d) => d is ImportDirective)) {
301 _merge(d, importPos);
302 }
303
304 int partPos = unit.directives.last.end;
305 for (var d in patch.unit.directives.where((d) => d is PartDirective)) {
306 _merge(d, partPos);
307 }
308
309 // Merge declarations from the patch
310 int declPos = edits.original.length;
311 for (var d in patch.mergeDeclarations) {
312 _merge(d, declPos);
313 }
314 }
315
316 @override
317 visitPartOfDirective(PartOfDirective node) {
318 _isLibrary = false;
319 }
320
321 @override
322 visitFunctionDeclaration(FunctionDeclaration node) {
323 _maybePatch(node);
324 }
325
326 /// Merge patches and extensions into the class
327 @override
328 visitClassDeclaration(ClassDeclaration node) {
329 node.members.forEach(_maybePatch);
330
331 var mergeMembers = patch.mergeMembers[_qualifiedName(node)];
332 if (mergeMembers == null) return;
333
334 // Merge members from the patch
335 var pos = node.members.last.end;
336 for (var member in mergeMembers) {
337 var code = patch.contents.substring(member.offset, member.end);
338 edits.insert(pos, '\n\n ' + code);
339 }
340 }
341
342 void _maybePatch(AstNode node) {
343 if (node is FieldDeclaration) return;
344
345 var externalKeyword = (node as dynamic).externalKeyword;
346 if (externalKeyword == null) return;
347
348 var name = _qualifiedName(node);
349 var patchNode = patch.patches[name];
350 if (patchNode == null) {
351 print('warning: patch not found for $name: $node');
352 return;
353 }
354
355 Annotation patchMeta = patchNode.metadata.lastWhere(_isPatchAnnotation);
356 int start = patchMeta.endToken.next.offset;
357 var code = patch.contents.substring(start, patchNode.end);
358
359 // For some node like static fields, the node's offset doesn't include
360 // the external keyword. Also starting from the keyword lets us preserve
361 // documentation comments.
362 edits.replace(externalKeyword.offset, node.end, code);
363 }
364 }
365
366 class PatchFinder extends GeneralizingAstVisitor {
367 final String contents;
368 final CompilationUnit unit;
369
370 final Map patches = <String, Declaration>{};
371 final Map mergeMembers = <String, List<ClassMember>>{};
372 final List mergeDeclarations = <CompilationUnitMember>[];
373
374 PatchFinder.parseAndVisit(String name, String contents)
375 : contents = contents,
376 unit = parseCompilationUnit(contents, name: name) {
377 visitCompilationUnit(unit);
378 }
379
380 @override
381 visitCompilationUnitMember(CompilationUnitMember node) {
382 mergeDeclarations.add(node);
383 }
384
385 @override
386 visitClassDeclaration(ClassDeclaration node) {
387 if (_isPatch(node)) {
388 var members = <ClassMember>[];
389 for (var member in node.members) {
390 if (_isPatch(member)) {
391 patches[_qualifiedName(member)] = member;
392 } else {
393 members.add(member);
394 }
395 }
396 if (members.isNotEmpty) {
397 mergeMembers[_qualifiedName(node)] = members;
398 }
399 } else {
400 mergeDeclarations.add(node);
401 }
402 }
403
404 @override
405 visitFunctionDeclaration(FunctionDeclaration node) {
406 if (_isPatch(node)) {
407 patches[_qualifiedName(node)] = node;
408 } else {
409 mergeDeclarations.add(node);
410 }
411 }
412
413 @override
414 visitFunctionBody(node) {} // skip method bodies
415 }
416
417 String _qualifiedName(Declaration node) {
418 var parent = node.parent;
419 var className = '';
420 if (parent is ClassDeclaration) {
421 className = parent.name.name + '.';
422 }
423 var name = (node as dynamic).name;
424 name = (name != null ? name.name : '');
425
426 var accessor = '';
427 if (node is MethodDeclaration) {
428 if (node.isGetter) accessor = 'get:';
429 else if (node.isSetter) accessor = 'set:';
430 }
431 return className + accessor + name;
432 }
433
434 bool _isPatch(AnnotatedNode node) => node.metadata.any(_isPatchAnnotation);
435
436 bool _isPatchAnnotation(Annotation m) =>
437 m.name.name == 'patch' && m.constructorName == null && m.arguments == null;
438
439 /// Editable string buffer.
440 ///
441 /// Applies a series of edits (insertions, removals, replacements) using
442 /// original location information, and composes them into the edited string.
443 ///
444 /// For example, starting with a parsed AST with original source locations,
445 /// this type allows edits to be made without regards to other edits.
446 class StringEditBuffer {
447 final String original;
448 final _edits = <_StringEdit>[];
449
450 /// Creates a new transaction.
451 StringEditBuffer(this.original);
452
453 bool get hasEdits => _edits.length > 0;
454
455 /// Edit the original text, replacing text on the range [begin] and
456 /// exclusive [end] with the [replacement] string.
457 void replace(int begin, int end, String replacement) {
458 _edits.add(new _StringEdit(begin, end, replacement));
459 }
460
461 /// Insert [string] at [offset].
462 /// Equivalent to `replace(offset, offset, string)`.
463 void insert(int offset, String string) => replace(offset, offset, string);
464
465 /// Remove text from the range [begin] to exclusive [end].
466 /// Equivalent to `replace(begin, end, '')`.
467 void remove(int begin, int end) => replace(begin, end, '');
468
469 /// Applies all pending [edit]s and returns a new string.
470 ///
471 /// This method is non-destructive: it does not discard existing edits or
472 /// change the [original] string. Further edits can be added and this method
473 /// can be called again.
474 ///
475 /// Throws [UnsupportedError] if the edits were overlapping. If no edits were
476 /// made, the original string will be returned.
477 String toString() {
478 var sb = new StringBuffer();
479 if (_edits.length == 0) return original;
480
481 // Sort edits by start location.
482 _edits.sort();
483
484 int consumed = 0;
485 for (var edit in _edits) {
486 if (consumed > edit.begin) {
487 sb = new StringBuffer();
488 sb.write('overlapping edits. Insert at offset ');
489 sb.write(edit.begin);
490 sb.write(' but have consumed ');
491 sb.write(consumed);
492 sb.write(' input characters. List of edits:');
493 for (var e in _edits) {
494 sb.write('\n ');
495 sb.write(e);
496 }
497 throw new UnsupportedError(sb.toString());
498 }
499
500 // Add characters from the original string between this edit and the last
501 // one, if any.
502 var betweenEdits = original.substring(consumed, edit.begin);
503 sb.write(betweenEdits);
504 sb.write(edit.replace);
505 consumed = edit.end;
506 }
507
508 // Add any text from the end of the original string that was not replaced.
509 sb.write(original.substring(consumed));
510 return sb.toString();
511 }
512 }
513
514 class _StringEdit implements Comparable<_StringEdit> {
515 final int begin;
516 final int end;
517 final String replace;
518
519 _StringEdit(this.begin, this.end, this.replace);
520
521 int get length => end - begin;
522
523 String toString() => '(Edit @ $begin,$end: "$replace")';
524
525 int compareTo(_StringEdit other) {
526 int diff = begin - other.begin;
527 if (diff != 0) return diff;
528 return end - other.end;
529 }
530 }
531
532 List<SdkLibrary> _getSdkLibraries(String contents) {
533 var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(true);
534 parseCompilationUnit(contents).accept(libraryBuilder);
535 return libraryBuilder.librariesMap.sdkLibraries;
536 }
OLDNEW
« no previous file with comments | « tests/language/language_kernel.status ('k') | tools/patch_sdk.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698