OLD | NEW |
| (Empty) |
1 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file | |
2 // for details. All rights reserved. Use of this source code is governed by a | |
3 // BSD-style license that can be found in the LICENSE file. | |
4 | |
5 /// Tracks the shape of the import/export graph and dependencies between files. | |
6 library dev_compiler.src.dependency_graph; | |
7 | |
8 import 'dart:collection' show HashSet, HashMap; | |
9 | |
10 import 'package:analyzer/analyzer.dart' show parseDirectives; | |
11 import 'package:analyzer/src/generated/ast.dart' | |
12 show | |
13 AstNode, | |
14 CompilationUnit, | |
15 ExportDirective, | |
16 Identifier, | |
17 ImportDirective, | |
18 LibraryDirective, | |
19 PartDirective, | |
20 PartOfDirective, | |
21 UriBasedDirective; | |
22 import 'package:analyzer/src/generated/engine.dart' | |
23 show ParseDartTask, AnalysisContext; | |
24 import 'package:analyzer/src/generated/error.dart'; | |
25 import 'package:analyzer/src/generated/source.dart' show Source, SourceKind; | |
26 import 'package:html/dom.dart' show Document, Node, Element; | |
27 import 'package:html/parser.dart' as html; | |
28 import 'package:logging/logging.dart' show Logger, Level; | |
29 import 'package:path/path.dart' as path; | |
30 | |
31 import 'info.dart'; | |
32 import 'options.dart'; | |
33 import 'report.dart'; | |
34 | |
35 /// Holds references to all source nodes in the import graph. This is mainly | |
36 /// used as a level of indirection to ensure that each source has a canonical | |
37 /// representation. | |
38 class SourceGraph { | |
39 /// All nodes in the source graph. Used to get a canonical representation for | |
40 /// any node. | |
41 final Map<Uri, SourceNode> nodes = {}; | |
42 | |
43 /// Resources included by default on any application. | |
44 final runtimeDeps = new Set<ResourceSourceNode>(); | |
45 | |
46 /// Analyzer used to resolve source files. | |
47 final AnalysisContext _context; | |
48 final AnalysisErrorListener _reporter; | |
49 final CompilerOptions _options; | |
50 | |
51 SourceGraph(this._context, this._reporter, this._options) { | |
52 var dir = _options.runtimeDir; | |
53 if (dir == null) { | |
54 _log.severe('Runtime dir could not be determined automatically, ' | |
55 'please specify the --runtime-dir flag on the command line.'); | |
56 return; | |
57 } | |
58 var prefix = path.absolute(dir); | |
59 var files = _options.serverMode | |
60 ? runtimeFilesForServerMode(_options.widget) | |
61 : defaultRuntimeFiles; | |
62 for (var file in files) { | |
63 runtimeDeps.add(nodeFromUri(path.toUri(path.join(prefix, file)))); | |
64 } | |
65 } | |
66 | |
67 /// Node associated with a resolved [uri]. | |
68 SourceNode nodeFromUri(Uri uri) { | |
69 var uriString = Uri.encodeFull('$uri'); | |
70 return nodes.putIfAbsent(uri, () { | |
71 var source = _context.sourceFactory.forUri(uriString); | |
72 var extension = path.extension(uriString); | |
73 if (extension == '.html') { | |
74 return new HtmlSourceNode(this, uri, source); | |
75 } else if (extension == '.dart' || uriString.startsWith('dart:')) { | |
76 return new DartSourceNode(this, uri, source); | |
77 } else { | |
78 return new ResourceSourceNode(this, uri, source); | |
79 } | |
80 }); | |
81 } | |
82 | |
83 List<String> get resources => _options.sourceOptions.resources; | |
84 } | |
85 | |
86 /// A node in the import graph representing a source file. | |
87 abstract class SourceNode { | |
88 final SourceGraph graph; | |
89 | |
90 /// Resolved URI for this node. | |
91 final Uri uri; | |
92 | |
93 /// Resolved source from the analyzer. We let the analyzer internally track | |
94 /// for modifications to the source files. | |
95 Source _source; | |
96 Source get source => _source; | |
97 | |
98 String get contents => graph._context.getContents(_source).data; | |
99 | |
100 /// Last stamp read from `source.modificationStamp`. | |
101 /// This starts at -1, because analyzer uses that for files that don't exist. | |
102 int _lastStamp = -1; | |
103 | |
104 /// A hash used to help browsers cache the output that would be produced from | |
105 /// building this node. | |
106 String cachingHash; | |
107 | |
108 /// Whether we need to rebuild this source file. | |
109 bool needsRebuild = false; | |
110 | |
111 /// Whether the structure of dependencies from this node (scripts, imports, | |
112 /// exports, or parts) changed after we reparsed its contents. | |
113 bool structureChanged = false; | |
114 | |
115 /// Direct dependencies in the [SourceGraph]. These include script tags for | |
116 /// [HtmlSourceNode]s; and imports, exports and parts for [DartSourceNode]s. | |
117 Iterable<SourceNode> get allDeps => const []; | |
118 | |
119 /// Like [allDeps] but excludes parts for [DartSourceNode]s. For many | |
120 /// operations we mainly care about dependencies at the library level, so | |
121 /// parts are excluded from this list. | |
122 Iterable<SourceNode> get depsWithoutParts => const []; | |
123 | |
124 SourceNode(this.graph, this.uri, this._source); | |
125 | |
126 /// Check for whether the file has changed and, if so, mark [needsRebuild] and | |
127 /// [structureChanged] as necessary. | |
128 void update() { | |
129 if (_source == null) { | |
130 _source = graph._context.sourceFactory.forUri(Uri.encodeFull('$uri')); | |
131 if (_source == null) return; | |
132 } | |
133 | |
134 int newStamp = _source.exists() ? _source.modificationStamp : -1; | |
135 if (newStamp > _lastStamp || newStamp == -1 && _lastStamp != -1) { | |
136 // If the timestamp changed, read the file from disk and cache it. | |
137 // We don't want the source text to change during compilation. | |
138 saveUpdatedContents(); | |
139 _lastStamp = newStamp; | |
140 needsRebuild = true; | |
141 } | |
142 } | |
143 | |
144 void saveUpdatedContents() {} | |
145 | |
146 String toString() { | |
147 var simpleUri = uri.scheme == 'file' ? path.relative(uri.path) : "$uri"; | |
148 return '[$runtimeType: $simpleUri]'; | |
149 } | |
150 } | |
151 | |
152 /// A node representing an entry HTML source file. | |
153 class HtmlSourceNode extends SourceNode { | |
154 /// Resources included by default on any application. | |
155 final runtimeDeps; | |
156 | |
157 /// Libraries referred to via script tags. | |
158 Set<DartSourceNode> scripts = new Set<DartSourceNode>(); | |
159 | |
160 /// Link-rel stylesheets, images, and other specified files. | |
161 Set<SourceNode> resources = new Set<SourceNode>(); | |
162 | |
163 @override | |
164 Iterable<SourceNode> get allDeps => | |
165 [scripts, resources, runtimeDeps].expand((e) => e); | |
166 | |
167 @override | |
168 Iterable<SourceNode> get depsWithoutParts => allDeps; | |
169 | |
170 /// Parsed document, updated whenever [update] is invoked. | |
171 Document document; | |
172 | |
173 /// Tracks resource files referenced from HTML nodes, e.g. | |
174 /// `<link rel=stylesheet href=...>` and `<img src=...>` | |
175 final htmlResourceNodes = new HashMap<Element, ResourceSourceNode>(); | |
176 | |
177 HtmlSourceNode(SourceGraph graph, Uri uri, Source source) | |
178 : runtimeDeps = graph.runtimeDeps, | |
179 super(graph, uri, source); | |
180 | |
181 @override | |
182 void update() { | |
183 super.update(); | |
184 if (needsRebuild) { | |
185 var reporter = graph._reporter; | |
186 if (reporter is SummaryReporter) { | |
187 reporter.clearHtml(uri); | |
188 } | |
189 document = html.parse(contents, generateSpans: true); | |
190 var newScripts = new Set<DartSourceNode>(); | |
191 var tags = document.querySelectorAll('script[type="application/dart"]'); | |
192 for (var script in tags) { | |
193 var src = script.attributes['src']; | |
194 if (src == null) { | |
195 _reportError(graph, 'inlined script tags not supported at this time ' | |
196 '(see https://github.com/dart-lang/dart-dev-compiler/issues/54).', | |
197 script); | |
198 continue; | |
199 } | |
200 var node = graph.nodeFromUri(uri.resolve(src)); | |
201 if (node == null || !node.source.exists()) { | |
202 _reportError(graph, 'Script file $src not found', script); | |
203 } | |
204 if (node != null) newScripts.add(node); | |
205 } | |
206 | |
207 if (!_same(newScripts, scripts)) { | |
208 structureChanged = true; | |
209 scripts = newScripts; | |
210 } | |
211 | |
212 // TODO(jmesserly): simplify the design here. Ideally we wouldn't need | |
213 // to track user-defined CSS, images, etc. Also we don't have a clear | |
214 // way to distinguish runtime injected resources, like messages.css, from | |
215 // user-defined files. | |
216 htmlResourceNodes.clear(); | |
217 var newResources = new Set<SourceNode>(); | |
218 for (var resource in graph.resources) { | |
219 newResources.add(graph.nodeFromUri(uri.resolve(resource))); | |
220 } | |
221 for (var tag in document.querySelectorAll('link[rel="stylesheet"]')) { | |
222 var res = graph.nodeFromUri(uri.resolve(tag.attributes['href'])); | |
223 htmlResourceNodes[tag] = res; | |
224 newResources.add(res); | |
225 } | |
226 for (var tag in document.querySelectorAll('img[src]')) { | |
227 var res = graph.nodeFromUri(uri.resolve(tag.attributes['src'])); | |
228 htmlResourceNodes[tag] = res; | |
229 newResources.add(res); | |
230 } | |
231 if (!_same(newResources, resources)) { | |
232 structureChanged = true; | |
233 resources = newResources; | |
234 } | |
235 } | |
236 } | |
237 | |
238 void _reportError(SourceGraph graph, String message, Node node) { | |
239 var span = node.sourceSpan; | |
240 | |
241 // TODO(jmesserly): should these be errors or warnings? | |
242 var errorCode = new HtmlWarningCode('dev_compiler.$runtimeType', message); | |
243 graph._reporter.onError( | |
244 new AnalysisError(_source, span.start.offset, span.length, errorCode)); | |
245 } | |
246 } | |
247 | |
248 /// A node representing a Dart library or part. | |
249 class DartSourceNode extends SourceNode { | |
250 /// Set of imported libraries (empty for part files). | |
251 Set<DartSourceNode> imports = new Set<DartSourceNode>(); | |
252 | |
253 /// Set of exported libraries (empty for part files). | |
254 Set<DartSourceNode> exports = new Set<DartSourceNode>(); | |
255 | |
256 /// Parts of this library (empty for part files). | |
257 Set<DartSourceNode> parts = new Set<DartSourceNode>(); | |
258 | |
259 /// How many times this file is included as a part. | |
260 int includedAsPart = 0; | |
261 | |
262 DartSourceNode(graph, uri, source) : super(graph, uri, source); | |
263 | |
264 @override | |
265 Iterable<SourceNode> get allDeps => | |
266 [imports, exports, parts].expand((e) => e); | |
267 | |
268 @override | |
269 Iterable<SourceNode> get depsWithoutParts => | |
270 [imports, exports].expand((e) => e); | |
271 | |
272 LibraryInfo info; | |
273 | |
274 // TODO(jmesserly): it would be nice to not keep all sources in memory at | |
275 // once, but how else can we ensure a consistent view across a given | |
276 // compile? One different from dev_compiler vs analyzer is that our | |
277 // messages later in the compiler need the original source text to print | |
278 // spans. We also read source text ourselves to parse directives. | |
279 // But we could discard it after that point. | |
280 void saveUpdatedContents() { | |
281 graph._context.setContents(_source, _source.contents.data); | |
282 } | |
283 | |
284 @override | |
285 void update() { | |
286 super.update(); | |
287 | |
288 if (needsRebuild) { | |
289 var reporter = graph._reporter; | |
290 if (reporter is SummaryReporter) { | |
291 reporter.clearLibrary(uri); | |
292 } | |
293 | |
294 // If the defining compilation-unit changed, the structure might have | |
295 // changed. | |
296 var unit = parseDirectives(contents, name: _source.fullName); | |
297 var newImports = new Set<DartSourceNode>(); | |
298 var newExports = new Set<DartSourceNode>(); | |
299 var newParts = new Set<DartSourceNode>(); | |
300 for (var d in unit.directives) { | |
301 // Nothing to do for parts. | |
302 if (d is PartOfDirective) return; | |
303 if (d is LibraryDirective) continue; | |
304 | |
305 var directiveUri = (d as UriBasedDirective).uri; | |
306 | |
307 // `dart:core` and other similar URLs only contain a name, but it is | |
308 // meant to be a folder when resolving relative paths from it. | |
309 var targetUri = uri.scheme == 'dart' && uri.pathSegments.length == 1 | |
310 ? Uri.parse('$uri/').resolve(directiveUri.stringValue) | |
311 : uri.resolve(directiveUri.stringValue); | |
312 var target = | |
313 ParseDartTask.resolveDirective(graph._context, _source, d, null); | |
314 var node = graph.nodes.putIfAbsent( | |
315 targetUri, () => new DartSourceNode(graph, targetUri, target)); | |
316 //var node = graph.nodeFromUri(targetUri); | |
317 if (node._source == null || !node._source.exists()) { | |
318 _reportError(graph, 'File $targetUri not found', d); | |
319 } | |
320 | |
321 if (d is ImportDirective) { | |
322 newImports.add(node); | |
323 } else if (d is ExportDirective) { | |
324 newExports.add(node); | |
325 } else if (d is PartDirective) { | |
326 newParts.add(node); | |
327 } | |
328 } | |
329 | |
330 if (!_same(newImports, imports)) { | |
331 structureChanged = true; | |
332 imports = newImports; | |
333 } | |
334 | |
335 if (!_same(newExports, exports)) { | |
336 structureChanged = true; | |
337 exports = newExports; | |
338 } | |
339 | |
340 if (!_same(newParts, parts)) { | |
341 structureChanged = true; | |
342 | |
343 // When parts are removed, it's possible they were updated to be | |
344 // imported as a library | |
345 for (var p in parts) { | |
346 if (newParts.contains(p)) continue; | |
347 if (--p.includedAsPart == 0) { | |
348 p.needsRebuild = true; | |
349 } | |
350 } | |
351 | |
352 for (var p in newParts) { | |
353 if (parts.contains(p)) continue; | |
354 p.includedAsPart++; | |
355 } | |
356 parts = newParts; | |
357 } | |
358 } | |
359 | |
360 // The library should be marked as needing rebuild if a part changed | |
361 // internally: | |
362 for (var p in parts) { | |
363 // Technically for parts we don't need to look at the contents. If they | |
364 // contain imports, exports, or parts, we'll ignore them in our crawling. | |
365 // However we do a full update to make it easier to adjust when users | |
366 // switch a file from a part to a library. | |
367 p.update(); | |
368 if (p.needsRebuild) needsRebuild = true; | |
369 } | |
370 } | |
371 | |
372 void _reportError(SourceGraph graph, String message, AstNode node) { | |
373 graph._reporter.onError(new AnalysisError(_source, node.offset, node.length, | |
374 new CompileTimeErrorCode('dev_compiler.$runtimeType', message))); | |
375 } | |
376 } | |
377 | |
378 /// Represents a runtime resource from our compiler that is needed to run an | |
379 /// application. | |
380 class ResourceSourceNode extends SourceNode { | |
381 ResourceSourceNode(graph, uri, source) : super(graph, uri, source); | |
382 } | |
383 | |
384 /// Updates the structure and `needsRebuild` marks in nodes of [graph] reachable | |
385 /// from [start]. | |
386 /// | |
387 /// That is, staring from [start], we update the graph by detecting file changes | |
388 /// and rebuilding the structure of the graph wherever it changed (an import was | |
389 /// added or removed, etc). | |
390 /// | |
391 /// After calling this function a node is marked with `needsRebuild` only if it | |
392 /// contained local changes. Rebuild decisions that derive from transitive | |
393 /// changes (e.g. when the API of a dependency changed) are handled later in | |
394 /// [rebuild]. | |
395 void refreshStructureAndMarks(SourceNode start) { | |
396 visitInPreOrder(start, (n) => n.update(), includeParts: false); | |
397 } | |
398 | |
399 /// Clears all the `needsRebuild` and `structureChanged` marks in nodes | |
400 /// reachable from [start]. | |
401 void clearMarks(SourceNode start) { | |
402 visitInPreOrder(start, (n) => n.needsRebuild = n.structureChanged = false, | |
403 includeParts: true); | |
404 } | |
405 | |
406 /// Traverses from [start] with the purpose of building any source that needs to | |
407 /// be rebuilt. | |
408 /// | |
409 /// This function will call [build] in a post-order fashion, on a subset of the | |
410 /// reachable nodes. There are four rules used to decide when to rebuild a node | |
411 /// (call [build] on a node): | |
412 /// | |
413 /// * Only rebuild Dart libraries ([DartSourceNode]) or HTML files | |
414 /// ([HtmlSourceNode]), but skip part files. That is because those are | |
415 /// built as part of some library. | |
416 /// | |
417 /// * Always rebuild [DartSourceNode]s and [HtmlSourceNode]s with local | |
418 /// changes or changes in a part of the library. Internally this function | |
419 /// calls [refreshStructureAndMarks] to ensure that the graph structure is | |
420 /// up-to-date and that these nodes with local changes contain the | |
421 /// `needsRebuild` bit. | |
422 /// | |
423 /// * Rebuild [HtmlSourceNode]s if there were structural changes somewhere | |
424 /// down its reachable subgraph. This is done because HTML files embed the | |
425 /// transitive closure of the import graph in their output. | |
426 /// | |
427 /// * Rebuild [DartSourceNode]s that depend on other [DartSourceNode]s | |
428 /// whose API may have changed. The result of [build] is used to determine | |
429 /// whether other nodes need to be rebuilt. The function [build] is expected | |
430 /// to return `true` on a node `n` if it detemines other nodes that import | |
431 /// `n` may need to be rebuilt as well. | |
432 rebuild(SourceNode start, bool build(SourceNode node)) { | |
433 refreshStructureAndMarks(start); | |
434 // Hold which source nodes may have changed their public API, this includes | |
435 // libraries that were modified or libraries that export other modified APIs. | |
436 // TODO(sigmund): consider removing this special support for exports? Many | |
437 // cases anways require using summaries to understand what parts of the public | |
438 // API may be affected by transitive changes. The re-export case is just one | |
439 // of those transitive cases, but is not sufficient. See | |
440 // https://github.com/dart-lang/dev_compiler/issues/76 | |
441 var apiChangeDetected = new HashSet<SourceNode>(); | |
442 bool htmlNeedsRebuild = false; | |
443 | |
444 bool shouldBuildNode(SourceNode n) { | |
445 if (n.needsRebuild) return true; | |
446 if (n is HtmlSourceNode) return htmlNeedsRebuild; | |
447 if (n is ResourceSourceNode) return false; | |
448 return (n as DartSourceNode).imports | |
449 .any((i) => apiChangeDetected.contains(i)); | |
450 } | |
451 | |
452 visitInPostOrder(start, (n) { | |
453 if (n.structureChanged) htmlNeedsRebuild = true; | |
454 if (shouldBuildNode(n)) { | |
455 var oldHash = n.cachingHash; | |
456 if (build(n)) apiChangeDetected.add(n); | |
457 if (oldHash != n.cachingHash) htmlNeedsRebuild = true; | |
458 } else if (n is DartSourceNode && | |
459 n.exports.any((e) => apiChangeDetected.contains(e))) { | |
460 apiChangeDetected.add(n); | |
461 } | |
462 n.needsRebuild = false; | |
463 n.structureChanged = false; | |
464 if (n is DartSourceNode) { | |
465 // Note: clearing out flags in the parts could be a problem if someone | |
466 // tries to use a file both as a part and a library at the same time. | |
467 // In that case, we might not correctly propagate changes in the places | |
468 // where it is used as a library. Technically it's not allowed to have a | |
469 // file as a part and a library at once, and the analyzer should report an | |
470 // error in that case. | |
471 n.parts.forEach((p) => p.needsRebuild = p.structureChanged = false); | |
472 } | |
473 }, includeParts: false); | |
474 } | |
475 | |
476 /// Helper that runs [action] on nodes reachable from [start] in pre-order. | |
477 visitInPreOrder(SourceNode start, void action(SourceNode node), | |
478 {bool includeParts: false}) { | |
479 var seen = new HashSet<SourceNode>(); | |
480 helper(SourceNode node) { | |
481 if (!seen.add(node)) return; | |
482 action(node); | |
483 var deps = includeParts ? node.allDeps : node.depsWithoutParts; | |
484 deps.forEach(helper); | |
485 } | |
486 helper(start); | |
487 } | |
488 | |
489 /// Helper that runs [action] on nodes reachable from [start] in post-order. | |
490 visitInPostOrder(SourceNode start, void action(SourceNode node), | |
491 {bool includeParts: false}) { | |
492 var seen = new HashSet<SourceNode>(); | |
493 helper(SourceNode node) { | |
494 if (!seen.add(node)) return; | |
495 var deps = includeParts ? node.allDeps : node.depsWithoutParts; | |
496 deps.forEach(helper); | |
497 action(node); | |
498 } | |
499 helper(start); | |
500 } | |
501 | |
502 bool _same(Set a, Set b) => a.length == b.length && a.containsAll(b); | |
503 | |
504 /// Runtime files added to all applications when running the compiler in the | |
505 /// command line. | |
506 final defaultRuntimeFiles = () { | |
507 var files = [ | |
508 'harmony_feature_check.js', | |
509 'dart_utils.js', | |
510 'dart_library.js', | |
511 '_errors.js', | |
512 '_types.js', | |
513 '_rtti.js', | |
514 '_classes.js', | |
515 '_operations.js', | |
516 'dart_runtime.js', | |
517 ]; | |
518 files.addAll(corelibOrder.map((l) => l.replaceAll('.', '/') + '.js')); | |
519 return files; | |
520 }(); | |
521 | |
522 /// Curated order to minimize lazy classes needed by dart:core and its | |
523 /// transitive SDK imports. | |
524 const corelibOrder = const [ | |
525 'dart.core', | |
526 'dart.collection', | |
527 'dart._internal', | |
528 'dart.math', | |
529 'dart._interceptors', | |
530 'dart.async', | |
531 'dart._foreign_helper', | |
532 'dart._js_embedded_names', | |
533 'dart._js_helper', | |
534 'dart.isolate', | |
535 'dart.typed_data', | |
536 'dart._native_typed_data', | |
537 'dart._isolate_helper', | |
538 'dart._js_primitives', | |
539 'dart.convert', | |
540 'dart.mirrors', | |
541 'dart._js_mirrors', | |
542 'dart.js' | |
543 // _foreign_helper is not included, as it only defines the JS builtin that | |
544 // the compiler handles at compile time. | |
545 ]; | |
546 | |
547 /// Runtime files added to applications when running in server mode. | |
548 List<String> runtimeFilesForServerMode([bool includeWidget = true]) => | |
549 new List<String>.from(defaultRuntimeFiles) | |
550 ..addAll(includeWidget ? const ['messages_widget.js', 'messages.css'] : []); | |
551 | |
552 final _log = new Logger('dev_compiler.dependency_graph'); | |
OLD | NEW |