OLD | NEW |
---|---|
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library barback.phase; | 5 library barback.phase; |
6 | 6 |
7 import 'dart:async'; | 7 import 'dart:async'; |
8 | 8 |
9 import 'asset.dart'; | 9 import 'asset.dart'; |
10 import 'asset_cascade.dart'; | 10 import 'asset_cascade.dart'; |
11 import 'asset_id.dart'; | 11 import 'asset_id.dart'; |
12 import 'asset_node.dart'; | 12 import 'asset_node.dart'; |
13 import 'asset_set.dart'; | 13 import 'asset_set.dart'; |
14 import 'errors.dart'; | 14 import 'errors.dart'; |
15 import 'transform_node.dart'; | 15 import 'transform_node.dart'; |
16 import 'transformer.dart'; | 16 import 'transformer.dart'; |
17 import 'utils.dart'; | |
17 | 18 |
18 /// One phase in the ordered series of transformations in an [AssetCascade]. | 19 /// One phase in the ordered series of transformations in an [AssetCascade]. |
19 /// | 20 /// |
20 /// Each phase can access outputs from previous phases and can in turn pass | 21 /// Each phase can access outputs from previous phases and can in turn pass |
21 /// outputs to later phases. Phases are processed strictly serially. All | 22 /// outputs to later phases. Phases are processed strictly serially. All |
22 /// transforms in a phase will be complete before moving on to the next phase. | 23 /// transforms in a phase will be complete before moving on to the next phase. |
23 /// Within a single phase, all transforms will be run in parallel. | 24 /// Within a single phase, all transforms will be run in parallel. |
24 /// | 25 /// |
25 /// Building can be interrupted between phases. For example, a source is added | 26 /// Building can be interrupted between phases. For example, a source is added |
26 /// which starts the background process. Sometime during, say, phase 2 (which | 27 /// which starts the background process. Sometime during, say, phase 2 (which |
(...skipping 11 matching lines...) Expand all Loading... | |
38 /// | 39 /// |
39 /// Their outputs will be available to the next phase. | 40 /// Their outputs will be available to the next phase. |
40 final List<Transformer> _transformers; | 41 final List<Transformer> _transformers; |
41 | 42 |
42 /// The inputs that are available for transforms in this phase to consume. | 43 /// The inputs that are available for transforms in this phase to consume. |
43 /// | 44 /// |
44 /// For the first phase, these will be the source assets. For all other | 45 /// For the first phase, these will be the source assets. For all other |
45 /// phases, they will be the outputs from the previous phase. | 46 /// phases, they will be the outputs from the previous phase. |
46 final inputs = new Map<AssetId, AssetNode>(); | 47 final inputs = new Map<AssetId, AssetNode>(); |
47 | 48 |
48 /// The transforms currently applicable to assets in [inputs]. | 49 /// The transforms currently applicable to assets in [inputs], indexed by |
50 /// the ids of their primary inputs. | |
49 /// | 51 /// |
50 /// These are the transforms that have been "wired up": they represent a | 52 /// These are the transforms that have been "wired up": they represent a |
51 /// repeatable transformation of a single concrete set of inputs. "dart2js" | 53 /// repeatable transformation of a single concrete set of inputs. "dart2js" |
52 /// is a transformer. "dart2js on web/main.dart" is a transform. | 54 /// is a transformer. "dart2js on web/main.dart" is a transform. |
53 final _transforms = new Set<TransformNode>(); | 55 final _transforms = new Map<AssetId, Set<TransformNode>>(); |
54 | 56 |
55 /// The nodes that are new in this phase since the last time [process] was | 57 /// Futures that will complete once the transformers that can consume a given |
56 /// called. | 58 /// asset are determined. |
57 /// | 59 /// |
58 /// When we process, we'll check these to see if we can hang new transforms | 60 /// Whenever an asset is added or modified, we need to asynchronously |
59 /// off them. | 61 /// determine which transformers can use it as their primary input. We can't |
60 final _newInputs = new Set<AssetNode>(); | 62 /// start processing until we know which transformers to run, and this allows |
63 /// us to wait until we do. | |
64 var _adjustTransformersFutures = new Map<AssetId, Future>(); | |
Bob Nystrom
2013/07/31 20:05:41
"adjust" is a bit confusing to me. How about "poss
nweiz
2013/07/31 22:47:53
The name comes from the fact that these are future
| |
65 | |
66 /// New asset nodes that were added while [_adjustTransformers] was still | |
67 /// being run on an old version of that asset. | |
68 var _pendingNewInputs = new Map<AssetId, AssetNode>(); | |
69 | |
70 /// The ids of assets that are emitted by transforms in this phase. | |
71 /// | |
72 /// This is used to detect collisions where multiple transforms emit the same | |
73 /// output. | |
74 final _outputs = new Set<AssetId>(); | |
61 | 75 |
62 /// The phase after this one. | 76 /// The phase after this one. |
63 /// | 77 /// |
64 /// Outputs from this phase will be passed to it. | 78 /// Outputs from this phase will be passed to it. |
65 final Phase _next; | 79 final Phase _next; |
66 | 80 |
67 Phase(this.cascade, this._index, this._transformers, this._next); | 81 Phase(this.cascade, this._index, this._transformers, this._next); |
68 | 82 |
69 /// Updates the phase's inputs with [updated] and removes [removed]. | 83 /// Adds a new asset as an input for this phase. |
70 /// | 84 /// |
71 /// This marks any affected [transforms] as dirty or discards them if their | 85 /// [node] doesn't have to be [AssetState.AVAILABLE]. Once it is, the phase |
72 /// inputs are removed. | 86 /// will automatically begin determining which transforms can consume it as a |
73 void updateInputs(AssetSet updated, Set<AssetId> removed) { | 87 /// primary input. The transforms themselves won't be applied until [process] |
74 // Remove any nodes that are no longer being output. Handle removals first | 88 /// is called, however. |
75 // in case there are assets that were removed by one transform but updated | 89 /// |
76 // by another. In that case, the update should win. | 90 /// This should only be used for brand-new assets or assets that have been |
77 for (var id in removed) { | 91 /// removed and re-created. The phase will automatically handle updated assets |
78 var node = inputs.remove(id); | 92 /// using the [AssetNode.onStateChange] stream. |
79 | 93 void addInput(AssetNode node) { |
80 // Every transform that was using it is dirty now. | 94 // We remove [node.id] from [inputs] as soon as the node is removed rather |
81 if (node != null) { | 95 // than at the same time [node.id] is removed from [_transforms] so we don't |
82 node.consumers.forEach((consumer) => consumer.dirty()); | 96 // have to wait on [_adjustTransformers]. It's important that [inputs] is |
83 } | 97 // always up-to-date so that the [AssetCascade] can look there for available |
98 // assets. | |
99 inputs[node.id] = node; | |
100 node.whenRemoved.then((_) => inputs.remove(node.id)); | |
101 | |
102 if (_adjustTransformersFutures.containsKey(node.id)) { | |
Bob Nystrom
2013/07/31 20:05:41
Switch the order of cases here and do if (!_adjust
nweiz
2013/07/31 22:47:53
Done.
| |
103 // If an input is added while the same input is still being processed, | |
104 // that means that the asset was removed and recreated while | |
105 // [_adjustTransformers] was being run on the old value. We have to wait | |
106 // until that finishes, then run it again on whatever the newest version | |
107 // of that asset is. | |
Bob Nystrom
2013/07/31 20:05:41
This comment is confused compared to the code. Is
nweiz
2013/07/31 22:47:53
The comment is notionally attached to the _pending
| |
108 var containedKey = _pendingNewInputs.containsKey(node.id); | |
109 _pendingNewInputs[node.id] = node; | |
110 if (containedKey) return; | |
111 | |
112 _adjustTransformersFutures[node.id].then((_) { | |
113 assert(!_adjustTransformersFutures.containsKey(node.id)); | |
114 assert(_pendingNewInputs.containsKey(node.id)); | |
115 _transforms[node.id] = new Set<TransformNode>(); | |
116 _adjustTransformers(_pendingNewInputs.remove(node.id)); | |
117 }, onError: (_) { | |
118 // If there was a programmatic error while processing the old input, | |
119 // we don't want to just ignore it; it may have left the system in an | |
120 // inconsistent state. We also don't want to top-level it, so we | |
121 // ignore it here but don't start processing the new input. That way | |
122 // when [process] is called, the error will be piped through its | |
123 // return value. | |
124 }).catchError((e) { | |
125 // If our code above has a programmatic error, ensure it will be piped | |
126 // through [process] by putting it into [_adjustTransformersFutures]. | |
127 _adjustTransformersFutures[node.id] = new Future.error(e); | |
128 }); | |
129 } else { | |
130 _transforms[node.id] = new Set<TransformNode>(); | |
131 _adjustTransformers(node); | |
84 } | 132 } |
85 | 133 } |
86 // Update and new or modified assets. | 134 |
87 for (var asset in updated) { | 135 /// Returns the input for this phase with the given [id], but only if that |
88 var node = inputs[asset.id]; | 136 /// input is known not to be consumed as a transformer's primary input. |
89 if (node == null) { | 137 /// |
90 // It's a new node. Add it and remember it so we can see if any new | 138 /// If the input is unavailable, or if the phase hasn't determined whether or |
91 // transforms will consume it. | 139 /// not any transformers will consume it as a primary input, null will be |
92 node = new AssetNode(asset); | 140 /// returned instead. This means that the return value is guaranteed to always |
93 inputs[asset.id] = node; | 141 /// be [AssetState.AVAILABLE]. |
94 _newInputs.add(node); | 142 AssetNode getUnconsumedInput(AssetId id) { |
95 } else { | 143 if (!inputs.containsKey(id)) return null; |
Bob Nystrom
2013/07/31 20:05:41
How about some blank lines above the comments in h
nweiz
2013/07/31 22:47:53
Done.
| |
96 node.updateAsset(asset); | 144 // If the asset has inputs but no _transforms set, that means that |
Bob Nystrom
2013/07/31 20:05:41
means what?
nweiz
2013/07/31 22:47:53
This can't actually happen any more; removed the c
| |
97 } | 145 if (!_transforms.containsKey(id)) return null; |
Bob Nystrom
2013/07/31 20:05:41
I don't understand this. If there's no transformer
nweiz
2013/07/31 22:47:53
There's a difference between no transformer being
| |
98 } | 146 // If the asset has transforms, it's not unconsumed. |
147 if (!_transforms[id].isEmpty) return null; | |
148 // If we're working on figuring out if the asset has transforms, we can't | |
149 // prove that it's unconsumed. | |
150 if (_adjustTransformersFutures.containsKey(id)) return null; | |
151 // The asset should be available. If it were removed, it wouldn't be in | |
152 // _inputs, and if it were dirty, it'd be in _adjustTransformersFutures. | |
153 assert(inputs[id].state.isAvailable); | |
154 return inputs[id]; | |
155 } | |
156 | |
157 /// Asynchronously determines which transformers can consume [node] as a | |
158 /// primary input and creates transforms for them. | |
159 /// | |
160 /// This ensures that if [node] is modified or removed during or after the | |
161 /// time it takes to adjust its transformers, they're appropriately | |
162 /// re-adjusted. Its progress can be tracked in [_adjustTransformersFutures]. | |
163 void _adjustTransformers(AssetNode node) { | |
Bob Nystrom
2013/07/31 20:05:41
This method name isn't very clear. How about _find
nweiz
2013/07/31 22:47:53
It doesn't just find new transforms. It also check
| |
164 // Once the input is available, hook up transformers for it. If it changes | |
165 // while that's happening, try again. | |
166 _adjustTransformersFutures[node.id] = node.tryUntilStable((asset) { | |
167 var oldTransformers = _transforms[node.id] | |
168 .map((transform) => transform.transformer).toSet(); | |
169 | |
170 return _removeStaleTransforms(asset) | |
171 .then((_) => _addNewTransforms(node, oldTransformers)); | |
172 }).then((_) { | |
173 // Now all the transforms are set up correctly and the asset is available | |
174 // for the time being. Set up handlers for when the asset changes in the | |
175 // future. | |
176 node.onStateChange.first.then((state) { | |
Bob Nystrom
2013/07/31 20:05:41
Do we need to handle the .first future having an e
nweiz
2013/07/31 22:47:53
Done. It now pipes the error so that it will be em
| |
177 if (state.isRemoved) { | |
178 _transforms.remove(node.id); | |
179 } else { | |
180 _adjustTransformers(node); | |
181 } | |
182 }); | |
183 }).catchError((error) { | |
184 if (error is! AssetNotFoundException || error.id != node.id) throw error; | |
185 | |
186 // If the asset is removed, [tryUntilStable] will throw an | |
187 // [AssetNotFoundException]. In that case, just remove all transforms for | |
188 // the node. | |
189 _transforms.remove(node.id); | |
190 }).whenComplete(() { | |
191 _adjustTransformersFutures.remove(node.id); | |
192 }); | |
193 | |
194 // Don't top-level errors coming from the input processing. Any errors will | |
195 // eventually be piped through [process]'s returned Future. | |
196 _adjustTransformersFutures[node.id].catchError((_) {}); | |
197 } | |
198 | |
199 // Remove any old transforms that used to have [asset] as a primary asset but | |
200 // no longer apply to its new contents. | |
201 Future _removeStaleTransforms(Asset asset) { | |
202 return Future.wait(_transforms[asset.id].map((transform) { | |
203 // TODO(rnystrom): Catch all errors from isPrimary() and redirect to | |
204 // results. | |
205 return transform.transformer.isPrimary(asset).then((isPrimary) { | |
206 if (isPrimary) return; | |
207 _transforms[asset.id].remove(transform); | |
208 transform.remove(); | |
209 }); | |
210 })); | |
211 } | |
212 | |
213 // Add new transforms for transformers that consider [asset] to be a primary | |
Bob Nystrom
2013/07/31 20:05:41
"[asset]" -> "[node]'s asset"
nweiz
2013/07/31 22:47:53
Done.
nweiz
2013/07/31 22:47:53
Done.
| |
214 // input. | |
215 // | |
216 // [oldTransformers] is the set of transformers that had [node] as a primary | |
217 // input prior to this. They don't need to be checked, since they were removed | |
218 // or preserved in [_removeStaleTransforms]. | |
219 Future _addNewTransforms(AssetNode node, Set<Transformer> oldTransformers) { | |
Bob Nystrom
2013/07/31 20:05:41
"new" -> "fresh" to correspond with "stale" above?
nweiz
2013/07/31 22:47:53
Done.
| |
220 return Future.wait(_transformers.map((transformer) { | |
221 if (oldTransformers.contains(transformer)) return new Future.value(); | |
222 | |
223 // If the asset is unavailable, the results of this [_adjustTransformers] | |
224 // run will be discarded, so we can just short-circuit. | |
225 if (node.asset == null) return new Future.value(); | |
226 | |
227 // We can safely access [node.asset] here even though it might have | |
228 // changed since (as above) if it has, [_adjustTransformers] will just be | |
229 // re-run. | |
230 // TODO(rnystrom): Catch all errors from isPrimary() and redirect to | |
231 // results. | |
232 return transformer.isPrimary(node.asset).then((isPrimary) { | |
233 if (!isPrimary) return; | |
234 _transforms[node.id].add(new TransformNode(this, transformer, node)); | |
235 }); | |
236 })); | |
99 } | 237 } |
100 | 238 |
101 /// Processes this phase. | 239 /// Processes this phase. |
102 /// | 240 /// |
103 /// For all new inputs, it tries to see if there are transformers that can | |
104 /// consume them. Then all applicable transforms are applied. | |
105 /// | |
106 /// Returns a future that completes when processing is done. If there is | 241 /// Returns a future that completes when processing is done. If there is |
107 /// nothing to process, returns `null`. | 242 /// nothing to process, returns `null`. |
108 Future process() { | 243 Future process() { |
109 var future = _processNewInputs(); | 244 if (_adjustTransformersFutures.isEmpty) return _processTransforms(); |
110 if (future == null) { | 245 return _waitForInputs().then((_) => _processTransforms()); |
111 return _processTransforms(); | 246 } |
112 } | 247 |
113 | 248 Future _waitForInputs() { |
114 return future.then((_) => _processTransforms()); | 249 if (_adjustTransformersFutures.isEmpty) return new Future.value(); |
115 } | 250 return Future.wait(_adjustTransformersFutures.values) |
116 | 251 .then((_) => _waitForInputs()); |
117 /// Creates new transforms for any new inputs that are applicable. | |
118 Future _processNewInputs() { | |
119 if (_newInputs.isEmpty) return null; | |
120 | |
121 var futures = []; | |
122 for (var node in _newInputs) { | |
123 for (var transformer in _transformers) { | |
124 // TODO(rnystrom): Catch all errors from isPrimary() and redirect | |
125 // to results. | |
126 futures.add(transformer.isPrimary(node.asset).then((isPrimary) { | |
127 if (!isPrimary) return; | |
128 var transform = new TransformNode(this, transformer, node); | |
129 node.consumers.add(transform); | |
130 _transforms.add(transform); | |
131 })); | |
132 } | |
133 } | |
134 | |
135 _newInputs.clear(); | |
136 | |
137 return Future.wait(futures); | |
138 } | 252 } |
139 | 253 |
140 /// Applies all currently wired up and dirty transforms. | 254 /// Applies all currently wired up and dirty transforms. |
141 /// | |
142 /// Passes their outputs to the next phase. | |
143 Future _processTransforms() { | 255 Future _processTransforms() { |
144 // Convert this to a list so we can safely modify _transforms while | 256 // Convert this to a list so we can safely modify _transforms while |
145 // iterating over it. | 257 // iterating over it. |
146 var dirtyTransforms = _transforms.where((transform) => transform.isDirty) | 258 var dirtyTransforms = |
147 .toList(); | 259 flatten(_transforms.values.map((transforms) => transforms.toList())) |
260 .where((transform) => transform.isDirty).toList(); | |
148 if (dirtyTransforms.isEmpty) return null; | 261 if (dirtyTransforms.isEmpty) return null; |
149 | 262 |
150 return Future.wait(dirtyTransforms.map((transform) { | 263 return Future.wait(dirtyTransforms.map((transform) => transform.apply())) |
151 if (inputs.containsKey(transform.primary.id)) return transform.apply(); | 264 .then((allNewOutputs) { |
152 | 265 var newOutputs = allNewOutputs.reduce((set1, set2) => set1.union(set2)); |
153 // If the primary input for the transform has been removed, get rid of it | 266 |
154 // and all its outputs. | |
155 _transforms.remove(transform); | |
156 return new Future.value( | |
157 new TransformOutputs(new AssetSet(), transform.outputs)); | |
158 })).then((transformOutputs) { | |
159 // Collect all of the outputs. Since the transforms are run in parallel, | |
160 // we have to be careful here to ensure that the result is deterministic | |
161 // and not influenced by the order that transforms complete. | |
162 var updated = new AssetSet(); | |
163 var removed = new Set<AssetId>(); | |
164 var collisions = new Set<AssetId>(); | 267 var collisions = new Set<AssetId>(); |
165 | 268 for (var newOutput in newOutputs) { |
166 // Handle the generated outputs of all transforms first. | 269 if (_outputs.contains(newOutput.id)) { |
167 for (var outputs in transformOutputs) { | 270 collisions.add(newOutput.id); |
168 // Collect the outputs of all transformers together. | 271 } else { |
169 for (var asset in outputs.updated) { | 272 _next.addInput(newOutput); |
170 if (updated.containsId(asset.id)) { | 273 _outputs.add(newOutput.id); |
171 // Report a collision. | 274 newOutput.whenRemoved.then((_) => _outputs.remove(newOutput.id)); |
172 collisions.add(asset.id); | |
173 } else { | |
174 // TODO(rnystrom): In the case of a collision, the asset that | |
175 // "wins" is chosen non-deterministically. Do something better. | |
176 updated.add(asset); | |
177 } | |
178 } | 275 } |
179 | |
180 // Track any assets no longer output by this transform. We don't | |
181 // handle the case where *another* transform generates the asset | |
182 // no longer generated by this one. updateInputs() handles that. | |
183 removed.addAll(outputs.removed); | |
184 } | 276 } |
185 | 277 |
186 // Report any collisions in deterministic order. | 278 // Report collisions in a deterministic order. |
187 collisions = collisions.toList(); | 279 collisions = collisions.toList(); |
188 collisions.sort((a, b) => a.toString().compareTo(b.toString())); | 280 collisions.sort((a, b) => a.toString().compareTo(b.toString())); |
189 for (var collision in collisions) { | 281 for (var collision in collisions) { |
190 cascade.reportError(new AssetCollisionException(collision)); | 282 cascade.reportError(new AssetCollisionException(collision)); |
191 // TODO(rnystrom): Define what happens after a collision occurs. | 283 // TODO(rnystrom): Define what happens after a collision occurs. |
192 } | 284 } |
193 | |
194 // Pass the outputs to the next phase. | |
195 _next.updateInputs(updated, removed); | |
196 }); | 285 }); |
197 } | 286 } |
198 } | 287 } |
OLD | NEW |