OLD | NEW |
1 # | 1 # |
2 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 The SCons Foundat
ion | 2 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 The SCons Foundat
ion |
3 # | 3 # |
4 # Permission is hereby granted, free of charge, to any person obtaining | 4 # Permission is hereby granted, free of charge, to any person obtaining |
5 # a copy of this software and associated documentation files (the | 5 # a copy of this software and associated documentation files (the |
6 # "Software"), to deal in the Software without restriction, including | 6 # "Software"), to deal in the Software without restriction, including |
7 # without limitation the rights to use, copy, modify, merge, publish, | 7 # without limitation the rights to use, copy, modify, merge, publish, |
8 # distribute, sublicense, and/or sell copies of the Software, and to | 8 # distribute, sublicense, and/or sell copies of the Software, and to |
9 # permit persons to whom the Software is furnished to do so, subject to | 9 # permit persons to whom the Software is furnished to do so, subject to |
10 # the following conditions: | 10 # the following conditions: |
(...skipping 30 matching lines...) Expand all Loading... |
41 which has Task subclasses that handle its specific behavior, | 41 which has Task subclasses that handle its specific behavior, |
42 like printing "`foo' is up to date" when a top-level target | 42 like printing "`foo' is up to date" when a top-level target |
43 doesn't need to be built, and handling the -c option by removing | 43 doesn't need to be built, and handling the -c option by removing |
44 targets as its "build" action. There is also a separate subclass | 44 targets as its "build" action. There is also a separate subclass |
45 for suppressing this output when the -q option is used. | 45 for suppressing this output when the -q option is used. |
46 | 46 |
47 The Taskmaster instantiates a Task object for each (set of) | 47 The Taskmaster instantiates a Task object for each (set of) |
48 target(s) that it decides need to be evaluated and/or built. | 48 target(s) that it decides need to be evaluated and/or built. |
49 """ | 49 """ |
50 | 50 |
51 __revision__ = "src/engine/SCons/Taskmaster.py 3603 2008/10/10 05:46:45 scons" | 51 __revision__ = "src/engine/SCons/Taskmaster.py 3842 2008/12/20 22:59:52 scons" |
52 | 52 |
53 from itertools import chain | 53 from itertools import chain |
54 import operator | 54 import operator |
55 import string | 55 import string |
56 import sys | 56 import sys |
57 import traceback | 57 import traceback |
58 | 58 |
59 import SCons.Errors | 59 import SCons.Errors |
60 import SCons.Node | 60 import SCons.Node |
61 | 61 |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
131 these methods explicitly to update state, etc., rather than | 131 these methods explicitly to update state, etc., rather than |
132 roll their own interaction with Taskmaster from scratch. | 132 roll their own interaction with Taskmaster from scratch. |
133 """ | 133 """ |
134 def __init__(self, tm, targets, top, node): | 134 def __init__(self, tm, targets, top, node): |
135 self.tm = tm | 135 self.tm = tm |
136 self.targets = targets | 136 self.targets = targets |
137 self.top = top | 137 self.top = top |
138 self.node = node | 138 self.node = node |
139 self.exc_clear() | 139 self.exc_clear() |
140 | 140 |
| 141 def trace_message(self, method, node, description='node'): |
| 142 fmt = '%-20s %s %s\n' |
| 143 return fmt % (method + ':', description, self.tm.trace_node(node)) |
| 144 |
141 def display(self, message): | 145 def display(self, message): |
142 """ | 146 """ |
143 Hook to allow the calling interface to display a message. | 147 Hook to allow the calling interface to display a message. |
144 | 148 |
145 This hook gets called as part of preparing a task for execution | 149 This hook gets called as part of preparing a task for execution |
146 (that is, a Node to be built). As part of figuring out what Node | 150 (that is, a Node to be built). As part of figuring out what Node |
147 should be built next, the actually target list may be altered, | 151 should be built next, the actually target list may be altered, |
148 along with a message describing the alteration. The calling | 152 along with a message describing the alteration. The calling |
149 interface can subclass Task and provide a concrete implementation | 153 interface can subclass Task and provide a concrete implementation |
150 of this method to see those messages. | 154 of this method to see those messages. |
151 """ | 155 """ |
152 pass | 156 pass |
153 | 157 |
154 def prepare(self): | 158 def prepare(self): |
155 """ | 159 """ |
156 Called just before the task is executed. | 160 Called just before the task is executed. |
157 | 161 |
158 This is mainly intended to give the target Nodes a chance to | 162 This is mainly intended to give the target Nodes a chance to |
159 unlink underlying files and make all necessary directories before | 163 unlink underlying files and make all necessary directories before |
160 the Action is actually called to build the targets. | 164 the Action is actually called to build the targets. |
161 """ | 165 """ |
| 166 T = self.tm.trace |
| 167 if T: T.write(self.trace_message('Task.prepare()', self.node)) |
162 | 168 |
163 # Now that it's the appropriate time, give the TaskMaster a | 169 # Now that it's the appropriate time, give the TaskMaster a |
164 # chance to raise any exceptions it encountered while preparing | 170 # chance to raise any exceptions it encountered while preparing |
165 # this task. | 171 # this task. |
166 self.exception_raise() | 172 self.exception_raise() |
167 | 173 |
168 if self.tm.message: | 174 if self.tm.message: |
169 self.display(self.tm.message) | 175 self.display(self.tm.message) |
170 self.tm.message = None | 176 self.tm.message = None |
171 | 177 |
(...skipping 30 matching lines...) Expand all Loading... |
202 return True | 208 return True |
203 | 209 |
204 def execute(self): | 210 def execute(self): |
205 """ | 211 """ |
206 Called to execute the task. | 212 Called to execute the task. |
207 | 213 |
208 This method is called from multiple threads in a parallel build, | 214 This method is called from multiple threads in a parallel build, |
209 so only do thread safe stuff here. Do thread unsafe stuff in | 215 so only do thread safe stuff here. Do thread unsafe stuff in |
210 prepare(), executed() or failed(). | 216 prepare(), executed() or failed(). |
211 """ | 217 """ |
| 218 T = self.tm.trace |
| 219 if T: T.write(self.trace_message('Task.execute()', self.node)) |
212 | 220 |
213 try: | 221 try: |
214 everything_was_cached = 1 | 222 everything_was_cached = 1 |
215 for t in self.targets: | 223 for t in self.targets: |
216 if not t.retrieve_from_cache(): | 224 if not t.retrieve_from_cache(): |
217 everything_was_cached = 0 | 225 everything_was_cached = 0 |
218 break | 226 break |
219 if not everything_was_cached: | 227 if not everything_was_cached: |
220 self.targets[0].build() | 228 self.targets[0].build() |
221 except SystemExit: | 229 except SystemExit: |
222 exc_value = sys.exc_info()[1] | 230 exc_value = sys.exc_info()[1] |
223 raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) | 231 raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) |
224 except SCons.Errors.UserError: | 232 except SCons.Errors.UserError: |
225 raise | 233 raise |
226 except SCons.Errors.BuildError: | 234 except SCons.Errors.BuildError: |
227 raise | 235 raise |
228 except: | 236 except Exception, e: |
229 raise SCons.Errors.TaskmasterException(self.targets[0], | 237 buildError = SCons.Errors.convert_to_BuildError(e) |
230 sys.exc_info()) | 238 buildError.node = self.targets[0] |
| 239 buildError.exc_info = sys.exc_info() |
| 240 raise buildError |
231 | 241 |
232 def executed_without_callbacks(self): | 242 def executed_without_callbacks(self): |
233 """ | 243 """ |
234 Called when the task has been successfully executed | 244 Called when the task has been successfully executed |
235 and the Taskmaster instance doesn't want to call | 245 and the Taskmaster instance doesn't want to call |
236 the Node's callback methods. | 246 the Node's callback methods. |
237 """ | 247 """ |
| 248 T = self.tm.trace |
| 249 if T: T.write(self.trace_message('Task.executed_without_callbacks()', |
| 250 self.node)) |
| 251 |
238 for t in self.targets: | 252 for t in self.targets: |
239 if t.get_state() == NODE_EXECUTING: | 253 if t.get_state() == NODE_EXECUTING: |
240 for side_effect in t.side_effects: | 254 for side_effect in t.side_effects: |
241 side_effect.set_state(NODE_NO_STATE) | 255 side_effect.set_state(NODE_NO_STATE) |
242 t.set_state(NODE_EXECUTED) | 256 t.set_state(NODE_EXECUTED) |
243 | 257 |
244 def executed_with_callbacks(self): | 258 def executed_with_callbacks(self): |
245 """ | 259 """ |
246 Called when the task has been successfully executed and | 260 Called when the task has been successfully executed and |
247 the Taskmaster instance wants to call the Node's callback | 261 the Taskmaster instance wants to call the Node's callback |
248 methods. | 262 methods. |
249 | 263 |
250 This may have been a do-nothing operation (to preserve build | 264 This may have been a do-nothing operation (to preserve build |
251 order), so we must check the node's state before deciding whether | 265 order), so we must check the node's state before deciding whether |
252 it was "built", in which case we call the appropriate Node method. | 266 it was "built", in which case we call the appropriate Node method. |
253 In any event, we always call "visited()", which will handle any | 267 In any event, we always call "visited()", which will handle any |
254 post-visit actions that must take place regardless of whether | 268 post-visit actions that must take place regardless of whether |
255 or not the target was an actual built target or a source Node. | 269 or not the target was an actual built target or a source Node. |
256 """ | 270 """ |
| 271 T = self.tm.trace |
| 272 if T: T.write(self.trace_message('Task.executed_with_callbacks()', |
| 273 self.node)) |
| 274 |
257 for t in self.targets: | 275 for t in self.targets: |
258 if t.get_state() == NODE_EXECUTING: | 276 if t.get_state() == NODE_EXECUTING: |
259 for side_effect in t.side_effects: | 277 for side_effect in t.side_effects: |
260 side_effect.set_state(NODE_NO_STATE) | 278 side_effect.set_state(NODE_NO_STATE) |
261 t.set_state(NODE_EXECUTED) | 279 t.set_state(NODE_EXECUTED) |
262 t.built() | 280 t.built() |
263 t.visited() | 281 t.visited() |
264 | 282 |
265 executed = executed_with_callbacks | 283 executed = executed_with_callbacks |
266 | 284 |
267 def failed(self): | 285 def failed(self): |
268 """ | 286 """ |
269 Default action when a task fails: stop the build. | 287 Default action when a task fails: stop the build. |
| 288 |
| 289 Note: Although this function is normally invoked on nodes in |
| 290 the executing state, it might also be invoked on up-to-date |
| 291 nodes when using Configure(). |
270 """ | 292 """ |
271 self.fail_stop() | 293 self.fail_stop() |
272 | 294 |
273 def fail_stop(self): | 295 def fail_stop(self): |
274 """ | 296 """ |
275 Explicit stop-the-build failure. | 297 Explicit stop-the-build failure. |
| 298 |
| 299 This sets failure status on the target nodes and all of |
| 300 their dependent parent nodes. |
| 301 |
| 302 Note: Although this function is normally invoked on nodes in |
| 303 the executing state, it might also be invoked on up-to-date |
| 304 nodes when using Configure(). |
276 """ | 305 """ |
277 | 306 T = self.tm.trace |
| 307 if T: T.write(self.trace_message('Task.failed_stop()', self.node)) |
| 308 |
278 # Invoke will_not_build() to clean-up the pending children | 309 # Invoke will_not_build() to clean-up the pending children |
279 # list. | 310 # list. |
280 self.tm.will_not_build(self.targets) | 311 self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) |
281 | 312 |
282 # Tell the taskmaster to not start any new tasks | 313 # Tell the taskmaster to not start any new tasks |
283 self.tm.stop() | 314 self.tm.stop() |
284 | 315 |
285 # We're stopping because of a build failure, but give the | 316 # We're stopping because of a build failure, but give the |
286 # calling Task class a chance to postprocess() the top-level | 317 # calling Task class a chance to postprocess() the top-level |
287 # target under which the build failure occurred. | 318 # target under which the build failure occurred. |
288 self.targets = [self.tm.current_top] | 319 self.targets = [self.tm.current_top] |
289 self.top = 1 | 320 self.top = 1 |
290 | 321 |
291 def fail_continue(self): | 322 def fail_continue(self): |
292 """ | 323 """ |
293 Explicit continue-the-build failure. | 324 Explicit continue-the-build failure. |
294 | 325 |
295 This sets failure status on the target nodes and all of | 326 This sets failure status on the target nodes and all of |
296 their dependent parent nodes. | 327 their dependent parent nodes. |
| 328 |
| 329 Note: Although this function is normally invoked on nodes in |
| 330 the executing state, it might also be invoked on up-to-date |
| 331 nodes when using Configure(). |
297 """ | 332 """ |
298 self.tm.will_not_build(self.targets) | 333 T = self.tm.trace |
299 | 334 if T: T.write(self.trace_message('Task.failed_continue()', self.node)) |
| 335 |
| 336 self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) |
| 337 |
300 def make_ready_all(self): | 338 def make_ready_all(self): |
301 """ | 339 """ |
302 Marks all targets in a task ready for execution. | 340 Marks all targets in a task ready for execution. |
303 | 341 |
304 This is used when the interface needs every target Node to be | 342 This is used when the interface needs every target Node to be |
305 visited--the canonical example being the "scons -c" option. | 343 visited--the canonical example being the "scons -c" option. |
306 """ | 344 """ |
| 345 T = self.tm.trace |
| 346 if T: T.write(self.trace_message('Task.make_ready_all()', self.node)) |
| 347 |
307 self.out_of_date = self.targets[:] | 348 self.out_of_date = self.targets[:] |
308 for t in self.targets: | 349 for t in self.targets: |
309 t.disambiguate().set_state(NODE_EXECUTING) | 350 t.disambiguate().set_state(NODE_EXECUTING) |
310 for s in t.side_effects: | 351 for s in t.side_effects: |
311 s.set_state(NODE_EXECUTING) | 352 s.set_state(NODE_EXECUTING) |
312 | 353 |
313 def make_ready_current(self): | 354 def make_ready_current(self): |
314 """ | 355 """ |
315 Marks all targets in a task ready for execution if any target | 356 Marks all targets in a task ready for execution if any target |
316 is not current. | 357 is not current. |
317 | 358 |
318 This is the default behavior for building only what's necessary. | 359 This is the default behavior for building only what's necessary. |
319 """ | 360 """ |
| 361 T = self.tm.trace |
| 362 if T: T.write(self.trace_message('Task.make_ready_current()', |
| 363 self.node)) |
| 364 |
320 self.out_of_date = [] | 365 self.out_of_date = [] |
321 needs_executing = False | 366 needs_executing = False |
322 for t in self.targets: | 367 for t in self.targets: |
323 try: | 368 try: |
324 t.disambiguate().make_ready() | 369 t.disambiguate().make_ready() |
325 is_up_to_date = not t.has_builder() or \ | 370 is_up_to_date = not t.has_builder() or \ |
326 (not t.always_build and t.is_up_to_date()) | 371 (not t.always_build and t.is_up_to_date()) |
327 except EnvironmentError, e: | 372 except EnvironmentError, e: |
328 raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filenam
e=e.filename) | 373 raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filenam
e=e.filename) |
329 | 374 |
330 if not is_up_to_date: | 375 if not is_up_to_date: |
331 self.out_of_date.append(t) | 376 self.out_of_date.append(t) |
332 needs_executing = True | 377 needs_executing = True |
333 | 378 |
334 if needs_executing: | 379 if needs_executing: |
335 for t in self.targets: | 380 for t in self.targets: |
336 t.set_state(NODE_EXECUTING) | 381 t.set_state(NODE_EXECUTING) |
337 for s in t.side_effects: | 382 for s in t.side_effects: |
338 s.set_state(NODE_EXECUTING) | 383 s.set_state(NODE_EXECUTING) |
339 else: | 384 else: |
340 for t in self.targets: | 385 for t in self.targets: |
341 # We must invoke visited() to ensure that the node | 386 # We must invoke visited() to ensure that the node |
342 # information has been computed before allowing the | 387 # information has been computed before allowing the |
343 # parent nodes to execute. (That could occur in a | 388 # parent nodes to execute. (That could occur in a |
344 # parallel build...) | 389 # parallel build...) |
345 t.visited() | 390 t.visited() |
346 t.set_state(NODE_UP_TO_DATE) | 391 t.set_state(NODE_UP_TO_DATE) |
347 | 392 |
348 make_ready = make_ready_current | 393 make_ready = make_ready_current |
349 | 394 |
350 def postprocess(self): | 395 def postprocess(self): |
351 """ | 396 """ |
352 Post-processes a task after it's been executed. | 397 Post-processes a task after it's been executed. |
353 | 398 |
354 This examines all the targets just built (or not, we don't care | 399 This examines all the targets just built (or not, we don't care |
355 if the build was successful, or even if there was no build | 400 if the build was successful, or even if there was no build |
356 because everything was up-to-date) to see if they have any | 401 because everything was up-to-date) to see if they have any |
357 waiting parent Nodes, or Nodes waiting on a common side effect, | 402 waiting parent Nodes, or Nodes waiting on a common side effect, |
358 that can be put back on the candidates list. | 403 that can be put back on the candidates list. |
359 """ | 404 """ |
| 405 T = self.tm.trace |
| 406 if T: T.write(self.trace_message('Task.postprocess()', self.node)) |
360 | 407 |
361 # We may have built multiple targets, some of which may have | 408 # We may have built multiple targets, some of which may have |
362 # common parents waiting for this build. Count up how many | 409 # common parents waiting for this build. Count up how many |
363 # targets each parent was waiting for so we can subtract the | 410 # targets each parent was waiting for so we can subtract the |
364 # values later, and so we *don't* put waiting side-effect Nodes | 411 # values later, and so we *don't* put waiting side-effect Nodes |
365 # back on the candidates list if the Node is also a waiting | 412 # back on the candidates list if the Node is also a waiting |
366 # parent. | 413 # parent. |
367 | 414 |
368 targets = set(self.targets) | 415 targets = set(self.targets) |
369 | 416 |
| 417 pending_children = self.tm.pending_children |
370 parents = {} | 418 parents = {} |
371 for t in targets: | 419 for t in targets: |
| 420 # A node can only be in the pending_children set if it has |
| 421 # some waiting_parents. |
| 422 if t.waiting_parents: |
| 423 if T: T.write(self.trace_message('Task.postprocess()', |
| 424 t, |
| 425 'removing')) |
| 426 pending_children.discard(t) |
372 for p in t.waiting_parents: | 427 for p in t.waiting_parents: |
373 parents[p] = parents.get(p, 0) + 1 | 428 parents[p] = parents.get(p, 0) + 1 |
374 | 429 |
375 for t in targets: | 430 for t in targets: |
376 for s in t.side_effects: | 431 for s in t.side_effects: |
377 if s.get_state() == NODE_EXECUTING: | 432 if s.get_state() == NODE_EXECUTING: |
378 s.set_state(NODE_NO_STATE) | 433 s.set_state(NODE_NO_STATE) |
379 for p in s.waiting_parents: | 434 for p in s.waiting_parents: |
380 parents[p] = parents.get(p, 0) + 1 | 435 parents[p] = parents.get(p, 0) + 1 |
381 for p in s.waiting_s_e: | 436 for p in s.waiting_s_e: |
382 if p.ref_count == 0: | 437 if p.ref_count == 0: |
383 self.tm.candidates.append(p) | 438 self.tm.candidates.append(p) |
384 self.tm.pending_children.discard(p) | |
385 | 439 |
386 for p, subtract in parents.items(): | 440 for p, subtract in parents.items(): |
387 p.ref_count = p.ref_count - subtract | 441 p.ref_count = p.ref_count - subtract |
| 442 if T: T.write(self.trace_message('Task.postprocess()', |
| 443 p, |
| 444 'adjusted parent ref count')) |
388 if p.ref_count == 0: | 445 if p.ref_count == 0: |
389 self.tm.candidates.append(p) | 446 self.tm.candidates.append(p) |
390 self.tm.pending_children.discard(p) | |
391 | 447 |
392 for t in targets: | 448 for t in targets: |
393 t.postprocess() | 449 t.postprocess() |
394 | 450 |
395 # Exception handling subsystem. | 451 # Exception handling subsystem. |
396 # | 452 # |
397 # Exceptions that occur while walking the DAG or examining Nodes | 453 # Exceptions that occur while walking the DAG or examining Nodes |
398 # must be raised, but must be raised at an appropriate time and in | 454 # must be raised, but must be raised at an appropriate time and in |
399 # a controlled manner so we can, if necessary, recover gracefully, | 455 # a controlled manner so we can, if necessary, recover gracefully, |
400 # possibly write out signature information for Nodes we've updated, | 456 # possibly write out signature information for Nodes we've updated, |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
472 self.candidates = [] | 528 self.candidates = [] |
473 self.tasker = tasker | 529 self.tasker = tasker |
474 if not order: | 530 if not order: |
475 order = lambda l: l | 531 order = lambda l: l |
476 self.order = order | 532 self.order = order |
477 self.message = None | 533 self.message = None |
478 self.trace = trace | 534 self.trace = trace |
479 self.next_candidate = self.find_next_candidate | 535 self.next_candidate = self.find_next_candidate |
480 self.pending_children = set() | 536 self.pending_children = set() |
481 | 537 |
482 | |
483 def find_next_candidate(self): | 538 def find_next_candidate(self): |
484 """ | 539 """ |
485 Returns the next candidate Node for (potential) evaluation. | 540 Returns the next candidate Node for (potential) evaluation. |
486 | 541 |
487 The candidate list (really a stack) initially consists of all of | 542 The candidate list (really a stack) initially consists of all of |
488 the top-level (command line) targets provided when the Taskmaster | 543 the top-level (command line) targets provided when the Taskmaster |
489 was initialized. While we walk the DAG, visiting Nodes, all the | 544 was initialized. While we walk the DAG, visiting Nodes, all the |
490 children that haven't finished processing get pushed on to the | 545 children that haven't finished processing get pushed on to the |
491 candidate list. Each child can then be popped and examined in | 546 candidate list. Each child can then be popped and examined in |
492 turn for whether *their* children are all up-to-date, in which | 547 turn for whether *their* children are all up-to-date, in which |
(...skipping 19 matching lines...) Expand all Loading... |
512 if alt: | 567 if alt: |
513 self.message = message | 568 self.message = message |
514 self.candidates.append(node) | 569 self.candidates.append(node) |
515 self.candidates.extend(self.order(alt)) | 570 self.candidates.extend(self.order(alt)) |
516 node = self.candidates.pop() | 571 node = self.candidates.pop() |
517 return node | 572 return node |
518 | 573 |
519 def no_next_candidate(self): | 574 def no_next_candidate(self): |
520 """ | 575 """ |
521 Stops Taskmaster processing by not returning a next candidate. | 576 Stops Taskmaster processing by not returning a next candidate. |
522 | 577 |
523 Note that we have to clean-up the Taskmaster candidate list | 578 Note that we have to clean-up the Taskmaster candidate list |
524 because the cycle detection depends on the fact all nodes have | 579 because the cycle detection depends on the fact all nodes have |
525 been processed somehow. | 580 been processed somehow. |
526 """ | 581 """ |
527 while self.candidates: | 582 while self.candidates: |
528 candidates = self.candidates | 583 candidates = self.candidates |
529 self.candidates = [] | 584 self.candidates = [] |
530 self.will_not_build(candidates, lambda n: n.state < NODE_UP_TO_DATE) | 585 self.will_not_build(candidates) |
531 return None | 586 return None |
532 | 587 |
| 588 def _validate_pending_children(self): |
| 589 """ |
| 590 Validate the content of the pending_children set. Assert if an |
| 591 internal error is found. |
| 592 |
| 593 This function is used strictly for debugging the taskmaster by |
| 594 checking that no invariants are violated. It is not used in |
| 595 normal operation. |
| 596 |
| 597 The pending_children set is used to detect cycles in the |
| 598 dependency graph. We call a "pending child" a child that is |
| 599 found in the "pending" state when checking the dependencies of |
| 600 its parent node. |
| 601 |
| 602 A pending child can occur when the Taskmaster completes a loop |
| 603 through a cycle. For example, lets imagine a graph made of |
| 604 three node (A, B and C) making a cycle. The evaluation starts |
| 605 at node A. The taskmaster first consider whether node A's |
| 606 child B is up-to-date. Then, recursively, node B needs to |
| 607 check whether node C is up-to-date. This leaves us with a |
| 608 dependency graph looking like: |
| 609 |
| 610 Next candidate \ |
| 611 \ |
| 612 Node A (Pending) --> Node B(Pending) --> Node C (NoState) |
| 613 ^ | |
| 614 | | |
| 615 +-------------------------------------+ |
| 616 |
| 617 Now, when the Taskmaster examines the Node C's child Node A, |
| 618 it finds that Node A is in the "pending" state. Therefore, |
| 619 Node A is a pending child of node C. |
| 620 |
| 621 Pending children indicate that the Taskmaster has potentially |
| 622 loop back through a cycle. We say potentially because it could |
| 623 also occur when a DAG is evaluated in parallel. For example, |
| 624 consider the following graph: |
| 625 |
| 626 |
| 627 Node A (Pending) --> Node B(Pending) --> Node C (Pending) --> ... |
| 628 | ^ |
| 629 | | |
| 630 +----------> Node D (NoState) --------+ |
| 631 / |
| 632 Next candidate / |
| 633 |
| 634 The Taskmaster first evaluates the nodes A, B, and C and |
| 635 starts building some children of node C. Assuming, that the |
| 636 maximum parallel level has not been reached, the Taskmaster |
| 637 will examine Node D. It will find that Node C is a pending |
| 638 child of Node D. |
| 639 |
| 640 In summary, evaluating a graph with a cycle will always |
| 641 involve a pending child at one point. A pending child might |
| 642 indicate either a cycle or a diamond-shaped DAG. Only a |
| 643 fraction of the nodes ends-up being a "pending child" of |
| 644 another node. This keeps the pending_children set small in |
| 645 practice. |
| 646 |
| 647 We can differentiate between the two cases if we wait until |
| 648 the end of the build. At this point, all the pending children |
| 649 nodes due to a diamond-shaped DAG will have been properly |
| 650 built (or will have failed to build). But, the pending |
| 651 children involved in a cycle will still be in the pending |
| 652 state. |
| 653 |
| 654 The taskmaster removes nodes from the pending_children set as |
| 655 soon as a pending_children node moves out of the pending |
| 656 state. This also helps to keep the pending_children set small. |
| 657 """ |
| 658 |
| 659 for n in self.pending_children: |
| 660 assert n.state in (NODE_PENDING, NODE_EXECUTING), \ |
| 661 (str(n), StateString[n.state]) |
| 662 assert len(n.waiting_parents) != 0, (str(n), len(n.waiting_parents)) |
| 663 for p in n.waiting_parents: |
| 664 assert p.ref_count > 0, (str(n), str(p), p.ref_count) |
| 665 |
| 666 |
| 667 def trace_message(self, message): |
| 668 return 'Taskmaster: %s\n' % message |
| 669 |
| 670 def trace_node(self, node): |
| 671 return '<%-10s %-3s %s>' % (StateString[node.get_state()], |
| 672 node.ref_count, |
| 673 repr(str(node))) |
| 674 |
533 def _find_next_ready_node(self): | 675 def _find_next_ready_node(self): |
534 """ | 676 """ |
535 Finds the next node that is ready to be built. | 677 Finds the next node that is ready to be built. |
536 | 678 |
537 This is *the* main guts of the DAG walk. We loop through the | 679 This is *the* main guts of the DAG walk. We loop through the |
538 list of candidates, looking for something that has no un-built | 680 list of candidates, looking for something that has no un-built |
539 children (i.e., that is a leaf Node or has dependencies that are | 681 children (i.e., that is a leaf Node or has dependencies that are |
540 all leaf Nodes or up-to-date). Candidate Nodes are re-scanned | 682 all leaf Nodes or up-to-date). Candidate Nodes are re-scanned |
541 (both the target Node itself and its sources, which are always | 683 (both the target Node itself and its sources, which are always |
542 scanned in the context of a given target) to discover implicit | 684 scanned in the context of a given target) to discover implicit |
543 dependencies. A Node that must wait for some children to be | 685 dependencies. A Node that must wait for some children to be |
544 built will be put back on the candidates list after the children | 686 built will be put back on the candidates list after the children |
545 have finished building. A Node that has been put back on the | 687 have finished building. A Node that has been put back on the |
546 candidates list in this way may have itself (or its sources) | 688 candidates list in this way may have itself (or its sources) |
547 re-scanned, in order to handle generated header files (e.g.) and | 689 re-scanned, in order to handle generated header files (e.g.) and |
548 the implicit dependencies therein. | 690 the implicit dependencies therein. |
549 | 691 |
550 Note that this method does not do any signature calculation or | 692 Note that this method does not do any signature calculation or |
551 up-to-date check itself. All of that is handled by the Task | 693 up-to-date check itself. All of that is handled by the Task |
552 class. This is purely concerned with the dependency graph walk. | 694 class. This is purely concerned with the dependency graph walk. |
553 """ | 695 """ |
554 | 696 |
555 self.ready_exc = None | 697 self.ready_exc = None |
556 | 698 |
557 T = self.trace | 699 T = self.trace |
558 if T: T.write('\nTaskmaster: Looking for a node to evaluate\n') | 700 if T: T.write('\n' + self.trace_message('Looking for a node to evaluate'
)) |
559 | 701 |
560 while 1: | 702 while 1: |
561 node = self.next_candidate() | 703 node = self.next_candidate() |
562 if node is None: | 704 if node is None: |
563 if T: T.write('Taskmaster: No candidate anymore.\n\n') | 705 if T: T.write(self.trace_message('No candidate anymore.') + '\n'
) |
564 return None | 706 return None |
565 | 707 |
566 node = node.disambiguate() | 708 node = node.disambiguate() |
567 state = node.get_state() | 709 state = node.get_state() |
568 | 710 |
| 711 # For debugging only: |
| 712 # |
| 713 # try: |
| 714 # self._validate_pending_children() |
| 715 # except: |
| 716 # self.ready_exc = sys.exc_info() |
| 717 # return node |
| 718 |
569 if CollectStats: | 719 if CollectStats: |
570 if not hasattr(node, 'stats'): | 720 if not hasattr(node, 'stats'): |
571 node.stats = Stats() | 721 node.stats = Stats() |
572 StatsNodes.append(node) | 722 StatsNodes.append(node) |
573 S = node.stats | 723 S = node.stats |
574 S.considered = S.considered + 1 | 724 S.considered = S.considered + 1 |
575 else: | 725 else: |
576 S = None | 726 S = None |
577 | 727 |
578 if T: T.write('Taskmaster: Considering node <%-10s %-3s %s> and
its children:\n' % | 728 if T: T.write(self.trace_message(' Considering node %s and its ch
ildren:' % self.trace_node(node))) |
579 (StateString[node.get_state()], node.ref_count, repr(s
tr(node)))) | |
580 | 729 |
581 if state == NODE_NO_STATE: | 730 if state == NODE_NO_STATE: |
582 # Mark this node as being on the execution stack: | 731 # Mark this node as being on the execution stack: |
583 node.set_state(NODE_PENDING) | 732 node.set_state(NODE_PENDING) |
584 elif state > NODE_PENDING: | 733 elif state > NODE_PENDING: |
585 # Skip this node if it has already been evaluated: | 734 # Skip this node if it has already been evaluated: |
586 if S: S.already_handled = S.already_handled + 1 | 735 if S: S.already_handled = S.already_handled + 1 |
587 if T: T.write('Taskmaster: already handled (executed)\n') | 736 if T: T.write(self.trace_message(' already handled (execut
ed)')) |
588 continue | 737 continue |
589 | 738 |
590 try: | 739 try: |
591 children = node.children() | 740 children = node.children() |
592 except SystemExit: | 741 except SystemExit: |
593 exc_value = sys.exc_info()[1] | 742 exc_value = sys.exc_info()[1] |
594 e = SCons.Errors.ExplicitExit(node, exc_value.code) | 743 e = SCons.Errors.ExplicitExit(node, exc_value.code) |
595 self.ready_exc = (SCons.Errors.ExplicitExit, e) | 744 self.ready_exc = (SCons.Errors.ExplicitExit, e) |
596 if T: T.write('Taskmaster: SystemExit\n') | 745 if T: T.write(self.trace_message(' SystemExit')) |
597 return node | 746 return node |
598 except Exception, e: | 747 except Exception, e: |
599 # We had a problem just trying to figure out the | 748 # We had a problem just trying to figure out the |
600 # children (like a child couldn't be linked in to a | 749 # children (like a child couldn't be linked in to a |
601 # VariantDir, or a Scanner threw something). Arrange to | 750 # VariantDir, or a Scanner threw something). Arrange to |
602 # raise the exception when the Task is "executed." | 751 # raise the exception when the Task is "executed." |
603 self.ready_exc = sys.exc_info() | 752 self.ready_exc = sys.exc_info() |
604 if S: S.problem = S.problem + 1 | 753 if S: S.problem = S.problem + 1 |
605 if T: T.write('Taskmaster: exception %s while scanning ch
ildren.\n'%s) | 754 if T: T.write(self.trace_message(' exception %s while scan
ning children.\n' % e)) |
606 return node | 755 return node |
607 | 756 |
608 children_not_visited = [] | 757 children_not_visited = [] |
609 children_pending = set() | 758 children_pending = set() |
610 children_not_ready = [] | 759 children_not_ready = [] |
611 children_failed = False | 760 children_failed = False |
612 | 761 |
613 for child in chain(children,node.prerequisites): | 762 for child in chain(children,node.prerequisites): |
614 childstate = child.get_state() | 763 childstate = child.get_state() |
615 | 764 |
616 if T: T.write('Taskmaster: <%-10s %-3s %s>\n' % | 765 if T: T.write(self.trace_message(' ' + self.trace_node(chi
ld))) |
617 (StateString[childstate], child.ref_count, repr(st
r(child)))) | |
618 | 766 |
619 if childstate == NODE_NO_STATE: | 767 if childstate == NODE_NO_STATE: |
620 children_not_visited.append(child) | 768 children_not_visited.append(child) |
621 elif childstate == NODE_PENDING: | 769 elif childstate == NODE_PENDING: |
622 children_pending.add(child) | 770 children_pending.add(child) |
623 elif childstate == NODE_FAILED: | 771 elif childstate == NODE_FAILED: |
624 children_failed = True | 772 children_failed = True |
625 | 773 |
626 if childstate <= NODE_EXECUTING: | 774 if childstate <= NODE_EXECUTING: |
627 children_not_ready.append(child) | 775 children_not_ready.append(child) |
628 | 776 |
629 | 777 |
630 # These nodes have not even been visited yet. Add | 778 # These nodes have not even been visited yet. Add |
631 # them to the list so that on some next pass we can | 779 # them to the list so that on some next pass we can |
632 # take a stab at evaluating them (or their children). | 780 # take a stab at evaluating them (or their children). |
633 children_not_visited.reverse() | 781 children_not_visited.reverse() |
634 self.candidates.extend(self.order(children_not_visited)) | 782 self.candidates.extend(self.order(children_not_visited)) |
635 #if T and children_not_visited: | 783 #if T and children_not_visited: |
636 # T.write('Taskmaster: adding to candidates: %s\n' % map(str
, children_not_visited)) | 784 # T.write(self.trace_message(' adding to candidates: %s' % ma
p(str, children_not_visited))) |
637 # T.write('Taskmaster: candidates now: %s\n' % map(str, self
.candidates)) | 785 # T.write(self.trace_message(' candidates now: %s\n' % map(st
r, self.candidates))) |
638 | 786 |
639 # Skip this node if any of its children have failed. | 787 # Skip this node if any of its children have failed. |
640 # | 788 # |
641 # This catches the case where we're descending a top-level | 789 # This catches the case where we're descending a top-level |
642 # target and one of our children failed while trying to be | 790 # target and one of our children failed while trying to be |
643 # built by a *previous* descent of an earlier top-level | 791 # built by a *previous* descent of an earlier top-level |
644 # target. | 792 # target. |
645 # | 793 # |
646 # It can also occur if a node is reused in multiple | 794 # It can also occur if a node is reused in multiple |
647 # targets. One first descends though the one of the | 795 # targets. One first descends though the one of the |
648 # target, the next time occurs through the other target. | 796 # target, the next time occurs through the other target. |
649 # | 797 # |
650 # Note that we can only have failed_children if the | 798 # Note that we can only have failed_children if the |
651 # --keep-going flag was used, because without it the build | 799 # --keep-going flag was used, because without it the build |
652 # will stop before diving in the other branch. | 800 # will stop before diving in the other branch. |
653 # | 801 # |
654 # Note that even if one of the children fails, we still | 802 # Note that even if one of the children fails, we still |
655 # added the other children to the list of candidate nodes | 803 # added the other children to the list of candidate nodes |
656 # to keep on building (--keep-going). | 804 # to keep on building (--keep-going). |
657 if children_failed: | 805 if children_failed: |
658 node.set_state(NODE_FAILED) | 806 node.set_state(NODE_FAILED) |
659 | 807 |
660 if S: S.child_failed = S.child_failed + 1 | 808 if S: S.child_failed = S.child_failed + 1 |
661 if T: T.write('Taskmaster:****** <%-10s %-3s %s>\n' % | 809 if T: T.write(self.trace_message('****** %s\n' % self.trace_node
(node))) |
662 (StateString[node.get_state()], node.ref_count, re
pr(str(node)))) | |
663 continue | 810 continue |
664 | 811 |
665 if children_not_ready: | 812 if children_not_ready: |
666 for child in children_not_ready: | 813 for child in children_not_ready: |
667 # We're waiting on one or more derived targets | 814 # We're waiting on one or more derived targets |
668 # that have not yet finished building. | 815 # that have not yet finished building. |
669 if S: S.not_built = S.not_built + 1 | 816 if S: S.not_built = S.not_built + 1 |
670 | 817 |
671 # Add this node to the waiting parents lists of | 818 # Add this node to the waiting parents lists of |
672 # anything we're waiting on, with a reference | 819 # anything we're waiting on, with a reference |
673 # count so we can be put back on the list for | 820 # count so we can be put back on the list for |
674 # re-evaluation when they've all finished. | 821 # re-evaluation when they've all finished. |
675 node.ref_count = node.ref_count + child.add_to_waiting_pare
nts(node) | 822 node.ref_count = node.ref_count + child.add_to_waiting_pare
nts(node) |
676 if T: T.write('Taskmaster: adjusting ref count: <%-10s
%-3s %s>\n' % | 823 if T: T.write(self.trace_message(' adjusted ref count: %
s, child %s' % |
677 (StateString[node.get_state()], node.ref_count
, repr(str(node)))) | 824 (self.trace_node(node), repr(str(child))))) |
678 | 825 |
| 826 if T: |
| 827 for pc in children_pending: |
| 828 T.write(self.trace_message(' adding %s to the pend
ing children set\n' % |
| 829 self.trace_node(pc))) |
679 self.pending_children = self.pending_children | children_pending | 830 self.pending_children = self.pending_children | children_pending |
680 | 831 |
681 continue | 832 continue |
682 | 833 |
683 # Skip this node if it has side-effects that are | 834 # Skip this node if it has side-effects that are |
684 # currently being built: | 835 # currently being built: |
685 wait_side_effects = False | 836 wait_side_effects = False |
686 for se in node.side_effects: | 837 for se in node.side_effects: |
687 if se.get_state() == NODE_EXECUTING: | 838 if se.get_state() == NODE_EXECUTING: |
688 se.add_to_waiting_s_e(node) | 839 se.add_to_waiting_s_e(node) |
689 wait_side_effects = True | 840 wait_side_effects = True |
690 | 841 |
691 if wait_side_effects: | 842 if wait_side_effects: |
692 if S: S.side_effects = S.side_effects + 1 | 843 if S: S.side_effects = S.side_effects + 1 |
693 continue | 844 continue |
694 | 845 |
695 # The default when we've gotten through all of the checks above: | 846 # The default when we've gotten through all of the checks above: |
696 # this node is ready to be built. | 847 # this node is ready to be built. |
697 if S: S.build = S.build + 1 | 848 if S: S.build = S.build + 1 |
698 if T: T.write('Taskmaster: Evaluating <%-10s %-3s %s>\n' % | 849 if T: T.write(self.trace_message('Evaluating %s\n' % |
699 (StateString[node.get_state()], node.ref_count, repr(s
tr(node)))) | 850 self.trace_node(node))) |
| 851 |
| 852 # For debugging only: |
| 853 # |
| 854 # try: |
| 855 # self._validate_pending_children() |
| 856 # except: |
| 857 # self.ready_exc = sys.exc_info() |
| 858 # return node |
| 859 |
700 return node | 860 return node |
701 | 861 |
702 return None | 862 return None |
703 | 863 |
704 def next_task(self): | 864 def next_task(self): |
705 """ | 865 """ |
706 Returns the next task to be executed. | 866 Returns the next task to be executed. |
707 | 867 |
708 This simply asks for the next Node to be evaluated, and then wraps | 868 This simply asks for the next Node to be evaluated, and then wraps |
709 it in the specific Task subclass with which we were initialized. | 869 it in the specific Task subclass with which we were initialized. |
(...skipping 15 matching lines...) Expand all Loading... |
725 # exception when the Task is "executed." | 885 # exception when the Task is "executed." |
726 self.ready_exc = sys.exc_info() | 886 self.ready_exc = sys.exc_info() |
727 | 887 |
728 if self.ready_exc: | 888 if self.ready_exc: |
729 task.exception_set(self.ready_exc) | 889 task.exception_set(self.ready_exc) |
730 | 890 |
731 self.ready_exc = None | 891 self.ready_exc = None |
732 | 892 |
733 return task | 893 return task |
734 | 894 |
735 def will_not_build(self, nodes, mark_fail=lambda n: n.state != NODE_FAILED): | 895 def will_not_build(self, nodes, node_func=lambda n: None): |
736 """ | 896 """ |
737 Perform clean-up about nodes that will never be built. | 897 Perform clean-up about nodes that will never be built. Invokes |
| 898 a user defined function on all of these nodes (including all |
| 899 of their parents). |
738 """ | 900 """ |
739 | 901 |
| 902 T = self.trace |
| 903 |
740 pending_children = self.pending_children | 904 pending_children = self.pending_children |
741 | 905 |
742 to_visit = set() | 906 to_visit = set(nodes) |
743 for node in nodes: | 907 pending_children = pending_children - to_visit |
744 # Set failure state on all of the parents that were dependent | |
745 # on this failed build. | |
746 if mark_fail(node): | |
747 node.set_state(NODE_FAILED) | |
748 parents = node.waiting_parents | |
749 to_visit = to_visit | parents | |
750 pending_children = pending_children - parents | |
751 | 908 |
| 909 if T: |
| 910 for n in nodes: |
| 911 T.write(self.trace_message(' removing node %s from the pen
ding children set\n' % |
| 912 self.trace_node(n))) |
752 try: | 913 try: |
753 while 1: | 914 while 1: |
754 try: | 915 try: |
755 node = to_visit.pop() | 916 node = to_visit.pop() |
756 except AttributeError: | 917 except AttributeError: |
757 # Python 1.5.2 | 918 # Python 1.5.2 |
758 if len(to_visit): | 919 if len(to_visit): |
759 node = to_visit[0] | 920 node = to_visit[0] |
760 to_visit.remove(node) | 921 to_visit.remove(node) |
761 else: | 922 else: |
762 break | 923 break |
763 if mark_fail(node): | 924 |
764 node.set_state(NODE_FAILED) | 925 node_func(node) |
765 parents = node.waiting_parents | 926 |
766 to_visit = to_visit | parents | 927 # Prune recursion by flushing the waiting children |
767 pending_children = pending_children - parents | 928 # list immediately. |
| 929 parents = node.waiting_parents |
| 930 node.waiting_parents = set() |
| 931 |
| 932 to_visit = to_visit | parents |
| 933 pending_children = pending_children - parents |
| 934 |
| 935 for p in parents: |
| 936 p.ref_count = p.ref_count - 1 |
| 937 if T: T.write(self.trace_message(' removing parent %s
from the pending children set\n' % |
| 938 self.trace_node(p))) |
768 except KeyError: | 939 except KeyError: |
769 # The container to_visit has been emptied. | 940 # The container to_visit has been emptied. |
770 pass | 941 pass |
771 | 942 |
772 # We have the stick back the pending_children list into the | 943 # We have the stick back the pending_children list into the |
773 # task master because the python 1.5.2 compatibility does not | 944 # task master because the python 1.5.2 compatibility does not |
774 # allow us to use in-place updates | 945 # allow us to use in-place updates |
775 self.pending_children = pending_children | 946 self.pending_children = pending_children |
776 | 947 |
777 def stop(self): | 948 def stop(self): |
778 """ | 949 """ |
779 Stops the current build completely. | 950 Stops the current build completely. |
780 """ | 951 """ |
781 self.next_candidate = self.no_next_candidate | 952 self.next_candidate = self.no_next_candidate |
782 | 953 |
783 def cleanup(self): | 954 def cleanup(self): |
784 """ | 955 """ |
785 Check for dependency cycles. | 956 Check for dependency cycles. |
786 """ | 957 """ |
787 if self.pending_children: | 958 if not self.pending_children: |
788 desc = 'Found dependency cycle(s):\n' | 959 return |
789 for node in self.pending_children: | |
790 cycle = find_cycle([node], set()) | |
791 if cycle: | |
792 desc = desc + " " + string.join(map(str, cycle), " -> ") +
"\n" | |
793 else: | |
794 desc = desc + \ | |
795 " Internal Error: no cycle found for node %s (%s) in st
ate %s\n" % \ | |
796 (node, repr(node), StateString[node.get_state()]) | |
797 | 960 |
798 raise SCons.Errors.UserError, desc | 961 # TODO(1.5) |
| 962 #nclist = [ (n, find_cycle([n], set())) for n in self.pending_children ] |
| 963 nclist = map(lambda n: (n, find_cycle([n], set())), self.pending_childre
n) |
| 964 |
| 965 # TODO(1.5) |
| 966 #genuine_cycles = [ |
| 967 # node for node, cycle in nclist |
| 968 # if cycle or node.get_state() != NODE_EXECUTED |
| 969 #] |
| 970 genuine_cycles = filter(lambda t: t[1] or t[0].get_state() != NODE_EXECU
TED, nclist) |
| 971 if not genuine_cycles: |
| 972 # All of the "cycles" found were single nodes in EXECUTED state, |
| 973 # which is to say, they really weren't cycles. Just return. |
| 974 return |
| 975 |
| 976 desc = 'Found dependency cycle(s):\n' |
| 977 for node, cycle in nclist: |
| 978 if cycle: |
| 979 desc = desc + " " + string.join(map(str, cycle), " -> ") + "\n" |
| 980 else: |
| 981 desc = desc + \ |
| 982 " Internal Error: no cycle found for node %s (%s) in state
%s\n" % \ |
| 983 (node, repr(node), StateString[node.get_state()]) |
| 984 |
| 985 raise SCons.Errors.UserError, desc |
OLD | NEW |