Index: scons-2.0.1/engine/SCons/Taskmaster.py |
=================================================================== |
--- scons-2.0.1/engine/SCons/Taskmaster.py (revision 0) |
+++ scons-2.0.1/engine/SCons/Taskmaster.py (revision 0) |
@@ -0,0 +1,1017 @@ |
+# |
+# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 The SCons Foundation |
+# |
+# Permission is hereby granted, free of charge, to any person obtaining |
+# a copy of this software and associated documentation files (the |
+# "Software"), to deal in the Software without restriction, including |
+# without limitation the rights to use, copy, modify, merge, publish, |
+# distribute, sublicense, and/or sell copies of the Software, and to |
+# permit persons to whom the Software is furnished to do so, subject to |
+# the following conditions: |
+# |
+# The above copyright notice and this permission notice shall be included |
+# in all copies or substantial portions of the Software. |
+# |
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY |
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE |
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
+ |
+__doc__ = """ |
+Generic Taskmaster module for the SCons build engine. |
+ |
+This module contains the primary interface(s) between a wrapping user |
+interface and the SCons build engine. There are two key classes here: |
+ |
+ Taskmaster |
+ This is the main engine for walking the dependency graph and |
+ calling things to decide what does or doesn't need to be built. |
+ |
+ Task |
+ This is the base class for allowing a wrapping interface to |
+ decide what does or doesn't actually need to be done. The |
+ intention is for a wrapping interface to subclass this as |
+ appropriate for different types of behavior it may need. |
+ |
+ The canonical example is the SCons native Python interface, |
+ which has Task subclasses that handle its specific behavior, |
+ like printing "`foo' is up to date" when a top-level target |
+ doesn't need to be built, and handling the -c option by removing |
+ targets as its "build" action. There is also a separate subclass |
+ for suppressing this output when the -q option is used. |
+ |
+ The Taskmaster instantiates a Task object for each (set of) |
+ target(s) that it decides need to be evaluated and/or built. |
+""" |
+ |
+__revision__ = "src/engine/SCons/Taskmaster.py 5134 2010/08/16 23:02:40 bdeegan" |
+ |
+from itertools import chain |
+import operator |
+import sys |
+import traceback |
+ |
+import SCons.Errors |
+import SCons.Node |
+import SCons.Warnings |
+ |
+StateString = SCons.Node.StateString |
+NODE_NO_STATE = SCons.Node.no_state |
+NODE_PENDING = SCons.Node.pending |
+NODE_EXECUTING = SCons.Node.executing |
+NODE_UP_TO_DATE = SCons.Node.up_to_date |
+NODE_EXECUTED = SCons.Node.executed |
+NODE_FAILED = SCons.Node.failed |
+ |
+ |
+# A subsystem for recording stats about how different Nodes are handled by |
+# the main Taskmaster loop. There's no external control here (no need for |
+# a --debug= option); enable it by changing the value of CollectStats. |
+ |
+CollectStats = None |
+ |
+class Stats(object): |
+ """ |
+ A simple class for holding statistics about the disposition of a |
+ Node by the Taskmaster. If we're collecting statistics, each Node |
+ processed by the Taskmaster gets one of these attached, in which case |
+ the Taskmaster records its decision each time it processes the Node. |
+ (Ideally, that's just once per Node.) |
+ """ |
+ def __init__(self): |
+ """ |
+ Instantiates a Taskmaster.Stats object, initializing all |
+ appropriate counters to zero. |
+ """ |
+ self.considered = 0 |
+ self.already_handled = 0 |
+ self.problem = 0 |
+ self.child_failed = 0 |
+ self.not_built = 0 |
+ self.side_effects = 0 |
+ self.build = 0 |
+ |
+StatsNodes = [] |
+ |
+fmt = "%(considered)3d "\ |
+ "%(already_handled)3d " \ |
+ "%(problem)3d " \ |
+ "%(child_failed)3d " \ |
+ "%(not_built)3d " \ |
+ "%(side_effects)3d " \ |
+ "%(build)3d " |
+ |
+def dump_stats(): |
+ for n in sorted(StatsNodes, key=lambda a: str(a)): |
+ print (fmt % n.stats.__dict__) + str(n) |
+ |
+ |
+ |
+class Task(object): |
+ """ |
+ Default SCons build engine task. |
+ |
+ This controls the interaction of the actual building of node |
+ and the rest of the engine. |
+ |
+ This is expected to handle all of the normally-customizable |
+ aspects of controlling a build, so any given application |
+ *should* be able to do what it wants by sub-classing this |
+ class and overriding methods as appropriate. If an application |
+ needs to customze something by sub-classing Taskmaster (or |
+ some other build engine class), we should first try to migrate |
+ that functionality into this class. |
+ |
+ Note that it's generally a good idea for sub-classes to call |
+ these methods explicitly to update state, etc., rather than |
+ roll their own interaction with Taskmaster from scratch. |
+ """ |
+ def __init__(self, tm, targets, top, node): |
+ self.tm = tm |
+ self.targets = targets |
+ self.top = top |
+ self.node = node |
+ self.exc_clear() |
+ |
+ def trace_message(self, method, node, description='node'): |
+ fmt = '%-20s %s %s\n' |
+ return fmt % (method + ':', description, self.tm.trace_node(node)) |
+ |
+ def display(self, message): |
+ """ |
+ Hook to allow the calling interface to display a message. |
+ |
+ This hook gets called as part of preparing a task for execution |
+ (that is, a Node to be built). As part of figuring out what Node |
+ should be built next, the actually target list may be altered, |
+ along with a message describing the alteration. The calling |
+ interface can subclass Task and provide a concrete implementation |
+ of this method to see those messages. |
+ """ |
+ pass |
+ |
+ def prepare(self): |
+ """ |
+ Called just before the task is executed. |
+ |
+ This is mainly intended to give the target Nodes a chance to |
+ unlink underlying files and make all necessary directories before |
+ the Action is actually called to build the targets. |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message(u'Task.prepare()', self.node)) |
+ |
+ # Now that it's the appropriate time, give the TaskMaster a |
+ # chance to raise any exceptions it encountered while preparing |
+ # this task. |
+ self.exception_raise() |
+ |
+ if self.tm.message: |
+ self.display(self.tm.message) |
+ self.tm.message = None |
+ |
+ # Let the targets take care of any necessary preparations. |
+ # This includes verifying that all of the necessary sources |
+ # and dependencies exist, removing the target file(s), etc. |
+ # |
+ # As of April 2008, the get_executor().prepare() method makes |
+ # sure that all of the aggregate sources necessary to build this |
+ # Task's target(s) exist in one up-front check. The individual |
+ # target t.prepare() methods check that each target's explicit |
+ # or implicit dependencies exists, and also initialize the |
+ # .sconsign info. |
+ executor = self.targets[0].get_executor() |
+ executor.prepare() |
+ for t in executor.get_action_targets(): |
+ t.prepare() |
+ for s in t.side_effects: |
+ s.prepare() |
+ |
+ def get_target(self): |
+ """Fetch the target being built or updated by this task. |
+ """ |
+ return self.node |
+ |
+ def needs_execute(self): |
+ # TODO(deprecate): "return True" is the old default behavior; |
+ # change it to NotImplementedError (after running through the |
+ # Deprecation Cycle) so the desired behavior is explicitly |
+ # determined by which concrete subclass is used. |
+ #raise NotImplementedError |
+ msg = ('Taskmaster.Task is an abstract base class; instead of\n' |
+ '\tusing it directly, ' |
+ 'derive from it and override the abstract methods.') |
+ SCons.Warnings.warn(SCons.Warnings.TaskmasterNeedsExecuteWarning, msg) |
+ return True |
+ |
+ def execute(self): |
+ """ |
+ Called to execute the task. |
+ |
+ This method is called from multiple threads in a parallel build, |
+ so only do thread safe stuff here. Do thread unsafe stuff in |
+ prepare(), executed() or failed(). |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message(u'Task.execute()', self.node)) |
+ |
+ try: |
+ everything_was_cached = 1 |
+ for t in self.targets: |
+ if t.retrieve_from_cache(): |
+ # Call the .built() method without calling the |
+ # .push_to_cache() method, since we just got the |
+ # target from the cache and don't need to push |
+ # it back there. |
+ t.set_state(NODE_EXECUTED) |
+ t.built() |
+ else: |
+ everything_was_cached = 0 |
+ break |
+ if not everything_was_cached: |
+ self.targets[0].build() |
+ except SystemExit: |
+ exc_value = sys.exc_info()[1] |
+ raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) |
+ except SCons.Errors.UserError: |
+ raise |
+ except SCons.Errors.BuildError: |
+ raise |
+ except Exception, e: |
+ buildError = SCons.Errors.convert_to_BuildError(e) |
+ buildError.node = self.targets[0] |
+ buildError.exc_info = sys.exc_info() |
+ raise buildError |
+ |
+ def executed_without_callbacks(self): |
+ """ |
+ Called when the task has been successfully executed |
+ and the Taskmaster instance doesn't want to call |
+ the Node's callback methods. |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message('Task.executed_without_callbacks()', |
+ self.node)) |
+ |
+ for t in self.targets: |
+ if t.get_state() == NODE_EXECUTING: |
+ for side_effect in t.side_effects: |
+ side_effect.set_state(NODE_NO_STATE) |
+ t.set_state(NODE_EXECUTED) |
+ |
+ def executed_with_callbacks(self): |
+ """ |
+ Called when the task has been successfully executed and |
+ the Taskmaster instance wants to call the Node's callback |
+ methods. |
+ |
+ This may have been a do-nothing operation (to preserve build |
+ order), so we must check the node's state before deciding whether |
+ it was "built", in which case we call the appropriate Node method. |
+ In any event, we always call "visited()", which will handle any |
+ post-visit actions that must take place regardless of whether |
+ or not the target was an actual built target or a source Node. |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message('Task.executed_with_callbacks()', |
+ self.node)) |
+ |
+ for t in self.targets: |
+ if t.get_state() == NODE_EXECUTING: |
+ for side_effect in t.side_effects: |
+ side_effect.set_state(NODE_NO_STATE) |
+ t.set_state(NODE_EXECUTED) |
+ t.push_to_cache() |
+ t.built() |
+ t.visited() |
+ |
+ executed = executed_with_callbacks |
+ |
+ def failed(self): |
+ """ |
+ Default action when a task fails: stop the build. |
+ |
+ Note: Although this function is normally invoked on nodes in |
+ the executing state, it might also be invoked on up-to-date |
+ nodes when using Configure(). |
+ """ |
+ self.fail_stop() |
+ |
+ def fail_stop(self): |
+ """ |
+ Explicit stop-the-build failure. |
+ |
+ This sets failure status on the target nodes and all of |
+ their dependent parent nodes. |
+ |
+ Note: Although this function is normally invoked on nodes in |
+ the executing state, it might also be invoked on up-to-date |
+ nodes when using Configure(). |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message('Task.failed_stop()', self.node)) |
+ |
+ # Invoke will_not_build() to clean-up the pending children |
+ # list. |
+ self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) |
+ |
+ # Tell the taskmaster to not start any new tasks |
+ self.tm.stop() |
+ |
+ # We're stopping because of a build failure, but give the |
+ # calling Task class a chance to postprocess() the top-level |
+ # target under which the build failure occurred. |
+ self.targets = [self.tm.current_top] |
+ self.top = 1 |
+ |
+ def fail_continue(self): |
+ """ |
+ Explicit continue-the-build failure. |
+ |
+ This sets failure status on the target nodes and all of |
+ their dependent parent nodes. |
+ |
+ Note: Although this function is normally invoked on nodes in |
+ the executing state, it might also be invoked on up-to-date |
+ nodes when using Configure(). |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message('Task.failed_continue()', self.node)) |
+ |
+ self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) |
+ |
+ def make_ready_all(self): |
+ """ |
+ Marks all targets in a task ready for execution. |
+ |
+ This is used when the interface needs every target Node to be |
+ visited--the canonical example being the "scons -c" option. |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message('Task.make_ready_all()', self.node)) |
+ |
+ self.out_of_date = self.targets[:] |
+ for t in self.targets: |
+ t.disambiguate().set_state(NODE_EXECUTING) |
+ for s in t.side_effects: |
+ # add disambiguate here to mirror the call on targets above |
+ s.disambiguate().set_state(NODE_EXECUTING) |
+ |
+ def make_ready_current(self): |
+ """ |
+ Marks all targets in a task ready for execution if any target |
+ is not current. |
+ |
+ This is the default behavior for building only what's necessary. |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message(u'Task.make_ready_current()', |
+ self.node)) |
+ |
+ self.out_of_date = [] |
+ needs_executing = False |
+ for t in self.targets: |
+ try: |
+ t.disambiguate().make_ready() |
+ is_up_to_date = not t.has_builder() or \ |
+ (not t.always_build and t.is_up_to_date()) |
+ except EnvironmentError, e: |
+ raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filename=e.filename) |
+ |
+ if not is_up_to_date: |
+ self.out_of_date.append(t) |
+ needs_executing = True |
+ |
+ if needs_executing: |
+ for t in self.targets: |
+ t.set_state(NODE_EXECUTING) |
+ for s in t.side_effects: |
+ # add disambiguate here to mirror the call on targets in first loop above |
+ s.disambiguate().set_state(NODE_EXECUTING) |
+ else: |
+ for t in self.targets: |
+ # We must invoke visited() to ensure that the node |
+ # information has been computed before allowing the |
+ # parent nodes to execute. (That could occur in a |
+ # parallel build...) |
+ t.visited() |
+ t.set_state(NODE_UP_TO_DATE) |
+ |
+ make_ready = make_ready_current |
+ |
+ def postprocess(self): |
+ """ |
+ Post-processes a task after it's been executed. |
+ |
+ This examines all the targets just built (or not, we don't care |
+ if the build was successful, or even if there was no build |
+ because everything was up-to-date) to see if they have any |
+ waiting parent Nodes, or Nodes waiting on a common side effect, |
+ that can be put back on the candidates list. |
+ """ |
+ T = self.tm.trace |
+ if T: T.write(self.trace_message(u'Task.postprocess()', self.node)) |
+ |
+ # We may have built multiple targets, some of which may have |
+ # common parents waiting for this build. Count up how many |
+ # targets each parent was waiting for so we can subtract the |
+ # values later, and so we *don't* put waiting side-effect Nodes |
+ # back on the candidates list if the Node is also a waiting |
+ # parent. |
+ |
+ targets = set(self.targets) |
+ |
+ pending_children = self.tm.pending_children |
+ parents = {} |
+ for t in targets: |
+ # A node can only be in the pending_children set if it has |
+ # some waiting_parents. |
+ if t.waiting_parents: |
+ if T: T.write(self.trace_message(u'Task.postprocess()', |
+ t, |
+ 'removing')) |
+ pending_children.discard(t) |
+ for p in t.waiting_parents: |
+ parents[p] = parents.get(p, 0) + 1 |
+ |
+ for t in targets: |
+ for s in t.side_effects: |
+ if s.get_state() == NODE_EXECUTING: |
+ s.set_state(NODE_NO_STATE) |
+ for p in s.waiting_parents: |
+ parents[p] = parents.get(p, 0) + 1 |
+ for p in s.waiting_s_e: |
+ if p.ref_count == 0: |
+ self.tm.candidates.append(p) |
+ |
+ for p, subtract in parents.items(): |
+ p.ref_count = p.ref_count - subtract |
+ if T: T.write(self.trace_message(u'Task.postprocess()', |
+ p, |
+ 'adjusted parent ref count')) |
+ if p.ref_count == 0: |
+ self.tm.candidates.append(p) |
+ |
+ for t in targets: |
+ t.postprocess() |
+ |
+ # Exception handling subsystem. |
+ # |
+ # Exceptions that occur while walking the DAG or examining Nodes |
+ # must be raised, but must be raised at an appropriate time and in |
+ # a controlled manner so we can, if necessary, recover gracefully, |
+ # possibly write out signature information for Nodes we've updated, |
+ # etc. This is done by having the Taskmaster tell us about the |
+ # exception, and letting |
+ |
+ def exc_info(self): |
+ """ |
+ Returns info about a recorded exception. |
+ """ |
+ return self.exception |
+ |
+ def exc_clear(self): |
+ """ |
+ Clears any recorded exception. |
+ |
+ This also changes the "exception_raise" attribute to point |
+ to the appropriate do-nothing method. |
+ """ |
+ self.exception = (None, None, None) |
+ self.exception_raise = self._no_exception_to_raise |
+ |
+ def exception_set(self, exception=None): |
+ """ |
+ Records an exception to be raised at the appropriate time. |
+ |
+ This also changes the "exception_raise" attribute to point |
+ to the method that will, in fact |
+ """ |
+ if not exception: |
+ exception = sys.exc_info() |
+ self.exception = exception |
+ self.exception_raise = self._exception_raise |
+ |
+ def _no_exception_to_raise(self): |
+ pass |
+ |
+ def _exception_raise(self): |
+ """ |
+ Raises a pending exception that was recorded while getting a |
+ Task ready for execution. |
+ """ |
+ exc = self.exc_info()[:] |
+ try: |
+ exc_type, exc_value, exc_traceback = exc |
+ except ValueError: |
+ exc_type, exc_value = exc |
+ exc_traceback = None |
+ raise exc_type, exc_value, exc_traceback |
+ |
+class AlwaysTask(Task): |
+ def needs_execute(self): |
+ """ |
+ Always returns True (indicating this Task should always |
+ be executed). |
+ |
+ Subclasses that need this behavior (as opposed to the default |
+ of only executing Nodes that are out of date w.r.t. their |
+ dependencies) can use this as follows: |
+ |
+ class MyTaskSubclass(SCons.Taskmaster.Task): |
+ needs_execute = SCons.Taskmaster.Task.execute_always |
+ """ |
+ return True |
+ |
+class OutOfDateTask(Task): |
+ def needs_execute(self): |
+ """ |
+ Returns True (indicating this Task should be executed) if this |
+ Task's target state indicates it needs executing, which has |
+ already been determined by an earlier up-to-date check. |
+ """ |
+ return self.targets[0].get_state() == SCons.Node.executing |
+ |
+ |
+def find_cycle(stack, visited): |
+ if stack[-1] in visited: |
+ return None |
+ visited.add(stack[-1]) |
+ for n in stack[-1].waiting_parents: |
+ stack.append(n) |
+ if stack[0] == stack[-1]: |
+ return stack |
+ if find_cycle(stack, visited): |
+ return stack |
+ stack.pop() |
+ return None |
+ |
+ |
+class Taskmaster(object): |
+ """ |
+ The Taskmaster for walking the dependency DAG. |
+ """ |
+ |
+ def __init__(self, targets=[], tasker=None, order=None, trace=None): |
+ self.original_top = targets |
+ self.top_targets_left = targets[:] |
+ self.top_targets_left.reverse() |
+ self.candidates = [] |
+ if tasker is None: |
+ tasker = OutOfDateTask |
+ self.tasker = tasker |
+ if not order: |
+ order = lambda l: l |
+ self.order = order |
+ self.message = None |
+ self.trace = trace |
+ self.next_candidate = self.find_next_candidate |
+ self.pending_children = set() |
+ |
+ def find_next_candidate(self): |
+ """ |
+ Returns the next candidate Node for (potential) evaluation. |
+ |
+ The candidate list (really a stack) initially consists of all of |
+ the top-level (command line) targets provided when the Taskmaster |
+ was initialized. While we walk the DAG, visiting Nodes, all the |
+ children that haven't finished processing get pushed on to the |
+ candidate list. Each child can then be popped and examined in |
+ turn for whether *their* children are all up-to-date, in which |
+ case a Task will be created for their actual evaluation and |
+ potential building. |
+ |
+ Here is where we also allow candidate Nodes to alter the list of |
+ Nodes that should be examined. This is used, for example, when |
+ invoking SCons in a source directory. A source directory Node can |
+ return its corresponding build directory Node, essentially saying, |
+ "Hey, you really need to build this thing over here instead." |
+ """ |
+ try: |
+ return self.candidates.pop() |
+ except IndexError: |
+ pass |
+ try: |
+ node = self.top_targets_left.pop() |
+ except IndexError: |
+ return None |
+ self.current_top = node |
+ alt, message = node.alter_targets() |
+ if alt: |
+ self.message = message |
+ self.candidates.append(node) |
+ self.candidates.extend(self.order(alt)) |
+ node = self.candidates.pop() |
+ return node |
+ |
+ def no_next_candidate(self): |
+ """ |
+ Stops Taskmaster processing by not returning a next candidate. |
+ |
+ Note that we have to clean-up the Taskmaster candidate list |
+ because the cycle detection depends on the fact all nodes have |
+ been processed somehow. |
+ """ |
+ while self.candidates: |
+ candidates = self.candidates |
+ self.candidates = [] |
+ self.will_not_build(candidates) |
+ return None |
+ |
+ def _validate_pending_children(self): |
+ """ |
+ Validate the content of the pending_children set. Assert if an |
+ internal error is found. |
+ |
+ This function is used strictly for debugging the taskmaster by |
+ checking that no invariants are violated. It is not used in |
+ normal operation. |
+ |
+ The pending_children set is used to detect cycles in the |
+ dependency graph. We call a "pending child" a child that is |
+ found in the "pending" state when checking the dependencies of |
+ its parent node. |
+ |
+ A pending child can occur when the Taskmaster completes a loop |
+ through a cycle. For example, lets imagine a graph made of |
+ three node (A, B and C) making a cycle. The evaluation starts |
+ at node A. The taskmaster first consider whether node A's |
+ child B is up-to-date. Then, recursively, node B needs to |
+ check whether node C is up-to-date. This leaves us with a |
+ dependency graph looking like: |
+ |
+ Next candidate \ |
+ \ |
+ Node A (Pending) --> Node B(Pending) --> Node C (NoState) |
+ ^ | |
+ | | |
+ +-------------------------------------+ |
+ |
+ Now, when the Taskmaster examines the Node C's child Node A, |
+ it finds that Node A is in the "pending" state. Therefore, |
+ Node A is a pending child of node C. |
+ |
+ Pending children indicate that the Taskmaster has potentially |
+ loop back through a cycle. We say potentially because it could |
+ also occur when a DAG is evaluated in parallel. For example, |
+ consider the following graph: |
+ |
+ |
+ Node A (Pending) --> Node B(Pending) --> Node C (Pending) --> ... |
+ | ^ |
+ | | |
+ +----------> Node D (NoState) --------+ |
+ / |
+ Next candidate / |
+ |
+ The Taskmaster first evaluates the nodes A, B, and C and |
+ starts building some children of node C. Assuming, that the |
+ maximum parallel level has not been reached, the Taskmaster |
+ will examine Node D. It will find that Node C is a pending |
+ child of Node D. |
+ |
+ In summary, evaluating a graph with a cycle will always |
+ involve a pending child at one point. A pending child might |
+ indicate either a cycle or a diamond-shaped DAG. Only a |
+ fraction of the nodes ends-up being a "pending child" of |
+ another node. This keeps the pending_children set small in |
+ practice. |
+ |
+ We can differentiate between the two cases if we wait until |
+ the end of the build. At this point, all the pending children |
+ nodes due to a diamond-shaped DAG will have been properly |
+ built (or will have failed to build). But, the pending |
+ children involved in a cycle will still be in the pending |
+ state. |
+ |
+ The taskmaster removes nodes from the pending_children set as |
+ soon as a pending_children node moves out of the pending |
+ state. This also helps to keep the pending_children set small. |
+ """ |
+ |
+ for n in self.pending_children: |
+ assert n.state in (NODE_PENDING, NODE_EXECUTING), \ |
+ (str(n), StateString[n.state]) |
+ assert len(n.waiting_parents) != 0, (str(n), len(n.waiting_parents)) |
+ for p in n.waiting_parents: |
+ assert p.ref_count > 0, (str(n), str(p), p.ref_count) |
+ |
+ |
+ def trace_message(self, message): |
+ return 'Taskmaster: %s\n' % message |
+ |
+ def trace_node(self, node): |
+ return '<%-10s %-3s %s>' % (StateString[node.get_state()], |
+ node.ref_count, |
+ repr(str(node))) |
+ |
+ def _find_next_ready_node(self): |
+ """ |
+ Finds the next node that is ready to be built. |
+ |
+ This is *the* main guts of the DAG walk. We loop through the |
+ list of candidates, looking for something that has no un-built |
+ children (i.e., that is a leaf Node or has dependencies that are |
+ all leaf Nodes or up-to-date). Candidate Nodes are re-scanned |
+ (both the target Node itself and its sources, which are always |
+ scanned in the context of a given target) to discover implicit |
+ dependencies. A Node that must wait for some children to be |
+ built will be put back on the candidates list after the children |
+ have finished building. A Node that has been put back on the |
+ candidates list in this way may have itself (or its sources) |
+ re-scanned, in order to handle generated header files (e.g.) and |
+ the implicit dependencies therein. |
+ |
+ Note that this method does not do any signature calculation or |
+ up-to-date check itself. All of that is handled by the Task |
+ class. This is purely concerned with the dependency graph walk. |
+ """ |
+ |
+ self.ready_exc = None |
+ |
+ T = self.trace |
+ if T: T.write(u'\n' + self.trace_message('Looking for a node to evaluate')) |
+ |
+ while True: |
+ node = self.next_candidate() |
+ if node is None: |
+ if T: T.write(self.trace_message('No candidate anymore.') + u'\n') |
+ return None |
+ |
+ node = node.disambiguate() |
+ state = node.get_state() |
+ |
+ # For debugging only: |
+ # |
+ # try: |
+ # self._validate_pending_children() |
+ # except: |
+ # self.ready_exc = sys.exc_info() |
+ # return node |
+ |
+ if CollectStats: |
+ if not hasattr(node, 'stats'): |
+ node.stats = Stats() |
+ StatsNodes.append(node) |
+ S = node.stats |
+ S.considered = S.considered + 1 |
+ else: |
+ S = None |
+ |
+ if T: T.write(self.trace_message(u' Considering node %s and its children:' % self.trace_node(node))) |
+ |
+ if state == NODE_NO_STATE: |
+ # Mark this node as being on the execution stack: |
+ node.set_state(NODE_PENDING) |
+ elif state > NODE_PENDING: |
+ # Skip this node if it has already been evaluated: |
+ if S: S.already_handled = S.already_handled + 1 |
+ if T: T.write(self.trace_message(u' already handled (executed)')) |
+ continue |
+ |
+ executor = node.get_executor() |
+ |
+ try: |
+ children = executor.get_all_children() |
+ except SystemExit: |
+ exc_value = sys.exc_info()[1] |
+ e = SCons.Errors.ExplicitExit(node, exc_value.code) |
+ self.ready_exc = (SCons.Errors.ExplicitExit, e) |
+ if T: T.write(self.trace_message(' SystemExit')) |
+ return node |
+ except Exception, e: |
+ # We had a problem just trying to figure out the |
+ # children (like a child couldn't be linked in to a |
+ # VariantDir, or a Scanner threw something). Arrange to |
+ # raise the exception when the Task is "executed." |
+ self.ready_exc = sys.exc_info() |
+ if S: S.problem = S.problem + 1 |
+ if T: T.write(self.trace_message(' exception %s while scanning children.\n' % e)) |
+ return node |
+ |
+ children_not_visited = [] |
+ children_pending = set() |
+ children_not_ready = [] |
+ children_failed = False |
+ |
+ for child in chain(executor.get_all_prerequisites(), children): |
+ childstate = child.get_state() |
+ |
+ if T: T.write(self.trace_message(u' ' + self.trace_node(child))) |
+ |
+ if childstate == NODE_NO_STATE: |
+ children_not_visited.append(child) |
+ elif childstate == NODE_PENDING: |
+ children_pending.add(child) |
+ elif childstate == NODE_FAILED: |
+ children_failed = True |
+ |
+ if childstate <= NODE_EXECUTING: |
+ children_not_ready.append(child) |
+ |
+ |
+ # These nodes have not even been visited yet. Add |
+ # them to the list so that on some next pass we can |
+ # take a stab at evaluating them (or their children). |
+ children_not_visited.reverse() |
+ self.candidates.extend(self.order(children_not_visited)) |
+ #if T and children_not_visited: |
+ # T.write(self.trace_message(' adding to candidates: %s' % map(str, children_not_visited))) |
+ # T.write(self.trace_message(' candidates now: %s\n' % map(str, self.candidates))) |
+ |
+ # Skip this node if any of its children have failed. |
+ # |
+ # This catches the case where we're descending a top-level |
+ # target and one of our children failed while trying to be |
+ # built by a *previous* descent of an earlier top-level |
+ # target. |
+ # |
+ # It can also occur if a node is reused in multiple |
+ # targets. One first descends though the one of the |
+ # target, the next time occurs through the other target. |
+ # |
+ # Note that we can only have failed_children if the |
+ # --keep-going flag was used, because without it the build |
+ # will stop before diving in the other branch. |
+ # |
+ # Note that even if one of the children fails, we still |
+ # added the other children to the list of candidate nodes |
+ # to keep on building (--keep-going). |
+ if children_failed: |
+ for n in executor.get_action_targets(): |
+ n.set_state(NODE_FAILED) |
+ |
+ if S: S.child_failed = S.child_failed + 1 |
+ if T: T.write(self.trace_message('****** %s\n' % self.trace_node(node))) |
+ continue |
+ |
+ if children_not_ready: |
+ for child in children_not_ready: |
+ # We're waiting on one or more derived targets |
+ # that have not yet finished building. |
+ if S: S.not_built = S.not_built + 1 |
+ |
+ # Add this node to the waiting parents lists of |
+ # anything we're waiting on, with a reference |
+ # count so we can be put back on the list for |
+ # re-evaluation when they've all finished. |
+ node.ref_count = node.ref_count + child.add_to_waiting_parents(node) |
+ if T: T.write(self.trace_message(u' adjusted ref count: %s, child %s' % |
+ (self.trace_node(node), repr(str(child))))) |
+ |
+ if T: |
+ for pc in children_pending: |
+ T.write(self.trace_message(' adding %s to the pending children set\n' % |
+ self.trace_node(pc))) |
+ self.pending_children = self.pending_children | children_pending |
+ |
+ continue |
+ |
+ # Skip this node if it has side-effects that are |
+ # currently being built: |
+ wait_side_effects = False |
+ for se in executor.get_action_side_effects(): |
+ if se.get_state() == NODE_EXECUTING: |
+ se.add_to_waiting_s_e(node) |
+ wait_side_effects = True |
+ |
+ if wait_side_effects: |
+ if S: S.side_effects = S.side_effects + 1 |
+ continue |
+ |
+ # The default when we've gotten through all of the checks above: |
+ # this node is ready to be built. |
+ if S: S.build = S.build + 1 |
+ if T: T.write(self.trace_message(u'Evaluating %s\n' % |
+ self.trace_node(node))) |
+ |
+ # For debugging only: |
+ # |
+ # try: |
+ # self._validate_pending_children() |
+ # except: |
+ # self.ready_exc = sys.exc_info() |
+ # return node |
+ |
+ return node |
+ |
+ return None |
+ |
+ def next_task(self): |
+ """ |
+ Returns the next task to be executed. |
+ |
+ This simply asks for the next Node to be evaluated, and then wraps |
+ it in the specific Task subclass with which we were initialized. |
+ """ |
+ node = self._find_next_ready_node() |
+ |
+ if node is None: |
+ return None |
+ |
+ tlist = node.get_executor().get_all_targets() |
+ |
+ task = self.tasker(self, tlist, node in self.original_top, node) |
+ try: |
+ task.make_ready() |
+ except: |
+ # We had a problem just trying to get this task ready (like |
+ # a child couldn't be linked in to a VariantDir when deciding |
+ # whether this node is current). Arrange to raise the |
+ # exception when the Task is "executed." |
+ self.ready_exc = sys.exc_info() |
+ |
+ if self.ready_exc: |
+ task.exception_set(self.ready_exc) |
+ |
+ self.ready_exc = None |
+ |
+ return task |
+ |
+ def will_not_build(self, nodes, node_func=lambda n: None): |
+ """ |
+ Perform clean-up about nodes that will never be built. Invokes |
+ a user defined function on all of these nodes (including all |
+ of their parents). |
+ """ |
+ |
+ T = self.trace |
+ |
+ pending_children = self.pending_children |
+ |
+ to_visit = set(nodes) |
+ pending_children = pending_children - to_visit |
+ |
+ if T: |
+ for n in nodes: |
+ T.write(self.trace_message(' removing node %s from the pending children set\n' % |
+ self.trace_node(n))) |
+ try: |
+ while len(to_visit): |
+ node = to_visit.pop() |
+ node_func(node) |
+ |
+ # Prune recursion by flushing the waiting children |
+ # list immediately. |
+ parents = node.waiting_parents |
+ node.waiting_parents = set() |
+ |
+ to_visit = to_visit | parents |
+ pending_children = pending_children - parents |
+ |
+ for p in parents: |
+ p.ref_count = p.ref_count - 1 |
+ if T: T.write(self.trace_message(' removing parent %s from the pending children set\n' % |
+ self.trace_node(p))) |
+ except KeyError: |
+ # The container to_visit has been emptied. |
+ pass |
+ |
+ # We have the stick back the pending_children list into the |
+ # taskmaster because the python 1.5.2 compatibility does not |
+ # allow us to use in-place updates |
+ self.pending_children = pending_children |
+ |
+ def stop(self): |
+ """ |
+ Stops the current build completely. |
+ """ |
+ self.next_candidate = self.no_next_candidate |
+ |
+ def cleanup(self): |
+ """ |
+ Check for dependency cycles. |
+ """ |
+ if not self.pending_children: |
+ return |
+ |
+ nclist = [(n, find_cycle([n], set())) for n in self.pending_children] |
+ |
+ genuine_cycles = [ |
+ node for node,cycle in nclist |
+ if cycle or node.get_state() != NODE_EXECUTED |
+ ] |
+ if not genuine_cycles: |
+ # All of the "cycles" found were single nodes in EXECUTED state, |
+ # which is to say, they really weren't cycles. Just return. |
+ return |
+ |
+ desc = 'Found dependency cycle(s):\n' |
+ for node, cycle in nclist: |
+ if cycle: |
+ desc = desc + " " + " -> ".join(map(str, cycle)) + "\n" |
+ else: |
+ desc = desc + \ |
+ " Internal Error: no cycle found for node %s (%s) in state %s\n" % \ |
+ (node, repr(node), StateString[node.get_state()]) |
+ |
+ raise SCons.Errors.UserError(desc) |
+ |
+# Local Variables: |
+# tab-width:4 |
+# indent-tabs-mode:nil |
+# End: |
+# vim: set expandtab tabstop=4 shiftwidth=4: |
Property changes on: scons-2.0.1/engine/SCons/Taskmaster.py |
___________________________________________________________________ |
Added: svn:eol-style |
+ LF |