Adding cython v0.20.2 in third-party.

third_party/cython/src/Cython/Compiler/Pipeline.py

+import itertools
+from time import time
+
+import Errors
+import DebugFlags
+import Options
+from Visitor import CythonTransform
+from Errors import CompileError, InternalError, AbortError
+import Naming
+
+#
+# Really small pipeline stages
+#
+def dumptree(t):
+    # For quick debugging in pipelines
+    print t.dump()
+    return t
+
+def abort_on_errors(node):
+    # Stop the pipeline if there are any errors.
+    if Errors.num_errors != 0:
+        raise AbortError("pipeline break")
+    return node
+
+def parse_stage_factory(context):
+    def parse(compsrc):
+        source_desc = compsrc.source_desc
+        full_module_name = compsrc.full_module_name
+        initial_pos = (source_desc, 1, 0)
+        saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False
+        scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0,
+                                    check_module_name = not Options.embed)
+        Options.cimport_from_pyx = saved_cimport_from_pyx
+        tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
+        tree.compilation_source = compsrc
+        tree.scope = scope
+        tree.is_pxd = False
+        return tree
+    return parse
+
+def parse_pxd_stage_factory(context, scope, module_name):
+    def parse(source_desc):
+        tree = context.parse(source_desc, scope, pxd=True,
+                             full_module_name=module_name)
+        tree.scope = scope
+        tree.is_pxd = True
+        return tree
+    return parse
+
+def generate_pyx_code_stage_factory(options, result):
+    def generate_pyx_code_stage(module_node):
+        module_node.process_implementation(options, result)
+        result.compilation_source = module_node.compilation_source
+        return result
+    return generate_pyx_code_stage
+
+def inject_pxd_code_stage_factory(context):
+    def inject_pxd_code_stage(module_node):
+        from textwrap import dedent
+        stats = module_node.body.stats
+        for name, (statlistnode, scope) in context.pxds.iteritems():
+            module_node.merge_in(statlistnode, scope)
+        return module_node
+    return inject_pxd_code_stage
+
+def use_utility_code_definitions(scope, target, seen=None):
+    if seen is None:
+        seen = set()
+
+    for entry in scope.entries.itervalues():
+        if entry in seen:
+            continue
+
+        seen.add(entry)
+        if entry.used and entry.utility_code_definition:
+            target.use_utility_code(entry.utility_code_definition)
+            for required_utility in entry.utility_code_definition.requires:
+                target.use_utility_code(required_utility)
+        elif entry.as_module:
+            use_utility_code_definitions(entry.as_module, target, seen)
+
+def inject_utility_code_stage_factory(context):
+    def inject_utility_code_stage(module_node):
+        use_utility_code_definitions(context.cython_scope, module_node.scope)
+        added = []
+        # Note: the list might be extended inside the loop (if some utility code
+        # pulls in other utility code, explicitly or implicitly)
+        for utilcode in module_node.scope.utility_code_list:
+            if utilcode in added: continue
+            added.append(utilcode)
+            if utilcode.requires:
+                for dep in utilcode.requires:
+                    if not dep in added and not dep in module_node.scope.utility_code_list:
+                        module_node.scope.utility_code_list.append(dep)
+            tree = utilcode.get_tree()
+            if tree:
+                module_node.merge_in(tree.body, tree.scope, merge_scope=True)
+        return module_node
+    return inject_utility_code_stage
+
+class UseUtilityCodeDefinitions(CythonTransform):
+    # Temporary hack to use any utility code in nodes' "utility_code_definitions".
+    # This should be moved to the code generation phase of the relevant nodes once
+    # it is safe to generate CythonUtilityCode at code generation time.
+    def __call__(self, node):
+        self.scope = node.scope
+        return super(UseUtilityCodeDefinitions, self).__call__(node)
+
+    def process_entry(self, entry):
+        if entry:
+            for utility_code in (entry.utility_code, entry.utility_code_definition):
+                if utility_code:
+                    self.scope.use_utility_code(utility_code)
+
+    def visit_AttributeNode(self, node):
+        self.process_entry(node.entry)
+        return node
+
+    def visit_NameNode(self, node):
+        self.process_entry(node.entry)
+        self.process_entry(node.type_entry)
+        return node
+
+#
+# Pipeline factories
+#
+
+def create_pipeline(context, mode, exclude_classes=()):
+    assert mode in ('pyx', 'py', 'pxd')
+    from Visitor import PrintTree
+    from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
+    from ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform
+    from ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes
+    from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
+    from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
+    from ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform
+    from ParseTreeTransforms import CalculateQualifiedNamesTransform
+    from TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
+    from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
+    from ParseTreeTransforms import RemoveUnreachableCode, GilCheck
+    from FlowControl import ControlFlowAnalysis
+    from AnalysedTreeTransforms import AutoTestDictTransform
+    from AutoDocTransforms import EmbedSignature
+    from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
+    from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
+    from Optimize import InlineDefNodeCalls
+    from Optimize import ConstantFolding, FinalOptimizePhase
+    from Optimize import DropRefcountingTransform
+    from Optimize import ConsolidateOverflowCheck
+    from Buffer import IntroduceBufferAuxiliaryVars
+    from ModuleNode import check_c_declarations, check_c_declarations_pxd
+
+
+    if mode == 'pxd':
+        _check_c_declarations = check_c_declarations_pxd
+        _specific_post_parse = PxdPostParse(context)
+    else:
+        _check_c_declarations = check_c_declarations
+        _specific_post_parse = None
+
+    if mode == 'py':
+        _align_function_definitions = AlignFunctionDefinitions(context)
+    else:
+        _align_function_definitions = None
+
+    # NOTE: This is the "common" parts of the pipeline, which is also
+    # code in pxd files. So it will be run multiple times in a
+    # compilation stage.
+    stages = [
+        NormalizeTree(context),
+        PostParse(context),
+        _specific_post_parse,
+        InterpretCompilerDirectives(context, context.compiler_directives),
+        ParallelRangeTransform(context),
+        AdjustDefByDirectives(context),
+        MarkClosureVisitor(context),
+        _align_function_definitions,
+        RemoveUnreachableCode(context),
+        ConstantFolding(),
+        FlattenInListTransform(),
+        WithTransform(context),
+        DecoratorTransform(context),
+        ForwardDeclareTypes(context),
+        AnalyseDeclarationsTransform(context),
+        AutoTestDictTransform(context),
+        EmbedSignature(context),
+        EarlyReplaceBuiltinCalls(context),  ## Necessary?
+        TransformBuiltinMethods(context),  ## Necessary?
+        MarkParallelAssignments(context),
+        ControlFlowAnalysis(context),
+        RemoveUnreachableCode(context),
+        # MarkParallelAssignments(context),
+        MarkOverflowingArithmetic(context),
+        IntroduceBufferAuxiliaryVars(context),
+        _check_c_declarations,
+        InlineDefNodeCalls(context),
+        AnalyseExpressionsTransform(context),
+        FindInvalidUseOfFusedTypes(context),
+        ExpandInplaceOperators(context),
+        OptimizeBuiltinCalls(context),  ## Necessary?
+        CreateClosureClasses(context),  ## After all lookups and type inference
+        CalculateQualifiedNamesTransform(context),
+        ConsolidateOverflowCheck(context),
+        IterationTransform(context),
+        SwitchTransform(),
+        DropRefcountingTransform(),
+        FinalOptimizePhase(context),
+        GilCheck(),
+        UseUtilityCodeDefinitions(context),
+        ]
+    filtered_stages = []
+    for s in stages:
+        if s.__class__ not in exclude_classes:
+            filtered_stages.append(s)
+    return filtered_stages
+
+def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
+    if py:
+        mode = 'py'
+    else:
+        mode = 'pyx'
+    test_support = []
+    if options.evaluate_tree_assertions:
+        from Cython.TestUtils import TreeAssertVisitor
+        test_support.append(TreeAssertVisitor())
+
+    if options.gdb_debug:
+        from Cython.Debugger import DebugWriter # requires Py2.5+
+        from ParseTreeTransforms import DebugTransform
+        context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter(
+            options.output_dir)
+        debug_transform = [DebugTransform(context, options, result)]
+    else:
+        debug_transform = []
+
+    return list(itertools.chain(
+        [parse_stage_factory(context)],
+        create_pipeline(context, mode, exclude_classes=exclude_classes),
+        test_support,
+        [inject_pxd_code_stage_factory(context),
+         inject_utility_code_stage_factory(context),
+         abort_on_errors],
+        debug_transform,
+        [generate_pyx_code_stage_factory(options, result)]))
+
+def create_pxd_pipeline(context, scope, module_name):
+    from CodeGeneration import ExtractPxdCode
+
+    # The pxd pipeline ends up with a CCodeWriter containing the
+    # code of the pxd, as well as a pxd scope.
+    return [
+        parse_pxd_stage_factory(context, scope, module_name)
+        ] + create_pipeline(context, 'pxd') + [
+        ExtractPxdCode()
+        ]
+
+def create_py_pipeline(context, options, result):
+    return create_pyx_pipeline(context, options, result, py=True)
+
+def create_pyx_as_pxd_pipeline(context, result):
+    from ParseTreeTransforms import AlignFunctionDefinitions, \
+        MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform
+    from Optimize import ConstantFolding, FlattenInListTransform
+    from Nodes import StatListNode
+    pipeline = []
+    pyx_pipeline = create_pyx_pipeline(context, context.options, result,
+                                       exclude_classes=[
+                                           AlignFunctionDefinitions,
+                                           MarkClosureVisitor,
+                                           ConstantFolding,
+                                           FlattenInListTransform,
+                                           WithTransform
+                                           ])
+    for stage in pyx_pipeline:
+        pipeline.append(stage)
+        if isinstance(stage, AnalyseDeclarationsTransform):
+            # This is the last stage we need.
+            break
+    def fake_pxd(root):
+        for entry in root.scope.entries.values():
+            if not entry.in_cinclude:
+                entry.defined_in_pxd = 1
+                if entry.name == entry.cname and entry.visibility != 'extern':
+                    # Always mangle non-extern cimported entries.
+                    entry.cname = entry.scope.mangle(Naming.func_prefix, entry.name)
+        return StatListNode(root.pos, stats=[]), root.scope
+    pipeline.append(fake_pxd)
+    return pipeline
+
+def insert_into_pipeline(pipeline, transform, before=None, after=None):
+    """
+    Insert a new transform into the pipeline after or before an instance of
+    the given class. e.g.
+
+        pipeline = insert_into_pipeline(pipeline, transform,
+                                        after=AnalyseDeclarationsTransform)
+    """
+    assert before or after
+
+    cls = before or after
+    for i, t in enumerate(pipeline):
+        if isinstance(t, cls):
+            break
+
+    if after:
+        i += 1
+
+    return pipeline[:i] + [transform] + pipeline[i:]
+
+#
+# Running a pipeline
+#
+
+def run_pipeline(pipeline, source, printtree=True):
+    from Cython.Compiler.Visitor import PrintTree
+
+    error = None
+    data = source
+    try:
+        try:
+            for phase in pipeline:
+                if phase is not None:
+                    if DebugFlags.debug_verbose_pipeline:
+                        t = time()
+                        print "Entering pipeline phase %r" % phase
+                    if not printtree and isinstance(phase, PrintTree):
+                        continue
+                    data = phase(data)
+                    if DebugFlags.debug_verbose_pipeline:
+                        print "    %.3f seconds" % (time() - t)
+        except CompileError, err:
+            # err is set
+            Errors.report_error(err)
+            error = err
+    except InternalError, err:
+        # Only raise if there was not an earlier error
+        if Errors.num_errors == 0:
+            raise
+        error = err
+    except AbortError, err:
+        error = err
+    return (error, data)