OLD | NEW |
1 # copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. | 1 # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr | 2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
| 3 # copyright 2003-2010 Sylvain Thenault, all rights reserved. |
| 4 # contact mailto:thenault@gmail.com |
3 # | 5 # |
4 # This file is part of astroid. | 6 # This file is part of logilab-astng. |
5 # | 7 # |
6 # astroid is free software: you can redistribute it and/or modify it | 8 # logilab-astng is free software: you can redistribute it and/or modify it |
7 # under the terms of the GNU Lesser General Public License as published by the | 9 # under the terms of the GNU Lesser General Public License as published by the |
8 # Free Software Foundation, either version 2.1 of the License, or (at your | 10 # Free Software Foundation, either version 2.1 of the License, or (at your |
9 # option) any later version. | 11 # option) any later version. |
10 # | 12 # |
11 # astroid is distributed in the hope that it will be useful, but | 13 # logilab-astng is distributed in the hope that it will be useful, but |
12 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | 14 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License | 15 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License |
14 # for more details. | 16 # for more details. |
15 # | 17 # |
16 # You should have received a copy of the GNU Lesser General Public License along | 18 # You should have received a copy of the GNU Lesser General Public License along |
17 # with astroid. If not, see <http://www.gnu.org/licenses/>. | 19 # with logilab-astng. If not, see <http://www.gnu.org/licenses/>. |
18 """The AstroidBuilder makes astroid from living object and / or from _ast | 20 """The ASTNGBuilder makes astng from living object and / or from _ast |
19 | 21 |
20 The builder is not thread safe and can't be used to parse different sources | 22 The builder is not thread safe and can't be used to parse different sources |
21 at the same time. | 23 at the same time. |
22 """ | 24 """ |
23 from __future__ import with_statement | |
24 | 25 |
25 __docformat__ = "restructuredtext en" | 26 __docformat__ = "restructuredtext en" |
26 | 27 |
27 import sys | 28 import sys, re |
28 from os.path import splitext, basename, exists, abspath | 29 from os.path import splitext, basename, dirname, exists, abspath |
29 | 30 |
30 from astroid.exceptions import AstroidBuildingException, InferenceError | 31 from logilab.common.modutils import modpath_from_file |
31 from astroid.raw_building import InspectBuilder | 32 |
32 from astroid.rebuilder import TreeRebuilder | 33 from logilab.astng.exceptions import ASTNGBuildingException, InferenceError |
33 from astroid.manager import AstroidManager | 34 from logilab.astng.raw_building import InspectBuilder |
34 from astroid.bases import YES, Instance | 35 from logilab.astng.rebuilder import TreeRebuilder |
35 from astroid.modutils import modpath_from_file | 36 from logilab.astng.manager import ASTNGManager |
| 37 from logilab.astng.bases import YES, Instance |
36 | 38 |
37 from _ast import PyCF_ONLY_AST | 39 from _ast import PyCF_ONLY_AST |
38 def parse(string): | 40 def parse(string): |
39 return compile(string, "<string>", 'exec', PyCF_ONLY_AST) | 41 return compile(string, "<string>", 'exec', PyCF_ONLY_AST) |
40 | 42 |
41 if sys.version_info >= (3, 0): | 43 if sys.version_info >= (3, 0): |
42 from tokenize import detect_encoding | 44 from tokenize import detect_encoding |
43 | 45 |
44 def open_source_file(filename): | 46 def open_source_file(filename): |
45 with open(filename, 'rb') as byte_stream: | 47 byte_stream = open(filename, 'bU') |
46 encoding = detect_encoding(byte_stream.readline)[0] | 48 encoding = detect_encoding(byte_stream.readline)[0] |
47 stream = open(filename, 'rU', encoding=encoding) | 49 stream = open(filename, 'U', encoding=encoding) |
48 try: | 50 try: |
49 data = stream.read() | 51 data = stream.read() |
50 except UnicodeError: # wrong encodingg | 52 except UnicodeError, uex: # wrong encodingg |
51 # detect_encoding returns utf-8 if no encoding specified | 53 # detect_encoding returns utf-8 if no encoding specified |
52 msg = 'Wrong (%s) or no encoding specified' % encoding | 54 msg = 'Wrong (%s) or no encoding specified' % encoding |
53 raise AstroidBuildingException(msg) | 55 raise ASTNGBuildingException(msg) |
54 return stream, encoding, data | 56 return stream, encoding, data |
55 | 57 |
56 else: | 58 else: |
57 import re | 59 import re |
58 | 60 |
59 _ENCODING_RGX = re.compile(r"\s*#+.*coding[:=]\s*([-\w.]+)") | 61 _ENCODING_RGX = re.compile("\s*#+.*coding[:=]\s*([-\w.]+)") |
60 | 62 |
61 def _guess_encoding(string): | 63 def _guess_encoding(string): |
62 """get encoding from a python file as string or return None if not found | 64 """get encoding from a python file as string or return None if not found |
63 """ | 65 """ |
64 # check for UTF-8 byte-order mark | 66 # check for UTF-8 byte-order mark |
65 if string.startswith('\xef\xbb\xbf'): | 67 if string.startswith('\xef\xbb\xbf'): |
66 return 'UTF-8' | 68 return 'UTF-8' |
67 for line in string.split('\n', 2)[:2]: | 69 for line in string.split('\n', 2)[:2]: |
68 # check for encoding declaration | 70 # check for encoding declaration |
69 match = _ENCODING_RGX.match(line) | 71 match = _ENCODING_RGX.match(line) |
70 if match is not None: | 72 if match is not None: |
71 return match.group(1) | 73 return match.group(1) |
72 | 74 |
73 def open_source_file(filename): | 75 def open_source_file(filename): |
74 """get data for parsing a file""" | 76 """get data for parsing a file""" |
75 stream = open(filename, 'U') | 77 stream = open(filename, 'U') |
76 data = stream.read() | 78 data = stream.read() |
77 encoding = _guess_encoding(data) | 79 encoding = _guess_encoding(data) |
78 return stream, encoding, data | 80 return stream, encoding, data |
79 | 81 |
80 # ast NG builder ############################################################## | 82 # ast NG builder ############################################################## |
81 | 83 |
82 MANAGER = AstroidManager() | 84 MANAGER = ASTNGManager() |
83 | 85 |
84 class AstroidBuilder(InspectBuilder): | 86 class ASTNGBuilder(InspectBuilder): |
85 """provide astroid building methods""" | 87 """provide astng building methods""" |
| 88 rebuilder = TreeRebuilder() |
86 | 89 |
87 def __init__(self, manager=None): | 90 def __init__(self, manager=None): |
88 InspectBuilder.__init__(self) | |
89 self._manager = manager or MANAGER | 91 self._manager = manager or MANAGER |
90 | 92 |
91 def module_build(self, module, modname=None): | 93 def module_build(self, module, modname=None): |
92 """build an astroid from a living module instance | 94 """build an astng from a living module instance |
93 """ | 95 """ |
94 node = None | 96 node = None |
95 path = getattr(module, '__file__', None) | 97 path = getattr(module, '__file__', None) |
96 if path is not None: | 98 if path is not None: |
97 path_, ext = splitext(module.__file__) | 99 path_, ext = splitext(module.__file__) |
98 if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'): | 100 if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'): |
99 node = self.file_build(path_ + '.py', modname) | 101 node = self.file_build(path_ + '.py', modname) |
100 if node is None: | 102 if node is None: |
101 # this is a built-in module | 103 # this is a built-in module |
102 # get a partial representation by introspection | 104 # get a partial representation by introspection |
103 node = self.inspect_build(module, modname=modname, path=path) | 105 node = self.inspect_build(module, modname=modname, path=path) |
104 # we have to handle transformation by ourselves since the rebuilder | |
105 # isn't called for builtin nodes | |
106 # | |
107 # XXX it's then only called for Module nodes, not for underlying | |
108 # nodes | |
109 node = self._manager.transform(node) | |
110 return node | 106 return node |
111 | 107 |
112 def file_build(self, path, modname=None): | 108 def file_build(self, path, modname=None): |
113 """build astroid from a source code file (i.e. from an ast) | 109 """build astng from a source code file (i.e. from an ast) |
114 | 110 |
115 path is expected to be a python source file | 111 path is expected to be a python source file |
116 """ | 112 """ |
117 try: | 113 try: |
118 _, encoding, data = open_source_file(path) | 114 stream, encoding, data = open_source_file(path) |
119 except IOError, exc: | 115 except IOError, exc: |
120 msg = 'Unable to load file %r (%s)' % (path, exc) | 116 msg = 'Unable to load file %r (%s)' % (path, exc) |
121 raise AstroidBuildingException(msg) | 117 raise ASTNGBuildingException(msg) |
122 except SyntaxError, exc: # py3k encoding specification error | 118 except SyntaxError, exc: # py3k encoding specification error |
123 raise AstroidBuildingException(exc) | 119 raise ASTNGBuildingException(exc) |
124 except LookupError, exc: # unknown encoding | 120 except LookupError, exc: # unknown encoding |
125 raise AstroidBuildingException(exc) | 121 raise ASTNGBuildingException(exc) |
126 # get module name if necessary | 122 # get module name if necessary |
127 if modname is None: | 123 if modname is None: |
128 try: | 124 try: |
129 modname = '.'.join(modpath_from_file(path)) | 125 modname = '.'.join(modpath_from_file(path)) |
130 except ImportError: | 126 except ImportError: |
131 modname = splitext(basename(path))[0] | 127 modname = splitext(basename(path))[0] |
132 # build astroid representation | 128 # build astng representation |
133 module = self._data_build(data, modname, path) | 129 node = self.string_build(data, modname, path) |
134 return self._post_build(module, encoding) | 130 node.file_encoding = encoding |
| 131 return node |
135 | 132 |
136 def string_build(self, data, modname='', path=None): | 133 def string_build(self, data, modname='', path=None): |
137 """build astroid from source code string and return rebuilded astroid""" | 134 """build astng from source code string and return rebuilded astng""" |
138 module = self._data_build(data, modname, path) | 135 module = self._data_build(data, modname, path) |
139 module.file_bytes = data.encode('utf-8') | 136 self._manager.astng_cache[module.name] = module |
140 return self._post_build(module, 'utf-8') | |
141 | |
142 def _post_build(self, module, encoding): | |
143 """handles encoding and delayed nodes | |
144 after a module has been built | |
145 """ | |
146 module.file_encoding = encoding | |
147 self._manager.cache_module(module) | |
148 # post tree building steps after we stored the module in the cache: | 137 # post tree building steps after we stored the module in the cache: |
149 for from_node in module._from_nodes: | 138 for from_node in module._from_nodes: |
150 if from_node.modname == '__future__': | |
151 for symbol, _ in from_node.names: | |
152 module.future_imports.add(symbol) | |
153 self.add_from_names_to_locals(from_node) | 139 self.add_from_names_to_locals(from_node) |
154 # handle delayed assattr nodes | 140 # handle delayed assattr nodes |
155 for delayed in module._delayed_assattr: | 141 for delayed in module._delayed_assattr: |
156 self.delayed_assattr(delayed) | 142 self.delayed_assattr(delayed) |
| 143 if modname: |
| 144 for transformer in self._manager.transformers: |
| 145 transformer(module) |
157 return module | 146 return module |
158 | 147 |
159 def _data_build(self, data, modname, path): | 148 def _data_build(self, data, modname, path): |
160 """build tree node from data and add some informations""" | 149 """build tree node from data and add some informations""" |
161 # this method could be wrapped with a pickle/cache function | 150 # this method could be wrapped with a pickle/cache function |
162 node = parse(data + '\n') | 151 node = parse(data + '\n') |
163 if path is not None: | 152 if path is not None: |
164 node_file = abspath(path) | 153 node_file = abspath(path) |
165 else: | 154 else: |
166 node_file = '<?>' | 155 node_file = '<?>' |
167 if modname.endswith('.__init__'): | 156 if modname.endswith('.__init__'): |
168 modname = modname[:-9] | 157 modname = modname[:-9] |
169 package = True | 158 package = True |
170 else: | 159 else: |
171 package = path and path.find('__init__.py') > -1 or False | 160 package = path and path.find('__init__.py') > -1 or False |
172 rebuilder = TreeRebuilder(self._manager) | 161 self.rebuilder.init() |
173 module = rebuilder.visit_module(node, modname, package) | 162 module = self.rebuilder.visit_module(node, modname, package) |
174 module.file = module.path = node_file | 163 module.file = module.path = node_file |
175 module._from_nodes = rebuilder._from_nodes | 164 module._from_nodes = self.rebuilder._from_nodes |
176 module._delayed_assattr = rebuilder._delayed_assattr | 165 module._delayed_assattr = self.rebuilder._delayed_assattr |
177 return module | 166 return module |
178 | 167 |
179 def add_from_names_to_locals(self, node): | 168 def add_from_names_to_locals(self, node): |
180 """store imported names to the locals; | 169 """store imported names to the locals; |
181 resort the locals if coming from a delayed node | 170 resort the locals if coming from a delayed node |
182 """ | 171 """ |
183 | 172 |
184 _key_func = lambda node: node.fromlineno | 173 _key_func = lambda node: node.fromlineno |
185 def sort_locals(my_list): | 174 def sort_locals(my_list): |
186 my_list.sort(key=_key_func) | 175 my_list.sort(key=_key_func) |
187 for (name, asname) in node.names: | 176 for (name, asname) in node.names: |
188 if name == '*': | 177 if name == '*': |
189 try: | 178 try: |
190 imported = node.do_import_module() | 179 imported = node.root().import_module(node.modname) |
191 except InferenceError: | 180 except ASTNGBuildingException: |
192 continue | 181 continue |
193 for name in imported.wildcard_import_names(): | 182 for name in imported.wildcard_import_names(): |
194 node.parent.set_local(name, node) | 183 node.parent.set_local(name, node) |
195 sort_locals(node.parent.scope().locals[name]) | 184 sort_locals(node.parent.scope().locals[name]) |
196 else: | 185 else: |
197 node.parent.set_local(asname or name, node) | 186 node.parent.set_local(asname or name, node) |
198 sort_locals(node.parent.scope().locals[asname or name]) | 187 sort_locals(node.parent.scope().locals[asname or name]) |
199 | 188 |
200 def delayed_assattr(self, node): | 189 def delayed_assattr(self, node): |
201 """visit a AssAttr node -> add name to locals, handle members | 190 """visit a AssAttr node -> add name to locals, handle members |
(...skipping 26 matching lines...) Expand all Loading... |
228 continue | 217 continue |
229 # get assign in __init__ first XXX useful ? | 218 # get assign in __init__ first XXX useful ? |
230 if frame.name == '__init__' and values and not \ | 219 if frame.name == '__init__' and values and not \ |
231 values[0].frame().name == '__init__': | 220 values[0].frame().name == '__init__': |
232 values.insert(0, node) | 221 values.insert(0, node) |
233 else: | 222 else: |
234 values.append(node) | 223 values.append(node) |
235 except InferenceError: | 224 except InferenceError: |
236 pass | 225 pass |
237 | 226 |
OLD | NEW |