OLD | NEW |
1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. | 1 # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr | 2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
| 3 # copyright 2003-2010 Sylvain Thenault, all rights reserved. |
| 4 # contact mailto:thenault@gmail.com |
3 # | 5 # |
4 # This file is part of astroid. | 6 # This file is part of logilab-astng. |
5 # | 7 # |
6 # astroid is free software: you can redistribute it and/or modify it | 8 # logilab-astng is free software: you can redistribute it and/or modify it |
7 # under the terms of the GNU Lesser General Public License as published by the | 9 # under the terms of the GNU Lesser General Public License as published by the |
8 # Free Software Foundation, either version 2.1 of the License, or (at your | 10 # Free Software Foundation, either version 2.1 of the License, or (at your |
9 # option) any later version. | 11 # option) any later version. |
10 # | 12 # |
11 # astroid is distributed in the hope that it will be useful, but | 13 # logilab-astng is distributed in the hope that it will be useful, but |
12 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | 14 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License | 15 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License |
14 # for more details. | 16 # for more details. |
15 # | 17 # |
16 # You should have received a copy of the GNU Lesser General Public License along | 18 # You should have received a copy of the GNU Lesser General Public License along |
17 # with astroid. If not, see <http://www.gnu.org/licenses/>. | 19 # with logilab-astng. If not, see <http://www.gnu.org/licenses/>. |
18 """astroid manager: avoid multiple astroid build of a same module when | 20 """astng manager: avoid multiple astng build of a same module when |
19 possible by providing a class responsible to get astroid representation | 21 possible by providing a class responsible to get astng representation |
20 from various source and using a cache of built modules) | 22 from various source and using a cache of built modules) |
21 """ | 23 """ |
22 | 24 |
23 __docformat__ = "restructuredtext en" | 25 __docformat__ = "restructuredtext en" |
24 | 26 |
| 27 import sys |
25 import os | 28 import os |
26 from os.path import dirname, join, isdir, exists | 29 from os.path import dirname, basename, abspath, join, isdir, exists |
27 from warnings import warn | |
28 | 30 |
| 31 from logilab.common.modutils import NoSourceFile, is_python_source, \ |
| 32 file_from_modpath, load_module_from_name, modpath_from_file, \ |
| 33 get_module_files, get_source_file, zipimport |
29 from logilab.common.configuration import OptionsProviderMixIn | 34 from logilab.common.configuration import OptionsProviderMixIn |
30 | 35 |
31 from astroid.exceptions import AstroidBuildingException | 36 from logilab.astng.exceptions import ASTNGBuildingException |
32 from astroid.modutils import NoSourceFile, is_python_source, \ | |
33 file_from_modpath, load_module_from_name, modpath_from_file, \ | |
34 get_module_files, get_source_file, zipimport | |
35 | 37 |
36 | 38 def astng_wrapper(func, modname): |
37 def astroid_wrapper(func, modname): | 39 """wrapper to give to ASTNGManager.project_from_files""" |
38 """wrapper to give to AstroidManager.project_from_files""" | |
39 print 'parsing %s...' % modname | 40 print 'parsing %s...' % modname |
40 try: | 41 try: |
41 return func(modname) | 42 return func(modname) |
42 except AstroidBuildingException, exc: | 43 except ASTNGBuildingException, exc: |
43 print exc | 44 print exc |
44 except Exception, exc: | 45 except Exception, exc: |
45 import traceback | 46 import traceback |
46 traceback.print_exc() | 47 traceback.print_exc() |
47 | 48 |
48 def _silent_no_wrap(func, modname): | 49 def _silent_no_wrap(func, modname): |
49 """silent wrapper that doesn't do anything; can be used for tests""" | 50 """silent wrapper that doesn't do anything; can be used for tests""" |
50 return func(modname) | 51 return func(modname) |
51 | 52 |
52 def safe_repr(obj): | 53 def safe_repr(obj): |
53 try: | 54 try: |
54 return repr(obj) | 55 return repr(obj) |
55 except: | 56 except: |
56 return '???' | 57 return '???' |
57 | 58 |
58 | 59 |
59 | 60 |
60 class AstroidManager(OptionsProviderMixIn): | 61 class ASTNGManager(OptionsProviderMixIn): |
61 """the astroid manager, responsible to build astroid from files | 62 """the astng manager, responsible to build astng from files |
62 or modules. | 63 or modules. |
63 | 64 |
64 Use the Borg pattern. | 65 Use the Borg pattern. |
65 """ | 66 """ |
66 | 67 |
67 name = 'astroid loader' | 68 name = 'astng loader' |
68 options = (("ignore", | 69 options = (("ignore", |
69 {'type' : "csv", 'metavar' : "<file>", | 70 {'type' : "csv", 'metavar' : "<file>", |
70 'dest' : "black_list", "default" : ('CVS',), | 71 'dest' : "black_list", "default" : ('CVS',), |
71 'help' : "add <file> (may be a directory) to the black list\ | 72 'help' : "add <file> (may be a directory) to the black list\ |
72 . It should be a base name, not a path. You may set this option multiple times\ | 73 . It should be a base name, not a path. You may set this option multiple times\ |
73 ."}), | 74 ."}), |
74 ("project", | 75 ("project", |
75 {'default': "No Name", 'type' : 'string', 'short': 'p', | 76 {'default': "No Name", 'type' : 'string', 'short': 'p', |
76 'metavar' : '<project name>', | 77 'metavar' : '<project name>', |
77 'help' : 'set the project name.'}), | 78 'help' : 'set the project name.'}), |
78 ) | 79 ) |
79 brain = {} | 80 brain = {} |
80 def __init__(self): | 81 def __init__(self): |
81 self.__dict__ = AstroidManager.brain | 82 self.__dict__ = ASTNGManager.brain |
82 if not self.__dict__: | 83 if not self.__dict__: |
83 OptionsProviderMixIn.__init__(self) | 84 OptionsProviderMixIn.__init__(self) |
84 self.load_defaults() | 85 self.load_defaults() |
85 # NOTE: cache entries are added by the [re]builder | 86 # NOTE: cache entries are added by the [re]builder |
86 self.astroid_cache = {} | 87 self.astng_cache = {} |
87 self._mod_file_cache = {} | 88 self._mod_file_cache = {} |
88 self.transforms = {} | 89 self.transformers = [] |
89 | 90 |
90 def ast_from_file(self, filepath, modname=None, fallback=True, source=False)
: | 91 def astng_from_file(self, filepath, modname=None, fallback=True, source=Fals
e): |
91 """given a module name, return the astroid object""" | 92 """given a module name, return the astng object""" |
92 try: | 93 try: |
93 filepath = get_source_file(filepath, include_no_ext=True) | 94 filepath = get_source_file(filepath, include_no_ext=True) |
94 source = True | 95 source = True |
95 except NoSourceFile: | 96 except NoSourceFile: |
96 pass | 97 pass |
97 if modname is None: | 98 if modname is None: |
98 try: | 99 try: |
99 modname = '.'.join(modpath_from_file(filepath)) | 100 modname = '.'.join(modpath_from_file(filepath)) |
100 except ImportError: | 101 except ImportError: |
101 modname = filepath | 102 modname = filepath |
102 if modname in self.astroid_cache and self.astroid_cache[modname].file ==
filepath: | 103 if modname in self.astng_cache: |
103 return self.astroid_cache[modname] | 104 return self.astng_cache[modname] |
104 if source: | 105 if source: |
105 from astroid.builder import AstroidBuilder | 106 from logilab.astng.builder import ASTNGBuilder |
106 return AstroidBuilder(self).file_build(filepath, modname) | 107 return ASTNGBuilder(self).file_build(filepath, modname) |
107 elif fallback and modname: | 108 elif fallback and modname: |
108 return self.ast_from_module_name(modname) | 109 return self.astng_from_module_name(modname) |
109 raise AstroidBuildingException('unable to get astroid for file %s' % | 110 raise ASTNGBuildingException('unable to get astng for file %s' % |
110 filepath) | 111 filepath) |
111 | 112 |
112 def ast_from_module_name(self, modname, context_file=None): | 113 def astng_from_module_name(self, modname, context_file=None): |
113 """given a module name, return the astroid object""" | 114 """given a module name, return the astng object""" |
114 if modname in self.astroid_cache: | 115 if modname in self.astng_cache: |
115 return self.astroid_cache[modname] | 116 return self.astng_cache[modname] |
116 if modname == '__main__': | 117 if modname == '__main__': |
117 from astroid.builder import AstroidBuilder | 118 from logilab.astng.builder import ASTNGBuilder |
118 return AstroidBuilder(self).string_build('', modname) | 119 return ASTNGBuilder(self).string_build('', modname) |
119 old_cwd = os.getcwd() | 120 old_cwd = os.getcwd() |
120 if context_file: | 121 if context_file: |
121 os.chdir(dirname(context_file)) | 122 os.chdir(dirname(context_file)) |
122 try: | 123 try: |
123 filepath = self.file_from_module_name(modname, context_file) | 124 filepath = self.file_from_module_name(modname, context_file) |
124 if filepath is not None and not is_python_source(filepath): | 125 if filepath is not None and not is_python_source(filepath): |
125 module = self.zip_import_data(filepath) | 126 module = self.zip_import_data(filepath) |
126 if module is not None: | 127 if module is not None: |
127 return module | 128 return module |
128 if filepath is None or not is_python_source(filepath): | 129 if filepath is None or not is_python_source(filepath): |
129 try: | 130 try: |
130 module = load_module_from_name(modname) | 131 module = load_module_from_name(modname) |
131 except Exception, ex: | 132 except Exception, ex: |
132 msg = 'Unable to load module %s (%s)' % (modname, ex) | 133 msg = 'Unable to load module %s (%s)' % (modname, ex) |
133 raise AstroidBuildingException(msg) | 134 raise ASTNGBuildingException(msg) |
134 return self.ast_from_module(module, modname) | 135 return self.astng_from_module(module, modname) |
135 return self.ast_from_file(filepath, modname, fallback=False) | 136 return self.astng_from_file(filepath, modname, fallback=False) |
136 finally: | 137 finally: |
137 os.chdir(old_cwd) | 138 os.chdir(old_cwd) |
138 | 139 |
139 def zip_import_data(self, filepath): | 140 def zip_import_data(self, filepath): |
140 if zipimport is None: | 141 if zipimport is None: |
141 return None | 142 return None |
142 from astroid.builder import AstroidBuilder | 143 from logilab.astng.builder import ASTNGBuilder |
143 builder = AstroidBuilder(self) | 144 builder = ASTNGBuilder(self) |
144 for ext in ('.zip', '.egg'): | 145 for ext in ('.zip', '.egg'): |
145 try: | 146 try: |
146 eggpath, resource = filepath.rsplit(ext + '/', 1) | 147 eggpath, resource = filepath.rsplit(ext + '/', 1) |
147 except ValueError: | 148 except ValueError: |
148 continue | 149 continue |
149 try: | 150 try: |
150 importer = zipimport.zipimporter(eggpath + ext) | 151 importer = zipimport.zipimporter(eggpath + ext) |
151 zmodname = resource.replace('/', '.') | 152 zmodname = resource.replace('/', '.') |
152 if importer.is_package(resource): | 153 if importer.is_package(resource): |
153 zmodname = zmodname + '.__init__' | 154 zmodname = zmodname + '.__init__' |
154 module = builder.string_build(importer.get_source(resource), | 155 module = builder.string_build(importer.get_source(resource), |
155 zmodname, filepath) | 156 zmodname, filepath) |
156 return module | 157 return module |
157 except: | 158 except: |
158 continue | 159 continue |
159 return None | 160 return None |
160 | 161 |
161 def file_from_module_name(self, modname, contextfile): | 162 def file_from_module_name(self, modname, contextfile): |
162 try: | 163 try: |
163 value = self._mod_file_cache[(modname, contextfile)] | 164 value = self._mod_file_cache[(modname, contextfile)] |
164 except KeyError: | 165 except KeyError: |
165 try: | 166 try: |
166 value = file_from_modpath(modname.split('.'), | 167 value = file_from_modpath(modname.split('.'), |
167 context_file=contextfile) | 168 context_file=contextfile) |
168 except ImportError, ex: | 169 except ImportError, ex: |
169 msg = 'Unable to load module %s (%s)' % (modname, ex) | 170 msg = 'Unable to load module %s (%s)' % (modname, ex) |
170 value = AstroidBuildingException(msg) | 171 value = ASTNGBuildingException(msg) |
171 self._mod_file_cache[(modname, contextfile)] = value | 172 self._mod_file_cache[(modname, contextfile)] = value |
172 if isinstance(value, AstroidBuildingException): | 173 if isinstance(value, ASTNGBuildingException): |
173 raise value | 174 raise value |
174 return value | 175 return value |
175 | 176 |
176 def ast_from_module(self, module, modname=None): | 177 def astng_from_module(self, module, modname=None): |
177 """given an imported module, return the astroid object""" | 178 """given an imported module, return the astng object""" |
178 modname = modname or module.__name__ | 179 modname = modname or module.__name__ |
179 if modname in self.astroid_cache: | 180 if modname in self.astng_cache: |
180 return self.astroid_cache[modname] | 181 return self.astng_cache[modname] |
181 try: | 182 try: |
182 # some builtin modules don't have __file__ attribute | 183 # some builtin modules don't have __file__ attribute |
183 filepath = module.__file__ | 184 filepath = module.__file__ |
184 if is_python_source(filepath): | 185 if is_python_source(filepath): |
185 return self.ast_from_file(filepath, modname) | 186 return self.astng_from_file(filepath, modname) |
186 except AttributeError: | 187 except AttributeError: |
187 pass | 188 pass |
188 from astroid.builder import AstroidBuilder | 189 from logilab.astng.builder import ASTNGBuilder |
189 return AstroidBuilder(self).module_build(module, modname) | 190 return ASTNGBuilder(self).module_build(module, modname) |
190 | 191 |
191 def ast_from_class(self, klass, modname=None): | 192 def astng_from_class(self, klass, modname=None): |
192 """get astroid for the given class""" | 193 """get astng for the given class""" |
193 if modname is None: | 194 if modname is None: |
194 try: | 195 try: |
195 modname = klass.__module__ | 196 modname = klass.__module__ |
196 except AttributeError: | 197 except AttributeError: |
197 raise AstroidBuildingException( | 198 raise ASTNGBuildingException( |
198 'Unable to get module for class %s' % safe_repr(klass)) | 199 'Unable to get module for class %s' % safe_repr(klass)) |
199 modastroid = self.ast_from_module_name(modname) | 200 modastng = self.astng_from_module_name(modname) |
200 return modastroid.getattr(klass.__name__)[0] # XXX | 201 return modastng.getattr(klass.__name__)[0] # XXX |
201 | 202 |
202 | 203 |
203 def infer_ast_from_something(self, obj, context=None): | 204 def infer_astng_from_something(self, obj, context=None): |
204 """infer astroid for the given class""" | 205 """infer astng for the given class""" |
205 if hasattr(obj, '__class__') and not isinstance(obj, type): | 206 if hasattr(obj, '__class__') and not isinstance(obj, type): |
206 klass = obj.__class__ | 207 klass = obj.__class__ |
207 else: | 208 else: |
208 klass = obj | 209 klass = obj |
209 try: | 210 try: |
210 modname = klass.__module__ | 211 modname = klass.__module__ |
211 except AttributeError: | 212 except AttributeError: |
212 raise AstroidBuildingException( | 213 raise ASTNGBuildingException( |
213 'Unable to get module for %s' % safe_repr(klass)) | 214 'Unable to get module for %s' % safe_repr(klass)) |
214 except Exception, ex: | 215 except Exception, ex: |
215 raise AstroidBuildingException( | 216 raise ASTNGBuildingException( |
216 'Unexpected error while retrieving module for %s: %s' | 217 'Unexpected error while retrieving module for %s: %s' |
217 % (safe_repr(klass), ex)) | 218 % (safe_repr(klass), ex)) |
218 try: | 219 try: |
219 name = klass.__name__ | 220 name = klass.__name__ |
220 except AttributeError: | 221 except AttributeError: |
221 raise AstroidBuildingException( | 222 raise ASTNGBuildingException( |
222 'Unable to get name for %s' % safe_repr(klass)) | 223 'Unable to get name for %s' % safe_repr(klass)) |
223 except Exception, ex: | 224 except Exception, ex: |
224 raise AstroidBuildingException( | 225 raise ASTNGBuildingException( |
225 'Unexpected error while retrieving name for %s: %s' | 226 'Unexpected error while retrieving name for %s: %s' |
226 % (safe_repr(klass), ex)) | 227 % (safe_repr(klass), ex)) |
227 # take care, on living object __module__ is regularly wrong :( | 228 # take care, on living object __module__ is regularly wrong :( |
228 modastroid = self.ast_from_module_name(modname) | 229 modastng = self.astng_from_module_name(modname) |
229 if klass is obj: | 230 if klass is obj: |
230 for infered in modastroid.igetattr(name, context): | 231 for infered in modastng.igetattr(name, context): |
231 yield infered | 232 yield infered |
232 else: | 233 else: |
233 for infered in modastroid.igetattr(name, context): | 234 for infered in modastng.igetattr(name, context): |
234 yield infered.instanciate_class() | 235 yield infered.instanciate_class() |
235 | 236 |
236 def project_from_files(self, files, func_wrapper=astroid_wrapper, | 237 def project_from_files(self, files, func_wrapper=astng_wrapper, |
237 project_name=None, black_list=None): | 238 project_name=None, black_list=None): |
238 """return a Project from a list of files or modules""" | 239 """return a Project from a list of files or modules""" |
239 # build the project representation | 240 # build the project representation |
240 project_name = project_name or self.config.project | 241 project_name = project_name or self.config.project |
241 black_list = black_list or self.config.black_list | 242 black_list = black_list or self.config.black_list |
242 project = Project(project_name) | 243 project = Project(project_name) |
243 for something in files: | 244 for something in files: |
244 if not exists(something): | 245 if not exists(something): |
245 fpath = file_from_modpath(something.split('.')) | 246 fpath = file_from_modpath(something.split('.')) |
246 elif isdir(something): | 247 elif isdir(something): |
247 fpath = join(something, '__init__.py') | 248 fpath = join(something, '__init__.py') |
248 else: | 249 else: |
249 fpath = something | 250 fpath = something |
250 astroid = func_wrapper(self.ast_from_file, fpath) | 251 astng = func_wrapper(self.astng_from_file, fpath) |
251 if astroid is None: | 252 if astng is None: |
252 continue | 253 continue |
253 # XXX why is first file defining the project.path ? | 254 # XXX why is first file defining the project.path ? |
254 project.path = project.path or astroid.file | 255 project.path = project.path or astng.file |
255 project.add_module(astroid) | 256 project.add_module(astng) |
256 base_name = astroid.name | 257 base_name = astng.name |
257 # recurse in package except if __init__ was explicitly given | 258 # recurse in package except if __init__ was explicitly given |
258 if astroid.package and something.find('__init__') == -1: | 259 if astng.package and something.find('__init__') == -1: |
259 # recurse on others packages / modules if this is a package | 260 # recurse on others packages / modules if this is a package |
260 for fpath in get_module_files(dirname(astroid.file), | 261 for fpath in get_module_files(dirname(astng.file), |
261 black_list): | 262 black_list): |
262 astroid = func_wrapper(self.ast_from_file, fpath) | 263 astng = func_wrapper(self.astng_from_file, fpath) |
263 if astroid is None or astroid.name == base_name: | 264 if astng is None or astng.name == base_name: |
264 continue | 265 continue |
265 project.add_module(astroid) | 266 project.add_module(astng) |
266 return project | 267 return project |
267 | 268 |
268 def register_transform(self, node_class, transform, predicate=None): | 269 def register_transformer(self, transformer): |
269 """Register `transform(node)` function to be applied on the given | 270 self.transformers.append(transformer) |
270 Astroid's `node_class` if `predicate` is None or return a true value | |
271 when called with the node as argument. | |
272 | 271 |
273 The transform function may return a value which is then used to | 272 class Project: |
274 substitute the original node in the tree. | |
275 """ | |
276 self.transforms.setdefault(node_class, []).append((transform, predicate)
) | |
277 | |
278 def unregister_transform(self, node_class, transform, predicate=None): | |
279 """Unregister the given transform.""" | |
280 self.transforms[node_class].remove((transform, predicate)) | |
281 | |
282 def transform(self, node): | |
283 """Call matching transforms for the given node if any and return the | |
284 transformed node. | |
285 """ | |
286 cls = node.__class__ | |
287 if cls not in self.transforms: | |
288 # no transform registered for this class of node | |
289 return node | |
290 | |
291 transforms = self.transforms[cls] | |
292 orig_node = node # copy the reference | |
293 for transform_func, predicate in transforms: | |
294 if predicate is None or predicate(node): | |
295 ret = transform_func(node) | |
296 # if the transformation function returns something, it's | |
297 # expected to be a replacement for the node | |
298 if ret is not None: | |
299 if node is not orig_node: | |
300 # node has already be modified by some previous | |
301 # transformation, warn about it | |
302 warn('node %s substituted multiple times' % node) | |
303 node = ret | |
304 return node | |
305 | |
306 def cache_module(self, module): | |
307 """Cache a module if no module with the same name is known yet.""" | |
308 self.astroid_cache.setdefault(module.name, module) | |
309 | |
310 def clear_cache(self): | |
311 self.astroid_cache.clear() | |
312 # force bootstrap again, else we may ends up with cache inconsistency | |
313 # between the manager and CONST_PROXY, making | |
314 # unittest_lookup.LookupTC.test_builtin_lookup fail depending on the | |
315 # test order | |
316 from astroid.raw_building import astroid_bootstrapping | |
317 astroid_bootstrapping() | |
318 | |
319 | |
320 class Project(object): | |
321 """a project handle a set of modules / packages""" | 273 """a project handle a set of modules / packages""" |
322 def __init__(self, name=''): | 274 def __init__(self, name=''): |
323 self.name = name | 275 self.name = name |
324 self.path = None | 276 self.path = None |
325 self.modules = [] | 277 self.modules = [] |
326 self.locals = {} | 278 self.locals = {} |
327 self.__getitem__ = self.locals.__getitem__ | 279 self.__getitem__ = self.locals.__getitem__ |
328 self.__iter__ = self.locals.__iter__ | 280 self.__iter__ = self.locals.__iter__ |
329 self.values = self.locals.values | 281 self.values = self.locals.values |
330 self.keys = self.locals.keys | 282 self.keys = self.locals.keys |
331 self.items = self.locals.items | 283 self.items = self.locals.items |
332 | 284 |
333 def add_module(self, node): | 285 def add_module(self, node): |
334 self.locals[node.name] = node | 286 self.locals[node.name] = node |
335 self.modules.append(node) | 287 self.modules.append(node) |
336 | 288 |
337 def get_module(self, name): | 289 def get_module(self, name): |
338 return self.locals[name] | 290 return self.locals[name] |
339 | 291 |
340 def get_children(self): | 292 def get_children(self): |
341 return self.modules | 293 return self.modules |
342 | 294 |
343 def __repr__(self): | 295 def __repr__(self): |
344 return '<Project %r at %s (%s modules)>' % (self.name, id(self), | 296 return '<Project %r at %s (%s modules)>' % (self.name, id(self), |
345 len(self.modules)) | 297 len(self.modules)) |
346 | 298 |
347 | 299 |
OLD | NEW |