OLD | NEW |
(Empty) | |
| 1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
| 2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
| 3 # |
| 4 # This file is part of astroid. |
| 5 # |
| 6 # astroid is free software: you can redistribute it and/or modify it |
| 7 # under the terms of the GNU Lesser General Public License as published by the |
| 8 # Free Software Foundation, either version 2.1 of the License, or (at your |
| 9 # option) any later version. |
| 10 # |
| 11 # astroid is distributed in the hope that it will be useful, but |
| 12 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License |
| 14 # for more details. |
| 15 # |
| 16 # You should have received a copy of the GNU Lesser General Public License along |
| 17 # with astroid. If not, see <http://www.gnu.org/licenses/>. |
| 18 """astroid manager: avoid multiple astroid build of a same module when |
| 19 possible by providing a class responsible to get astroid representation |
| 20 from various source and using a cache of built modules) |
| 21 """ |
| 22 from __future__ import print_function |
| 23 |
| 24 __docformat__ = "restructuredtext en" |
| 25 |
| 26 import collections |
| 27 import imp |
| 28 import os |
| 29 from os.path import dirname, join, isdir, exists |
| 30 from warnings import warn |
| 31 import zipimport |
| 32 |
| 33 from logilab.common.configuration import OptionsProviderMixIn |
| 34 |
| 35 from astroid.exceptions import AstroidBuildingException |
| 36 from astroid import modutils |
| 37 |
| 38 |
| 39 def astroid_wrapper(func, modname): |
| 40 """wrapper to give to AstroidManager.project_from_files""" |
| 41 print('parsing %s...' % modname) |
| 42 try: |
| 43 return func(modname) |
| 44 except AstroidBuildingException as exc: |
| 45 print(exc) |
| 46 except Exception as exc: |
| 47 import traceback |
| 48 traceback.print_exc() |
| 49 |
| 50 def _silent_no_wrap(func, modname): |
| 51 """silent wrapper that doesn't do anything; can be used for tests""" |
| 52 return func(modname) |
| 53 |
| 54 def safe_repr(obj): |
| 55 try: |
| 56 return repr(obj) |
| 57 except: |
| 58 return '???' |
| 59 |
| 60 |
| 61 |
| 62 class AstroidManager(OptionsProviderMixIn): |
| 63 """the astroid manager, responsible to build astroid from files |
| 64 or modules. |
| 65 |
| 66 Use the Borg pattern. |
| 67 """ |
| 68 |
| 69 name = 'astroid loader' |
| 70 options = (("ignore", |
| 71 {'type' : "csv", 'metavar' : "<file>", |
| 72 'dest' : "black_list", "default" : ('CVS',), |
| 73 'help' : "add <file> (may be a directory) to the black list\ |
| 74 . It should be a base name, not a path. You may set this option multiple times\ |
| 75 ."}), |
| 76 ("project", |
| 77 {'default': "No Name", 'type' : 'string', 'short': 'p', |
| 78 'metavar' : '<project name>', |
| 79 'help' : 'set the project name.'}), |
| 80 ) |
| 81 brain = {} |
| 82 def __init__(self): |
| 83 self.__dict__ = AstroidManager.brain |
| 84 if not self.__dict__: |
| 85 OptionsProviderMixIn.__init__(self) |
| 86 self.load_defaults() |
| 87 # NOTE: cache entries are added by the [re]builder |
| 88 self.astroid_cache = {} |
| 89 self._mod_file_cache = {} |
| 90 self.transforms = collections.defaultdict(list) |
| 91 self._failed_import_hooks = [] |
| 92 self.always_load_extensions = False |
| 93 self.extension_package_whitelist = set() |
| 94 |
| 95 def ast_from_file(self, filepath, modname=None, fallback=True, source=False)
: |
| 96 """given a module name, return the astroid object""" |
| 97 try: |
| 98 filepath = modutils.get_source_file(filepath, include_no_ext=True) |
| 99 source = True |
| 100 except modutils.NoSourceFile: |
| 101 pass |
| 102 if modname is None: |
| 103 try: |
| 104 modname = '.'.join(modutils.modpath_from_file(filepath)) |
| 105 except ImportError: |
| 106 modname = filepath |
| 107 if modname in self.astroid_cache and self.astroid_cache[modname].file ==
filepath: |
| 108 return self.astroid_cache[modname] |
| 109 if source: |
| 110 from astroid.builder import AstroidBuilder |
| 111 return AstroidBuilder(self).file_build(filepath, modname) |
| 112 elif fallback and modname: |
| 113 return self.ast_from_module_name(modname) |
| 114 raise AstroidBuildingException('unable to get astroid for file %s' % |
| 115 filepath) |
| 116 |
| 117 def _build_stub_module(self, modname): |
| 118 from astroid.builder import AstroidBuilder |
| 119 return AstroidBuilder(self).string_build('', modname) |
| 120 |
| 121 def _can_load_extension(self, modname): |
| 122 if self.always_load_extensions: |
| 123 return True |
| 124 if modutils.is_standard_module(modname): |
| 125 return True |
| 126 parts = modname.split('.') |
| 127 return any( |
| 128 '.'.join(parts[:x]) in self.extension_package_whitelist |
| 129 for x in range(1, len(parts) + 1)) |
| 130 |
| 131 def ast_from_module_name(self, modname, context_file=None): |
| 132 """given a module name, return the astroid object""" |
| 133 if modname in self.astroid_cache: |
| 134 return self.astroid_cache[modname] |
| 135 if modname == '__main__': |
| 136 return self._build_stub_module(modname) |
| 137 old_cwd = os.getcwd() |
| 138 if context_file: |
| 139 os.chdir(dirname(context_file)) |
| 140 try: |
| 141 filepath, mp_type = self.file_from_module_name(modname, context_file
) |
| 142 if mp_type == modutils.PY_ZIPMODULE: |
| 143 module = self.zip_import_data(filepath) |
| 144 if module is not None: |
| 145 return module |
| 146 elif mp_type in (imp.C_BUILTIN, imp.C_EXTENSION): |
| 147 if mp_type == imp.C_EXTENSION and not self._can_load_extension(m
odname): |
| 148 return self._build_stub_module(modname) |
| 149 try: |
| 150 module = modutils.load_module_from_name(modname) |
| 151 except Exception as ex: |
| 152 msg = 'Unable to load module %s (%s)' % (modname, ex) |
| 153 raise AstroidBuildingException(msg) |
| 154 return self.ast_from_module(module, modname) |
| 155 elif mp_type == imp.PY_COMPILED: |
| 156 raise AstroidBuildingException("Unable to load compiled module %
s" % (modname,)) |
| 157 if filepath is None: |
| 158 raise AstroidBuildingException("Unable to load module %s" % (mod
name,)) |
| 159 return self.ast_from_file(filepath, modname, fallback=False) |
| 160 except AstroidBuildingException as e: |
| 161 for hook in self._failed_import_hooks: |
| 162 try: |
| 163 return hook(modname) |
| 164 except AstroidBuildingException: |
| 165 pass |
| 166 raise e |
| 167 finally: |
| 168 os.chdir(old_cwd) |
| 169 |
| 170 def zip_import_data(self, filepath): |
| 171 if zipimport is None: |
| 172 return None |
| 173 from astroid.builder import AstroidBuilder |
| 174 builder = AstroidBuilder(self) |
| 175 for ext in ('.zip', '.egg'): |
| 176 try: |
| 177 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1) |
| 178 except ValueError: |
| 179 continue |
| 180 try: |
| 181 importer = zipimport.zipimporter(eggpath + ext) |
| 182 zmodname = resource.replace(os.path.sep, '.') |
| 183 if importer.is_package(resource): |
| 184 zmodname = zmodname + '.__init__' |
| 185 module = builder.string_build(importer.get_source(resource), |
| 186 zmodname, filepath) |
| 187 return module |
| 188 except: |
| 189 continue |
| 190 return None |
| 191 |
| 192 def file_from_module_name(self, modname, contextfile): |
| 193 try: |
| 194 value = self._mod_file_cache[(modname, contextfile)] |
| 195 except KeyError: |
| 196 try: |
| 197 value = modutils.file_info_from_modpath( |
| 198 modname.split('.'), context_file=contextfile) |
| 199 except ImportError as ex: |
| 200 msg = 'Unable to load module %s (%s)' % (modname, ex) |
| 201 value = AstroidBuildingException(msg) |
| 202 self._mod_file_cache[(modname, contextfile)] = value |
| 203 if isinstance(value, AstroidBuildingException): |
| 204 raise value |
| 205 return value |
| 206 |
| 207 def ast_from_module(self, module, modname=None): |
| 208 """given an imported module, return the astroid object""" |
| 209 modname = modname or module.__name__ |
| 210 if modname in self.astroid_cache: |
| 211 return self.astroid_cache[modname] |
| 212 try: |
| 213 # some builtin modules don't have __file__ attribute |
| 214 filepath = module.__file__ |
| 215 if modutils.is_python_source(filepath): |
| 216 return self.ast_from_file(filepath, modname) |
| 217 except AttributeError: |
| 218 pass |
| 219 from astroid.builder import AstroidBuilder |
| 220 return AstroidBuilder(self).module_build(module, modname) |
| 221 |
| 222 def ast_from_class(self, klass, modname=None): |
| 223 """get astroid for the given class""" |
| 224 if modname is None: |
| 225 try: |
| 226 modname = klass.__module__ |
| 227 except AttributeError: |
| 228 raise AstroidBuildingException( |
| 229 'Unable to get module for class %s' % safe_repr(klass)) |
| 230 modastroid = self.ast_from_module_name(modname) |
| 231 return modastroid.getattr(klass.__name__)[0] # XXX |
| 232 |
| 233 |
| 234 def infer_ast_from_something(self, obj, context=None): |
| 235 """infer astroid for the given class""" |
| 236 if hasattr(obj, '__class__') and not isinstance(obj, type): |
| 237 klass = obj.__class__ |
| 238 else: |
| 239 klass = obj |
| 240 try: |
| 241 modname = klass.__module__ |
| 242 except AttributeError: |
| 243 raise AstroidBuildingException( |
| 244 'Unable to get module for %s' % safe_repr(klass)) |
| 245 except Exception as ex: |
| 246 raise AstroidBuildingException( |
| 247 'Unexpected error while retrieving module for %s: %s' |
| 248 % (safe_repr(klass), ex)) |
| 249 try: |
| 250 name = klass.__name__ |
| 251 except AttributeError: |
| 252 raise AstroidBuildingException( |
| 253 'Unable to get name for %s' % safe_repr(klass)) |
| 254 except Exception as ex: |
| 255 raise AstroidBuildingException( |
| 256 'Unexpected error while retrieving name for %s: %s' |
| 257 % (safe_repr(klass), ex)) |
| 258 # take care, on living object __module__ is regularly wrong :( |
| 259 modastroid = self.ast_from_module_name(modname) |
| 260 if klass is obj: |
| 261 for infered in modastroid.igetattr(name, context): |
| 262 yield infered |
| 263 else: |
| 264 for infered in modastroid.igetattr(name, context): |
| 265 yield infered.instanciate_class() |
| 266 |
| 267 def project_from_files(self, files, func_wrapper=astroid_wrapper, |
| 268 project_name=None, black_list=None): |
| 269 """return a Project from a list of files or modules""" |
| 270 # build the project representation |
| 271 project_name = project_name or self.config.project |
| 272 black_list = black_list or self.config.black_list |
| 273 project = Project(project_name) |
| 274 for something in files: |
| 275 if not exists(something): |
| 276 fpath = modutils.file_from_modpath(something.split('.')) |
| 277 elif isdir(something): |
| 278 fpath = join(something, '__init__.py') |
| 279 else: |
| 280 fpath = something |
| 281 astroid = func_wrapper(self.ast_from_file, fpath) |
| 282 if astroid is None: |
| 283 continue |
| 284 # XXX why is first file defining the project.path ? |
| 285 project.path = project.path or astroid.file |
| 286 project.add_module(astroid) |
| 287 base_name = astroid.name |
| 288 # recurse in package except if __init__ was explicitly given |
| 289 if astroid.package and something.find('__init__') == -1: |
| 290 # recurse on others packages / modules if this is a package |
| 291 for fpath in modutils.get_module_files(dirname(astroid.file), |
| 292 black_list): |
| 293 astroid = func_wrapper(self.ast_from_file, fpath) |
| 294 if astroid is None or astroid.name == base_name: |
| 295 continue |
| 296 project.add_module(astroid) |
| 297 return project |
| 298 |
| 299 def register_transform(self, node_class, transform, predicate=None): |
| 300 """Register `transform(node)` function to be applied on the given |
| 301 Astroid's `node_class` if `predicate` is None or returns true |
| 302 when called with the node as argument. |
| 303 |
| 304 The transform function may return a value which is then used to |
| 305 substitute the original node in the tree. |
| 306 """ |
| 307 self.transforms[node_class].append((transform, predicate)) |
| 308 |
| 309 def unregister_transform(self, node_class, transform, predicate=None): |
| 310 """Unregister the given transform.""" |
| 311 self.transforms[node_class].remove((transform, predicate)) |
| 312 |
| 313 def register_failed_import_hook(self, hook): |
| 314 """Registers a hook to resolve imports that cannot be found otherwise. |
| 315 |
| 316 `hook` must be a function that accepts a single argument `modname` which |
| 317 contains the name of the module or package that could not be imported. |
| 318 If `hook` can resolve the import, must return a node of type `astroid.Mo
dule`, |
| 319 otherwise, it must raise `AstroidBuildingException`. |
| 320 """ |
| 321 self._failed_import_hooks.append(hook) |
| 322 |
| 323 def transform(self, node): |
| 324 """Call matching transforms for the given node if any and return the |
| 325 transformed node. |
| 326 """ |
| 327 cls = node.__class__ |
| 328 if cls not in self.transforms: |
| 329 # no transform registered for this class of node |
| 330 return node |
| 331 |
| 332 transforms = self.transforms[cls] |
| 333 orig_node = node # copy the reference |
| 334 for transform_func, predicate in transforms: |
| 335 if predicate is None or predicate(node): |
| 336 ret = transform_func(node) |
| 337 # if the transformation function returns something, it's |
| 338 # expected to be a replacement for the node |
| 339 if ret is not None: |
| 340 if node is not orig_node: |
| 341 # node has already be modified by some previous |
| 342 # transformation, warn about it |
| 343 warn('node %s substituted multiple times' % node) |
| 344 node = ret |
| 345 return node |
| 346 |
| 347 def cache_module(self, module): |
| 348 """Cache a module if no module with the same name is known yet.""" |
| 349 self.astroid_cache.setdefault(module.name, module) |
| 350 |
| 351 def clear_cache(self, astroid_builtin=None): |
| 352 # XXX clear transforms |
| 353 self.astroid_cache.clear() |
| 354 # force bootstrap again, else we may ends up with cache inconsistency |
| 355 # between the manager and CONST_PROXY, making |
| 356 # unittest_lookup.LookupTC.test_builtin_lookup fail depending on the |
| 357 # test order |
| 358 import astroid.raw_building |
| 359 astroid.raw_building._astroid_bootstrapping( |
| 360 astroid_builtin=astroid_builtin) |
| 361 |
| 362 |
| 363 class Project(object): |
| 364 """a project handle a set of modules / packages""" |
| 365 def __init__(self, name=''): |
| 366 self.name = name |
| 367 self.path = None |
| 368 self.modules = [] |
| 369 self.locals = {} |
| 370 self.__getitem__ = self.locals.__getitem__ |
| 371 self.__iter__ = self.locals.__iter__ |
| 372 self.values = self.locals.values |
| 373 self.keys = self.locals.keys |
| 374 self.items = self.locals.items |
| 375 |
| 376 def add_module(self, node): |
| 377 self.locals[node.name] = node |
| 378 self.modules.append(node) |
| 379 |
| 380 def get_module(self, name): |
| 381 return self.locals[name] |
| 382 |
| 383 def get_children(self): |
| 384 return self.modules |
| 385 |
| 386 def __repr__(self): |
| 387 return '<Project %r at %s (%s modules)>' % (self.name, id(self), |
| 388 len(self.modules)) |
| 389 |
| 390 |
OLD | NEW |