Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(64)

Side by Side Diff: third_party/logilab/logilab/astroid/manager.py

Issue 1920403002: [content/test/gpu] Run pylint check of gpu tests in unittest instead of PRESUBMIT (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Update path to LICENSE.txt of logilab/README.chromium Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
3 #
4 # This file is part of astroid.
5 #
6 # astroid is free software: you can redistribute it and/or modify it
7 # under the terms of the GNU Lesser General Public License as published by the
8 # Free Software Foundation, either version 2.1 of the License, or (at your
9 # option) any later version.
10 #
11 # astroid is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
14 # for more details.
15 #
16 # You should have received a copy of the GNU Lesser General Public License along
17 # with astroid. If not, see <http://www.gnu.org/licenses/>.
18 """astroid manager: avoid multiple astroid build of a same module when
19 possible by providing a class responsible to get astroid representation
20 from various source and using a cache of built modules)
21 """
22 from __future__ import print_function
23
24 __docformat__ = "restructuredtext en"
25
26 import collections
27 import imp
28 import os
29 from os.path import dirname, join, isdir, exists
30 from warnings import warn
31 import zipimport
32
33 from logilab.common.configuration import OptionsProviderMixIn
34
35 from astroid.exceptions import AstroidBuildingException
36 from astroid import modutils
37
38
39 def astroid_wrapper(func, modname):
40 """wrapper to give to AstroidManager.project_from_files"""
41 print('parsing %s...' % modname)
42 try:
43 return func(modname)
44 except AstroidBuildingException as exc:
45 print(exc)
46 except Exception as exc:
47 import traceback
48 traceback.print_exc()
49
50 def _silent_no_wrap(func, modname):
51 """silent wrapper that doesn't do anything; can be used for tests"""
52 return func(modname)
53
54 def safe_repr(obj):
55 try:
56 return repr(obj)
57 except:
58 return '???'
59
60
61
62 class AstroidManager(OptionsProviderMixIn):
63 """the astroid manager, responsible to build astroid from files
64 or modules.
65
66 Use the Borg pattern.
67 """
68
69 name = 'astroid loader'
70 options = (("ignore",
71 {'type' : "csv", 'metavar' : "<file>",
72 'dest' : "black_list", "default" : ('CVS',),
73 'help' : "add <file> (may be a directory) to the black list\
74 . It should be a base name, not a path. You may set this option multiple times\
75 ."}),
76 ("project",
77 {'default': "No Name", 'type' : 'string', 'short': 'p',
78 'metavar' : '<project name>',
79 'help' : 'set the project name.'}),
80 )
81 brain = {}
82 def __init__(self):
83 self.__dict__ = AstroidManager.brain
84 if not self.__dict__:
85 OptionsProviderMixIn.__init__(self)
86 self.load_defaults()
87 # NOTE: cache entries are added by the [re]builder
88 self.astroid_cache = {}
89 self._mod_file_cache = {}
90 self.transforms = collections.defaultdict(list)
91 self._failed_import_hooks = []
92 self.always_load_extensions = False
93 self.extension_package_whitelist = set()
94
95 def ast_from_file(self, filepath, modname=None, fallback=True, source=False) :
96 """given a module name, return the astroid object"""
97 try:
98 filepath = modutils.get_source_file(filepath, include_no_ext=True)
99 source = True
100 except modutils.NoSourceFile:
101 pass
102 if modname is None:
103 try:
104 modname = '.'.join(modutils.modpath_from_file(filepath))
105 except ImportError:
106 modname = filepath
107 if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
108 return self.astroid_cache[modname]
109 if source:
110 from astroid.builder import AstroidBuilder
111 return AstroidBuilder(self).file_build(filepath, modname)
112 elif fallback and modname:
113 return self.ast_from_module_name(modname)
114 raise AstroidBuildingException('unable to get astroid for file %s' %
115 filepath)
116
117 def _build_stub_module(self, modname):
118 from astroid.builder import AstroidBuilder
119 return AstroidBuilder(self).string_build('', modname)
120
121 def _can_load_extension(self, modname):
122 if self.always_load_extensions:
123 return True
124 if modutils.is_standard_module(modname):
125 return True
126 parts = modname.split('.')
127 return any(
128 '.'.join(parts[:x]) in self.extension_package_whitelist
129 for x in range(1, len(parts) + 1))
130
131 def ast_from_module_name(self, modname, context_file=None):
132 """given a module name, return the astroid object"""
133 if modname in self.astroid_cache:
134 return self.astroid_cache[modname]
135 if modname == '__main__':
136 return self._build_stub_module(modname)
137 old_cwd = os.getcwd()
138 if context_file:
139 os.chdir(dirname(context_file))
140 try:
141 filepath, mp_type = self.file_from_module_name(modname, context_file )
142 if mp_type == modutils.PY_ZIPMODULE:
143 module = self.zip_import_data(filepath)
144 if module is not None:
145 return module
146 elif mp_type in (imp.C_BUILTIN, imp.C_EXTENSION):
147 if mp_type == imp.C_EXTENSION and not self._can_load_extension(m odname):
148 return self._build_stub_module(modname)
149 try:
150 module = modutils.load_module_from_name(modname)
151 except Exception as ex:
152 msg = 'Unable to load module %s (%s)' % (modname, ex)
153 raise AstroidBuildingException(msg)
154 return self.ast_from_module(module, modname)
155 elif mp_type == imp.PY_COMPILED:
156 raise AstroidBuildingException("Unable to load compiled module % s" % (modname,))
157 if filepath is None:
158 raise AstroidBuildingException("Unable to load module %s" % (mod name,))
159 return self.ast_from_file(filepath, modname, fallback=False)
160 except AstroidBuildingException as e:
161 for hook in self._failed_import_hooks:
162 try:
163 return hook(modname)
164 except AstroidBuildingException:
165 pass
166 raise e
167 finally:
168 os.chdir(old_cwd)
169
170 def zip_import_data(self, filepath):
171 if zipimport is None:
172 return None
173 from astroid.builder import AstroidBuilder
174 builder = AstroidBuilder(self)
175 for ext in ('.zip', '.egg'):
176 try:
177 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
178 except ValueError:
179 continue
180 try:
181 importer = zipimport.zipimporter(eggpath + ext)
182 zmodname = resource.replace(os.path.sep, '.')
183 if importer.is_package(resource):
184 zmodname = zmodname + '.__init__'
185 module = builder.string_build(importer.get_source(resource),
186 zmodname, filepath)
187 return module
188 except:
189 continue
190 return None
191
192 def file_from_module_name(self, modname, contextfile):
193 try:
194 value = self._mod_file_cache[(modname, contextfile)]
195 except KeyError:
196 try:
197 value = modutils.file_info_from_modpath(
198 modname.split('.'), context_file=contextfile)
199 except ImportError as ex:
200 msg = 'Unable to load module %s (%s)' % (modname, ex)
201 value = AstroidBuildingException(msg)
202 self._mod_file_cache[(modname, contextfile)] = value
203 if isinstance(value, AstroidBuildingException):
204 raise value
205 return value
206
207 def ast_from_module(self, module, modname=None):
208 """given an imported module, return the astroid object"""
209 modname = modname or module.__name__
210 if modname in self.astroid_cache:
211 return self.astroid_cache[modname]
212 try:
213 # some builtin modules don't have __file__ attribute
214 filepath = module.__file__
215 if modutils.is_python_source(filepath):
216 return self.ast_from_file(filepath, modname)
217 except AttributeError:
218 pass
219 from astroid.builder import AstroidBuilder
220 return AstroidBuilder(self).module_build(module, modname)
221
222 def ast_from_class(self, klass, modname=None):
223 """get astroid for the given class"""
224 if modname is None:
225 try:
226 modname = klass.__module__
227 except AttributeError:
228 raise AstroidBuildingException(
229 'Unable to get module for class %s' % safe_repr(klass))
230 modastroid = self.ast_from_module_name(modname)
231 return modastroid.getattr(klass.__name__)[0] # XXX
232
233
234 def infer_ast_from_something(self, obj, context=None):
235 """infer astroid for the given class"""
236 if hasattr(obj, '__class__') and not isinstance(obj, type):
237 klass = obj.__class__
238 else:
239 klass = obj
240 try:
241 modname = klass.__module__
242 except AttributeError:
243 raise AstroidBuildingException(
244 'Unable to get module for %s' % safe_repr(klass))
245 except Exception as ex:
246 raise AstroidBuildingException(
247 'Unexpected error while retrieving module for %s: %s'
248 % (safe_repr(klass), ex))
249 try:
250 name = klass.__name__
251 except AttributeError:
252 raise AstroidBuildingException(
253 'Unable to get name for %s' % safe_repr(klass))
254 except Exception as ex:
255 raise AstroidBuildingException(
256 'Unexpected error while retrieving name for %s: %s'
257 % (safe_repr(klass), ex))
258 # take care, on living object __module__ is regularly wrong :(
259 modastroid = self.ast_from_module_name(modname)
260 if klass is obj:
261 for infered in modastroid.igetattr(name, context):
262 yield infered
263 else:
264 for infered in modastroid.igetattr(name, context):
265 yield infered.instanciate_class()
266
267 def project_from_files(self, files, func_wrapper=astroid_wrapper,
268 project_name=None, black_list=None):
269 """return a Project from a list of files or modules"""
270 # build the project representation
271 project_name = project_name or self.config.project
272 black_list = black_list or self.config.black_list
273 project = Project(project_name)
274 for something in files:
275 if not exists(something):
276 fpath = modutils.file_from_modpath(something.split('.'))
277 elif isdir(something):
278 fpath = join(something, '__init__.py')
279 else:
280 fpath = something
281 astroid = func_wrapper(self.ast_from_file, fpath)
282 if astroid is None:
283 continue
284 # XXX why is first file defining the project.path ?
285 project.path = project.path or astroid.file
286 project.add_module(astroid)
287 base_name = astroid.name
288 # recurse in package except if __init__ was explicitly given
289 if astroid.package and something.find('__init__') == -1:
290 # recurse on others packages / modules if this is a package
291 for fpath in modutils.get_module_files(dirname(astroid.file),
292 black_list):
293 astroid = func_wrapper(self.ast_from_file, fpath)
294 if astroid is None or astroid.name == base_name:
295 continue
296 project.add_module(astroid)
297 return project
298
299 def register_transform(self, node_class, transform, predicate=None):
300 """Register `transform(node)` function to be applied on the given
301 Astroid's `node_class` if `predicate` is None or returns true
302 when called with the node as argument.
303
304 The transform function may return a value which is then used to
305 substitute the original node in the tree.
306 """
307 self.transforms[node_class].append((transform, predicate))
308
309 def unregister_transform(self, node_class, transform, predicate=None):
310 """Unregister the given transform."""
311 self.transforms[node_class].remove((transform, predicate))
312
313 def register_failed_import_hook(self, hook):
314 """Registers a hook to resolve imports that cannot be found otherwise.
315
316 `hook` must be a function that accepts a single argument `modname` which
317 contains the name of the module or package that could not be imported.
318 If `hook` can resolve the import, must return a node of type `astroid.Mo dule`,
319 otherwise, it must raise `AstroidBuildingException`.
320 """
321 self._failed_import_hooks.append(hook)
322
323 def transform(self, node):
324 """Call matching transforms for the given node if any and return the
325 transformed node.
326 """
327 cls = node.__class__
328 if cls not in self.transforms:
329 # no transform registered for this class of node
330 return node
331
332 transforms = self.transforms[cls]
333 orig_node = node # copy the reference
334 for transform_func, predicate in transforms:
335 if predicate is None or predicate(node):
336 ret = transform_func(node)
337 # if the transformation function returns something, it's
338 # expected to be a replacement for the node
339 if ret is not None:
340 if node is not orig_node:
341 # node has already be modified by some previous
342 # transformation, warn about it
343 warn('node %s substituted multiple times' % node)
344 node = ret
345 return node
346
347 def cache_module(self, module):
348 """Cache a module if no module with the same name is known yet."""
349 self.astroid_cache.setdefault(module.name, module)
350
351 def clear_cache(self, astroid_builtin=None):
352 # XXX clear transforms
353 self.astroid_cache.clear()
354 # force bootstrap again, else we may ends up with cache inconsistency
355 # between the manager and CONST_PROXY, making
356 # unittest_lookup.LookupTC.test_builtin_lookup fail depending on the
357 # test order
358 import astroid.raw_building
359 astroid.raw_building._astroid_bootstrapping(
360 astroid_builtin=astroid_builtin)
361
362
363 class Project(object):
364 """a project handle a set of modules / packages"""
365 def __init__(self, name=''):
366 self.name = name
367 self.path = None
368 self.modules = []
369 self.locals = {}
370 self.__getitem__ = self.locals.__getitem__
371 self.__iter__ = self.locals.__iter__
372 self.values = self.locals.values
373 self.keys = self.locals.keys
374 self.items = self.locals.items
375
376 def add_module(self, node):
377 self.locals[node.name] = node
378 self.modules.append(node)
379
380 def get_module(self, name):
381 return self.locals[name]
382
383 def get_children(self):
384 return self.modules
385
386 def __repr__(self):
387 return '<Project %r at %s (%s modules)>' % (self.name, id(self),
388 len(self.modules))
389
390
OLDNEW
« no previous file with comments | « third_party/logilab/logilab/astroid/inspector.py ('k') | third_party/logilab/logilab/astroid/mixins.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698