OLD | NEW |
(Empty) | |
| 1 # Copyright (c) 2003-2013 LOGILAB S.A. (Paris, FRANCE). |
| 2 # http://www.logilab.fr/ -- mailto:contact@logilab.fr |
| 3 # |
| 4 # This program is free software; you can redistribute it and/or modify it under |
| 5 # the terms of the GNU General Public License as published by the Free Software |
| 6 # Foundation; either version 2 of the License, or (at your option) any later |
| 7 # version. |
| 8 # |
| 9 # This program is distributed in the hope that it will be useful, but WITHOUT |
| 10 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| 11 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
| 12 # |
| 13 # You should have received a copy of the GNU General Public License along with |
| 14 # this program; if not, write to the Free Software Foundation, Inc., |
| 15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 16 """imports checkers for Python code""" |
| 17 |
| 18 import sys |
| 19 from collections import defaultdict |
| 20 |
| 21 import six |
| 22 from six.moves import map # pylint: disable=redefined-builtin |
| 23 |
| 24 from logilab.common.graph import get_cycles, DotBackend |
| 25 from logilab.common.ureports import VerbatimText, Paragraph |
| 26 |
| 27 import astroid |
| 28 from astroid import are_exclusive |
| 29 from astroid.modutils import get_module_part, is_standard_module |
| 30 |
| 31 from pylint.interfaces import IAstroidChecker |
| 32 from pylint.utils import EmptyReport |
| 33 from pylint.checkers import BaseChecker |
| 34 from pylint.checkers.utils import check_messages, is_import_error |
| 35 |
| 36 def _except_import_error(node): |
| 37 """ |
| 38 Check if the try-except node has an ImportError handler. |
| 39 Return True if an ImportError handler was infered, False otherwise. |
| 40 """ |
| 41 if not isinstance(node, astroid.TryExcept): |
| 42 return |
| 43 return any(map(is_import_error, node.handlers)) |
| 44 |
| 45 def get_first_import(node, context, name, base, level): |
| 46 """return the node where [base.]<name> is imported or None if not found |
| 47 """ |
| 48 fullname = '%s.%s' % (base, name) if base else name |
| 49 |
| 50 first = None |
| 51 found = False |
| 52 for first in context.body: |
| 53 if first is node: |
| 54 continue |
| 55 if first.scope() is node.scope() and first.fromlineno > node.fromlineno: |
| 56 continue |
| 57 if isinstance(first, astroid.Import): |
| 58 if any(fullname == iname[0] for iname in first.names): |
| 59 found = True |
| 60 break |
| 61 elif isinstance(first, astroid.From): |
| 62 if level == first.level and any( |
| 63 fullname == '%s.%s' % (first.modname, iname[0]) |
| 64 for iname in first.names): |
| 65 found = True |
| 66 break |
| 67 if found and not are_exclusive(first, node): |
| 68 return first |
| 69 |
| 70 # utilities to represents import dependencies as tree and dot graph ########### |
| 71 |
| 72 def make_tree_defs(mod_files_list): |
| 73 """get a list of 2-uple (module, list_of_files_which_import_this_module), |
| 74 it will return a dictionary to represent this as a tree |
| 75 """ |
| 76 tree_defs = {} |
| 77 for mod, files in mod_files_list: |
| 78 node = (tree_defs, ()) |
| 79 for prefix in mod.split('.'): |
| 80 node = node[0].setdefault(prefix, [{}, []]) |
| 81 node[1] += files |
| 82 return tree_defs |
| 83 |
| 84 def repr_tree_defs(data, indent_str=None): |
| 85 """return a string which represents imports as a tree""" |
| 86 lines = [] |
| 87 nodes = data.items() |
| 88 for i, (mod, (sub, files)) in enumerate(sorted(nodes, key=lambda x: x[0])): |
| 89 if not files: |
| 90 files = '' |
| 91 else: |
| 92 files = '(%s)' % ','.join(files) |
| 93 if indent_str is None: |
| 94 lines.append('%s %s' % (mod, files)) |
| 95 sub_indent_str = ' ' |
| 96 else: |
| 97 lines.append(r'%s\-%s %s' % (indent_str, mod, files)) |
| 98 if i == len(nodes)-1: |
| 99 sub_indent_str = '%s ' % indent_str |
| 100 else: |
| 101 sub_indent_str = '%s| ' % indent_str |
| 102 if sub: |
| 103 lines.append(repr_tree_defs(sub, sub_indent_str)) |
| 104 return '\n'.join(lines) |
| 105 |
| 106 |
| 107 def dependencies_graph(filename, dep_info): |
| 108 """write dependencies as a dot (graphviz) file |
| 109 """ |
| 110 done = {} |
| 111 printer = DotBackend(filename[:-4], rankdir='LR') |
| 112 printer.emit('URL="." node[shape="box"]') |
| 113 for modname, dependencies in sorted(six.iteritems(dep_info)): |
| 114 done[modname] = 1 |
| 115 printer.emit_node(modname) |
| 116 for modname in dependencies: |
| 117 if modname not in done: |
| 118 done[modname] = 1 |
| 119 printer.emit_node(modname) |
| 120 for depmodname, dependencies in sorted(six.iteritems(dep_info)): |
| 121 for modname in dependencies: |
| 122 printer.emit_edge(modname, depmodname) |
| 123 printer.generate(filename) |
| 124 |
| 125 |
| 126 def make_graph(filename, dep_info, sect, gtype): |
| 127 """generate a dependencies graph and add some information about it in the |
| 128 report's section |
| 129 """ |
| 130 dependencies_graph(filename, dep_info) |
| 131 sect.append(Paragraph('%simports graph has been written to %s' |
| 132 % (gtype, filename))) |
| 133 |
| 134 |
| 135 # the import checker itself ################################################### |
| 136 |
| 137 MSGS = { |
| 138 'F0401': ('Unable to import %s', |
| 139 'import-error', |
| 140 'Used when pylint has been unable to import a module.'), |
| 141 'R0401': ('Cyclic import (%s)', |
| 142 'cyclic-import', |
| 143 'Used when a cyclic import between two or more modules is \ |
| 144 detected.'), |
| 145 |
| 146 'W0401': ('Wildcard import %s', |
| 147 'wildcard-import', |
| 148 'Used when `from module import *` is detected.'), |
| 149 'W0402': ('Uses of a deprecated module %r', |
| 150 'deprecated-module', |
| 151 'Used a module marked as deprecated is imported.'), |
| 152 'W0403': ('Relative import %r, should be %r', |
| 153 'relative-import', |
| 154 'Used when an import relative to the package directory is ' |
| 155 'detected.', |
| 156 {'maxversion': (3, 0)}), |
| 157 'W0404': ('Reimport %r (imported line %s)', |
| 158 'reimported', |
| 159 'Used when a module is reimported multiple times.'), |
| 160 'W0406': ('Module import itself', |
| 161 'import-self', |
| 162 'Used when a module is importing itself.'), |
| 163 |
| 164 'W0410': ('__future__ import is not the first non docstring statement', |
| 165 'misplaced-future', |
| 166 'Python 2.5 and greater require __future__ import to be the \ |
| 167 first non docstring statement in the module.', |
| 168 {'maxversion': (3, 0)}), |
| 169 } |
| 170 |
| 171 class ImportsChecker(BaseChecker): |
| 172 """checks for |
| 173 * external modules dependencies |
| 174 * relative / wildcard imports |
| 175 * cyclic imports |
| 176 * uses of deprecated modules |
| 177 """ |
| 178 |
| 179 __implements__ = IAstroidChecker |
| 180 |
| 181 name = 'imports' |
| 182 msgs = MSGS |
| 183 priority = -2 |
| 184 |
| 185 if sys.version_info < (3,): |
| 186 deprecated_modules = ('regsub', 'TERMIOS', 'Bastion', 'rexec') |
| 187 else: |
| 188 deprecated_modules = ('stringprep', 'optparse') |
| 189 options = (('deprecated-modules', |
| 190 {'default' : deprecated_modules, |
| 191 'type' : 'csv', |
| 192 'metavar' : '<modules>', |
| 193 'help' : 'Deprecated modules which should not be used, \ |
| 194 separated by a comma'} |
| 195 ), |
| 196 ('import-graph', |
| 197 {'default' : '', |
| 198 'type' : 'string', |
| 199 'metavar' : '<file.dot>', |
| 200 'help' : 'Create a graph of every (i.e. internal and \ |
| 201 external) dependencies in the given file (report RP0402 must not be disabled)'} |
| 202 ), |
| 203 ('ext-import-graph', |
| 204 {'default' : '', |
| 205 'type' : 'string', |
| 206 'metavar' : '<file.dot>', |
| 207 'help' : 'Create a graph of external dependencies in the \ |
| 208 given file (report RP0402 must not be disabled)'} |
| 209 ), |
| 210 ('int-import-graph', |
| 211 {'default' : '', |
| 212 'type' : 'string', |
| 213 'metavar' : '<file.dot>', |
| 214 'help' : 'Create a graph of internal dependencies in the \ |
| 215 given file (report RP0402 must not be disabled)'} |
| 216 ), |
| 217 ) |
| 218 |
| 219 def __init__(self, linter=None): |
| 220 BaseChecker.__init__(self, linter) |
| 221 self.stats = None |
| 222 self.import_graph = None |
| 223 self.__int_dep_info = self.__ext_dep_info = None |
| 224 self.reports = (('RP0401', 'External dependencies', |
| 225 self.report_external_dependencies), |
| 226 ('RP0402', 'Modules dependencies graph', |
| 227 self.report_dependencies_graph), |
| 228 ) |
| 229 |
| 230 def open(self): |
| 231 """called before visiting project (i.e set of modules)""" |
| 232 self.linter.add_stats(dependencies={}) |
| 233 self.linter.add_stats(cycles=[]) |
| 234 self.stats = self.linter.stats |
| 235 self.import_graph = defaultdict(set) |
| 236 |
| 237 def close(self): |
| 238 """called before visiting project (i.e set of modules)""" |
| 239 # don't try to compute cycles if the associated message is disabled |
| 240 if self.linter.is_message_enabled('cyclic-import'): |
| 241 vertices = list(self.import_graph) |
| 242 for cycle in get_cycles(self.import_graph, vertices=vertices): |
| 243 self.add_message('cyclic-import', args=' -> '.join(cycle)) |
| 244 |
| 245 def visit_import(self, node): |
| 246 """triggered when an import statement is seen""" |
| 247 modnode = node.root() |
| 248 for name, _ in node.names: |
| 249 importedmodnode = self.get_imported_module(node, name) |
| 250 if importedmodnode is None: |
| 251 continue |
| 252 self._check_relative_import(modnode, node, importedmodnode, name) |
| 253 self._add_imported_module(node, importedmodnode.name) |
| 254 self._check_deprecated_module(node, name) |
| 255 self._check_reimport(node, name) |
| 256 |
| 257 # TODO This appears to be the list of all messages of the checker... |
| 258 # @check_messages('W0410', 'W0401', 'W0403', 'W0402', 'W0404', 'W0406', 'F04
01') |
| 259 @check_messages(*(MSGS.keys())) |
| 260 def visit_from(self, node): |
| 261 """triggered when a from statement is seen""" |
| 262 basename = node.modname |
| 263 if basename == '__future__': |
| 264 # check if this is the first non-docstring statement in the module |
| 265 prev = node.previous_sibling() |
| 266 if prev: |
| 267 # consecutive future statements are possible |
| 268 if not (isinstance(prev, astroid.From) |
| 269 and prev.modname == '__future__'): |
| 270 self.add_message('misplaced-future', node=node) |
| 271 return |
| 272 for name, _ in node.names: |
| 273 if name == '*': |
| 274 self.add_message('wildcard-import', args=basename, node=node) |
| 275 modnode = node.root() |
| 276 importedmodnode = self.get_imported_module(node, basename) |
| 277 if importedmodnode is None: |
| 278 return |
| 279 self._check_relative_import(modnode, node, importedmodnode, basename) |
| 280 self._check_deprecated_module(node, basename) |
| 281 for name, _ in node.names: |
| 282 if name != '*': |
| 283 self._add_imported_module(node, '%s.%s' % (importedmodnode.name,
name)) |
| 284 self._check_reimport(node, name, basename, node.level) |
| 285 |
| 286 def get_imported_module(self, importnode, modname): |
| 287 try: |
| 288 return importnode.do_import_module(modname) |
| 289 except astroid.InferenceError as ex: |
| 290 if str(ex) != modname: |
| 291 args = '%r (%s)' % (modname, ex) |
| 292 else: |
| 293 args = repr(modname) |
| 294 if not _except_import_error(importnode.parent): |
| 295 self.add_message("import-error", args=args, node=importnode) |
| 296 |
| 297 def _check_relative_import(self, modnode, importnode, importedmodnode, |
| 298 importedasname): |
| 299 """check relative import. node is either an Import or From node, modname |
| 300 the imported module name. |
| 301 """ |
| 302 if not self.linter.is_message_enabled('relative-import'): |
| 303 return |
| 304 if importedmodnode.file is None: |
| 305 return False # built-in module |
| 306 if modnode is importedmodnode: |
| 307 return False # module importing itself |
| 308 if modnode.absolute_import_activated() or getattr(importnode, 'level', N
one): |
| 309 return False |
| 310 if importedmodnode.name != importedasname: |
| 311 # this must be a relative import... |
| 312 self.add_message('relative-import', |
| 313 args=(importedasname, importedmodnode.name), |
| 314 node=importnode) |
| 315 |
| 316 def _add_imported_module(self, node, importedmodname): |
| 317 """notify an imported module, used to analyze dependencies""" |
| 318 try: |
| 319 importedmodname = get_module_part(importedmodname) |
| 320 except ImportError: |
| 321 pass |
| 322 context_name = node.root().name |
| 323 if context_name == importedmodname: |
| 324 # module importing itself ! |
| 325 self.add_message('import-self', node=node) |
| 326 elif not is_standard_module(importedmodname): |
| 327 # handle dependencies |
| 328 importedmodnames = self.stats['dependencies'].setdefault( |
| 329 importedmodname, set()) |
| 330 if not context_name in importedmodnames: |
| 331 importedmodnames.add(context_name) |
| 332 # update import graph |
| 333 mgraph = self.import_graph[context_name] |
| 334 if importedmodname not in mgraph: |
| 335 mgraph.add(importedmodname) |
| 336 |
| 337 def _check_deprecated_module(self, node, mod_path): |
| 338 """check if the module is deprecated""" |
| 339 for mod_name in self.config.deprecated_modules: |
| 340 if mod_path == mod_name or mod_path.startswith(mod_name + '.'): |
| 341 self.add_message('deprecated-module', node=node, args=mod_path) |
| 342 |
| 343 def _check_reimport(self, node, name, basename=None, level=None): |
| 344 """check if the import is necessary (i.e. not already done)""" |
| 345 if not self.linter.is_message_enabled('reimported'): |
| 346 return |
| 347 frame = node.frame() |
| 348 root = node.root() |
| 349 contexts = [(frame, level)] |
| 350 if root is not frame: |
| 351 contexts.append((root, None)) |
| 352 for context, level in contexts: |
| 353 first = get_first_import(node, context, name, basename, level) |
| 354 if first is not None: |
| 355 self.add_message('reimported', node=node, |
| 356 args=(name, first.fromlineno)) |
| 357 |
| 358 |
| 359 def report_external_dependencies(self, sect, _, dummy): |
| 360 """return a verbatim layout for displaying dependencies""" |
| 361 dep_info = make_tree_defs(six.iteritems(self._external_dependencies_info
())) |
| 362 if not dep_info: |
| 363 raise EmptyReport() |
| 364 tree_str = repr_tree_defs(dep_info) |
| 365 sect.append(VerbatimText(tree_str)) |
| 366 |
| 367 def report_dependencies_graph(self, sect, _, dummy): |
| 368 """write dependencies as a dot (graphviz) file""" |
| 369 dep_info = self.stats['dependencies'] |
| 370 if not dep_info or not (self.config.import_graph |
| 371 or self.config.ext_import_graph |
| 372 or self.config.int_import_graph): |
| 373 raise EmptyReport() |
| 374 filename = self.config.import_graph |
| 375 if filename: |
| 376 make_graph(filename, dep_info, sect, '') |
| 377 filename = self.config.ext_import_graph |
| 378 if filename: |
| 379 make_graph(filename, self._external_dependencies_info(), |
| 380 sect, 'external ') |
| 381 filename = self.config.int_import_graph |
| 382 if filename: |
| 383 make_graph(filename, self._internal_dependencies_info(), |
| 384 sect, 'internal ') |
| 385 |
| 386 def _external_dependencies_info(self): |
| 387 """return cached external dependencies information or build and |
| 388 cache them |
| 389 """ |
| 390 if self.__ext_dep_info is None: |
| 391 package = self.linter.current_name |
| 392 self.__ext_dep_info = result = {} |
| 393 for importee, importers in six.iteritems(self.stats['dependencies'])
: |
| 394 if not importee.startswith(package): |
| 395 result[importee] = importers |
| 396 return self.__ext_dep_info |
| 397 |
| 398 def _internal_dependencies_info(self): |
| 399 """return cached internal dependencies information or build and |
| 400 cache them |
| 401 """ |
| 402 if self.__int_dep_info is None: |
| 403 package = self.linter.current_name |
| 404 self.__int_dep_info = result = {} |
| 405 for importee, importers in six.iteritems(self.stats['dependencies'])
: |
| 406 if importee.startswith(package): |
| 407 result[importee] = importers |
| 408 return self.__int_dep_info |
| 409 |
| 410 |
| 411 def register(linter): |
| 412 """required method to auto register this checker """ |
| 413 linter.register_checker(ImportsChecker(linter)) |
OLD | NEW |