Revert "Revert "pylint: upgrade to 1.3.1""

third_party/logilab/common/modutils.py

 # -*- coding: utf-8 -*-
-# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
 #
 # This file is part of logilab-common.
 #
 # logilab-common is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Lesser General Public License as published by the Free
 # Software Foundation, either version 2.1 of the License, or (at your option) any
 # later version.
 #
 # logilab-common is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
 # details.
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
 """Python modules manipulation utility functions.
 
 :type PY_SOURCE_EXTS: tuple(str)
 :var PY_SOURCE_EXTS: list of possible python source file extension
 
 :type STD_LIB_DIR: str
 :var STD_LIB_DIR: directory where standard modules are located
 
 :type BUILTIN_MODULES: dict
 :var BUILTIN_MODULES: dictionary with builtin module names has key
 """
+
 __docformat__ = "restructuredtext en"
 
 import sys
 import os
 from os.path import splitext, join, abspath, isdir, dirname, exists, basename
 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
+from distutils.errors import DistutilsPlatformError
+
+from six.moves import range
 
 try:
     import zipimport
 except ImportError:
     zipimport = None
 
 ZIPFILE = object()
 
 from logilab.common import STD_BLACKLIST, _handle_blacklist
 
 # Notes about STD_LIB_DIR
 # Consider arch-specific installation for STD_LIB_DIR definition
 # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on
 #
 # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_
 # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_
 if sys.platform.startswith('win'):
     PY_SOURCE_EXTS = ('py', 'pyw')
     PY_COMPILED_EXTS = ('dll', 'pyd')
-    STD_LIB_DIR = get_python_lib(standard_lib=1)
 else:
     PY_SOURCE_EXTS = ('py',)
     PY_COMPILED_EXTS = ('so',)
-    # extend lib dir with some arch-dependant paths
-    STD_LIB_DIR = join(get_config_var("LIBDIR"), "python%s" % get_python_version())
+
+try:
+    STD_LIB_DIR = get_python_lib(standard_lib=1)
+# get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to
+# non-valid path, see https://bugs.pypy.org/issue1164
+except DistutilsPlatformError:
+    STD_LIB_DIR = '//'
+
+EXT_LIB_DIR = get_python_lib()
 
 BUILTIN_MODULES = dict(zip(sys.builtin_module_names,
                            [1]*len(sys.builtin_module_names)))
 
 
 class NoSourceFile(Exception):
     """exception raised when we are not able to get a python
     source file for a precompiled file
     """
 
 class LazyObject(object):
     def __init__(self, module, obj):
         self.module = module
         self.obj = obj
         self._imported = None
 
     def _getobj(self):
         if self._imported is None:
            self._imported = getattr(load_module_from_name(self.module),
                                     self.obj)
         return self._imported
 
     def __getattribute__(self, attr):
         try:
             return super(LazyObject, self).__getattribute__(attr)
-        except AttributeError, ex:
+        except AttributeError as ex:
             return getattr(self._getobj(), attr)
 
     def __call__(self, *args, **kwargs):
         return self._getobj()(*args, **kwargs)
 
 
 def load_module_from_name(dotted_name, path=None, use_sys=1):
-    """Load a Python module from it's name.
+    """Load a Python module from its name.
 
     :type dotted_name: str
     :param dotted_name: python name of a module or package
 
     :type path: list or None
     :param path:
       optional list of path where the module or package should be
       searched (use sys.path if nothing or None is given)
 
     :type use_sys: bool
     :param use_sys:
       boolean indicating whether the sys.modules dictionary should be
       used or not
 
 
     :raise ImportError: if the module or package is not found
 
     :rtype: module
     :return: the loaded module
     """
     return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
 
 
 def load_module_from_modpath(parts, path=None, use_sys=1):
-    """Load a python module from it's splitted name.
+    """Load a python module from its splitted name.
 
     :type parts: list(str) or tuple(str)
     :param parts:
       python name of a module or package splitted on '.'
 
     :type path: list or None
     :param path:
       optional list of path where the module or package should be
       searched (use sys.path if nothing or None is given)
 
@@ skipping 13 matching lines @@
             pass
     modpath = []
     prevmodule = None
     for part in parts:
         modpath.append(part)
         curname = '.'.join(modpath)
         module = None
         if len(modpath) != len(parts):
             # even with use_sys=False, should try to get outer packages from sys.modules
             module = sys.modules.get(curname)
+        elif use_sys:
+            # because it may have been indirectly loaded through a parent
+            module = sys.modules.get(curname)
         if module is None:
             mp_file, mp_filename, mp_desc = find_module(part, path)
             module = load_module(curname, mp_file, mp_filename, mp_desc)
         if prevmodule:
             setattr(prevmodule, part, module)
         _file = getattr(module, '__file__', '')
         if not _file and len(modpath) != len(parts):
             raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
         path = [dirname( _file )]
         prevmodule = module
@@ skipping 59 matching lines @@
     if extrapath is not None:
         for path_ in extrapath:
             path = abspath(path_)
             if path and base[:len(path)] == path:
                 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
                               if pkg]
                 if _check_init(path, submodpath[:-1]):
                     return extrapath[path_].split('.') + submodpath
     for path in sys.path:
         path = abspath(path)
-        if path and base[:len(path)] == path:
-            if filename.find('site-packages') != -1 and \
-                   path.find('site-packages') == -1:
-                continue
+        if path and base.startswith(path):
             modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
             if _check_init(path, modpath[:-1]):
                 return modpath
     raise ImportError('Unable to find module for %s in %s' % (
         filename, ', \n'.join(sys.path)))
 
 
 
 def file_from_modpath(modpath, path=None, context_file=None):
     """given a mod path (i.e. splitted module / package name), return the
@@ skipping 192 matching lines @@
         source_path = '%s.%s' % (base, ext)
         if exists(source_path):
             return source_path
     if include_no_ext and not orig_ext and exists(base):
         return base
     raise NoSourceFile(filename)
 
 
 def cleanup_sys_modules(directories):
     """remove submodules of `directories` from `sys.modules`"""
-    for modname, module in sys.modules.items():
+    cleaned = []
+    for modname, module in list(sys.modules.items()):
         modfile = getattr(module, '__file__', None)
         if modfile:
             for directory in directories:
                 if modfile.startswith(directory):
+                    cleaned.append(modname)
                     del sys.modules[modname]
                     break
+    return cleaned
 
 
 def is_python_source(filename):
     """
     rtype: bool
     return: True if the filename is a python source file
     """
     return splitext(filename)[1][1:] in PY_SOURCE_EXTS
 
 
@@ skipping 11 matching lines @@
 
     :rtype: bool
     :return:
       true if the module:
       - is located on the path listed in one of the directory in `std_path`
       - is a built-in module
     """
     modname = modname.split('.')[0]
     try:
         filename = file_from_modpath([modname])
-    except ImportError, ex:
+    except ImportError as ex:
         # import failed, i'm probably not so wrong by supposing it's
         # not standard...
         return 0
     # modules which are not living in a file are considered standard
     # (sys and __builtin__ for instance)
     if filename is None:
         return 1
     filename = abspath(filename)
+    if filename.startswith(EXT_LIB_DIR):
+        return 0
     for path in std_path:
-        path = abspath(path)
-        if filename.startswith(path):
-            pfx_len = len(path)
-            if filename[pfx_len+1:pfx_len+14] != 'site-packages':
-                return 1
-            return 0
+        if filename.startswith(abspath(path)):
+            return 1
     return False
 
 
 
 def is_relative(modname, from_file):
     """return true if the given module name is relative to the given
     file name
 
     :type modname: str
     :param modname: name of the module we are interested in
@@ skipping 45 matching lines @@
     elif mtype == PKG_DIRECTORY:
         mp_filename = _has_init(mp_filename)
     return mp_filename
 
 def _search_zip(modpath, pic):
     for filepath, importer in pic.items():
         if importer is not None:
             if importer.find_module(modpath[0]):
                 if not importer.find_module('/'.join(modpath)):
                     raise ImportError('No module named %s in %s/%s' % (
-                        '.'.join(modpath[1:]), file, modpath))
+                        '.'.join(modpath[1:]), filepath, modpath))
                 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
     raise ImportError('No module named %s' % '.'.join(modpath))
 
+try:
+    import pkg_resources
+except ImportError:
+    pkg_resources = None
+
 def _module_file(modpath, path=None):
     """get a module type / file path
 
     :type modpath: list or tuple
     :param modpath:
       splitted module's name (i.e name of a module or package splitted
       on '.'), with leading empty strings for explicit relative import
 
     :type path: list or None
     :param path:
@@ skipping 10 matching lines @@
         _path = (path is None and sys.path or path)
         for __path in _path:
             if not __path in pic:
                 try:
                     pic[__path] = zipimport.zipimporter(__path)
                 except zipimport.ZipImportError:
                     pic[__path] = None
         checkeggs = True
     except AttributeError:
         checkeggs = False
+    # pkg_resources support (aka setuptools namespace packages)
+    if (pkg_resources is not None
+            and modpath[0] in pkg_resources._namespace_packages
+            and modpath[0] in sys.modules
+            and len(modpath) > 1):
+        # setuptools has added into sys.modules a module object with proper
+        # __path__, get back information from there
+        module = sys.modules[modpath.pop(0)]
+        path = module.__path__
     imported = []
     while modpath:
+        modname = modpath[0]
+        # take care to changes in find_module implementation wrt builtin modules
+        #
+        # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23)
+        # >>> imp.find_module('posix')
+        # (None, 'posix', ('', '', 6))
+        #
+        # Python 3.3.1 (default, Apr 26 2013, 12:08:46)
+        # >>> imp.find_module('posix')
+        # (None, None, ('', '', 6))
         try:
-            _, mp_filename, mp_desc = find_module(modpath[0], path)
+            _, mp_filename, mp_desc = find_module(modname, path)
         except ImportError:
             if checkeggs:
                 return _search_zip(modpath, pic)[:2]
             raise
         else:
-            if checkeggs:
+            if checkeggs and mp_filename:
                 fullabspath = [abspath(x) for x in _path]
                 try:
                     pathindex = fullabspath.index(dirname(abspath(mp_filename)))
                     emtype, emp_filename, zippath = _search_zip(modpath, pic)
                     if pathindex > _path.index(zippath):
                         # an egg takes priority
                         return emtype, emp_filename
                 except ValueError:
                     # XXX not in _path
                     pass
                 except ImportError:
                     pass
                 checkeggs = False
         imported.append(modpath.pop(0))
         mtype = mp_desc[2]
         if modpath:
             if mtype != PKG_DIRECTORY:
                 raise ImportError('No module %s in %s' % ('.'.join(modpath),
                                                           '.'.join(imported)))
-            path = [mp_filename]
+            # XXX guess if package is using pkgutil.extend_path by looking for
+            # those keywords in the first four Kbytes
+            try:
+                with open(join(mp_filename, '__init__.py')) as stream:
+                    data = stream.read(4096)
+            except IOError:
+                path = [mp_filename]
+            else:
+                if 'pkgutil' in data and 'extend_path' in data:
+                    # extend_path is called, search sys.path for module/packages
+                    # of this name see pkgutil.extend_path documentation
+                    path = [join(p, *imported) for p in sys.path
+                            if isdir(join(p, *imported))]
+                else:
+                    path = [mp_filename]
     return mtype, mp_filename
 
 def _is_python_file(filename):
     """return true if the given filename should be considered as a python file
 
     .pyc and .pyo are ignored
     """
     for ext in ('.py', '.so', '.pyd', '.pyw'):
         if filename.endswith(ext):
             return True
     return False
 
 
 def _has_init(directory):
     """if the given directory has a valid __init__ file, return its path,
     else return None
     """
     mod_or_pack = join(directory, '__init__')
     for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
         if exists(mod_or_pack + '.' + ext):
             return mod_or_pack + '.' + ext
     return None