Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names as key 
 29  """ 
 30   
 31  __docformat__ = "restructuredtext en" 
 32   
 33  import sys 
 34  import os 
 35  from os.path import splitext, join, abspath, isdir, dirname, exists, basename 
 36  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 37  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 38  from distutils.errors import DistutilsPlatformError 
 39   
 40  from six.moves import range 
 41   
 42  try: 
 43      import zipimport 
 44  except ImportError: 
 45      zipimport = None 
 46   
 47  ZIPFILE = object() 
 48   
 49  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 50   
 51  # Notes about STD_LIB_DIR 
 52  # Consider arch-specific installation for STD_LIB_DIR definition 
 53  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 54  # 
 55  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 56  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 57  if sys.platform.startswith('win'): 
 58      PY_SOURCE_EXTS = ('py', 'pyw') 
 59      PY_COMPILED_EXTS = ('dll', 'pyd') 
 60  else: 
 61      PY_SOURCE_EXTS = ('py',) 
 62      PY_COMPILED_EXTS = ('so',) 
 63   
 64  try: 
 65      STD_LIB_DIR = get_python_lib(standard_lib=True) 
 66  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 67  # non-valid path, see https://bugs.pypy.org/issue1164 
 68  except DistutilsPlatformError: 
 69      STD_LIB_DIR = '//' 
 70   
 71  EXT_LIB_DIR = get_python_lib() 
 72   
 73  BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) 
 74   
 75   
76 -class NoSourceFile(Exception):
77 """exception raised when we are not able to get a python 78 source file for a precompiled file 79 """
80
81 -class LazyObject(object):
82 - def __init__(self, module, obj):
83 self.module = module 84 self.obj = obj 85 self._imported = None
86
87 - def _getobj(self):
88 if self._imported is None: 89 self._imported = getattr(load_module_from_name(self.module), 90 self.obj) 91 return self._imported
92
93 - def __getattribute__(self, attr):
94 try: 95 return super(LazyObject, self).__getattribute__(attr) 96 except AttributeError as ex: 97 return getattr(self._getobj(), attr)
98
99 - def __call__(self, *args, **kwargs):
100 return self._getobj()(*args, **kwargs)
101 102
103 -def load_module_from_name(dotted_name, path=None, use_sys=True):
104 """Load a Python module from its name. 105 106 :type dotted_name: str 107 :param dotted_name: python name of a module or package 108 109 :type path: list or None 110 :param path: 111 optional list of path where the module or package should be 112 searched (use sys.path if nothing or None is given) 113 114 :type use_sys: bool 115 :param use_sys: 116 boolean indicating whether the sys.modules dictionary should be 117 used or not 118 119 120 :raise ImportError: if the module or package is not found 121 122 :rtype: module 123 :return: the loaded module 124 """ 125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126 127
128 -def load_module_from_modpath(parts, path=None, use_sys=True):
129 """Load a python module from its splitted name. 130 131 :type parts: list(str) or tuple(str) 132 :param parts: 133 python name of a module or package splitted on '.' 134 135 :type path: list or None 136 :param path: 137 optional list of path where the module or package should be 138 searched (use sys.path if nothing or None is given) 139 140 :type use_sys: bool 141 :param use_sys: 142 boolean indicating whether the sys.modules dictionary should be used or not 143 144 :raise ImportError: if the module or package is not found 145 146 :rtype: module 147 :return: the loaded module 148 """ 149 if use_sys: 150 try: 151 return sys.modules['.'.join(parts)] 152 except KeyError: 153 pass 154 modpath = [] 155 prevmodule = None 156 for part in parts: 157 modpath.append(part) 158 curname = '.'.join(modpath) 159 module = None 160 if len(modpath) != len(parts): 161 # even with use_sys=False, should try to get outer packages from sys.modules 162 module = sys.modules.get(curname) 163 elif use_sys: 164 # because it may have been indirectly loaded through a parent 165 module = sys.modules.get(curname) 166 if module is None: 167 mp_file, mp_filename, mp_desc = find_module(part, path) 168 try: 169 module = load_module(curname, mp_file, mp_filename, mp_desc) 170 finally: 171 if mp_file is not None: 172 mp_file.close() 173 if prevmodule: 174 setattr(prevmodule, part, module) 175 _file = getattr(module, '__file__', '') 176 prevmodule = module 177 if not _file and _is_namespace(curname): 178 continue 179 if not _file and len(modpath) != len(parts): 180 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 181 path = [dirname( _file )] 182 return module
183 184
185 -def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
186 """Load a Python module from it's path. 187 188 :type filepath: str 189 :param filepath: path to the python module or package 190 191 :type path: list or None 192 :param path: 193 optional list of path where the module or package should be 194 searched (use sys.path if nothing or None is given) 195 196 :type use_sys: bool 197 :param use_sys: 198 boolean indicating whether the sys.modules dictionary should be 199 used or not 200 201 202 :raise ImportError: if the module or package is not found 203 204 :rtype: module 205 :return: the loaded module 206 """ 207 modpath = modpath_from_file(filepath, extrapath) 208 return load_module_from_modpath(modpath, path, use_sys)
209 210
211 -def _check_init(path, mod_path):
212 """check there are some __init__.py all along the way""" 213 modpath = [] 214 for part in mod_path: 215 modpath.append(part) 216 path = join(path, part) 217 if not _is_namespace('.'.join(modpath)) and not _has_init(path): 218 return False 219 return True
220 221
222 -def modpath_from_file(filename, extrapath=None):
223 """given a file path return the corresponding splitted module's name 224 (i.e name of a module or package splitted on '.') 225 226 :type filename: str 227 :param filename: file's path for which we want the module's name 228 229 :type extrapath: dict 230 :param extrapath: 231 optional extra search path, with path as key and package name for the path 232 as value. This is usually useful to handle package splitted in multiple 233 directories using __path__ trick. 234 235 236 :raise ImportError: 237 if the corresponding module's name has not been found 238 239 :rtype: list(str) 240 :return: the corresponding splitted module's name 241 """ 242 base = splitext(abspath(filename))[0] 243 if extrapath is not None: 244 for path_ in extrapath: 245 path = abspath(path_) 246 if path and base[:len(path)] == path: 247 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 248 if pkg] 249 if _check_init(path, submodpath[:-1]): 250 return extrapath[path_].split('.') + submodpath 251 for path in sys.path: 252 path = abspath(path) 253 if path and base.startswith(path): 254 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 255 if _check_init(path, modpath[:-1]): 256 return modpath 257 raise ImportError('Unable to find module for %s in %s' % ( 258 filename, ', \n'.join(sys.path)))
259 260 261
262 -def file_from_modpath(modpath, path=None, context_file=None):
263 """given a mod path (i.e. splitted module / package name), return the 264 corresponding file, giving priority to source file over precompiled 265 file if it exists 266 267 :type modpath: list or tuple 268 :param modpath: 269 splitted module's name (i.e name of a module or package splitted 270 on '.') 271 (this means explicit relative imports that start with dots have 272 empty strings in this list!) 273 274 :type path: list or None 275 :param path: 276 optional list of path where the module or package should be 277 searched (use sys.path if nothing or None is given) 278 279 :type context_file: str or None 280 :param context_file: 281 context file to consider, necessary if the identifier has been 282 introduced using a relative import unresolvable in the actual 283 context (i.e. modutils) 284 285 :raise ImportError: if there is no such module in the directory 286 287 :rtype: str or None 288 :return: 289 the path to the module's file or None if it's an integrated 290 builtin module such as 'sys' 291 """ 292 if context_file is not None: 293 context = dirname(context_file) 294 else: 295 context = context_file 296 if modpath[0] == 'xml': 297 # handle _xmlplus 298 try: 299 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 300 except ImportError: 301 return _file_from_modpath(modpath, path, context) 302 elif modpath == ['os', 'path']: 303 # FIXME: currently ignoring search_path... 304 return os.path.__file__ 305 return _file_from_modpath(modpath, path, context)
306 307 308
309 -def get_module_part(dotted_name, context_file=None):
310 """given a dotted name return the module part of the name : 311 312 >>> get_module_part('logilab.common.modutils.get_module_part') 313 'logilab.common.modutils' 314 315 :type dotted_name: str 316 :param dotted_name: full name of the identifier we are interested in 317 318 :type context_file: str or None 319 :param context_file: 320 context file to consider, necessary if the identifier has been 321 introduced using a relative import unresolvable in the actual 322 context (i.e. modutils) 323 324 325 :raise ImportError: if there is no such module in the directory 326 327 :rtype: str or None 328 :return: 329 the module part of the name or None if we have not been able at 330 all to import the given name 331 332 XXX: deprecated, since it doesn't handle package precedence over module 333 (see #10066) 334 """ 335 # os.path trick 336 if dotted_name.startswith('os.path'): 337 return 'os.path' 338 parts = dotted_name.split('.') 339 if context_file is not None: 340 # first check for builtin module which won't be considered latter 341 # in that case (path != None) 342 if parts[0] in BUILTIN_MODULES: 343 if len(parts) > 2: 344 raise ImportError(dotted_name) 345 return parts[0] 346 # don't use += or insert, we want a new list to be created ! 347 path = None 348 starti = 0 349 if parts[0] == '': 350 assert context_file is not None, \ 351 'explicit relative import, but no context_file?' 352 path = [] # prevent resolving the import non-relatively 353 starti = 1 354 while parts[starti] == '': # for all further dots: change context 355 starti += 1 356 context_file = dirname(context_file) 357 for i in range(starti, len(parts)): 358 try: 359 file_from_modpath(parts[starti:i+1], 360 path=path, context_file=context_file) 361 except ImportError: 362 if not i >= max(1, len(parts) - 2): 363 raise 364 return '.'.join(parts[:i]) 365 return dotted_name
366 367
368 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
369 """given a package directory return a list of all available python 370 modules in the package and its subpackages 371 372 :type package: str 373 :param package: the python name for the package 374 375 :type src_directory: str 376 :param src_directory: 377 path of the directory corresponding to the package 378 379 :type blacklist: list or tuple 380 :param blacklist: 381 optional list of files or directory to ignore, default to 382 the value of `logilab.common.STD_BLACKLIST` 383 384 :rtype: list 385 :return: 386 the list of all available python modules in the package and its 387 subpackages 388 """ 389 modules = [] 390 for directory, dirnames, filenames in os.walk(src_directory): 391 _handle_blacklist(blacklist, dirnames, filenames) 392 # check for __init__.py 393 if not '__init__.py' in filenames: 394 dirnames[:] = () 395 continue 396 if directory != src_directory: 397 dir_package = directory[len(src_directory):].replace(os.sep, '.') 398 modules.append(package + dir_package) 399 for filename in filenames: 400 if _is_python_file(filename) and filename != '__init__.py': 401 src = join(directory, filename) 402 module = package + src[len(src_directory):-3] 403 modules.append(module.replace(os.sep, '.')) 404 return modules
405 406 407
408 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
409 """given a package directory return a list of all available python 410 module's files in the package and its subpackages 411 412 :type src_directory: str 413 :param src_directory: 414 path of the directory corresponding to the package 415 416 :type blacklist: list or tuple 417 :param blacklist: 418 optional list of files or directory to ignore, default to the value of 419 `logilab.common.STD_BLACKLIST` 420 421 :rtype: list 422 :return: 423 the list of all available python module's files in the package and 424 its subpackages 425 """ 426 files = [] 427 for directory, dirnames, filenames in os.walk(src_directory): 428 _handle_blacklist(blacklist, dirnames, filenames) 429 # check for __init__.py 430 if not '__init__.py' in filenames: 431 dirnames[:] = () 432 continue 433 for filename in filenames: 434 if _is_python_file(filename): 435 src = join(directory, filename) 436 files.append(src) 437 return files
438 439
440 -def get_source_file(filename, include_no_ext=False):
441 """given a python module's file name return the matching source file 442 name (the filename will be returned identically if it's a already an 443 absolute path to a python source file...) 444 445 :type filename: str 446 :param filename: python module's file name 447 448 449 :raise NoSourceFile: if no source file exists on the file system 450 451 :rtype: str 452 :return: the absolute path of the source file if it exists 453 """ 454 base, orig_ext = splitext(abspath(filename)) 455 for ext in PY_SOURCE_EXTS: 456 source_path = '%s.%s' % (base, ext) 457 if exists(source_path): 458 return source_path 459 if include_no_ext and not orig_ext and exists(base): 460 return base 461 raise NoSourceFile(filename)
462 463
464 -def cleanup_sys_modules(directories):
465 """remove submodules of `directories` from `sys.modules`""" 466 cleaned = [] 467 for modname, module in list(sys.modules.items()): 468 modfile = getattr(module, '__file__', None) 469 if modfile: 470 for directory in directories: 471 if modfile.startswith(directory): 472 cleaned.append(modname) 473 del sys.modules[modname] 474 break 475 return cleaned
476 477
478 -def is_python_source(filename):
479 """ 480 rtype: bool 481 return: True if the filename is a python source file 482 """ 483 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
484 485
486 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
487 """try to guess if a module is a standard python module (by default, 488 see `std_path` parameter's description) 489 490 :type modname: str 491 :param modname: name of the module we are interested in 492 493 :type std_path: list(str) or tuple(str) 494 :param std_path: list of path considered as standard 495 496 497 :rtype: bool 498 :return: 499 true if the module: 500 - is located on the path listed in one of the directory in `std_path` 501 - is a built-in module 502 503 Note: this function is known to return wrong values when inside virtualenv. 504 See https://www.logilab.org/ticket/294756. 505 """ 506 modname = modname.split('.')[0] 507 try: 508 filename = file_from_modpath([modname]) 509 except ImportError as ex: 510 # import failed, i'm probably not so wrong by supposing it's 511 # not standard... 512 return False 513 # modules which are not living in a file are considered standard 514 # (sys and __builtin__ for instance) 515 if filename is None: 516 # we assume there are no namespaces in stdlib 517 return not _is_namespace(modname) 518 filename = abspath(filename) 519 if filename.startswith(EXT_LIB_DIR): 520 return False 521 for path in std_path: 522 if filename.startswith(abspath(path)): 523 return True 524 return False
525 526 527
528 -def is_relative(modname, from_file):
529 """return true if the given module name is relative to the given 530 file name 531 532 :type modname: str 533 :param modname: name of the module we are interested in 534 535 :type from_file: str 536 :param from_file: 537 path of the module from which modname has been imported 538 539 :rtype: bool 540 :return: 541 true if the module has been imported relatively to `from_file` 542 """ 543 if not isdir(from_file): 544 from_file = dirname(from_file) 545 if from_file in sys.path: 546 return False 547 try: 548 find_module(modname.split('.')[0], [from_file]) 549 return True 550 except ImportError: 551 return False
552 553 554 # internal only functions ##################################################### 555
556 -def _file_from_modpath(modpath, path=None, context=None):
557 """given a mod path (i.e. splitted module / package name), return the 558 corresponding file 559 560 this function is used internally, see `file_from_modpath`'s 561 documentation for more information 562 """ 563 assert len(modpath) > 0 564 if context is not None: 565 try: 566 mtype, mp_filename = _module_file(modpath, [context]) 567 except ImportError: 568 mtype, mp_filename = _module_file(modpath, path) 569 else: 570 mtype, mp_filename = _module_file(modpath, path) 571 if mtype == PY_COMPILED: 572 try: 573 return get_source_file(mp_filename) 574 except NoSourceFile: 575 return mp_filename 576 elif mtype == C_BUILTIN: 577 # integrated builtin module 578 return None 579 elif mtype == PKG_DIRECTORY: 580 mp_filename = _has_init(mp_filename) 581 return mp_filename
582
583 -def _search_zip(modpath, pic):
584 for filepath, importer in pic.items(): 585 if importer is not None: 586 if importer.find_module(modpath[0]): 587 if not importer.find_module('/'.join(modpath)): 588 raise ImportError('No module named %s in %s/%s' % ( 589 '.'.join(modpath[1:]), filepath, modpath)) 590 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 591 raise ImportError('No module named %s' % '.'.join(modpath))
592 593 try: 594 import pkg_resources 595 except ImportError: 596 pkg_resources = None 597 598
599 -def _is_namespace(modname):
600 return (pkg_resources is not None 601 and modname in pkg_resources._namespace_packages)
602 603
604 -def _module_file(modpath, path=None):
605 """get a module type / file path 606 607 :type modpath: list or tuple 608 :param modpath: 609 splitted module's name (i.e name of a module or package splitted 610 on '.'), with leading empty strings for explicit relative import 611 612 :type path: list or None 613 :param path: 614 optional list of path where the module or package should be 615 searched (use sys.path if nothing or None is given) 616 617 618 :rtype: tuple(int, str) 619 :return: the module type flag and the file path for a module 620 """ 621 # egg support compat 622 try: 623 pic = sys.path_importer_cache 624 _path = (path is None and sys.path or path) 625 for __path in _path: 626 if not __path in pic: 627 try: 628 pic[__path] = zipimport.zipimporter(__path) 629 except zipimport.ZipImportError: 630 pic[__path] = None 631 checkeggs = True 632 except AttributeError: 633 checkeggs = False 634 # pkg_resources support (aka setuptools namespace packages) 635 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules): 636 # setuptools has added into sys.modules a module object with proper 637 # __path__, get back information from there 638 module = sys.modules[modpath.pop(0)] 639 path = module.__path__ 640 if not modpath: 641 return C_BUILTIN, None 642 imported = [] 643 while modpath: 644 modname = modpath[0] 645 # take care to changes in find_module implementation wrt builtin modules 646 # 647 # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23) 648 # >>> imp.find_module('posix') 649 # (None, 'posix', ('', '', 6)) 650 # 651 # Python 3.3.1 (default, Apr 26 2013, 12:08:46) 652 # >>> imp.find_module('posix') 653 # (None, None, ('', '', 6)) 654 try: 655 _, mp_filename, mp_desc = find_module(modname, path) 656 except ImportError: 657 if checkeggs: 658 return _search_zip(modpath, pic)[:2] 659 raise 660 else: 661 if checkeggs and mp_filename: 662 fullabspath = [abspath(x) for x in _path] 663 try: 664 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 665 emtype, emp_filename, zippath = _search_zip(modpath, pic) 666 if pathindex > _path.index(zippath): 667 # an egg takes priority 668 return emtype, emp_filename 669 except ValueError: 670 # XXX not in _path 671 pass 672 except ImportError: 673 pass 674 checkeggs = False 675 imported.append(modpath.pop(0)) 676 mtype = mp_desc[2] 677 if modpath: 678 if mtype != PKG_DIRECTORY: 679 raise ImportError('No module %s in %s' % ('.'.join(modpath), 680 '.'.join(imported))) 681 # XXX guess if package is using pkgutil.extend_path by looking for 682 # those keywords in the first four Kbytes 683 try: 684 with open(join(mp_filename, '__init__.py')) as stream: 685 data = stream.read(4096) 686 except IOError: 687 path = [mp_filename] 688 else: 689 if 'pkgutil' in data and 'extend_path' in data: 690 # extend_path is called, search sys.path for module/packages 691 # of this name see pkgutil.extend_path documentation 692 path = [join(p, *imported) for p in sys.path 693 if isdir(join(p, *imported))] 694 else: 695 path = [mp_filename] 696 return mtype, mp_filename
697
698 -def _is_python_file(filename):
699 """return true if the given filename should be considered as a python file 700 701 .pyc and .pyo are ignored 702 """ 703 for ext in ('.py', '.so', '.pyd', '.pyw'): 704 if filename.endswith(ext): 705 return True 706 return False
707 708
709 -def _has_init(directory):
710 """if the given directory has a valid __init__ file, return its path, 711 else return None 712 """ 713 mod_or_pack = join(directory, '__init__') 714 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 715 if exists(mod_or_pack + '.' + ext): 716 return mod_or_pack + '.' + ext 717 return None
718