Package common :: Module modutils
[frames] | no frames]

Source Code for Module common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 29  """ 
 30   
 31   
 32  __docformat__ = "restructuredtext en" 
 33   
 34  import sys 
 35  import os 
 36  from os.path import splitext, join, abspath, isdir, dirname, exists, basename 
 37  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 38  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 39  from distutils.errors import DistutilsPlatformError 
 40   
 41  try: 
 42      import zipimport 
 43  except ImportError: 
 44      zipimport = None 
 45   
 46  ZIPFILE = object() 
 47   
 48  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 49   
 50  # Notes about STD_LIB_DIR 
 51  # Consider arch-specific installation for STD_LIB_DIR definition 
 52  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 53  # 
 54  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 55  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 56  if sys.platform.startswith('win'): 
 57      PY_SOURCE_EXTS = ('py', 'pyw') 
 58      PY_COMPILED_EXTS = ('dll', 'pyd') 
 59  else: 
 60      PY_SOURCE_EXTS = ('py',) 
 61      PY_COMPILED_EXTS = ('so',) 
 62   
 63  try: 
 64      STD_LIB_DIR = get_python_lib(standard_lib=1) 
 65  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 66  # non-valid path, see https://bugs.pypy.org/issue1164 
 67  except DistutilsPlatformError: 
 68      STD_LIB_DIR = '//' 
 69   
 70  EXT_LIB_DIR = get_python_lib() 
 71   
 72  BUILTIN_MODULES = dict(list(zip(sys.builtin_module_names, 
 73                             [1]*len(sys.builtin_module_names)))) 
 74   
 75   
76 -class NoSourceFile(Exception):
77 """exception raised when we are not able to get a python 78 source file for a precompiled file 79 """
80
81 -class LazyObject(object):
82 - def __init__(self, module, obj):
83 self.module = module 84 self.obj = obj 85 self._imported = None
86
87 - def _getobj(self):
88 if self._imported is None: 89 self._imported = getattr(load_module_from_name(self.module), 90 self.obj) 91 return self._imported
92
93 - def __getattribute__(self, attr):
94 try: 95 return super(LazyObject, self).__getattribute__(attr) 96 except AttributeError as ex: 97 return getattr(self._getobj(), attr)
98
99 - def __call__(self, *args, **kwargs):
100 return self._getobj()(*args, **kwargs)
101 102
103 -def load_module_from_name(dotted_name, path=None, use_sys=1):
104 """Load a Python module from its name. 105 106 :type dotted_name: str 107 :param dotted_name: python name of a module or package 108 109 :type path: list or None 110 :param path: 111 optional list of path where the module or package should be 112 searched (use sys.path if nothing or None is given) 113 114 :type use_sys: bool 115 :param use_sys: 116 boolean indicating whether the sys.modules dictionary should be 117 used or not 118 119 120 :raise ImportError: if the module or package is not found 121 122 :rtype: module 123 :return: the loaded module 124 """ 125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126 127
128 -def load_module_from_modpath(parts, path=None, use_sys=1):
129 """Load a python module from its splitted name. 130 131 :type parts: list(str) or tuple(str) 132 :param parts: 133 python name of a module or package splitted on '.' 134 135 :type path: list or None 136 :param path: 137 optional list of path where the module or package should be 138 searched (use sys.path if nothing or None is given) 139 140 :type use_sys: bool 141 :param use_sys: 142 boolean indicating whether the sys.modules dictionary should be used or not 143 144 :raise ImportError: if the module or package is not found 145 146 :rtype: module 147 :return: the loaded module 148 """ 149 if use_sys: 150 try: 151 return sys.modules['.'.join(parts)] 152 except KeyError: 153 pass 154 modpath = [] 155 prevmodule = None 156 for part in parts: 157 modpath.append(part) 158 curname = '.'.join(modpath) 159 module = None 160 if len(modpath) != len(parts): 161 # even with use_sys=False, should try to get outer packages from sys.modules 162 module = sys.modules.get(curname) 163 elif use_sys: 164 # because it may have been indirectly loaded through a parent 165 module = sys.modules.get(curname) 166 if module is None: 167 mp_file, mp_filename, mp_desc = find_module(part, path) 168 module = load_module(curname, mp_file, mp_filename, mp_desc) 169 if prevmodule: 170 setattr(prevmodule, part, module) 171 _file = getattr(module, '__file__', '') 172 if not _file and len(modpath) != len(parts): 173 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 174 path = [dirname( _file )] 175 prevmodule = module 176 return module
177 178
179 -def load_module_from_file(filepath, path=None, use_sys=1, extrapath=None):
180 """Load a Python module from it's path. 181 182 :type filepath: str 183 :param filepath: path to the python module or package 184 185 :type path: list or None 186 :param path: 187 optional list of path where the module or package should be 188 searched (use sys.path if nothing or None is given) 189 190 :type use_sys: bool 191 :param use_sys: 192 boolean indicating whether the sys.modules dictionary should be 193 used or not 194 195 196 :raise ImportError: if the module or package is not found 197 198 :rtype: module 199 :return: the loaded module 200 """ 201 modpath = modpath_from_file(filepath, extrapath) 202 return load_module_from_modpath(modpath, path, use_sys)
203 204
205 -def _check_init(path, mod_path):
206 """check there are some __init__.py all along the way""" 207 for part in mod_path: 208 path = join(path, part) 209 if not _has_init(path): 210 return False 211 return True
212 213
214 -def modpath_from_file(filename, extrapath=None):
215 """given a file path return the corresponding splitted module's name 216 (i.e name of a module or package splitted on '.') 217 218 :type filename: str 219 :param filename: file's path for which we want the module's name 220 221 :type extrapath: dict 222 :param extrapath: 223 optional extra search path, with path as key and package name for the path 224 as value. This is usually useful to handle package splitted in multiple 225 directories using __path__ trick. 226 227 228 :raise ImportError: 229 if the corresponding module's name has not been found 230 231 :rtype: list(str) 232 :return: the corresponding splitted module's name 233 """ 234 base = splitext(abspath(filename))[0] 235 if extrapath is not None: 236 for path_ in extrapath: 237 path = abspath(path_) 238 if path and base[:len(path)] == path: 239 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 240 if pkg] 241 if _check_init(path, submodpath[:-1]): 242 return extrapath[path_].split('.') + submodpath 243 for path in sys.path: 244 path = abspath(path) 245 if path and base.startswith(path): 246 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 247 if _check_init(path, modpath[:-1]): 248 return modpath 249 raise ImportError('Unable to find module for %s in %s' % ( 250 filename, ', \n'.join(sys.path)))
251 252 253
254 -def file_from_modpath(modpath, path=None, context_file=None):
255 """given a mod path (i.e. splitted module / package name), return the 256 corresponding file, giving priority to source file over precompiled 257 file if it exists 258 259 :type modpath: list or tuple 260 :param modpath: 261 splitted module's name (i.e name of a module or package splitted 262 on '.') 263 (this means explicit relative imports that start with dots have 264 empty strings in this list!) 265 266 :type path: list or None 267 :param path: 268 optional list of path where the module or package should be 269 searched (use sys.path if nothing or None is given) 270 271 :type context_file: str or None 272 :param context_file: 273 context file to consider, necessary if the identifier has been 274 introduced using a relative import unresolvable in the actual 275 context (i.e. modutils) 276 277 :raise ImportError: if there is no such module in the directory 278 279 :rtype: str or None 280 :return: 281 the path to the module's file or None if it's an integrated 282 builtin module such as 'sys' 283 """ 284 if context_file is not None: 285 context = dirname(context_file) 286 else: 287 context = context_file 288 if modpath[0] == 'xml': 289 # handle _xmlplus 290 try: 291 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 292 except ImportError: 293 return _file_from_modpath(modpath, path, context) 294 elif modpath == ['os', 'path']: 295 # FIXME: currently ignoring search_path... 296 return os.path.__file__ 297 return _file_from_modpath(modpath, path, context)
298 299 300
301 -def get_module_part(dotted_name, context_file=None):
302 """given a dotted name return the module part of the name : 303 304 >>> get_module_part('logilab.common.modutils.get_module_part') 305 'logilab.common.modutils' 306 307 :type dotted_name: str 308 :param dotted_name: full name of the identifier we are interested in 309 310 :type context_file: str or None 311 :param context_file: 312 context file to consider, necessary if the identifier has been 313 introduced using a relative import unresolvable in the actual 314 context (i.e. modutils) 315 316 317 :raise ImportError: if there is no such module in the directory 318 319 :rtype: str or None 320 :return: 321 the module part of the name or None if we have not been able at 322 all to import the given name 323 324 XXX: deprecated, since it doesn't handle package precedence over module 325 (see #10066) 326 """ 327 # os.path trick 328 if dotted_name.startswith('os.path'): 329 return 'os.path' 330 parts = dotted_name.split('.') 331 if context_file is not None: 332 # first check for builtin module which won't be considered latter 333 # in that case (path != None) 334 if parts[0] in BUILTIN_MODULES: 335 if len(parts) > 2: 336 raise ImportError(dotted_name) 337 return parts[0] 338 # don't use += or insert, we want a new list to be created ! 339 path = None 340 starti = 0 341 if parts[0] == '': 342 assert context_file is not None, \ 343 'explicit relative import, but no context_file?' 344 path = [] # prevent resolving the import non-relatively 345 starti = 1 346 while parts[starti] == '': # for all further dots: change context 347 starti += 1 348 context_file = dirname(context_file) 349 for i in range(starti, len(parts)): 350 try: 351 file_from_modpath(parts[starti:i+1], 352 path=path, context_file=context_file) 353 except ImportError: 354 if not i >= max(1, len(parts) - 2): 355 raise 356 return '.'.join(parts[:i]) 357 return dotted_name
358 359
360 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
361 """given a package directory return a list of all available python 362 modules in the package and its subpackages 363 364 :type package: str 365 :param package: the python name for the package 366 367 :type src_directory: str 368 :param src_directory: 369 path of the directory corresponding to the package 370 371 :type blacklist: list or tuple 372 :param blacklist: 373 optional list of files or directory to ignore, default to 374 the value of `logilab.common.STD_BLACKLIST` 375 376 :rtype: list 377 :return: 378 the list of all available python modules in the package and its 379 subpackages 380 """ 381 modules = [] 382 for directory, dirnames, filenames in os.walk(src_directory): 383 _handle_blacklist(blacklist, dirnames, filenames) 384 # check for __init__.py 385 if not '__init__.py' in filenames: 386 dirnames[:] = () 387 continue 388 if directory != src_directory: 389 dir_package = directory[len(src_directory):].replace(os.sep, '.') 390 modules.append(package + dir_package) 391 for filename in filenames: 392 if _is_python_file(filename) and filename != '__init__.py': 393 src = join(directory, filename) 394 module = package + src[len(src_directory):-3] 395 modules.append(module.replace(os.sep, '.')) 396 return modules
397 398 399
400 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
401 """given a package directory return a list of all available python 402 module's files in the package and its subpackages 403 404 :type src_directory: str 405 :param src_directory: 406 path of the directory corresponding to the package 407 408 :type blacklist: list or tuple 409 :param blacklist: 410 optional list of files or directory to ignore, default to the value of 411 `logilab.common.STD_BLACKLIST` 412 413 :rtype: list 414 :return: 415 the list of all available python module's files in the package and 416 its subpackages 417 """ 418 files = [] 419 for directory, dirnames, filenames in os.walk(src_directory): 420 _handle_blacklist(blacklist, dirnames, filenames) 421 # check for __init__.py 422 if not '__init__.py' in filenames: 423 dirnames[:] = () 424 continue 425 for filename in filenames: 426 if _is_python_file(filename): 427 src = join(directory, filename) 428 files.append(src) 429 return files
430 431
432 -def get_source_file(filename, include_no_ext=False):
433 """given a python module's file name return the matching source file 434 name (the filename will be returned identically if it's a already an 435 absolute path to a python source file...) 436 437 :type filename: str 438 :param filename: python module's file name 439 440 441 :raise NoSourceFile: if no source file exists on the file system 442 443 :rtype: str 444 :return: the absolute path of the source file if it exists 445 """ 446 base, orig_ext = splitext(abspath(filename)) 447 for ext in PY_SOURCE_EXTS: 448 source_path = '%s.%s' % (base, ext) 449 if exists(source_path): 450 return source_path 451 if include_no_ext and not orig_ext and exists(base): 452 return base 453 raise NoSourceFile(filename)
454 455
456 -def cleanup_sys_modules(directories):
457 """remove submodules of `directories` from `sys.modules`""" 458 cleaned = [] 459 for modname, module in list(sys.modules.items()): 460 modfile = getattr(module, '__file__', None) 461 if modfile: 462 for directory in directories: 463 if modfile.startswith(directory): 464 cleaned.append(modname) 465 del sys.modules[modname] 466 break 467 return cleaned
468 469
470 -def is_python_source(filename):
471 """ 472 rtype: bool 473 return: True if the filename is a python source file 474 """ 475 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
476 477 478
479 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
480 """try to guess if a module is a standard python module (by default, 481 see `std_path` parameter's description) 482 483 :type modname: str 484 :param modname: name of the module we are interested in 485 486 :type std_path: list(str) or tuple(str) 487 :param std_path: list of path considered has standard 488 489 490 :rtype: bool 491 :return: 492 true if the module: 493 - is located on the path listed in one of the directory in `std_path` 494 - is a built-in module 495 """ 496 modname = modname.split('.')[0] 497 try: 498 filename = file_from_modpath([modname]) 499 except ImportError as ex: 500 # import failed, i'm probably not so wrong by supposing it's 501 # not standard... 502 return 0 503 # modules which are not living in a file are considered standard 504 # (sys and __builtin__ for instance) 505 if filename is None: 506 return 1 507 filename = abspath(filename) 508 if filename.startswith(EXT_LIB_DIR): 509 return 0 510 for path in std_path: 511 if filename.startswith(abspath(path)): 512 return 1 513 return False
514 515 516
517 -def is_relative(modname, from_file):
518 """return true if the given module name is relative to the given 519 file name 520 521 :type modname: str 522 :param modname: name of the module we are interested in 523 524 :type from_file: str 525 :param from_file: 526 path of the module from which modname has been imported 527 528 :rtype: bool 529 :return: 530 true if the module has been imported relatively to `from_file` 531 """ 532 if not isdir(from_file): 533 from_file = dirname(from_file) 534 if from_file in sys.path: 535 return False 536 try: 537 find_module(modname.split('.')[0], [from_file]) 538 return True 539 except ImportError: 540 return False
541 542 543 # internal only functions ##################################################### 544
545 -def _file_from_modpath(modpath, path=None, context=None):
546 """given a mod path (i.e. splitted module / package name), return the 547 corresponding file 548 549 this function is used internally, see `file_from_modpath`'s 550 documentation for more information 551 """ 552 assert len(modpath) > 0 553 if context is not None: 554 try: 555 mtype, mp_filename = _module_file(modpath, [context]) 556 except ImportError: 557 mtype, mp_filename = _module_file(modpath, path) 558 else: 559 mtype, mp_filename = _module_file(modpath, path) 560 if mtype == PY_COMPILED: 561 try: 562 return get_source_file(mp_filename) 563 except NoSourceFile: 564 return mp_filename 565 elif mtype == C_BUILTIN: 566 # integrated builtin module 567 return None 568 elif mtype == PKG_DIRECTORY: 569 mp_filename = _has_init(mp_filename) 570 return mp_filename
571
572 -def _search_zip(modpath, pic):
573 for filepath, importer in list(pic.items()): 574 if importer is not None: 575 if importer.find_module(modpath[0]): 576 if not importer.find_module('/'.join(modpath)): 577 raise ImportError('No module named %s in %s/%s' % ( 578 '.'.join(modpath[1:]), filepath, modpath)) 579 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 580 raise ImportError('No module named %s' % '.'.join(modpath))
581 582 try: 583 import pkg_resources 584 except ImportError: 585 pkg_resources = None 586
587 -def _module_file(modpath, path=None):
588 """get a module type / file path 589 590 :type modpath: list or tuple 591 :param modpath: 592 splitted module's name (i.e name of a module or package splitted 593 on '.'), with leading empty strings for explicit relative import 594 595 :type path: list or None 596 :param path: 597 optional list of path where the module or package should be 598 searched (use sys.path if nothing or None is given) 599 600 601 :rtype: tuple(int, str) 602 :return: the module type flag and the file path for a module 603 """ 604 # egg support compat 605 try: 606 pic = sys.path_importer_cache 607 _path = (path is None and sys.path or path) 608 for __path in _path: 609 if not __path in pic: 610 try: 611 pic[__path] = zipimport.zipimporter(__path) 612 except zipimport.ZipImportError: 613 pic[__path] = None 614 checkeggs = True 615 except AttributeError: 616 checkeggs = False 617 # pkg_resources support (aka setuptools namespace packages) 618 if pkg_resources is not None and modpath[0] in pkg_resources._namespace_packages and len(modpath) > 1: 619 # setuptools has added into sys.modules a module object with proper 620 # __path__, get back information from there 621 module = sys.modules[modpath.pop(0)] 622 path = module.__path__ 623 imported = [] 624 while modpath: 625 modname = modpath[0] 626 # take care to changes in find_module implementation wrt builtin modules 627 # 628 # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23) 629 # >>> imp.find_module('posix') 630 # (None, 'posix', ('', '', 6)) 631 # 632 # Python 3.3.1 (default, Apr 26 2013, 12:08:46) 633 # >>> imp.find_module('posix') 634 # (None, None, ('', '', 6)) 635 try: 636 _, mp_filename, mp_desc = find_module(modname, path) 637 except ImportError: 638 if checkeggs: 639 return _search_zip(modpath, pic)[:2] 640 raise 641 else: 642 if checkeggs and mp_filename: 643 fullabspath = [abspath(x) for x in _path] 644 try: 645 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 646 emtype, emp_filename, zippath = _search_zip(modpath, pic) 647 if pathindex > _path.index(zippath): 648 # an egg takes priority 649 return emtype, emp_filename 650 except ValueError: 651 # XXX not in _path 652 pass 653 except ImportError: 654 pass 655 checkeggs = False 656 imported.append(modpath.pop(0)) 657 mtype = mp_desc[2] 658 if modpath: 659 if mtype != PKG_DIRECTORY: 660 raise ImportError('No module %s in %s' % ('.'.join(modpath), 661 '.'.join(imported))) 662 # XXX guess if package is using pkgutil.extend_path by looking for 663 # those keywords in the first four Kbytes 664 try: 665 with open(join(mp_filename, '__init__.py')) as stream: 666 data = stream.read(4096) 667 except IOError: 668 path = [mp_filename] 669 else: 670 if 'pkgutil' in data and 'extend_path' in data: 671 # extend_path is called, search sys.path for module/packages 672 # of this name see pkgutil.extend_path documentation 673 path = [join(p, *imported) for p in sys.path 674 if isdir(join(p, *imported))] 675 else: 676 path = [mp_filename] 677 return mtype, mp_filename
678
679 -def _is_python_file(filename):
680 """return true if the given filename should be considered as a python file 681 682 .pyc and .pyo are ignored 683 """ 684 for ext in ('.py', '.so', '.pyd', '.pyw'): 685 if filename.endswith(ext): 686 return True 687 return False
688 689
690 -def _has_init(directory):
691 """if the given directory has a valid __init__ file, return its path, 692 else return None 693 """ 694 mod_or_pack = join(directory, '__init__') 695 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 696 if exists(mod_or_pack + '.' + ext): 697 return mod_or_pack + '.' + ext 698 return None
699