Edit File: modutils.py
# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt """Python modules manipulation utility functions. :type PY_SOURCE_EXTS: tuple(str) :var PY_SOURCE_EXTS: list of possible python source file extension :type STD_LIB_DIRS: set of str :var STD_LIB_DIRS: directories where standard modules are located :type BUILTIN_MODULES: dict :var BUILTIN_MODULES: dictionary with builtin module names has key """ from __future__ import annotations import importlib import importlib.machinery import importlib.util import io import itertools import logging import os import sys import sysconfig import types import warnings from collections.abc import Callable, Iterable, Sequence from contextlib import redirect_stderr, redirect_stdout from functools import lru_cache from pathlib import Path from astroid.const import IS_JYTHON, IS_PYPY, PY310_PLUS from astroid.interpreter._import import spec, util if PY310_PLUS: from sys import stdlib_module_names else: from astroid._backport_stdlib_names import stdlib_module_names logger = logging.getLogger(__name__) if sys.platform.startswith("win"): PY_SOURCE_EXTS = ("py", "pyw") PY_COMPILED_EXTS = ("dll", "pyd") else: PY_SOURCE_EXTS = ("py",) PY_COMPILED_EXTS = ("so",) # TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should # revisit whether this is still necessary. See https://github.com/PyCQA/astroid/pull/1323. STD_LIB_DIRS = {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")} if os.name == "nt": STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls")) try: # real_prefix is defined when running inside virtual environments, # created with the **virtualenv** library. # Deprecated in virtualenv==16.7.9 # See: https://github.com/pypa/virtualenv/issues/1622 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined] except AttributeError: # sys.base_exec_prefix is always defined, but in a virtual environment # created with the stdlib **venv** module, it points to the original # installation, if the virtual env is activated. try: STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls")) except AttributeError: pass if IS_PYPY and sys.version_info < (3, 8): # PyPy stores the stdlib in two places: sys.prefix/lib_pypy and sys.prefix/lib-python/3 # sysconfig.get_path on PyPy returns the first, but without an underscore so we patch this manually. # Beginning with 3.8 the stdlib is only stored in: sys.prefix/pypy{py_version_short} STD_LIB_DIRS.add(str(Path(sysconfig.get_path("stdlib")).parent / "lib_pypy")) STD_LIB_DIRS.add(str(Path(sysconfig.get_path("stdlib")).parent / "lib-python/3")) # TODO: This is a fix for a workaround in virtualenv. At some point we should revisit # whether this is still necessary. See https://github.com/PyCQA/astroid/pull/1324. STD_LIB_DIRS.add(str(Path(sysconfig.get_path("platstdlib")).parent / "lib_pypy")) STD_LIB_DIRS.add( str(Path(sysconfig.get_path("platstdlib")).parent / "lib-python/3") ) if os.name == "posix": # Need the real prefix if we're in a virtualenv, otherwise # the usual one will do. # Deprecated in virtualenv==16.7.9 # See: https://github.com/pypa/virtualenv/issues/1622 try: prefix: str = sys.real_prefix # type: ignore[attr-defined] except AttributeError: prefix = sys.prefix def _posix_path(path: str) -> str: base_python = "python%d.%d" % sys.version_info[:2] return os.path.join(prefix, path, base_python) STD_LIB_DIRS.add(_posix_path("lib")) if sys.maxsize > 2**32: # This tries to fix a problem with /usr/lib64 builds, # where systems are running both 32-bit and 64-bit code # on the same machine, which reflects into the places where # standard library could be found. More details can be found # here http://bugs.python.org/issue1294959. # An easy reproducing case would be # https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753 STD_LIB_DIRS.add(_posix_path("lib64")) EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")} BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) class NoSourceFile(Exception): """Exception raised when we are not able to get a python source file for a precompiled file. """ def _normalize_path(path: str) -> str: """Resolve symlinks in path and convert to absolute path. Note that environment variables and ~ in the path need to be expanded in advance. This can be cached by using _cache_normalize_path. """ return os.path.normcase(os.path.realpath(path)) def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str: if not is_jython: return filename head, has_pyclass, _ = filename.partition("$py.class") if has_pyclass: return head + ".py" return filename def _handle_blacklist( blacklist: Sequence[str], dirnames: list[str], filenames: list[str] ) -> None: """Remove files/directories in the black list. dirnames/filenames are usually from os.walk """ for norecurs in blacklist: if norecurs in dirnames: dirnames.remove(norecurs) elif norecurs in filenames: filenames.remove(norecurs) @lru_cache() def _cache_normalize_path_(path: str) -> str: return _normalize_path(path) def _cache_normalize_path(path: str) -> str: """Normalize path with caching.""" # _module_file calls abspath on every path in sys.path every time it's # called; on a larger codebase this easily adds up to half a second just # assembling path components. This cache alleviates that. if not path: # don't cache result for '' return _normalize_path(path) return _cache_normalize_path_(path) def load_module_from_name(dotted_name: str) -> types.ModuleType: """Load a Python module from its name. :type dotted_name: str :param dotted_name: python name of a module or package :raise ImportError: if the module or package is not found :rtype: module :return: the loaded module """ try: return sys.modules[dotted_name] except KeyError: pass # Capture and log anything emitted during import to avoid # contaminating JSON reports in pylint with redirect_stderr(io.StringIO()) as stderr, redirect_stdout( io.StringIO() ) as stdout: module = importlib.import_module(dotted_name) stderr_value = stderr.getvalue() if stderr_value: logger.error( "Captured stderr while importing %s:\n%s", dotted_name, stderr_value ) stdout_value = stdout.getvalue() if stdout_value: logger.info( "Captured stdout while importing %s:\n%s", dotted_name, stdout_value ) return module def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType: """Load a python module from its split name. :param parts: python name of a module or package split on '.' :raise ImportError: if the module or package is not found :return: the loaded module """ return load_module_from_name(".".join(parts)) def load_module_from_file(filepath: str) -> types.ModuleType: """Load a Python module from it's path. :type filepath: str :param filepath: path to the python module or package :raise ImportError: if the module or package is not found :rtype: module :return: the loaded module """ modpath = modpath_from_file(filepath) return load_module_from_modpath(modpath) def check_modpath_has_init(path: str, mod_path: list[str]) -> bool: """Check there are some __init__.py all along the way.""" modpath: list[str] = [] for part in mod_path: modpath.append(part) path = os.path.join(path, part) if not _has_init(path): old_namespace = util.is_namespace(".".join(modpath)) if not old_namespace: return False return True def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None: """Extracts the relative mod path of the file to import from. Check if a file is within the passed in path and if so, returns the relative mod path from the one passed in. If the filename is no in path_to_check, returns None Note this function will look for both abs and realpath of the file, this allows to find the relative base path even if the file is a symlink of a file in the passed in path Examples: _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"] _get_relative_base_path("/a/b/c/d.py", "/dev") -> None """ importable_path = None path_to_check = os.path.normcase(path_to_check) abs_filename = os.path.abspath(filename) if os.path.normcase(abs_filename).startswith(path_to_check): importable_path = abs_filename real_filename = os.path.realpath(filename) if os.path.normcase(real_filename).startswith(path_to_check): importable_path = real_filename # if "var" in path_to_check: # breakpoint() if importable_path: base_path = os.path.splitext(importable_path)[0] relative_base_path = base_path[len(path_to_check) :] return [pkg for pkg in relative_base_path.split(os.sep) if pkg] return None def modpath_from_file_with_callback( filename: str, path: Sequence[str] | None = None, is_package_cb: Callable[[str, list[str]], bool] | None = None, ) -> list[str]: filename = os.path.expanduser(_path_from_filename(filename)) paths_to_check = sys.path.copy() if path: paths_to_check += path for pathname in itertools.chain( paths_to_check, map(_cache_normalize_path, paths_to_check) ): if not pathname: continue modpath = _get_relative_base_path(filename, pathname) if not modpath: continue assert is_package_cb is not None if is_package_cb(pathname, modpath[:-1]): return modpath raise ImportError( "Unable to find module for {} in {}".format(filename, ", \n".join(sys.path)) ) def modpath_from_file(filename: str, path: Sequence[str] | None = None) -> list[str]: """Get the corresponding split module's name from a filename. This function will return the name of a module or package split on `.`. :type filename: str :param filename: file's path for which we want the module's name :type Optional[List[str]] path: Optional list of path where the module or package should be searched (use sys.path if nothing or None is given) :raise ImportError: if the corresponding module's name has not been found :rtype: list(str) :return: the corresponding split module's name """ return modpath_from_file_with_callback(filename, path, check_modpath_has_init) def file_from_modpath( modpath: list[str], path: Sequence[str] | None = None, context_file: str | None = None, ) -> str | None: return file_info_from_modpath(modpath, path, context_file).location def file_info_from_modpath( modpath: list[str], path: Sequence[str] | None = None, context_file: str | None = None, ) -> spec.ModuleSpec: """Given a mod path (i.e. split module / package name), return the corresponding file. Giving priority to source file over precompiled file if it exists. :param modpath: split module's name (i.e name of a module or package split on '.') (this means explicit relative imports that start with dots have empty strings in this list!) :param path: optional list of path where the module or package should be searched (use sys.path if nothing or None is given) :param context_file: context file to consider, necessary if the identifier has been introduced using a relative import unresolvable in the actual context (i.e. modutils) :raise ImportError: if there is no such module in the directory :return: the path to the module's file or None if it's an integrated builtin module such as 'sys' """ if context_file is not None: context: str | None = os.path.dirname(context_file) else: context = context_file if modpath[0] == "xml": # handle _xmlplus try: return _spec_from_modpath(["_xmlplus"] + modpath[1:], path, context) except ImportError: return _spec_from_modpath(modpath, path, context) elif modpath == ["os", "path"]: # FIXME: currently ignoring search_path... return spec.ModuleSpec( name="os.path", location=os.path.__file__, type=spec.ModuleType.PY_SOURCE, ) return _spec_from_modpath(modpath, path, context) def get_module_part(dotted_name: str, context_file: str | None = None) -> str: """Given a dotted name return the module part of the name : >>> get_module_part('astroid.as_string.dump') 'astroid.as_string' :param dotted_name: full name of the identifier we are interested in :param context_file: context file to consider, necessary if the identifier has been introduced using a relative import unresolvable in the actual context (i.e. modutils) :raise ImportError: if there is no such module in the directory :return: the module part of the name or None if we have not been able at all to import the given name XXX: deprecated, since it doesn't handle package precedence over module (see #10066) """ # os.path trick if dotted_name.startswith("os.path"): return "os.path" parts = dotted_name.split(".") if context_file is not None: # first check for builtin module which won't be considered latter # in that case (path != None) if parts[0] in BUILTIN_MODULES: if len(parts) > 2: raise ImportError(dotted_name) return parts[0] # don't use += or insert, we want a new list to be created ! path: list[str] | None = None starti = 0 if parts[0] == "": assert ( context_file is not None ), "explicit relative import, but no context_file?" path = [] # prevent resolving the import non-relatively starti = 1 # for all further dots: change context while starti < len(parts) and parts[starti] == "": starti += 1 assert ( context_file is not None ), "explicit relative import, but no context_file?" context_file = os.path.dirname(context_file) for i in range(starti, len(parts)): try: file_from_modpath( parts[starti : i + 1], path=path, context_file=context_file ) except ImportError: if i < max(1, len(parts) - 2): raise return ".".join(parts[:i]) return dotted_name def get_module_files( src_directory: str, blacklist: Sequence[str], list_all: bool = False ) -> list[str]: """Given a package directory return a list of all available python module's files in the package and its subpackages. :param src_directory: path of the directory corresponding to the package :param blacklist: iterable list of files or directories to ignore. :param list_all: get files from all paths, including ones without __init__.py :return: the list of all available python module's files in the package and its subpackages """ files: list[str] = [] for directory, dirnames, filenames in os.walk(src_directory): if directory in blacklist: continue _handle_blacklist(blacklist, dirnames, filenames) # check for __init__.py if not list_all and "__init__.py" not in filenames: dirnames[:] = () continue for filename in filenames: if _is_python_file(filename): src = os.path.join(directory, filename) files.append(src) return files def get_source_file(filename: str, include_no_ext: bool = False) -> str: """Given a python module's file name return the matching source file name (the filename will be returned identically if it's already an. absolute path to a python source file...) :param filename: python module's file name :raise NoSourceFile: if no source file exists on the file system :return: the absolute path of the source file if it exists """ filename = os.path.abspath(_path_from_filename(filename)) base, orig_ext = os.path.splitext(filename) for ext in PY_SOURCE_EXTS: source_path = f"{base}.{ext}" if os.path.exists(source_path): return source_path if include_no_ext and not orig_ext and os.path.exists(base): return base raise NoSourceFile(filename) def is_python_source(filename: str | None) -> bool: """Return: True if the filename is a python source file.""" if not filename: return False return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS def is_stdlib_module(modname: str) -> bool: """Return: True if the modname is in the standard library""" return modname.split(".")[0] in stdlib_module_names def module_in_path(modname: str, path: str | Iterable[str]) -> bool: """Try to determine if a module is imported from one of the specified paths :param modname: name of the module :param path: paths to consider :return: true if the module: - is located on the path listed in one of the directory in `paths` """ modname = modname.split(".")[0] try: filename = file_from_modpath([modname]) except ImportError: # Import failed, we can't check path if we don't know it return False if filename is None: # No filename likely means it's compiled in, or potentially a namespace return False filename = _normalize_path(filename) if isinstance(path, str): return filename.startswith(_cache_normalize_path(path)) return any(filename.startswith(_cache_normalize_path(entry)) for entry in path) def is_standard_module(modname: str, std_path: Iterable[str] | None = None) -> bool: """Try to guess if a module is a standard python module (by default, see `std_path` parameter's description). :param modname: name of the module we are interested in :param std_path: list of path considered has standard :return: true if the module: - is located on the path listed in one of the directory in `std_path` - is a built-in module """ warnings.warn( "is_standard_module() is deprecated. Use, is_stdlib_module() or module_in_path() instead", DeprecationWarning, stacklevel=2, ) modname = modname.split(".")[0] try: filename = file_from_modpath([modname]) except ImportError: # import failed, i'm probably not so wrong by supposing it's # not standard... return False # modules which are not living in a file are considered standard # (sys and __builtin__ for instance) if filename is None: # we assume there are no namespaces in stdlib return not util.is_namespace(modname) filename = _normalize_path(filename) for path in EXT_LIB_DIRS: if filename.startswith(_cache_normalize_path(path)): return False if std_path is None: std_path = STD_LIB_DIRS return any(filename.startswith(_cache_normalize_path(path)) for path in std_path) def is_relative(modname: str, from_file: str) -> bool: """Return true if the given module name is relative to the given file name. :param modname: name of the module we are interested in :param from_file: path of the module from which modname has been imported :return: true if the module has been imported relatively to `from_file` """ if not os.path.isdir(from_file): from_file = os.path.dirname(from_file) if from_file in sys.path: return False return bool( importlib.machinery.PathFinder.find_spec( modname.split(".", maxsplit=1)[0], [from_file] ) ) # internal only functions ##################################################### def _spec_from_modpath( modpath: list[str], path: Sequence[str] | None = None, context: str | None = None, ) -> spec.ModuleSpec: """Given a mod path (i.e. split module / package name), return the corresponding spec. this function is used internally, see `file_from_modpath`'s documentation for more information """ assert modpath location = None if context is not None: try: found_spec = spec.find_spec(modpath, [context]) location = found_spec.location except ImportError: found_spec = spec.find_spec(modpath, path) location = found_spec.location else: found_spec = spec.find_spec(modpath, path) if found_spec.type == spec.ModuleType.PY_COMPILED: try: assert found_spec.location is not None location = get_source_file(found_spec.location) return found_spec._replace( location=location, type=spec.ModuleType.PY_SOURCE ) except NoSourceFile: return found_spec._replace(location=location) elif found_spec.type == spec.ModuleType.C_BUILTIN: # integrated builtin module return found_spec._replace(location=None) elif found_spec.type == spec.ModuleType.PKG_DIRECTORY: assert found_spec.location is not None location = _has_init(found_spec.location) return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE) return found_spec def _is_python_file(filename: str) -> bool: """Return true if the given filename should be considered as a python file. .pyc and .pyo are ignored """ return filename.endswith((".py", ".so", ".pyd", ".pyw")) def _has_init(directory: str) -> str | None: """If the given directory has a valid __init__ file, return its path, else return None. """ mod_or_pack = os.path.join(directory, "__init__") for ext in PY_SOURCE_EXTS + ("pyc", "pyo"): if os.path.exists(mod_or_pack + "." + ext): return mod_or_pack + "." + ext return None def is_namespace(specobj: spec.ModuleSpec) -> bool: return specobj.type == spec.ModuleType.PY_NAMESPACE def is_directory(specobj: spec.ModuleSpec) -> bool: return specobj.type == spec.ModuleType.PKG_DIRECTORY def is_module_name_part_of_extension_package_whitelist( module_name: str, package_whitelist: set[str] ) -> bool: """ Returns True if one part of the module name is in the package whitelist. >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'}) True """ parts = module_name.split(".") return any( ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1) )