added completly new version for haslach 2025

This commit is contained in:
fotobox
2025-03-17 03:47:13 +01:00
parent 152832515c
commit 769ab91da8
2333 changed files with 409208 additions and 341 deletions

View File

@@ -0,0 +1,148 @@
"""Validation of dependencies of packages
"""
from collections import namedtuple
from pip._vendor.packaging.utils import canonicalize_name
from pip._internal.operations.prepare import make_abstract_dist
from pip._internal.utils.misc import get_installed_distributions
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from pip._internal.req.req_install import InstallRequirement # noqa: F401
from typing import ( # noqa: F401
Any, Callable, Dict, Iterator, Optional, Set, Tuple, List
)
# Shorthands
PackageSet = Dict[str, 'PackageDetails']
Missing = Tuple[str, Any]
Conflicting = Tuple[str, str, Any]
MissingDict = Dict[str, List[Missing]]
ConflictingDict = Dict[str, List[Conflicting]]
CheckResult = Tuple[MissingDict, ConflictingDict]
PackageDetails = namedtuple('PackageDetails', ['version', 'requires'])
def create_package_set_from_installed(**kwargs):
# type: (**Any) -> PackageSet
"""Converts a list of distributions into a PackageSet.
"""
# Default to using all packages installed on the system
if kwargs == {}:
kwargs = {"local_only": False, "skip": ()}
package_set = {}
for dist in get_installed_distributions(**kwargs):
name = canonicalize_name(dist.project_name)
package_set[name] = PackageDetails(dist.version, dist.requires())
return package_set
def check_package_set(package_set, should_ignore=None):
# type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult
"""Check if a package set is consistent
If should_ignore is passed, it should be a callable that takes a
package name and returns a boolean.
"""
if should_ignore is None:
def should_ignore(name):
return False
missing = dict()
conflicting = dict()
for package_name in package_set:
# Info about dependencies of package_name
missing_deps = set() # type: Set[Missing]
conflicting_deps = set() # type: Set[Conflicting]
if should_ignore(package_name):
continue
for req in package_set[package_name].requires:
name = canonicalize_name(req.project_name) # type: str
# Check if it's missing
if name not in package_set:
missed = True
if req.marker is not None:
missed = req.marker.evaluate()
if missed:
missing_deps.add((name, req))
continue
# Check if there's a conflict
version = package_set[name].version # type: str
if not req.specifier.contains(version, prereleases=True):
conflicting_deps.add((name, version, req))
if missing_deps:
missing[package_name] = sorted(missing_deps, key=str)
if conflicting_deps:
conflicting[package_name] = sorted(conflicting_deps, key=str)
return missing, conflicting
def check_install_conflicts(to_install):
# type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult]
"""For checking if the dependency graph would be consistent after \
installing given requirements
"""
# Start from the current state
package_set = create_package_set_from_installed()
# Install packages
would_be_installed = _simulate_installation_of(to_install, package_set)
# Only warn about directly-dependent packages; create a whitelist of them
whitelist = _create_whitelist(would_be_installed, package_set)
return (
package_set,
check_package_set(
package_set, should_ignore=lambda name: name not in whitelist
)
)
# NOTE from @pradyunsg
# This required a minor update in dependency link handling logic over at
# operations.prepare.IsSDist.dist() to get it working
def _simulate_installation_of(to_install, package_set):
# type: (List[InstallRequirement], PackageSet) -> Set[str]
"""Computes the version of packages after installing to_install.
"""
# Keep track of packages that were installed
installed = set()
# Modify it as installing requirement_set would (assuming no errors)
for inst_req in to_install:
dist = make_abstract_dist(inst_req).dist(finder=None)
name = canonicalize_name(dist.key)
package_set[name] = PackageDetails(dist.version, dist.requires())
installed.add(name)
return installed
def _create_whitelist(would_be_installed, package_set):
# type: (Set[str], PackageSet) -> Set[str]
packages_affected = set(would_be_installed)
for package_name in package_set:
if package_name in packages_affected:
continue
for req in package_set[package_name].requires:
if canonicalize_name(req.name) in packages_affected:
packages_affected.add(package_name)
break
return packages_affected

View File

@@ -0,0 +1,264 @@
from __future__ import absolute_import
import collections
import logging
import os
import re
from pip._vendor import pkg_resources, six
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.pkg_resources import RequirementParseError
from pip._internal.exceptions import InstallationError
from pip._internal.req.constructors import (
install_req_from_editable, install_req_from_line,
)
from pip._internal.req.req_file import COMMENT_RE
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.misc import (
dist_is_editable, get_installed_distributions, make_vcs_requirement_url,
)
logger = logging.getLogger(__name__)
def freeze(
requirement=None,
find_links=None, local_only=None, user_only=None, skip_regex=None,
isolated=False,
wheel_cache=None,
exclude_editable=False,
skip=()):
find_links = find_links or []
skip_match = None
if skip_regex:
skip_match = re.compile(skip_regex).search
dependency_links = []
for dist in pkg_resources.working_set:
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt')
)
for link in find_links:
if '#egg=' in link:
dependency_links.append(link)
for link in find_links:
yield '-f %s' % link
installations = {}
for dist in get_installed_distributions(local_only=local_only,
skip=(),
user_only=user_only):
try:
req = FrozenRequirement.from_dist(
dist,
dependency_links
)
except RequirementParseError:
logger.warning(
"Could not parse requirement: %s",
dist.project_name
)
continue
if exclude_editable and req.editable:
continue
installations[req.name] = req
if requirement:
# the options that don't get turned into an InstallRequirement
# should only be emitted once, even if the same option is in multiple
# requirements files, so we need to keep track of what has been emitted
# so that we don't emit it again if it's seen again
emitted_options = set()
# keep track of which files a requirement is in so that we can
# give an accurate warning if a requirement appears multiple times.
req_files = collections.defaultdict(list)
for req_file_path in requirement:
with open(req_file_path) as req_file:
for line in req_file:
if (not line.strip() or
line.strip().startswith('#') or
(skip_match and skip_match(line)) or
line.startswith((
'-r', '--requirement',
'-Z', '--always-unzip',
'-f', '--find-links',
'-i', '--index-url',
'--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url'))):
line = line.rstrip()
if line not in emitted_options:
emitted_options.add(line)
yield line
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = install_req_from_editable(
line,
isolated=isolated,
wheel_cache=wheel_cache,
)
else:
line_req = install_req_from_line(
COMMENT_RE.sub('', line).strip(),
isolated=isolated,
wheel_cache=wheel_cache,
)
if not line_req.name:
logger.info(
"Skipping line in requirement file [%s] because "
"it's not clear what it would install: %s",
req_file_path, line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)"
)
elif line_req.name not in installations:
# either it's not installed, or it is installed
# but has been processed already
if not req_files[line_req.name]:
logger.warning(
"Requirement file [%s] contains %s, but that "
"package is not installed",
req_file_path,
COMMENT_RE.sub('', line).strip(),
)
else:
req_files[line_req.name].append(req_file_path)
else:
yield str(installations[line_req.name]).rstrip()
del installations[line_req.name]
req_files[line_req.name].append(req_file_path)
# Warn about requirements that were included multiple times (in a
# single requirements file or in different requirements files).
for name, files in six.iteritems(req_files):
if len(files) > 1:
logger.warning("Requirement %s included multiple times [%s]",
name, ', '.join(sorted(set(files))))
yield(
'## The following requirements were added by '
'pip freeze:'
)
for installation in sorted(
installations.values(), key=lambda x: x.name.lower()):
if canonicalize_name(installation.name) not in skip:
yield str(installation).rstrip()
class FrozenRequirement(object):
def __init__(self, name, req, editable, comments=()):
self.name = name
self.req = req
self.editable = editable
self.comments = comments
_rev_re = re.compile(r'-r(\d+)$')
_date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
@classmethod
def _init_args_from_dist(cls, dist, dependency_links):
"""
Compute and return arguments (req, editable, comments) to pass to
FrozenRequirement.__init__().
This method is for use in FrozenRequirement.from_dist().
"""
location = os.path.normcase(os.path.abspath(dist.location))
comments = []
from pip._internal.vcs import vcs, get_src_requirement
if dist_is_editable(dist) and vcs.get_backend_name(location):
editable = True
try:
req = get_src_requirement(dist, location)
except InstallationError as exc:
logger.warning(
"Error when trying to get requirement for VCS system %s, "
"falling back to uneditable format", exc
)
req = None
if req is None:
logger.warning(
'Could not determine repository location of %s', location
)
comments.append(
'## !! Could not determine repository location'
)
req = dist.as_requirement()
editable = False
else:
editable = False
req = dist.as_requirement()
specs = req.specs
assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
'Expected 1 spec with == or ===; specs = %r; dist = %r' % \
(specs, dist)
version = specs[0][1]
ver_match = cls._rev_re.search(version)
date_match = cls._date_re.search(version)
if ver_match or date_match:
svn_backend = vcs.get_backend('svn')
if svn_backend:
svn_location = svn_backend().get_location(
dist,
dependency_links,
)
if not svn_location:
logger.warning(
'Warning: cannot find svn location for %s', req,
)
comments.append(
'## FIXME: could not find svn URL in dependency_links '
'for this package:'
)
else:
deprecated(
"SVN editable detection based on dependency links "
"will be dropped in the future.",
replacement=None,
gone_in="18.2",
issue=4187,
)
comments.append(
'# Installing as editable to satisfy requirement %s:' %
req
)
if ver_match:
rev = ver_match.group(1)
else:
rev = '{%s}' % date_match.group(1)
editable = True
egg_name = cls.egg_name(dist)
req = make_vcs_requirement_url(svn_location, rev, egg_name)
return (req, editable, comments)
@classmethod
def from_dist(cls, dist, dependency_links):
args = cls._init_args_from_dist(dist, dependency_links)
return cls(dist.project_name, *args)
@staticmethod
def egg_name(dist):
name = dist.egg_name()
match = re.search(r'-py\d\.\d$', name)
if match:
name = name[:match.start()]
return name
def __str__(self):
req = self.req
if self.editable:
req = '-e %s' % req
return '\n'.join(list(self.comments) + [str(req)]) + '\n'

View File

@@ -0,0 +1,355 @@
"""Prepares a distribution for installation
"""
import logging
import os
from pip._vendor import pkg_resources, requests
from pip._internal.build_env import BuildEnvironment
from pip._internal.download import (
is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path,
)
from pip._internal.exceptions import (
DirectoryUrlHashUnsupported, HashUnpinned, InstallationError,
PreviousBuildDirError, VcsHashUnsupported,
)
from pip._internal.utils.compat import expanduser
from pip._internal.utils.hashes import MissingHashes
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import display_path, normalize_path
from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
def make_abstract_dist(req):
"""Factory to make an abstract dist object.
Preconditions: Either an editable req with a source_dir, or satisfied_by or
a wheel link, or a non-editable req with a source_dir.
:return: A concrete DistAbstraction.
"""
if req.editable:
return IsSDist(req)
elif req.link and req.link.is_wheel:
return IsWheel(req)
else:
return IsSDist(req)
class DistAbstraction(object):
"""Abstracts out the wheel vs non-wheel Resolver.resolve() logic.
The requirements for anything installable are as follows:
- we must be able to determine the requirement name
(or we can't correctly handle the non-upgrade case).
- we must be able to generate a list of run-time dependencies
without installing any additional packages (or we would
have to either burn time by doing temporary isolated installs
or alternatively violate pips 'don't start installing unless
all requirements are available' rule - neither of which are
desirable).
- for packages with setup requirements, we must also be able
to determine their requirements without installing additional
packages (for the same reason as run-time dependencies)
- we must be able to create a Distribution object exposing the
above metadata.
"""
def __init__(self, req):
self.req = req
def dist(self, finder):
"""Return a setuptools Dist object."""
raise NotImplementedError(self.dist)
def prep_for_dist(self, finder, build_isolation):
"""Ensure that we can get a Dist for this requirement."""
raise NotImplementedError(self.dist)
class IsWheel(DistAbstraction):
def dist(self, finder):
return list(pkg_resources.find_distributions(
self.req.source_dir))[0]
def prep_for_dist(self, finder, build_isolation):
# FIXME:https://github.com/pypa/pip/issues/1112
pass
class IsSDist(DistAbstraction):
def dist(self, finder):
dist = self.req.get_dist()
# FIXME: shouldn't be globally added.
if finder and dist.has_metadata('dependency_links.txt'):
finder.add_dependency_links(
dist.get_metadata_lines('dependency_links.txt')
)
return dist
def prep_for_dist(self, finder, build_isolation):
# Prepare for building. We need to:
# 1. Load pyproject.toml (if it exists)
# 2. Set up the build environment
self.req.load_pyproject_toml()
should_isolate = self.req.use_pep517 and build_isolation
if should_isolate:
# Isolate in a BuildEnvironment and install the build-time
# requirements.
self.req.build_env = BuildEnvironment()
self.req.build_env.install_requirements(
finder, self.req.pyproject_requires,
"Installing build dependencies"
)
missing = []
if self.req.requirements_to_check:
check = self.req.requirements_to_check
missing = self.req.build_env.missing_requirements(check)
if missing:
logger.warning(
"Missing build requirements in pyproject.toml for %s.",
self.req,
)
logger.warning(
"The project does not specify a build backend, and pip "
"cannot fall back to setuptools without %s.",
" and ".join(map(repr, sorted(missing)))
)
self.req.run_egg_info()
self.req.assert_source_matches_version()
class Installed(DistAbstraction):
def dist(self, finder):
return self.req.satisfied_by
def prep_for_dist(self, finder, build_isolation):
pass
class RequirementPreparer(object):
"""Prepares a Requirement
"""
def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
progress_bar, build_isolation, req_tracker):
super(RequirementPreparer, self).__init__()
self.src_dir = src_dir
self.build_dir = build_dir
self.req_tracker = req_tracker
# Where still packed archives should be written to. If None, they are
# not saved, and are deleted immediately after unpacking.
self.download_dir = download_dir
# Where still-packed .whl files should be written to. If None, they are
# written to the download_dir parameter. Separate to download_dir to
# permit only keeping wheel archives for pip wheel.
if wheel_download_dir:
wheel_download_dir = normalize_path(wheel_download_dir)
self.wheel_download_dir = wheel_download_dir
# NOTE
# download_dir and wheel_download_dir overlap semantically and may
# be combined if we're willing to have non-wheel archives present in
# the wheelhouse output by 'pip wheel'.
self.progress_bar = progress_bar
# Is build isolation allowed?
self.build_isolation = build_isolation
@property
def _download_should_save(self):
# TODO: Modify to reduce indentation needed
if self.download_dir:
self.download_dir = expanduser(self.download_dir)
if os.path.exists(self.download_dir):
return True
else:
logger.critical('Could not find download directory')
raise InstallationError(
"Could not find or access download directory '%s'"
% display_path(self.download_dir))
return False
def prepare_linked_requirement(self, req, session, finder,
upgrade_allowed, require_hashes):
"""Prepare a requirement that would be obtained from req.link
"""
# TODO: Breakup into smaller functions
if req.link and req.link.scheme == 'file':
path = url_to_path(req.link.url)
logger.info('Processing %s', display_path(path))
else:
logger.info('Collecting %s', req)
with indent_log():
# @@ if filesystem packages are not marked
# editable in a req, a non deterministic error
# occurs when the script attempts to unpack the
# build directory
req.ensure_has_source_dir(self.build_dir)
# If a checkout exists, it's unwise to keep going. version
# inconsistencies are logged later, but do not fail the
# installation.
# FIXME: this won't upgrade when there's an existing
# package unpacked in `req.source_dir`
# package unpacked in `req.source_dir`
if os.path.exists(os.path.join(req.source_dir, 'setup.py')):
raise PreviousBuildDirError(
"pip can't proceed with requirements '%s' due to a"
" pre-existing build directory (%s). This is "
"likely due to a previous installation that failed"
". pip is being responsible and not assuming it "
"can delete this. Please delete it and try again."
% (req, req.source_dir)
)
req.populate_link(finder, upgrade_allowed, require_hashes)
# We can't hit this spot and have populate_link return None.
# req.satisfied_by is None here (because we're
# guarded) and upgrade has no impact except when satisfied_by
# is not None.
# Then inside find_requirement existing_applicable -> False
# If no new versions are found, DistributionNotFound is raised,
# otherwise a result is guaranteed.
assert req.link
link = req.link
# Now that we have the real link, we can tell what kind of
# requirements we have and raise some more informative errors
# than otherwise. (For example, we can raise VcsHashUnsupported
# for a VCS URL rather than HashMissing.)
if require_hashes:
# We could check these first 2 conditions inside
# unpack_url and save repetition of conditions, but then
# we would report less-useful error messages for
# unhashable requirements, complaining that there's no
# hash provided.
if is_vcs_url(link):
raise VcsHashUnsupported()
elif is_file_url(link) and is_dir_url(link):
raise DirectoryUrlHashUnsupported()
if not req.original_link and not req.is_pinned:
# Unpinned packages are asking for trouble when a new
# version is uploaded. This isn't a security check, but
# it saves users a surprising hash mismatch in the
# future.
#
# file:/// URLs aren't pinnable, so don't complain
# about them not being pinned.
raise HashUnpinned()
hashes = req.hashes(trust_internet=not require_hashes)
if require_hashes and not hashes:
# Known-good hashes are missing for this requirement, so
# shim it with a facade object that will provoke hash
# computation and then raise a HashMissing exception
# showing the user what the hash should be.
hashes = MissingHashes()
try:
download_dir = self.download_dir
# We always delete unpacked sdists after pip ran.
autodelete_unpacked = True
if req.link.is_wheel and self.wheel_download_dir:
# when doing 'pip wheel` we download wheels to a
# dedicated dir.
download_dir = self.wheel_download_dir
if req.link.is_wheel:
if download_dir:
# When downloading, we only unpack wheels to get
# metadata.
autodelete_unpacked = True
else:
# When installing a wheel, we use the unpacked
# wheel.
autodelete_unpacked = False
unpack_url(
req.link, req.source_dir,
download_dir, autodelete_unpacked,
session=session, hashes=hashes,
progress_bar=self.progress_bar
)
except requests.HTTPError as exc:
logger.critical(
'Could not install requirement %s because of error %s',
req,
exc,
)
raise InstallationError(
'Could not install requirement %s because of HTTP '
'error %s for URL %s' %
(req, exc, req.link)
)
abstract_dist = make_abstract_dist(req)
with self.req_tracker.track(req):
abstract_dist.prep_for_dist(finder, self.build_isolation)
if self._download_should_save:
# Make a .zip of the source_dir we already created.
if req.link.scheme in vcs.all_schemes:
req.archive(self.download_dir)
return abstract_dist
def prepare_editable_requirement(self, req, require_hashes, use_user_site,
finder):
"""Prepare an editable requirement
"""
assert req.editable, "cannot prepare a non-editable req as editable"
logger.info('Obtaining %s', req)
with indent_log():
if require_hashes:
raise InstallationError(
'The editable requirement %s cannot be installed when '
'requiring hashes, because there is no single file to '
'hash.' % req
)
req.ensure_has_source_dir(self.src_dir)
req.update_editable(not self._download_should_save)
abstract_dist = make_abstract_dist(req)
with self.req_tracker.track(req):
abstract_dist.prep_for_dist(finder, self.build_isolation)
if self._download_should_save:
req.archive(self.download_dir)
req.check_if_exists(use_user_site)
return abstract_dist
def prepare_installed_requirement(self, req, require_hashes, skip_reason):
"""Prepare an already-installed requirement
"""
assert req.satisfied_by, "req should have been satisfied but isn't"
assert skip_reason is not None, (
"did not get skip reason skipped but req.satisfied_by "
"is set to %r" % (req.satisfied_by,)
)
logger.info(
'Requirement %s: %s (%s)',
skip_reason, req, req.satisfied_by.version
)
with indent_log():
if require_hashes:
logger.debug(
'Since it is already installed, we are trusting this '
'package without checking its hash. To ensure a '
'completely repeatable environment, install into an '
'empty virtualenv.'
)
abstract_dist = Installed(req)
return abstract_dist