diff --git a/CHANGES.md b/CHANGES.md index edd645cb4..5c3bccf96 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ # Release Notes +## 2.103.0 + +This release adds `pex3 venv create --link-python`, which points the created venv's Python at a +given path instead of the interpreter that created the venv. Since that path need not exist yet, +this also unlocks creating a venv for a foreign platform: the venv is laid out by a local +interpreter that differs from the foreign platform in machine architecture alone and its Python is +then linked to where the foreign interpreter will live. + +* Support creating a venv for a foreign platform. (#3279) + ## 2.102.0 This release adds support for mixing target platforms specified via --python, diff --git a/pex/cli/commands/venv.py b/pex/cli/commands/venv.py index d7b59b0a0..8bb0785b5 100644 --- a/pex/cli/commands/venv.py +++ b/pex/cli/commands/venv.py @@ -6,7 +6,7 @@ import itertools import logging import os.path -from argparse import ArgumentParser, _ActionsContainer +from argparse import ArgumentParser, Namespace, _ActionsContainer from pex import dependency_configuration, pex_warnings from pex.cli.command import BuildTimeCommand @@ -18,8 +18,11 @@ from pex.executables import is_python_script, is_script from pex.executor import Executor from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.interpreter import PythonInterpreter from pex.orderedset import OrderedSet +from pex.os import Os from pex.pex import PEX +from pex.pex_bootstrapper import normalize_path from pex.pex_info import PexInfo from pex.resolve import configured_resolve, requirement_options, resolver_options, target_options from pex.resolve.resolver_configuration import ( @@ -39,7 +42,7 @@ from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Any, Dict, Iterable, Optional, Sequence + from typing import Any, Dict, Iterable, Optional, Sequence, Union logger = logging.getLogger(__name__) @@ -119,6 +122,20 @@ def _add_create_arguments(cls, parser): ) ), ) + parser.add_argument( + "--link-python", + dest="link_python", + metavar="PATH", + default=None, + help=( + "The path the venv's Python should link to instead of the interpreter used to " + "create the venv. The interpreter need not exist; it only has to be in place by " + "the time the venv is used. This is what allows creating a venv for a foreign " + "platform: the venv is laid out by a local interpreter differing from the foreign " + "platform in machine architecture alone and its Python is then linked to the " + "given path. Only applies to the {venv} layout.".format(venv=InstallLayout.VENV) + ), + ) installer_options.register(parser) target_options.register(parser, include_platforms=True) resolver_options.register( @@ -274,16 +291,33 @@ def _create(self): ) ) if layout is InstallLayout.VENV: + venv_interpreter = target.get_interpreter() if target.is_foreign: - return Error( - "Cannot create a local venv for foreign platform {platform}.".format( - platform=target.platform + if Os.CURRENT is Os.WINDOWS: + return Error( + "Cannot create a local venv for foreign platform {platform}.\n" + "This is only supported for Unix environments.".format( + platform=target.platform + ) ) - ) + elif not self.options.link_python: + return Error( + "Cannot create a local venv for foreign platform {platform}.\n" + "Specify --link-python to say where the venv's Python will live in " + "the foreign environment.".format(platform=target.platform) + ) + elif not target.python_version or target.python_version < (3, 3): + return Error( + "Cannot create a local venv for foreign platform {platform}.\n" + "This is only supported for Python 3.3 and newer.".format( + platform=target.platform + ) + ) + venv_interpreter = try_(_find_stand_in_interpreter(target, self.options)) venv = Virtualenv.create( venv_dir=dest_dir, - interpreter=target.get_interpreter(), + interpreter=venv_interpreter, force=installer_configuration.force, copies=installer_configuration.copies, system_site_packages=installer_configuration.system_site_packages, @@ -398,11 +432,17 @@ def _create(self): if installer_configuration.compile: with TRACER.timed("Compiling venv sources"): + compiler = venv.interpreter if venv else target.get_interpreter() try: - target.get_interpreter().execute(["-m", "compileall", dest_dir]) + compiler.execute(["-m", "compileall", dest_dir]) except Executor.NonZeroExit as non_zero_exit: pex_warnings.warn("ignoring compile error {}".format(repr(non_zero_exit))) + # N.B.: This must come last since it can leave the venv Python dangling until the venv + # reaches its final resting place. + if venv and self.options.link_python: + venv.link_python(self.options.link_python) + if layout is InstallLayout.FLAT_ZIPPED: paths = sorted( os.path.join(root, path) @@ -417,6 +457,40 @@ def _create(self): return Ok() +def _find_stand_in_interpreter( + target, # type: Target + options, # type: Namespace +): + # type: (...) -> Union[PythonInterpreter, Error] + """Find a local interpreter that lays a venv out just like `target` would. + + A venv's layout is fixed by the Python implementation, version and OS; the machine architecture + plays no part. That lets a local interpreter stand in for a foreign one it differs from in + architecture alone. + """ + python_path = target_options.configure_interpreters(options).python_path + target_tag = target.platform.tag + for interpreter in PythonInterpreter.iter(paths=normalize_path(python_path)): + tag = interpreter.platform.tag + if ( + tag.interpreter == target_tag.interpreter + and tag.abi == target_tag.abi + and interpreter.identity.env_markers.platform_system + == target.marker_environment.platform_system + ): + return interpreter + + return Error( + "Could not find a local interpreter to lay out a venv for {platform}.\n" + "A {interpreter} interpreter running on {system} is needed; searched: {search_path}".format( + platform=target.platform, + interpreter=target_tag.interpreter, + system=target.marker_environment.platform_system, + search_path=os.pathsep.join(python_path) if python_path else "$PATH", + ) + ) + + def _install_from_pex( pex, # type: PEX installer_configuration, # type: InstallerConfiguration diff --git a/pex/sysconfig.py b/pex/sysconfig.py index d21257dc2..880edf5f9 100644 --- a/pex/sysconfig.py +++ b/pex/sysconfig.py @@ -17,7 +17,7 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Optional, Text, TypeVar + from typing import Optional, Text, Tuple, TypeVar, Union EXE_EXTENSION = get_config_var("EXE") or "" EXE_EXTENSIONS = ( @@ -159,6 +159,14 @@ def venv_bin_dir(self): # type: () -> str return "Scripts" if self.os is Os.WINDOWS else "bin" + def venv_lib_dir(self, version): + # type: (Union[Tuple[int, int], Tuple[int, int, int]]) -> str + return ( + "Lib" + if self.os is Os.WINDOWS + else "lib/python{major}.{minor}".format(major=version[0], minor=version[1]) + ) + def binary_name(self, binary_name): # type: (_Text) -> _Text return "{binary_name}{extension}".format( diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py index a378b7d35..cb2e5a125 100644 --- a/pex/venv/virtualenv.py +++ b/pex/venv/virtualenv.py @@ -504,6 +504,35 @@ def iter_distributions(self, rescan=False): ): yield dist + def link_python(self, python): + # type: (str) -> None + """Point this venv's Python links at `python`. + + The `python` path need not exist; it only has to be in place by the time the venv is used. + This supports creating a venv destined for a final resting place, like an image layer, + where the interpreter lives at a known path but is not available where the venv is created. + """ + python_links = [ + path + for path in _iter_files(self._bin_dir) + if os.path.islink(path) and PythonInterpreter.matches_binary_name(path) + ] + for python_link in python_links: + os.unlink(python_link) + safe_symlink(python, python_link) + + pyvenv_cfg = self._pyvenv_cfg + if pyvenv_cfg: + with closing(FileInput(files=[pyvenv_cfg.path], inplace=True)) as fi: + for line in fi: + key, delimiter, _ = line.partition("=") + if delimiter and key.strip() == "home": + sys.stdout.write("home = {home}\n".format(home=os.path.dirname(python))) + elif delimiter and key.strip() == "executable": + sys.stdout.write("executable = {python}\n".format(python=python)) + else: + sys.stdout.write(line) + def _rewrite_base_scripts(self, real_venv_dir): # type: (str) -> Iterator[str] scripts = [ diff --git a/pex/version.py b/pex/version.py index 07d68fcb9..200f66235 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.102.0" +__version__ = "2.103.0" diff --git a/tests/integration/cli/commands/test_venv_create.py b/tests/integration/cli/commands/test_venv_create.py index 0361ac1fd..f515893ce 100644 --- a/tests/integration/cli/commands/test_venv_create.py +++ b/tests/integration/cli/commands/test_venv_create.py @@ -5,6 +5,7 @@ import glob import os.path +import re import shutil import sys from subprocess import CalledProcessError @@ -19,18 +20,22 @@ from pex.common import open_zip, safe_open from pex.compatibility import safe_commonpath from pex.dist_metadata import Distribution -from pex.interpreter import PythonInterpreter +from pex.interpreter import PythonInterpreter, PyVenvCfg from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.pex import PEX from pex.resolve import abbreviated_platforms +from pex.sysconfig import SysPlatform from pex.typing import TYPE_CHECKING from pex.venv.bin_path import BinPath from pex.venv.virtualenv import Virtualenv from testing import ( + IS_ARM_64, IS_MAC, + IS_WINDOWS, PY39, PY310, + IntegResults, ensure_python_interpreter, make_env, run_pex_command, @@ -499,12 +504,31 @@ def test_foreign_target( ) result.assert_failure() assert ( - "Cannot create a local venv for foreign platform {platform}.".format( - platform=abbreviated_platforms.create(foreign_platform) - ) + "Cannot create a local venv for foreign platform {platform}.\n" + "Specify --link-python to say where the venv's Python will live in the foreign " + "environment.".format(platform=abbreviated_platforms.create(foreign_platform)) == result.error.strip() ) + # The foreign_platform fixture is a foreign OS; so no local interpreter can stand in for it. + result = run_pex3( + "venv", + "create", + "psutil==5.9.5", + "-d", + dest, + "--platform", + foreign_platform, + "--link-python", + "/opt/python/bin/python3.10", + ) + result.assert_failure() + assert result.error.strip().startswith( + "Could not find a local interpreter to lay out a venv for {platform}.".format( + platform=abbreviated_platforms.create(foreign_platform) + ) + ), result.error + run_pex3( "venv", "create", @@ -525,6 +549,96 @@ def test_foreign_target( assert Version("5.9.5") == dist.metadata.version +def cross_arch_platform( + major, # type: int + minor, # type: int + abiflags="", # type: str +): + # type: (...) -> str + """A foreign platform differing from the local one in machine architecture alone.""" + pyver = "{major}{minor}".format(major=major, minor=minor) + if IS_MAC: + return "macosx_11_0_{machine}-cp-{pyver}-cp{pyver}{abiflags}".format( + machine="x86_64" if IS_ARM_64 else "arm64", pyver=pyver, abiflags=abiflags + ) + return "linux_{machine}-cp-{pyver}-cp{pyver}{abiflags}".format( + machine="x86_64" if IS_ARM_64 else "aarch64", pyver=pyver, abiflags=abiflags + ) + + +@pytest.mark.skipif(IS_WINDOWS, reason="This feature is not supported on Windows.") +def test_foreign_target_link_python( + tmpdir, # type: Tempdir + py310, # type: PythonInterpreter + py311, # type: PythonInterpreter +): + # type: (...) -> None + + venv_dir = tmpdir.join("venv") + link_python = "/opt/python/bin/bob" + + def create_venv(foreign_platform): + # type: (str) -> IntegResults + return run_pex3( + "venv", + "create", + "psutil==5.9.5", + "-d", + venv_dir, + "--platform", + foreign_platform, + "--python-path", + os.pathsep.join(interpreter.binary for interpreter in (py310, py311)), + "--link-python", + link_python, + "--force", + ) + + def assert_foreign_venv( + major, # type: int + minor, # type: int + expect_pyvenv_cfg_executable, # type: Optional[str] + ): + # type: (...) -> None + assert {link_python} == { + os.readlink(python) for python in glob.glob(os.path.join(venv_dir, "bin", "python*")) + } + + pyvenv_cfg = PyVenvCfg.parse(os.path.join(venv_dir, "pyvenv.cfg")) + assert os.path.dirname(link_python) == pyvenv_cfg.home + pyvenv_cfg_executable = pyvenv_cfg.config("executable") + if expect_pyvenv_cfg_executable: + assert pyvenv_cfg_executable == expect_pyvenv_cfg_executable + else: + assert pyvenv_cfg_executable is None + + site_packages = os.path.join( + venv_dir, SysPlatform.CURRENT.venv_lib_dir(version=(major, minor)), "site-packages" + ) + distributions = list(dist_metadata.find_distributions(search_path=[site_packages])) + assert 1 == len(distributions) + assert ProjectName("psutil") == distributions[0].metadata.project_name + + foreign_platform = cross_arch_platform(2, 7, "mu") + create_venv(foreign_platform=cross_arch_platform(2, 7, "mu")).assert_failure( + expected_error_re=r".*{msg}$".format( + msg=re.escape( + "Cannot create a local venv for foreign platform {foreign_platform}.\n" + "This is only supported for Python 3.3 and newer.".format( + foreign_platform=foreign_platform + ) + ) + ), + re_flags=re.DOTALL, + ) + + create_venv(foreign_platform=cross_arch_platform(3, 10)).assert_success() + assert_foreign_venv(3, 10, expect_pyvenv_cfg_executable=None) + + create_venv(foreign_platform=cross_arch_platform(3, 11)).assert_success() + assert_foreign_venv(3, 11, expect_pyvenv_cfg_executable=link_python) + + def test_venv_update_target_mismatch( tmpdir, # type: Any foreign_platform, # type: str