Support creating a venv for a foreign platform.

CHANGES.md

@@ -1,5 +1,15 @@
 # Release Notes
 
+## 2.103.0
+
+This release adds `pex3 venv create --link-python`, which points the created venv's Python at a
+given path instead of the interpreter that created the venv. Since that path need not exist yet,
+this also unlocks creating a venv for a foreign platform: the venv is laid out by a local
+interpreter that differs from the foreign platform in machine architecture alone and its Python is
+then linked to where the foreign interpreter will live.
+
+* Support creating a venv for a foreign platform. (#3279)
+
 ## 2.102.0
 
 This release adds support for mixing target platforms specified via --python,

pex/cli/commands/venv.py

@@ -6,7 +6,7 @@
 import itertools
 import logging
 import os.path
-from argparse import ArgumentParser, _ActionsContainer
+from argparse import ArgumentParser, Namespace, _ActionsContainer
 
 from pex import dependency_configuration, pex_warnings
 from pex.cli.command import BuildTimeCommand
@@ -18,8 +18,11 @@
 from pex.executables import is_python_script, is_script
 from pex.executor import Executor
 from pex.fingerprinted_distribution import FingerprintedDistribution
+from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
+from pex.os import Os
 from pex.pex import PEX
+from pex.pex_bootstrapper import normalize_path
 from pex.pex_info import PexInfo
 from pex.resolve import configured_resolve, requirement_options, resolver_options, target_options
 from pex.resolve.resolver_configuration import (
@@ -39,7 +42,7 @@
 from pex.venv.virtualenv import Virtualenv
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Iterable, Optional, Sequence
+    from typing import Any, Dict, Iterable, Optional, Sequence, Union
 
 
 logger = logging.getLogger(__name__)
@@ -119,6 +122,20 @@ def _add_create_arguments(cls, parser):
                 )
             ),
         )
+        parser.add_argument(
+            "--link-python",
+            dest="link_python",
+            metavar="PATH",
+            default=None,
+            help=(
+                "The path the venv's Python should link to instead of the interpreter used to "
+                "create the venv. The interpreter need not exist; it only has to be in place by "
+                "the time the venv is used. This is what allows creating a venv for a foreign "
+                "platform: the venv is laid out by a local interpreter differing from the foreign "
+                "platform in machine architecture alone and its Python is then linked to the "
+                "given path. Only applies to the {venv} layout.".format(venv=InstallLayout.VENV)
+            ),
+        )
         installer_options.register(parser)
         target_options.register(parser, include_platforms=True)
         resolver_options.register(
@@ -274,16 +291,33 @@ def _create(self):
                 )
             )
             if layout is InstallLayout.VENV:
+                venv_interpreter = target.get_interpreter()
                 if target.is_foreign:
-                    return Error(
-                        "Cannot create a local venv for foreign platform {platform}.".format(
-                            platform=target.platform
+                    if Os.CURRENT is Os.WINDOWS:
+                        return Error(
+                            "Cannot create a local venv for foreign platform {platform}.\n"
+                            "This is only supported for Unix environments.".format(
+                                platform=target.platform
+                            )
                         )
-                    )
+                    elif not self.options.link_python:
+                        return Error(
+                            "Cannot create a local venv for foreign platform {platform}.\n"
+                            "Specify --link-python to say where the venv's Python will live in "
+                            "the foreign environment.".format(platform=target.platform)
+                        )
+                    elif not target.python_version or target.python_version < (3, 3):
+                        return Error(
+                            "Cannot create a local venv for foreign platform {platform}.\n"
+                            "This is only supported for Python 3.3 and newer.".format(
+                                platform=target.platform
+                            )
+                        )
+                    venv_interpreter = try_(_find_stand_in_interpreter(target, self.options))
 
                 venv = Virtualenv.create(
                     venv_dir=dest_dir,
-                    interpreter=target.get_interpreter(),
+                    interpreter=venv_interpreter,
                     force=installer_configuration.force,
                     copies=installer_configuration.copies,
                     system_site_packages=installer_configuration.system_site_packages,
@@ -398,11 +432,17 @@ def _create(self):
 
         if installer_configuration.compile:
             with TRACER.timed("Compiling venv sources"):
+                compiler = venv.interpreter if venv else target.get_interpreter()
                 try:
-                    target.get_interpreter().execute(["-m", "compileall", dest_dir])
+                    compiler.execute(["-m", "compileall", dest_dir])
                 except Executor.NonZeroExit as non_zero_exit:
                     pex_warnings.warn("ignoring compile error {}".format(repr(non_zero_exit)))
 
+        # N.B.: This must come last since it can leave the venv Python dangling until the venv
+        # reaches its final resting place.
+        if venv and self.options.link_python:
+            venv.link_python(self.options.link_python)
+
         if layout is InstallLayout.FLAT_ZIPPED:
             paths = sorted(
                 os.path.join(root, path)
@@ -417,6 +457,40 @@ def _create(self):
         return Ok()
 
 
+def _find_stand_in_interpreter(
+    target,  # type: Target
+    options,  # type: Namespace
+):
+    # type: (...) -> Union[PythonInterpreter, Error]
+    """Find a local interpreter that lays a venv out just like `target` would.
+
+    A venv's layout is fixed by the Python implementation, version and OS; the machine architecture
+    plays no part. That lets a local interpreter stand in for a foreign one it differs from in
+    architecture alone.
+    """
+    python_path = target_options.configure_interpreters(options).python_path
+    target_tag = target.platform.tag
+    for interpreter in PythonInterpreter.iter(paths=normalize_path(python_path)):
+        tag = interpreter.platform.tag
+        if (
+            tag.interpreter == target_tag.interpreter
+            and tag.abi == target_tag.abi
+            and interpreter.identity.env_markers.platform_system
+            == target.marker_environment.platform_system
+        ):
+            return interpreter
+
+    return Error(
+        "Could not find a local interpreter to lay out a venv for {platform}.\n"
+        "A {interpreter} interpreter running on {system} is needed; searched: {search_path}".format(
+            platform=target.platform,
+            interpreter=target_tag.interpreter,
+            system=target.marker_environment.platform_system,
+            search_path=os.pathsep.join(python_path) if python_path else "$PATH",
+        )
+    )
+
+
 def _install_from_pex(
     pex,  # type: PEX
     installer_configuration,  # type: InstallerConfiguration

pex/sysconfig.py

@@ -17,7 +17,7 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Optional, Text, TypeVar
+    from typing import Optional, Text, Tuple, TypeVar, Union
 
 EXE_EXTENSION = get_config_var("EXE") or ""
 EXE_EXTENSIONS = (
@@ -159,6 +159,14 @@ def venv_bin_dir(self):
         # type: () -> str
         return "Scripts" if self.os is Os.WINDOWS else "bin"
 
+    def venv_lib_dir(self, version):
+        # type: (Union[Tuple[int, int], Tuple[int, int, int]]) -> str
+        return (
+            "Lib"
+            if self.os is Os.WINDOWS
+            else "lib/python{major}.{minor}".format(major=version[0], minor=version[1])
+        )
+
     def binary_name(self, binary_name):
         # type: (_Text) -> _Text
         return "{binary_name}{extension}".format(

pex/venv/virtualenv.py

@@ -504,6 +504,35 @@ def iter_distributions(self, rescan=False):
         ):
             yield dist
 
+    def link_python(self, python):
+        # type: (str) -> None
+        """Point this venv's Python links at `python`.
+
+        The `python` path need not exist; it only has to be in place by the time the venv is used.
+        This supports creating a venv destined for a final resting place, like an image layer,
+        where the interpreter lives at a known path but is not available where the venv is created.
+        """
+        python_links = [
+            path
+            for path in _iter_files(self._bin_dir)
+            if os.path.islink(path) and PythonInterpreter.matches_binary_name(path)
+        ]
+        for python_link in python_links:
+            os.unlink(python_link)
+            safe_symlink(python, python_link)
+
+        pyvenv_cfg = self._pyvenv_cfg
+        if pyvenv_cfg:
+            with closing(FileInput(files=[pyvenv_cfg.path], inplace=True)) as fi:
+                for line in fi:
+                    key, delimiter, _ = line.partition("=")
+                    if delimiter and key.strip() == "home":
+                        sys.stdout.write("home = {home}\n".format(home=os.path.dirname(python)))
+                    elif delimiter and key.strip() == "executable":
+                        sys.stdout.write("executable = {python}\n".format(python=python))
+                    else:
+                        sys.stdout.write(line)
+
     def _rewrite_base_scripts(self, real_venv_dir):
         # type: (str) -> Iterator[str]
         scripts = [

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.102.0"
+__version__ = "2.103.0"

tests/integration/cli/commands/test_venv_create.py

@@ -5,6 +5,7 @@
 
 import glob
 import os.path
+import re
 import shutil
 import sys
 from subprocess import CalledProcessError
@@ -19,18 +20,22 @@
 from pex.common import open_zip, safe_open
 from pex.compatibility import safe_commonpath
 from pex.dist_metadata import Distribution
-from pex.interpreter import PythonInterpreter
+from pex.interpreter import PythonInterpreter, PyVenvCfg
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pex import PEX
 from pex.resolve import abbreviated_platforms
+from pex.sysconfig import SysPlatform
 from pex.typing import TYPE_CHECKING
 from pex.venv.bin_path import BinPath
 from pex.venv.virtualenv import Virtualenv
 from testing import (
+    IS_ARM_64,
     IS_MAC,
+    IS_WINDOWS,
     PY39,
     PY310,
+    IntegResults,
     ensure_python_interpreter,
     make_env,
     run_pex_command,
@@ -499,12 +504,31 @@ def test_foreign_target(
     )
     result.assert_failure()
     assert (
-        "Cannot create a local venv for foreign platform {platform}.".format(
-            platform=abbreviated_platforms.create(foreign_platform)
-        )
+        "Cannot create a local venv for foreign platform {platform}.\n"
+        "Specify --link-python to say where the venv's Python will live in the foreign "
+        "environment.".format(platform=abbreviated_platforms.create(foreign_platform))
         == result.error.strip()
     )
 
+    # The foreign_platform fixture is a foreign OS; so no local interpreter can stand in for it.
+    result = run_pex3(
+        "venv",
+        "create",
+        "psutil==5.9.5",
+        "-d",
+        dest,
+        "--platform",
+        foreign_platform,
+        "--link-python",
+        "/opt/python/bin/python3.10",
+    )
+    result.assert_failure()
+    assert result.error.strip().startswith(
+        "Could not find a local interpreter to lay out a venv for {platform}.".format(
+            platform=abbreviated_platforms.create(foreign_platform)
+        )
+    ), result.error
+
     run_pex3(
         "venv",
         "create",
@@ -525,6 +549,96 @@ def test_foreign_target(
     assert Version("5.9.5") == dist.metadata.version
 
 
+def cross_arch_platform(
+    major,  # type: int
+    minor,  # type: int
+    abiflags="",  # type: str
+):
+    # type: (...) -> str
+    """A foreign platform differing from the local one in machine architecture alone."""
+    pyver = "{major}{minor}".format(major=major, minor=minor)
+    if IS_MAC:
+        return "macosx_11_0_{machine}-cp-{pyver}-cp{pyver}{abiflags}".format(
+            machine="x86_64" if IS_ARM_64 else "arm64", pyver=pyver, abiflags=abiflags
+        )
+    return "linux_{machine}-cp-{pyver}-cp{pyver}{abiflags}".format(
+        machine="x86_64" if IS_ARM_64 else "aarch64", pyver=pyver, abiflags=abiflags
+    )
+
+
+@pytest.mark.skipif(IS_WINDOWS, reason="This feature is not supported on Windows.")
+def test_foreign_target_link_python(
+    tmpdir,  # type: Tempdir
+    py310,  # type: PythonInterpreter
+    py311,  # type: PythonInterpreter
+):
+    # type: (...) -> None
+
+    venv_dir = tmpdir.join("venv")
+    link_python = "/opt/python/bin/bob"
+
+    def create_venv(foreign_platform):
+        # type: (str) -> IntegResults
+        return run_pex3(
+            "venv",
+            "create",
+            "psutil==5.9.5",
+            "-d",
+            venv_dir,
+            "--platform",
+            foreign_platform,
+            "--python-path",
+            os.pathsep.join(interpreter.binary for interpreter in (py310, py311)),
+            "--link-python",
+            link_python,
+            "--force",
+        )
+
+    def assert_foreign_venv(
+        major,  # type: int
+        minor,  # type: int
+        expect_pyvenv_cfg_executable,  # type: Optional[str]
+    ):
+        # type: (...) -> None
+        assert {link_python} == {
+            os.readlink(python) for python in glob.glob(os.path.join(venv_dir, "bin", "python*"))
+        }
+
+        pyvenv_cfg = PyVenvCfg.parse(os.path.join(venv_dir, "pyvenv.cfg"))
+        assert os.path.dirname(link_python) == pyvenv_cfg.home
+        pyvenv_cfg_executable = pyvenv_cfg.config("executable")
+        if expect_pyvenv_cfg_executable:
+            assert pyvenv_cfg_executable == expect_pyvenv_cfg_executable
+        else:
+            assert pyvenv_cfg_executable is None
+
+        site_packages = os.path.join(
+            venv_dir, SysPlatform.CURRENT.venv_lib_dir(version=(major, minor)), "site-packages"
+        )
+        distributions = list(dist_metadata.find_distributions(search_path=[site_packages]))
+        assert 1 == len(distributions)
+        assert ProjectName("psutil") == distributions[0].metadata.project_name
+
+    foreign_platform = cross_arch_platform(2, 7, "mu")
+    create_venv(foreign_platform=cross_arch_platform(2, 7, "mu")).assert_failure(
+        expected_error_re=r".*{msg}$".format(
+            msg=re.escape(
+                "Cannot create a local venv for foreign platform {foreign_platform}.\n"
+                "This is only supported for Python 3.3 and newer.".format(
+                    foreign_platform=foreign_platform
+                )
+            )
+        ),
+        re_flags=re.DOTALL,
+    )
+
+    create_venv(foreign_platform=cross_arch_platform(3, 10)).assert_success()
+    assert_foreign_venv(3, 10, expect_pyvenv_cfg_executable=None)
+
+    create_venv(foreign_platform=cross_arch_platform(3, 11)).assert_success()
+    assert_foreign_venv(3, 11, expect_pyvenv_cfg_executable=link_python)
+
+
 def test_venv_update_target_mismatch(
     tmpdir,  # type: Any
     foreign_platform,  # type: str