diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index 2f0a0d7f3..d3a2a8d88 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -78,6 +78,9 @@ layers override earlier ones: │ 4. Variant overrides (within package YAML) │ │ (env vars, pre_built, wheel_server_url) │ │ │ + │ 4a. Version-specific variant overrides │ + │ (per-version pre_built, wheel_server_url) │ + │ │ │ 5. Version-specific patches and changelog │ │ (patches/-/, changelog entries)│ │ │ diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index c4943096a..ce6078985 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -12,7 +12,7 @@ from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name -from packaging.version import Version +from packaging.version import InvalidVersion, Version from . import finders, resolver, sources, wheels from .dependency_graph import DependencyGraph @@ -24,6 +24,21 @@ logger = logging.getLogger(__name__) +def _extract_pinned_version(req: Requirement) -> Version | None: + """Return the version if *req* is pinned to exactly one (``==``). + + Returns ``None`` for range specifiers, wildcard pins (``==1.*``), + extras-only, or empty specifiers. + """ + specs = list(req.specifier) + if len(specs) == 1 and specs[0].operator == "==" and "*" not in specs[0].version: + try: + return Version(specs[0].version) + except InvalidVersion: + return None + return None + + class BootstrapRequirementResolver: """Resolve package requirements from PyPI or dependency graph during bootstrap. @@ -113,7 +128,8 @@ def resolve( # Determine pre_built if not specified (needed for cache key) if pre_built is None: pbi = self.ctx.package_build_info(req) - pre_built = pbi.pre_built + pinned = _extract_pinned_version(req) + pre_built = pbi.is_pre_built(pinned) rule_key = (str(req), pre_built) @@ -181,8 +197,12 @@ def _resolve_and_extend( results = cached_resolution elif pre_built: # Resolve prebuilt wheel + pinned = _extract_pinned_version(req) wheel_server_urls = wheels.get_wheel_server_urls( - self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL + self.ctx, + req, + cache_wheel_server_url=resolver.PYPI_SERVER_URL, + version=pinned, ) results = wheels.resolve_all_prebuilt_wheels( ctx=self.ctx, diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index 8d33ad899..99bade774 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -181,7 +181,7 @@ def _resolve_and_add_top_level( req=req, req_version=version, download_url=source_url, - pre_built=pbi.pre_built, + pre_built=pbi.is_pre_built(version), constraint=self.ctx.constraints.get_constraint(req.name), ) @@ -600,7 +600,7 @@ def add_to_graph( req=req, req_version=req_version, download_url=download_url, - pre_built=pbi.pre_built, + pre_built=pbi.is_pre_built(req_version), constraint=self.ctx.constraints.get_constraint(req.name), ) self._write_graph_async() diff --git a/src/fromager/bootstrapper/_process_install_deps.py b/src/fromager/bootstrapper/_process_install_deps.py index a6ed4b58d..3ae5dc338 100644 --- a/src/fromager/bootstrapper/_process_install_deps.py +++ b/src/fromager/bootstrapper/_process_install_deps.py @@ -154,7 +154,7 @@ def run(self, bt: Bootstrapper) -> list[Phase]: version=wi.resolved_version, source_url=wi.source_url, source_type=wi.build_result.source_type, - prebuilt=pbi.pre_built, + prebuilt=pbi.is_pre_built(wi.resolved_version), constraint=constraint, ) diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index 2db1eac92..febd9602e 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -3,17 +3,67 @@ import logging import typing +from packaging.requirements import Requirement +from packaging.version import Version + +from .. import resolver, sources, wheels from ..requirements_file import RequirementType from ._phase import Phase from ._prepare_source import PrepareSource from ._types import BootstrapPhase if typing.TYPE_CHECKING: + from .. import context from ._bootstrapper import Bootstrapper logger = logging.getLogger(__name__) +def _re_resolve_url( + ctx: context.WorkContext, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + pre_built: bool, + cache_wheel_server_url: str | None, +) -> str | None: + """Re-resolve the download URL when version-specific pre_built differs. + + Returns the new URL or ``None`` if re-resolution fails. + """ + pinned_req = Requirement(f"{req.name}=={resolved_version}") + if pre_built: + wheel_server_urls = wheels.get_wheel_server_urls( + ctx, + req, + cache_wheel_server_url=cache_wheel_server_url, + version=resolved_version, + ) + try: + url, _ = wheels.resolve_prebuilt_wheel( + ctx=ctx, + req=pinned_req, + wheel_server_urls=wheel_server_urls, + req_type=req_type, + ) + except ExceptionGroup: + return None + return str(url) + else: + pbi = ctx.package_build_info(req) + sdist_server = pbi.resolver_sdist_server_url(resolver.PYPI_SERVER_URL) + provider = sources.get_source_provider( + ctx=ctx, + req=pinned_req, + sdist_server_url=sdist_server, + req_type=req_type, + ) + results = resolver.find_all_matching_from_provider(provider, pinned_req) + if results: + return str(results[0][0]) + return None + + class Start(Phase): """Record a resolved requirement in the dependency graph and deduplicate. @@ -44,7 +94,46 @@ def run(self, bt: Bootstrapper) -> list[Phase]: assert wi.resolved_version is not None assert wi.source_url is not None - # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) + wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() + + # Must set pbi_pre_built before constructing PrepareSource so that + # PrepareSource.background_work() immediately sees the correct value. + pbi = bt.ctx.package_build_info(wi.req) + wi.pbi_pre_built = pbi.is_pre_built(wi.resolved_version) + wi.exclusive_build = pbi.exclusive_build + + # Re-resolve URL before graph insertion so the graph stores the + # final URL, and before the seen-check so duplicate parents still + # get their edge recorded with the correct URL. + version_url = pbi.get_wheel_server_url(wi.resolved_version) + variant_url = pbi.wheel_server_url + needs_re_resolve = wi.pbi_pre_built != pbi.pre_built or ( + wi.pbi_pre_built and version_url != variant_url + ) + if needs_re_resolve: + logger.info( + f"{wi.req} {wi.resolved_version}: version-specific override " + f"(pre_built={wi.pbi_pre_built}, url={version_url}) differs " + f"from variant default, re-resolving URL" + ) + new_url = _re_resolve_url( + bt.ctx, + wi.req, + wi.req_type, + wi.resolved_version, + wi.pbi_pre_built, + bt.cache_wheel_server_url, + ) + if new_url is not None: + wi.source_url = new_url + else: + logger.warning( + f"{wi.req} {wi.resolved_version}: could not re-resolve URL " + f"for pre_built={wi.pbi_pre_built}, using original" + ) + + # Add to graph after URL finalization but before the seen-check + # so every parent-to-dep edge is recorded with the correct URL. if wi.req_type != RequirementType.TOP_LEVEL: bt.add_to_graph( wi.req, @@ -54,8 +143,6 @@ def run(self, bt: Bootstrapper) -> list[Phase]: wi.parent, ) - wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() - if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): logger.debug( f"redundant {wi.req_type} dependency {wi.req} " @@ -69,9 +156,4 @@ def run(self, bt: Bootstrapper) -> list[Phase]: f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" ) - # Must set pbi_pre_built before constructing PrepareSource so that - # PrepareSource.background_work() immediately sees the correct value. - pbi = bt.ctx.package_build_info(wi.req) - wi.pbi_pre_built = pbi.pre_built - wi.exclusive_build = pbi.exclusive_build return [PrepareSource(wi)] diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 2e33f8f0e..8824d9b76 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -344,10 +344,13 @@ def _build( logger.info("starting processing") pbi = wkctx.package_build_info(req) - prebuilt = pbi.pre_built + prebuilt = pbi.is_pre_built(resolved_version) wheel_server_urls = wheels.get_wheel_server_urls( - wkctx, req, cache_wheel_server_url=cache_wheel_server_url + wkctx, + req, + cache_wheel_server_url=cache_wheel_server_url, + version=resolved_version, ) # See if we can reuse an existing wheel. diff --git a/src/fromager/packagesettings/__init__.py b/src/fromager/packagesettings/__init__.py index abca2f21d..fc1a769eb 100644 --- a/src/fromager/packagesettings/__init__.py +++ b/src/fromager/packagesettings/__init__.py @@ -12,6 +12,7 @@ ResolverDist, SbomSettings, VariantInfo, + VersionSpecificSettings, ) from ._pbi import PackageBuildInfo from ._resolver import ( @@ -88,6 +89,7 @@ "Variant", "VariantChangelog", "VariantInfo", + "VersionSpecificSettings", "default_update_extra_environ", "get_extra_environ", "pep440_tag_matcher", diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index e87230613..8ee2bc027 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -22,6 +22,7 @@ BuildDirectory, EnvVars, Package, + PackageVersion, PurlType, RawAnnotations, Template, @@ -450,6 +451,32 @@ def validate_update_build_requires(cls, v: list[str]) -> list[str]: return v +class VersionSpecificSettings(pydantic.BaseModel): + """Per-version overrides within a variant. + + Allows overriding ``pre_built`` and ``wheel_server_url`` for + specific package versions. When a field is ``None``, the + variant-wide default is used. + + .. versionadded:: 0.95.0 + + :: + + versions: + "2.9.0": + pre_built: true + wheel_server_url: https://gitlab.example.com/simple + """ + + model_config = MODEL_CONFIG + + wheel_server_url: str | None = None + """Alternative package index for this version's pre-built wheel""" + + pre_built: bool | None = None + """Override pre-built flag for this version (None = inherit variant default)""" + + class VariantInfo(pydantic.BaseModel): """Variant information for a package @@ -460,6 +487,10 @@ class VariantInfo(pydantic.BaseModel): VAR2: "2.0 wheel_server_url: https://pypi.org/simple/ pre_built: False + versions: + "2.9.0": + pre_built: true + wheel_server_url: https://gitlab.example.com/simple """ model_config = MODEL_CONFIG @@ -480,10 +511,33 @@ class VariantInfo(pydantic.BaseModel): pre_built: bool = False """Use pre-built wheel from index server?""" + versions: Mapping[PackageVersion, VersionSpecificSettings] = Field( + default_factory=dict + ) + """Per-version overrides for ``pre_built`` and ``wheel_server_url``. + + Version-specific settings take precedence over variant defaults + when present. + + .. versionadded:: 0.95.0 + """ + # TODO # source: SourceResolver | None # """Source resolver and downloader""" + @pydantic.field_validator("versions", mode="before") + @classmethod + def before_none_versions( + cls, + v: dict[str, typing.Any] | None, + info: core_schema.ValidationInfo, + ) -> dict[str, typing.Any]: + """Coerce ``None`` to empty dict for bare ``versions:`` YAML key.""" + if v is None: + return {} + return v + class GitOptions(pydantic.BaseModel): """Git repository cloning options diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 2e2620047..a318308df 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -162,19 +162,57 @@ def has_customizations(self) -> bool: @property def pre_built(self) -> bool: - """Does the variant use pre-build wheels?""" + """Does the variant use pre-build wheels? + + Delegates to :meth:`is_pre_built` with no version. + """ + return self.is_pre_built() + + def is_pre_built(self, version: Version | None = None) -> bool: + """Version-aware pre-built check. + + Resolution order: + + 1. Version-specific YAML setting + 2. Variant-wide default + + .. versionadded:: 0.95.0 + """ vi = self._ps.variants.get(self.variant) - if vi is not None: - return vi.pre_built - return False + if vi is None: + return False + if version is not None: + pv = typing.cast(PackageVersion, Version(version.public)) + vs = vi.versions.get(pv) + if vs is not None and vs.pre_built is not None: + return vs.pre_built + return vi.pre_built @property def wheel_server_url(self) -> str | None: - """Alternative package index for pre-build wheel""" + """Alternative package index for pre-build wheel. + + Delegates to :meth:`get_wheel_server_url` with no version. + """ + return self.get_wheel_server_url() + + def get_wheel_server_url(self, version: Version | None = None) -> str | None: + """Version-aware wheel server URL. + + Returns the version-specific URL if defined, otherwise + falls back to the variant-wide default. + + .. versionadded:: 0.95.0 + """ vi = self._ps.variants.get(self.variant) - if vi is not None and vi.wheel_server_url is not None: - return str(vi.wheel_server_url) - return None + if vi is None: + return None + if version is not None: + pv = typing.cast(PackageVersion, Version(version.public)) + vs = vi.versions.get(pv) + if vs is not None and vs.wheel_server_url is not None: + return str(vs.wheel_server_url) + return str(vi.wheel_server_url) if vi.wheel_server_url is not None else None @property def override_module_name(self) -> str: @@ -295,8 +333,7 @@ def build_tag(self, version: Version) -> BuildTag: the build tag from changelog, e.g. version `1.0.3+local.suffix` uses `1.0.3`. """ - if self.pre_built: - # pre-built wheels have no built tag + if self.is_pre_built(version): return () pv = typing.cast(PackageVersion, version) release = len(self.get_changelog(pv)) diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index dc9bd5241..477e5f0e3 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -459,13 +459,25 @@ def download_wheel( def get_wheel_server_urls( - ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None + ctx: context.WorkContext, + req: Requirement, + *, + cache_wheel_server_url: str | None, + version: Version | None = None, ) -> list[str]: + """Build ordered list of wheel server URLs for a package. + + When *version* is given, version-specific ``wheel_server_url`` + overrides are checked first. + + .. versionchanged:: 0.95.0 + Added *version* parameter for version-specific URL lookup. + """ pbi = ctx.package_build_info(req) + url = pbi.get_wheel_server_url(version) wheel_server_urls: list[str] = [] - if pbi.wheel_server_url: - # use only the wheel server from settings if it is defined. Do not fallback to other URLs - wheel_server_urls.append(pbi.wheel_server_url) + if url: + wheel_server_urls.append(url) else: if ctx.wheel_server_url: # local wheel server diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index 7222a6fbc..171dd4eb6 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -444,7 +444,9 @@ def test_resolve_auto_routes_to_prebuilt( # Mock package build info to return pre_built=True mock_pbi = MagicMock() mock_pbi.pre_built = True + mock_pbi.is_pre_built.return_value = True mock_pbi.wheel_server_url = None + mock_pbi.get_wheel_server_url.return_value = None mock_pbi.resolver_min_release_age = None with patch.object(tmp_context, "package_build_info", return_value=mock_pbi): @@ -485,6 +487,7 @@ def test_resolve_auto_routes_to_source( # Mock package build info to return pre_built=False mock_pbi = MagicMock() mock_pbi.pre_built = False + mock_pbi.is_pre_built.return_value = False mock_pbi.resolver_include_sdists = True mock_pbi.resolver_include_wheels = True mock_pbi.resolver_ignore_platform = True diff --git a/tests/test_bootstrap_requirement_resolver_multiple.py b/tests/test_bootstrap_requirement_resolver_multiple.py index ce9482def..c172919a2 100644 --- a/tests/test_bootstrap_requirement_resolver_multiple.py +++ b/tests/test_bootstrap_requirement_resolver_multiple.py @@ -25,6 +25,7 @@ def tmp_context(tmp_path: Path) -> WorkContext: ctx.package_build_info = MagicMock() pbi = MagicMock() pbi.pre_built = False + pbi.is_pre_built.return_value = False pbi.resolver_include_sdists = True pbi.resolver_include_wheels = False pbi.resolver_ignore_platform = False diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 79ff439ca..6ef366732 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -38,7 +38,7 @@ from fromager.bootstrapper._prepare_source import PrepareSource from fromager.bootstrapper._process_install_deps import ProcessInstallDeps from fromager.bootstrapper._resolve import Resolve -from fromager.bootstrapper._start import Start +from fromager.bootstrapper._start import Start, _re_resolve_url from fromager.bootstrapper._types import ( BootstrapPhase, PreparedSourceData, @@ -602,6 +602,52 @@ def test_skips_graph_for_toplevel(self, tmp_context: WorkContext) -> None: key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes + def test_graph_edge_recorded_for_already_seen_package( + self, tmp_context: WorkContext + ) -> None: + """Graph edge is recorded even when the package was already processed.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + + parent_a = (Requirement("parent-a"), Version("1.0")) + parent_b = (Requirement("parent-b"), Version("2.0")) + + # Add parent nodes so add_dependency can attach edges + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=parent_a[0], + req_version=parent_a[1], + ) + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=parent_b[0], + req_version=parent_b[1], + ) + + item1 = _make_start_item(req_type=RequirementType.INSTALL, parent=parent_a) + item2 = _make_start_item(req_type=RequirementType.INSTALL, parent=parent_b) + + result1 = item1.run(bt) + assert len(result1) == 1 + + result2 = item2.run(bt) + assert result2 == [] + + # Both parent edges should be in the graph + dep_key = f"{canonicalize_name('testpkg')}==1.0" + parent_a_key = f"{canonicalize_name('parent-a')}==1.0" + parent_b_key = f"{canonicalize_name('parent-b')}==2.0" + assert dep_key in tmp_context.dependency_graph.nodes + dep_node = tmp_context.dependency_graph.nodes[dep_key] + parent_a_node = tmp_context.dependency_graph.nodes[parent_a_key] + parent_b_node = tmp_context.dependency_graph.nodes[parent_b_key] + assert dep_node in [e.destination_node for e in parent_a_node.children] + assert dep_node in [e.destination_node for e in parent_b_node.children] + def test_sdist_only_set_for_non_build_requirement( self, tmp_context: WorkContext ) -> None: @@ -646,7 +692,12 @@ def test_sets_pbi_pre_built_before_prepare_source( with patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=True), + return_value=Mock( + pre_built=True, + is_pre_built=Mock(return_value=True), + wheel_server_url=None, + get_wheel_server_url=Mock(return_value=None), + ), ): result = item.run(bt) @@ -655,6 +706,202 @@ def test_sets_pbi_pre_built_before_prepare_source( assert result[0].work_item.pbi_pre_built is True +class TestReResolveUrl: + """Tests for _re_resolve_url used by Start.run for version-specific overrides.""" + + def test_prebuilt_returns_resolved_url(self, tmp_context: WorkContext) -> None: + """Pre-built path returns URL from resolve_prebuilt_wheel.""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.wheels.get_wheel_server_urls", + return_value=["https://wheels.test/simple/"], + ), + patch( + "fromager.bootstrapper._start.wheels.resolve_prebuilt_wheel", + return_value=( + "https://wheels.test/testpkg-1.0-py3-none-any.whl", + Version("1.0"), + ), + ), + ): + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=True, + cache_wheel_server_url=None, + ) + + assert result == "https://wheels.test/testpkg-1.0-py3-none-any.whl" + + def test_source_returns_resolved_url(self, tmp_context: WorkContext) -> None: + """Source path returns URL from find_all_matching_from_provider.""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.sources.get_source_provider", + ) as mock_provider, + patch( + "fromager.bootstrapper._start.resolver.find_all_matching_from_provider", + return_value=[("https://pypi.test/testpkg-1.0.tar.gz", Version("1.0"))], + ), + ): + mock_provider.return_value = Mock() + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=False, + cache_wheel_server_url=None, + ) + + assert result == "https://pypi.test/testpkg-1.0.tar.gz" + + def test_prebuilt_returns_none_on_exception_group( + self, tmp_context: WorkContext + ) -> None: + """Pre-built path returns None when no wheel found (ExceptionGroup).""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.wheels.get_wheel_server_urls", + return_value=["https://wheels.test/simple/"], + ), + patch( + "fromager.bootstrapper._start.wheels.resolve_prebuilt_wheel", + side_effect=ExceptionGroup( + "no wheel found", + [Exception("server 1 failed")], + ), + ), + ): + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=True, + cache_wheel_server_url=None, + ) + + assert result is None + + def test_source_returns_none_when_no_match(self, tmp_context: WorkContext) -> None: + """Source path returns None when find_all_matching returns empty.""" + req = Requirement("testpkg==1.0") + with ( + patch( + "fromager.bootstrapper._start.sources.get_source_provider", + ) as mock_provider, + patch( + "fromager.bootstrapper._start.resolver.find_all_matching_from_provider", + return_value=[], + ), + ): + mock_provider.return_value = Mock() + result = _re_resolve_url( + tmp_context, + req, + RequirementType.INSTALL, + Version("1.0"), + pre_built=False, + cache_wheel_server_url=None, + ) + + assert result is None + + def test_wheel_server_url_differs_triggers_re_resolve( + self, tmp_context: WorkContext + ) -> None: + """Start.run re-resolves when wheel_server_url differs but pre_built matches.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + + mock_pbi = Mock( + pre_built=True, + is_pre_built=Mock(return_value=True), + exclusive_build=False, + wheel_server_url="https://default.test/simple/", + get_wheel_server_url=Mock(return_value="https://version.test/simple/"), + ) + + with ( + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._start._re_resolve_url", + return_value="https://version.test/testpkg-1.0-py3-none-any.whl", + ) as mock_re_resolve, + ): + item.run(bt) + + mock_re_resolve.assert_called_once() + assert ( + item.work_item.source_url + == "https://version.test/testpkg-1.0-py3-none-any.whl" + ) + + def test_no_re_resolve_when_settings_match_defaults( + self, tmp_context: WorkContext + ) -> None: + """Start.run skips re-resolution when version settings match variant defaults.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + original_url = item.work_item.source_url + + mock_pbi = Mock( + pre_built=False, + is_pre_built=Mock(return_value=False), + exclusive_build=False, + wheel_server_url=None, + get_wheel_server_url=Mock(return_value=None), + ) + + with ( + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._start._re_resolve_url", + ) as mock_re_resolve, + ): + item.run(bt) + + mock_re_resolve.assert_not_called() + assert item.work_item.source_url == original_url + + def test_fallback_to_original_url_on_failed_re_resolve( + self, tmp_context: WorkContext + ) -> None: + """Start.run keeps original URL and logs warning when re-resolve returns None.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + original_url = item.work_item.source_url + + mock_pbi = Mock( + pre_built=False, + is_pre_built=Mock(return_value=True), + exclusive_build=False, + wheel_server_url=None, + get_wheel_server_url=Mock(return_value=None), + ) + + with ( + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._start._re_resolve_url", + return_value=None, + ) as mock_re_resolve, + ): + item.run(bt) + + mock_re_resolve.assert_called_once() + assert item.work_item.source_url == original_url + + class TestComplete: def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -1914,7 +2161,9 @@ def test_normal_path_returns_item_and_dep_items( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order") as mock_build_order, @@ -1954,7 +2203,9 @@ def test_hook_error_test_mode_records_and_continues( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order") as mock_build_order, @@ -1999,7 +2250,9 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order") as mock_build_order, @@ -2051,7 +2304,9 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=False), + return_value=Mock( + pre_built=False, is_pre_built=Mock(return_value=False) + ), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "add_to_build_order"), @@ -2080,7 +2335,7 @@ def test_build_order_called_with_correct_args( patch.object( tmp_context, "package_build_info", - return_value=Mock(pre_built=True), + return_value=Mock(pre_built=True, is_pre_built=Mock(return_value=True)), ), patch.object( tmp_context.constraints, diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index a5107ad98..8e5497c79 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -8,7 +8,7 @@ from packaging.utils import NormalizedName from packaging.version import Version -from fromager import build_environment, context +from fromager import build_environment, context, wheels from fromager.packagesettings import ( Annotations, BuildDirectory, @@ -98,6 +98,16 @@ "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"}, "wheel_server_url": "https://wheel.test/simple", "pre_built": False, + "versions": { + Version("2.9.0"): { + "wheel_server_url": "https://mirror.test/simple", + "pre_built": True, + }, + Version("2.8.0"): { + "wheel_server_url": None, + "pre_built": False, + }, + }, }, "rocm": { "annotations": { @@ -106,12 +116,19 @@ "env": {"SPAM": ""}, "wheel_server_url": None, "pre_built": True, + "versions": { + Version("1.0.0"): { + "wheel_server_url": None, + "pre_built": False, + }, + }, }, "cuda": { "annotations": None, "env": {}, "wheel_server_url": None, "pre_built": False, + "versions": {}, }, }, } @@ -199,6 +216,7 @@ "env": {}, "pre_built": True, "wheel_server_url": None, + "versions": {}, }, }, } @@ -584,6 +602,105 @@ def test_global_changelog(testdata_context: context.WorkContext) -> None: assert pbi.build_tag(Version("1.0.1")) == () +def test_is_pre_built_version_specific( + testdata_context: context.WorkContext, +) -> None: + """Version-specific pre_built overrides variant default.""" + # cpu variant: pre_built=False by default, but 2.9.0 is pre_built=True + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.variant == "cpu" + assert pbi.pre_built is False + assert pbi.is_pre_built() is False + assert pbi.is_pre_built(Version("2.9.0")) is True + assert pbi.is_pre_built(Version("2.9.0+cpu")) is True + assert pbi.is_pre_built(Version("2.8.0")) is False + assert pbi.is_pre_built(Version("2.8.0+local")) is False + assert pbi.is_pre_built(Version("3.0.0")) is False + + # rocm variant: pre_built=True by default, but 1.0.0 is pre_built=False + testdata_context.settings.variant = Variant("rocm") + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.pre_built is True + assert pbi.is_pre_built() is True + assert pbi.is_pre_built(Version("1.0.0")) is False + assert pbi.is_pre_built(Version("2.0.0")) is True + + # cuda variant: no version-specific settings + testdata_context.settings.variant = Variant("cuda") + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.is_pre_built() is False + assert pbi.is_pre_built(Version("2.9.0")) is False + + +def test_get_wheel_server_url_version_specific( + testdata_context: context.WorkContext, +) -> None: + """Version-specific wheel_server_url overrides variant default.""" + # cpu variant: default wheel_server_url, 2.9.0 has override + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.wheel_server_url == "https://wheel.test/simple" + assert pbi.get_wheel_server_url() == "https://wheel.test/simple" + assert pbi.get_wheel_server_url(Version("2.9.0")) == "https://mirror.test/simple" + assert ( + pbi.get_wheel_server_url(Version("2.9.0+cpu")) == "https://mirror.test/simple" + ) + assert pbi.get_wheel_server_url(Version("2.8.0")) == "https://wheel.test/simple" + assert pbi.get_wheel_server_url(Version("3.0.0")) == "https://wheel.test/simple" + + +def test_build_tag_version_specific_prebuilt( + testdata_context: context.WorkContext, +) -> None: + """Pre-built versions return empty build tag even when variant default is source.""" + pbi = testdata_context.settings.package_build_info(TEST_PKG) + assert pbi.variant == "cpu" + # 2.9.0 is version-specific pre_built=True, so no build tag + assert pbi.build_tag(Version("2.9.0")) == () + + +def test_get_wheel_server_urls_version_specific( + testdata_context: context.WorkContext, +) -> None: + """get_wheel_server_urls uses version-specific URL when version given.""" + req = Requirement("test-pkg") + # cpu variant: default URL is https://wheel.test/simple, + # version 2.9.0 overrides to https://mirror.test/simple + urls_default = wheels.get_wheel_server_urls( + testdata_context, req, cache_wheel_server_url=None + ) + assert urls_default == ["https://wheel.test/simple"] + + urls_versioned = wheels.get_wheel_server_urls( + testdata_context, req, cache_wheel_server_url=None, version=Version("2.9.0") + ) + assert urls_versioned == ["https://mirror.test/simple"] + + urls_other = wheels.get_wheel_server_urls( + testdata_context, req, cache_wheel_server_url=None, version=Version("3.0.0") + ) + assert urls_other == ["https://wheel.test/simple"] + + +def test_variant_info_versions_none_yaml() -> None: + """Bare ``versions:`` key in YAML (parsed as None) produces empty dict.""" + ps = PackageSettings.from_string( + "test-none-versions", + "variants:\n cpu:\n versions:\n", + ) + vi = ps.variants[Variant("cpu")] + assert vi.versions == {} + + +def test_variant_info_versions_omitted() -> None: + """Omitting ``versions`` entirely produces empty dict.""" + ps = PackageSettings.from_string( + "test-no-versions", + "variants:\n cpu:\n pre_built: true\n", + ) + vi = ps.variants[Variant("cpu")] + assert vi.versions == {} + + def test_settings_list(testdata_context: context.WorkContext) -> None: assert testdata_context.settings.list_overrides() == { TEST_COOLDOWN_PKG, diff --git a/tests/testdata/context/overrides/settings/test_pkg.yaml b/tests/testdata/context/overrides/settings/test_pkg.yaml index a1d11352c..1bb0c6302 100644 --- a/tests/testdata/context/overrides/settings/test_pkg.yaml +++ b/tests/testdata/context/overrides/settings/test_pkg.yaml @@ -50,10 +50,19 @@ variants: EGG: "spam ${EGG}" EGG_AGAIN: "$EGG" wheel_server_url: https://wheel.test/simple + versions: + "2.9.0": + pre_built: true + wheel_server_url: https://mirror.test/simple + "2.8.0": + pre_built: false rocm: annotations: fromager.test.override: amd override env: SPAM: "" pre_built: True + versions: + "1.0.0": + pre_built: false cuda: {}