From c97ec0d53f3b123ee4ac87100ae6f8166adb8b11 Mon Sep 17 00:00:00 2001 From: Ricardo Dias Date: Tue, 13 Oct 2026 16:52:26 +0100 Subject: [PATCH 1/2] project release: copy package-level qa lanes into release mirrors as . entries Signed-off-by: Ricardo Dias --- docs/PERCONA_OBS_TOOL.md | 6 +- ...-prerelease-project-release-flow-design.md | 11 +- percona_obs/cmd_project.py | 77 ++++++++-- tests/test_project_release.py | 131 ++++++++++++++++++ 4 files changed, 215 insertions(+), 10 deletions(-) diff --git a/docs/PERCONA_OBS_TOOL.md b/docs/PERCONA_OBS_TOOL.md index 6e7a87483..7d97f56a0 100644 --- a/docs/PERCONA_OBS_TOOL.md +++ b/docs/PERCONA_OBS_TOOL.md @@ -1087,7 +1087,11 @@ error. `project release ` does the following: (`containers/`, `extras/`, `extras/containers/`, `tarballs/`). All mirrors get `build: false`, `publish:` flags are carried over, and subproject paths — including intra-project sibling paths — are rewritten to the release namespace. A mirror - directory whose staging source no longer exists is **deleted**. + directory whose staging source no longer exists is **deleted**. Each mirror also carries + its staging project's `qa:` entries (macros rendered, project references rewritten to the + release project), plus the `qa:` entries of its direct packages' `package.yaml` as + project-level entries named `.` (segment = the entry's `name` or its + `pipeline`), since mirrors have no package directories; a duplicate name fails the cut. 6. **Builds `CHANGELOG.md`** — compares built package versions from the top-level staging project, `extras`, and `tarballs` against the release OBS project (for updates) and generates entries with upstream URLs. Container images are listed per flavor (`ubi8` diff --git a/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md b/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md index ca4dded3c..8eadbfbf9 100644 --- a/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md +++ b/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md @@ -161,7 +161,13 @@ of `main` whose staging tree the pre-release was built from. `OBS_PROJECT: ${OBS_ROOTPRJ}:ppg:releases:18`, `VERSION: ppg-18.6`); `ppg:staging:` inside parameter values is rewritten to `ppg:releases:` the same way subproject paths are. Only `${…}` env - tokens survive, resolved per profile at QA time. + tokens survive, resolved per profile at QA time. Package-level `qa:` entries + (from each direct package's `package.yaml`, `_shared/` links followed) are + appended to the mirror's list as project-level entries named + `.` (segment = the entry's `name` or its `pipeline`), + because mirrors have no package directories; on the pre-release they run as + project lanes with check names `OBS QA / / .`. + A duplicate name fails the cut. ### `sync prerelease ` (new command) @@ -354,3 +360,6 @@ re-dispatch of hop 2. entries (removed in a `finally`), as hop 1 does; OBS requires release targets for a project-level release, so the pre-release meta carries them only during the copy. +- Package-level `qa:` lanes reach the mirrors as `.` + project-level entries (mirrors have no package directories); duplicates + fail the cut. diff --git a/percona_obs/cmd_project.py b/percona_obs/cmd_project.py index 2d42d0939..0ed0d2c53 100644 --- a/percona_obs/cmd_project.py +++ b/percona_obs/cmd_project.py @@ -41,6 +41,7 @@ is_project, is_shared_source_dir, load_macros, + load_package_yaml, load_project_yaml, load_yaml, logger, @@ -1733,6 +1734,60 @@ def _rewrite_qa_project_refs(value, source_project_id: str, release_project_id: return value +def _mirror_qa_entries( + project_cfg: dict, + project_path: Path, + obs_id: str, + source_project_id: str, + release_project_id: str, +) -> "list[dict] | None": + """The ``qa:`` list a release mirror gets for one staging project. + + The project's own entries come first (rewritten to the release project), + then one entry per ``qa:`` entry of each direct package (``package.yaml``, + macros rendered from the package's own chain, ``${VAR}`` tokens kept), + named ``.`` where segment is the entry's ``name`` or its + ``pipeline``. Mirrors have no package directories, so package lanes can + only reach the pre-release as project lanes; the package prefix keeps the + check names unique and recognisable. Returns None when there is nothing + to carry. A duplicate name is a hard error at cut time. + """ + entries: list[dict] = [] + own = project_cfg.get("qa") + if own is not None: + own = _rewrite_qa_project_refs(own, source_project_id, release_project_id) + entries.extend(own if isinstance(own, list) else [own]) + for _obs, pkg_path in find_packages(project_path, obs_id, recursive=False): + pkg_yaml = pkg_path / "package.yaml" + if not pkg_yaml.is_file(): + continue + pkg_qa = load_package_yaml(pkg_yaml).get("qa") + if pkg_qa is None: + continue + for entry in pkg_qa if isinstance(pkg_qa, list) else [pkg_qa]: + if not isinstance(entry, dict): + raise SystemExit( + f"error: {pkg_path.name}/package.yaml: qa entries must be mappings" + ) + segment = entry.get("name") or entry.get("pipeline") or "" + copied: dict = _rewrite_qa_project_refs( # type: ignore[assignment] + entry, source_project_id, release_project_id + ) + copied.pop("name", None) + # name first, for readable mirrors + entries.append({"name": f"{pkg_path.name}.{segment}", **copied}) + seen: set[str] = set() + for entry in entries: + name = entry.get("name") or entry.get("pipeline") or "" + if name in seen: + raise SystemExit( + f"error: {source_project_id}: two qa entries resolve to the same " + f"name {name!r} in the release mirror; rename one of them" + ) + seen.add(name) + return entries or None + + def _write_release_tree( release_dir: Path, project_data: dict, @@ -1791,10 +1846,15 @@ def _write_release_tree( for key in ("debuginfo", "publish", "project-config"): if key in source_sub_config: sub_data[key] = source_sub_config[key] - if source_sub_config.get("qa") is not None: - sub_data["qa"] = _rewrite_qa_project_refs( - source_sub_config["qa"], source_project_id, release_project - ) + mirror_qa = _mirror_qa_entries( + source_sub_config, + sub_path, + sub_obs_id, + source_project_id, + release_project, + ) + if mirror_qa is not None: + sub_data["qa"] = mirror_qa release_sub_dir = release_dir / Path(*subproject_name.split(":")) expected_dirs.add(release_sub_dir) release_sub_dir.mkdir(parents=True, exist_ok=True) @@ -2611,10 +2671,11 @@ def _derive() -> str: top_cfg_no_env = _load_project_config_with_inheritance( source_path, repo_filter=RepositoryFilter.EMPTY ) - if top_cfg_no_env.get("qa") is not None: - project_data["qa"] = _rewrite_qa_project_refs( - top_cfg_no_env["qa"], args.project, release_project - ) + top_qa = _mirror_qa_entries( + top_cfg_no_env, source_path, args.project, args.project, release_project + ) + if top_qa is not None: + project_data["qa"] = top_qa if is_first_release: commit_msg = f"Release {release_project} ({release_id})" diff --git a/tests/test_project_release.py b/tests/test_project_release.py index 632bc65c4..86d091ab0 100644 --- a/tests/test_project_release.py +++ b/tests/test_project_release.py @@ -763,6 +763,137 @@ def test_write_release_tree_carries_rendered_qa_blocks(tmp_path, monkeypatch): assert "qa" not in tarballs +def _pkg_with_qa(project_dir, name, qa_yaml): + pkg = project_dir / name + (pkg / "obs").mkdir(parents=True) + (pkg / "obs" / "_service").write_text("\n") + (pkg / "package.yaml").write_text(qa_yaml) + return pkg + + +_AUDIT_QA = "qa:\n - pipeline: audit\n parameters:\n X: 1\n" + + +def test_mirror_qa_entries_appends_package_lanes(tmp_path, monkeypatch): + import percona_obs.common as common + from percona_obs.cmd_project import _mirror_qa_entries + + src = _staging_tree(tmp_path) + monkeypatch.setattr(cmd_project, "_REPO_DIR", tmp_path) + monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root") + (tmp_path / "root/macros.yaml").write_text("- PG_VERSION: 17.11\n") + _pkg_with_qa( + src, + "percona-pg_tde", + "qa:\n - name: pg_tde\n pipeline: tde-core-parallel\n parameters:\n" + " OBS_PROJECT: ${OBS_ROOTPRJ}:ppg:staging:17\n VERSION: ppg-%!{PG_VERSION}\n", + ) + _pkg_with_qa(src, "percona-pgaudit", _AUDIT_QA) + _pkg_with_qa(src, "percona-haproxy", "build:\n RockyLinux_9: true\n") + cfg = { + "qa": [ + {"name": "server", "pipeline": "p", "parameters": {"A": "ppg:staging:17"}} + ] + } + out = _mirror_qa_entries( + cfg, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17" + ) + assert [e["name"] for e in out] == [ + "server", + "percona-pg_tde.pg_tde", + "percona-pgaudit.audit", + ] + assert out[0]["parameters"]["A"] == "ppg:releases:17" + assert out[1]["parameters"] == { + "OBS_PROJECT": "${OBS_ROOTPRJ}:ppg:releases:17", + "VERSION": "ppg-17.11", + } + assert out[1]["pipeline"] == "tde-core-parallel" + + +def test_mirror_qa_entries_none_when_nothing_and_package_only(tmp_path, monkeypatch): + import percona_obs.common as common + from percona_obs.cmd_project import _mirror_qa_entries + + src = _staging_tree(tmp_path) + monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root") + ids = ("ppg:staging:17", "ppg:staging:17", "ppg:releases:17") + assert _mirror_qa_entries({}, src, *ids) is None + _pkg_with_qa(src, "percona-pgaudit", _AUDIT_QA) + out = _mirror_qa_entries({}, src, *ids) + assert [e["name"] for e in out] == ["percona-pgaudit.audit"] + + +def test_mirror_qa_entries_rejects_duplicate_names(tmp_path, monkeypatch): + import percona_obs.common as common + from percona_obs.cmd_project import _mirror_qa_entries + + src = _staging_tree(tmp_path) + monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root") + _pkg_with_qa(src, "percona-pgaudit", _AUDIT_QA) + cfg = { + "qa": [ + {"name": "percona-pgaudit.audit", "pipeline": "p", "parameters": {"A": 1}} + ] + } + with pytest.raises(SystemExit, match="percona-pgaudit.audit"): + _mirror_qa_entries( + cfg, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17" + ) + + +def test_mirror_qa_entries_follows_shared_symlinks(tmp_path, monkeypatch): + import percona_obs.common as common + from percona_obs.cmd_project import _mirror_qa_entries + + src = _staging_tree(tmp_path) + monkeypatch.setattr(cmd_project, "_REPO_DIR", tmp_path) + monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root") + shared = tmp_path / "root/ppg/staging/_shared" + shared.mkdir(parents=True) + _pkg_with_qa(shared, "percona-pgaudit", _AUDIT_QA) + (src / "percona-pgaudit").symlink_to("../_shared/percona-pgaudit") + out = _mirror_qa_entries( + {}, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17" + ) + assert [e["name"] for e in out] == ["percona-pgaudit.audit"] + + +def test_write_release_tree_carries_package_lanes_into_subproject_mirror( + tmp_path, monkeypatch +): + import percona_obs.common as common + + src = _staging_tree(tmp_path) + monkeypatch.setattr(cmd_project, "_REPO_DIR", tmp_path) + monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root") + (tmp_path / "root/macros.yaml").write_text("- PG_VERSION: 17.11\n") + _pkg_with_qa( + src / "containers", + "percona-postgresql-image", + "qa:\n - name: img\n pipeline: img-p\n parameters:\n" + " OBS_PROJECT: ${OBS_ROOTPRJ}:ppg:staging:17:containers\n", + ) + rel = tmp_path / "root/ppg/releases/17" + _write_release_tree( + rel, + {"build": False, "repositories": []}, + src, + "ppg:staging:17", + "ppg:releases:17", + "ppg", + "17", + ) + containers = yaml.safe_load((rel / "containers" / "project.yaml").read_text()) + assert containers["qa"] == [ + { + "name": "percona-postgresql-image.img", + "pipeline": "img-p", + "parameters": {"OBS_PROJECT": "${OBS_ROOTPRJ}:ppg:releases:17:containers"}, + } + ] + + def test_replace_changelog_section(): from percona_obs.cmd_project import _replace_changelog_section From 6a4d212bab5d413d14077c7b23a37a57980601de Mon Sep 17 00:00:00 2001 From: Ricardo Dias Date: Tue, 13 Oct 2026 16:56:43 +0100 Subject: [PATCH 2/2] project release: validate mirror qa via _normalise_qa_block, type _rewrite_qa_project_refs, clarify check names Signed-off-by: Ricardo Dias --- docs/PERCONA_OBS_TOOL.md | 3 ++- ...-prerelease-project-release-flow-design.md | 3 ++- percona_obs/cmd_project.py | 23 +++++++++---------- tests/test_project_release.py | 15 ++++++++++++ 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/PERCONA_OBS_TOOL.md b/docs/PERCONA_OBS_TOOL.md index 7d97f56a0..556a5265e 100644 --- a/docs/PERCONA_OBS_TOOL.md +++ b/docs/PERCONA_OBS_TOOL.md @@ -1091,7 +1091,8 @@ error. `project release ` does the following: its staging project's `qa:` entries (macros rendered, project references rewritten to the release project), plus the `qa:` entries of its direct packages' `package.yaml` as project-level entries named `.` (segment = the entry's `name` or its - `pipeline`), since mirrors have no package directories; a duplicate name fails the cut. + `pipeline`), since mirrors have no package directories (when a mirror has more than one entry, the entry + name becomes the check-name segment); a duplicate name fails the cut. 6. **Builds `CHANGELOG.md`** — compares built package versions from the top-level staging project, `extras`, and `tarballs` against the release OBS project (for updates) and generates entries with upstream URLs. Container images are listed per flavor (`ubi8` diff --git a/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md b/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md index 8eadbfbf9..62bdb7097 100644 --- a/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md +++ b/docs/superpowers/specs/2026-10-13-prerelease-project-release-flow-design.md @@ -166,7 +166,8 @@ of `main` whose staging tree the pre-release was built from. appended to the mirror's list as project-level entries named `.` (segment = the entry's `name` or its `pipeline`), because mirrors have no package directories; on the pre-release they run as - project lanes with check names `OBS QA / / .`. + project lanes; when the mirror has more than one entry, the entry name + becomes the check-name segment (`OBS QA / / .`). A duplicate name fails the cut. ### `sync prerelease ` (new command) diff --git a/percona_obs/cmd_project.py b/percona_obs/cmd_project.py index 0ed0d2c53..1a9e21498 100644 --- a/percona_obs/cmd_project.py +++ b/percona_obs/cmd_project.py @@ -10,6 +10,7 @@ import xml.etree.ElementTree as ET from collections.abc import Callable, Sequence from pathlib import Path +from typing import Any import osc.conf import osc.core @@ -1709,7 +1710,9 @@ def _is_relative_to(child: Path, parent: Path) -> bool: return False -def _rewrite_qa_project_refs(value, source_project_id: str, release_project_id: str): +def _rewrite_qa_project_refs( + value: Any, source_project_id: str, release_project_id: str +) -> Any: """Deep-copy a loaded ``qa:`` value replacing ````/``:...`` project references inside strings with the release project. Macros are already rendered by the loader; ``${VAR}`` tokens are left as they are.""" @@ -1770,22 +1773,18 @@ def _mirror_qa_entries( f"error: {pkg_path.name}/package.yaml: qa entries must be mappings" ) segment = entry.get("name") or entry.get("pipeline") or "" - copied: dict = _rewrite_qa_project_refs( # type: ignore[assignment] + copied: dict = _rewrite_qa_project_refs( entry, source_project_id, release_project_id ) copied.pop("name", None) # name first, for readable mirrors entries.append({"name": f"{pkg_path.name}.{segment}", **copied}) - seen: set[str] = set() - for entry in entries: - name = entry.get("name") or entry.get("pipeline") or "" - if name in seen: - raise SystemExit( - f"error: {source_project_id}: two qa entries resolve to the same " - f"name {name!r} in the release mirror; rename one of them" - ) - seen.add(name) - return entries or None + if not entries: + return None + # local import: cmd_qa imports cmd_project + from .cmd_qa import _normalise_qa_block + + return _normalise_qa_block(entries, release_project_id) def _write_release_tree( diff --git a/tests/test_project_release.py b/tests/test_project_release.py index 86d091ab0..d07e844bc 100644 --- a/tests/test_project_release.py +++ b/tests/test_project_release.py @@ -842,6 +842,21 @@ def test_mirror_qa_entries_rejects_duplicate_names(tmp_path, monkeypatch): ) +def test_mirror_qa_entries_rejects_entry_without_name_or_pipeline( + tmp_path, monkeypatch +): + import percona_obs.common as common + from percona_obs.cmd_project import _mirror_qa_entries + + src = _staging_tree(tmp_path) + monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root") + _pkg_with_qa(src, "percona-pgaudit", "qa:\n - parameters:\n X: 1\n") + with pytest.raises(SystemExit): + _mirror_qa_entries( + {}, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17" + ) + + def test_mirror_qa_entries_follows_shared_symlinks(tmp_path, monkeypatch): import percona_obs.common as common from percona_obs.cmd_project import _mirror_qa_entries