Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion docs/PERCONA_OBS_TOOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -1087,7 +1087,12 @@ error. `project release <source-project>` does the following:
(`containers/`, `extras/`, `extras/containers/`, `tarballs/`). All mirrors get
`build: false`, `publish:` flags are carried over, and subproject paths — including
intra-project sibling paths — are rewritten to the release namespace. A mirror
directory whose staging source no longer exists is **deleted**.
directory whose staging source no longer exists is **deleted**. Each mirror also carries
its staging project's `qa:` entries (macros rendered, project references rewritten to the
release project), plus the `qa:` entries of its direct packages' `package.yaml` as
project-level entries named `<package>.<segment>` (segment = the entry's `name` or its
`pipeline`), since mirrors have no package directories (when a mirror has more than one entry, the entry
name becomes the check-name segment); a duplicate name fails the cut.
6. **Builds `CHANGELOG.md`** — compares built package versions from the top-level staging
project, `extras`, and `tarballs` against the release OBS project (for updates) and
generates entries with upstream URLs. Container images are listed per flavor (`ubi8`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,14 @@ of `main` whose staging tree the pre-release was built from.
`OBS_PROJECT: ${OBS_ROOTPRJ}:ppg:releases:18`, `VERSION: ppg-18.6`);
`ppg:staging:<major>` inside parameter values is rewritten to
`ppg:releases:<major>` the same way subproject paths are. Only `${…}` env
tokens survive, resolved per profile at QA time.
tokens survive, resolved per profile at QA time. Package-level `qa:` entries
(from each direct package's `package.yaml`, `_shared/` links followed) are
appended to the mirror's list as project-level entries named
`<package>.<segment>` (segment = the entry's `name` or its `pipeline`),
because mirrors have no package directories; on the pre-release they run as
project lanes; when the mirror has more than one entry, the entry name
becomes the check-name segment (`OBS QA / <project> / <package>.<segment>`).
A duplicate name fails the cut.

### `sync prerelease <releases-project>` (new command)

Expand Down Expand Up @@ -354,3 +361,6 @@ re-dispatch of hop 2.
entries (removed in a `finally`), as hop 1 does; OBS requires release
targets for a project-level release, so the pre-release meta carries them
only during the copy.
- Package-level `qa:` lanes reach the mirrors as `<package>.<segment>`
project-level entries (mirrors have no package directories); duplicates
fail the cut.
78 changes: 69 additions & 9 deletions percona_obs/cmd_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import xml.etree.ElementTree as ET
from collections.abc import Callable, Sequence
from pathlib import Path
from typing import Any

import osc.conf
import osc.core
Expand Down Expand Up @@ -41,6 +42,7 @@
is_project,
is_shared_source_dir,
load_macros,
load_package_yaml,
load_project_yaml,
load_yaml,
logger,
Expand Down Expand Up @@ -1708,7 +1710,9 @@ def _is_relative_to(child: Path, parent: Path) -> bool:
return False


def _rewrite_qa_project_refs(value, source_project_id: str, release_project_id: str):
def _rewrite_qa_project_refs(
value: Any, source_project_id: str, release_project_id: str
) -> Any:
"""Deep-copy a loaded ``qa:`` value replacing ``<source>``/``<source>:...``
project references inside strings with the release project. Macros are
already rendered by the loader; ``${VAR}`` tokens are left as they are."""
Expand All @@ -1733,6 +1737,56 @@ def _rewrite_qa_project_refs(value, source_project_id: str, release_project_id:
return value


def _mirror_qa_entries(
project_cfg: dict,
project_path: Path,
obs_id: str,
source_project_id: str,
release_project_id: str,
) -> "list[dict] | None":
"""The ``qa:`` list a release mirror gets for one staging project.

The project's own entries come first (rewritten to the release project),
then one entry per ``qa:`` entry of each direct package (``package.yaml``,
macros rendered from the package's own chain, ``${VAR}`` tokens kept),
named ``<package>.<segment>`` where segment is the entry's ``name`` or its
``pipeline``. Mirrors have no package directories, so package lanes can
only reach the pre-release as project lanes; the package prefix keeps the
check names unique and recognisable. Returns None when there is nothing
to carry. A duplicate name is a hard error at cut time.
"""
entries: list[dict] = []
own = project_cfg.get("qa")
if own is not None:
own = _rewrite_qa_project_refs(own, source_project_id, release_project_id)
entries.extend(own if isinstance(own, list) else [own])
for _obs, pkg_path in find_packages(project_path, obs_id, recursive=False):
pkg_yaml = pkg_path / "package.yaml"
if not pkg_yaml.is_file():
continue
pkg_qa = load_package_yaml(pkg_yaml).get("qa")
if pkg_qa is None:
continue
for entry in pkg_qa if isinstance(pkg_qa, list) else [pkg_qa]:
if not isinstance(entry, dict):
raise SystemExit(
f"error: {pkg_path.name}/package.yaml: qa entries must be mappings"
)
segment = entry.get("name") or entry.get("pipeline") or ""
copied: dict = _rewrite_qa_project_refs(
entry, source_project_id, release_project_id
)
copied.pop("name", None)
# name first, for readable mirrors
entries.append({"name": f"{pkg_path.name}.{segment}", **copied})
if not entries:
return None
# local import: cmd_qa imports cmd_project
from .cmd_qa import _normalise_qa_block

return _normalise_qa_block(entries, release_project_id)


def _write_release_tree(
release_dir: Path,
project_data: dict,
Expand Down Expand Up @@ -1791,10 +1845,15 @@ def _write_release_tree(
for key in ("debuginfo", "publish", "project-config"):
if key in source_sub_config:
sub_data[key] = source_sub_config[key]
if source_sub_config.get("qa") is not None:
sub_data["qa"] = _rewrite_qa_project_refs(
source_sub_config["qa"], source_project_id, release_project
)
mirror_qa = _mirror_qa_entries(
source_sub_config,
sub_path,
sub_obs_id,
source_project_id,
release_project,
)
if mirror_qa is not None:
sub_data["qa"] = mirror_qa
release_sub_dir = release_dir / Path(*subproject_name.split(":"))
expected_dirs.add(release_sub_dir)
release_sub_dir.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -2611,10 +2670,11 @@ def _derive() -> str:
top_cfg_no_env = _load_project_config_with_inheritance(
source_path, repo_filter=RepositoryFilter.EMPTY
)
if top_cfg_no_env.get("qa") is not None:
project_data["qa"] = _rewrite_qa_project_refs(
top_cfg_no_env["qa"], args.project, release_project
)
top_qa = _mirror_qa_entries(
top_cfg_no_env, source_path, args.project, args.project, release_project
)
if top_qa is not None:
project_data["qa"] = top_qa

if is_first_release:
commit_msg = f"Release {release_project} ({release_id})"
Expand Down
146 changes: 146 additions & 0 deletions tests/test_project_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -763,6 +763,152 @@ def test_write_release_tree_carries_rendered_qa_blocks(tmp_path, monkeypatch):
assert "qa" not in tarballs


def _pkg_with_qa(project_dir, name, qa_yaml):
pkg = project_dir / name
(pkg / "obs").mkdir(parents=True)
(pkg / "obs" / "_service").write_text("<services/>\n")
(pkg / "package.yaml").write_text(qa_yaml)
return pkg


_AUDIT_QA = "qa:\n - pipeline: audit\n parameters:\n X: 1\n"


def test_mirror_qa_entries_appends_package_lanes(tmp_path, monkeypatch):
import percona_obs.common as common
from percona_obs.cmd_project import _mirror_qa_entries

src = _staging_tree(tmp_path)
monkeypatch.setattr(cmd_project, "_REPO_DIR", tmp_path)
monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root")
(tmp_path / "root/macros.yaml").write_text("- PG_VERSION: 17.11\n")
_pkg_with_qa(
src,
"percona-pg_tde",
"qa:\n - name: pg_tde\n pipeline: tde-core-parallel\n parameters:\n"
" OBS_PROJECT: ${OBS_ROOTPRJ}:ppg:staging:17\n VERSION: ppg-%!{PG_VERSION}\n",
)
_pkg_with_qa(src, "percona-pgaudit", _AUDIT_QA)
_pkg_with_qa(src, "percona-haproxy", "build:\n RockyLinux_9: true\n")
cfg = {
"qa": [
{"name": "server", "pipeline": "p", "parameters": {"A": "ppg:staging:17"}}
]
}
out = _mirror_qa_entries(
cfg, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17"
)
assert [e["name"] for e in out] == [
"server",
"percona-pg_tde.pg_tde",
"percona-pgaudit.audit",
]
assert out[0]["parameters"]["A"] == "ppg:releases:17"
assert out[1]["parameters"] == {
"OBS_PROJECT": "${OBS_ROOTPRJ}:ppg:releases:17",
"VERSION": "ppg-17.11",
}
assert out[1]["pipeline"] == "tde-core-parallel"


def test_mirror_qa_entries_none_when_nothing_and_package_only(tmp_path, monkeypatch):
import percona_obs.common as common
from percona_obs.cmd_project import _mirror_qa_entries

src = _staging_tree(tmp_path)
monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root")
ids = ("ppg:staging:17", "ppg:staging:17", "ppg:releases:17")
assert _mirror_qa_entries({}, src, *ids) is None
_pkg_with_qa(src, "percona-pgaudit", _AUDIT_QA)
out = _mirror_qa_entries({}, src, *ids)
assert [e["name"] for e in out] == ["percona-pgaudit.audit"]


def test_mirror_qa_entries_rejects_duplicate_names(tmp_path, monkeypatch):
import percona_obs.common as common
from percona_obs.cmd_project import _mirror_qa_entries

src = _staging_tree(tmp_path)
monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root")
_pkg_with_qa(src, "percona-pgaudit", _AUDIT_QA)
cfg = {
"qa": [
{"name": "percona-pgaudit.audit", "pipeline": "p", "parameters": {"A": 1}}
]
}
with pytest.raises(SystemExit, match="percona-pgaudit.audit"):
_mirror_qa_entries(
cfg, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17"
)


def test_mirror_qa_entries_rejects_entry_without_name_or_pipeline(
tmp_path, monkeypatch
):
import percona_obs.common as common
from percona_obs.cmd_project import _mirror_qa_entries

src = _staging_tree(tmp_path)
monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root")
_pkg_with_qa(src, "percona-pgaudit", "qa:\n - parameters:\n X: 1\n")
with pytest.raises(SystemExit):
_mirror_qa_entries(
{}, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17"
)


def test_mirror_qa_entries_follows_shared_symlinks(tmp_path, monkeypatch):
import percona_obs.common as common
from percona_obs.cmd_project import _mirror_qa_entries

src = _staging_tree(tmp_path)
monkeypatch.setattr(cmd_project, "_REPO_DIR", tmp_path)
monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root")
shared = tmp_path / "root/ppg/staging/_shared"
shared.mkdir(parents=True)
_pkg_with_qa(shared, "percona-pgaudit", _AUDIT_QA)
(src / "percona-pgaudit").symlink_to("../_shared/percona-pgaudit")
out = _mirror_qa_entries(
{}, src, "ppg:staging:17", "ppg:staging:17", "ppg:releases:17"
)
assert [e["name"] for e in out] == ["percona-pgaudit.audit"]


def test_write_release_tree_carries_package_lanes_into_subproject_mirror(
tmp_path, monkeypatch
):
import percona_obs.common as common

src = _staging_tree(tmp_path)
monkeypatch.setattr(cmd_project, "_REPO_DIR", tmp_path)
monkeypatch.setattr(common, "REPO_ROOT", tmp_path / "root")
(tmp_path / "root/macros.yaml").write_text("- PG_VERSION: 17.11\n")
_pkg_with_qa(
src / "containers",
"percona-postgresql-image",
"qa:\n - name: img\n pipeline: img-p\n parameters:\n"
" OBS_PROJECT: ${OBS_ROOTPRJ}:ppg:staging:17:containers\n",
)
rel = tmp_path / "root/ppg/releases/17"
_write_release_tree(
rel,
{"build": False, "repositories": []},
src,
"ppg:staging:17",
"ppg:releases:17",
"ppg",
"17",
)
containers = yaml.safe_load((rel / "containers" / "project.yaml").read_text())
assert containers["qa"] == [
{
"name": "percona-postgresql-image.img",
"pipeline": "img-p",
"parameters": {"OBS_PROJECT": "${OBS_ROOTPRJ}:ppg:releases:17:containers"},
}
]


def test_replace_changelog_section():
from percona_obs.cmd_project import _replace_changelog_section

Expand Down
Loading