diff --git a/.github/scripts/list_qa_matrix.py b/.github/scripts/list_qa_matrix.py index 4f74c1238..a174d4d85 100644 --- a/.github/scripts/list_qa_matrix.py +++ b/.github/scripts/list_qa_matrix.py @@ -20,6 +20,15 @@ ``package``) whose package is not in the OBS subproject. Set by PR runs, where the PR project holds only promoted packages; unset for the nightly. + OBS_INSTANCES JSON list of OBS instances (the repository variable). + When set (the nightly), projects come from this + checkout's qa: blocks instead of one OBS server's + subproject list, each resolved to the instance that + hosts it (as obs-qa-run.yml does), and every combo + carries that instance (instance_name, _apiurl, + _user, _registry, _rootprj, _env). One server + does not host every project: e.g. container + projects live on another instance than tarballs. QA_MAJOR a PostgreSQL major (e.g. "17"): keep only that major's combos (see ``belongs_to_major``). Set by the nightly's one-major-per-day schedule; unset = all. @@ -28,6 +37,7 @@ from __future__ import annotations import json +from pathlib import Path import os import subprocess import sys @@ -106,14 +116,91 @@ def belongs_to_major(project: str, name: str | None, major: str) -> bool: ``-pg``; image upgrades belong to the major they upgrade *into* (``...-16-to-17`` is PG 17's), so each upgrade runs exactly once a week. """ - if major in project.split(":"): - return True + project_majors = [part for part in project.split(":") if part.isdigit()] + if project_majors: # a major's own project: that major only, whatever the name + return major in project_majors name = name or "" return name.endswith(f"-pg{major}") or name.endswith(f"-to-{major}") +def local_qa_projects(root: Path) -> list[str]: + """Projects of this checkout (``root/``) with a qa: block, own or a package's.""" + found = [] + for y in sorted(root.rglob("project.yaml")): + rel = y.parent.relative_to(root) + if "_shared" in rel.parts: + continue + text = y.read_text(encoding="utf-8") + pkgs = [d / "package.yaml" for d in y.parent.iterdir() if (d / "package.yaml").is_file()] + if "\nqa:" in text or text.startswith("qa:") or any("\nqa:" in q.read_text(encoding="utf-8") for q in pkgs): + found.append(":".join(rel.parts)) + return found + + +def _qa_show(project: str, profile: str) -> list: + result = subprocess.run( + ["venv/bin/python", "-m", "percona_obs", "-P", profile, "qa", "show", project, "--json"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"warning: qa show {project}: {result.stderr.strip()}", file=sys.stderr) + return [] + for line in result.stderr.rstrip().splitlines(): + print(f" {project}: {line.strip()}", file=sys.stderr) + try: + return json.loads(result.stdout or "[]") + except json.JSONDecodeError as exc: + print(f"warning: qa show {project}: invalid JSON ({exc})", file=sys.stderr) + return [] + + +def matrix_per_instance(instances_json: str, root: Path, profile: str, major: str) -> list: + """Nightly matrix: every qa: project of the checkout, on the instance hosting it.""" + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from resolve_qa_instance import _load_instances, resolve_instance + + instances = _load_instances(instances_json) + matrix: list = [] + for project in local_qa_projects(root): + if is_release_tier("root:" + project): + continue + try: + inst = resolve_instance(instances, project) + except SystemExit as exc: # no instance (or several): cannot run, say so + print(f"warning: {project}: {exc}", file=sys.stderr) + continue + entries = _qa_show(project, profile) + if major: + entries = [e for e in entries if belongs_to_major(project, e.get("name"), major)] + if not entries: + continue + print(f" {project}: {len(entries)} combo(s) on instance {inst.get('name')}", file=sys.stderr) + fields = { + "instance_name": str(inst.get("name", "")), + "instance_apiurl": str(inst.get("apiurl", "")), + "instance_user": str(inst.get("user", "") or ""), + "instance_registry": str(inst.get("registry", "") or ""), + "instance_rootprj": str(inst.get("rootprj", "") or ""), + "instance_env": json.dumps(inst.get("env")) if inst.get("env") else "null", + } + for e in entries: + e.setdefault("project", project) + matrix.append({**normalize_entry(e), **fields}) + return matrix + + def main() -> None: + instances_json = os.environ.get("OBS_INSTANCES", "").strip() + if instances_json: + major = os.environ.get("QA_MAJOR", "").strip() + matrix = matrix_per_instance(instances_json, Path("root"), + os.environ.get("PERCONA_OBS_PROFILE", "ci"), major) + print(f"{('PG ' + major) if major else 'all majors'}: {len(matrix)} combo(s) in total", file=sys.stderr) + print(json.dumps(matrix)) + return + import osc.conf from percona_obs.obs_api import _fetch_obs_subproject_names diff --git a/.github/workflows/obs-nightly-qa.yml b/.github/workflows/obs-nightly-qa.yml index e09a77c8f..358815fbf 100644 --- a/.github/workflows/obs-nightly-qa.yml +++ b/.github/workflows/obs-nightly-qa.yml @@ -105,6 +105,9 @@ jobs: OBS_APIURL: ${{ vars.OBS_APIURL }} OBS_PROJECT: ${{ vars.OBS_ROOTPRJ }} QA_MAJOR: ${{ needs.plan.outputs.major }} + # Each project runs on the instance that hosts it (as obs-qa-run.yml): + # one server does not host every project. + OBS_INSTANCES: ${{ vars.OBS_INSTANCES }} PYTHONPATH: ${{ github.workspace }} run: | venv/bin/python .github/scripts/list_qa_matrix.py > /tmp/qa-matrix.json @@ -157,18 +160,23 @@ jobs: with: ref: main + # Connect to the instance hosting this combo's project (resolved by the + # detect job from OBS_INSTANCES), with that instance's credentials. - uses: ./.github/actions/obs-setup with: - obs-apiurl: ${{ vars.OBS_APIURL }} - obs-user: ${{ vars.OBS_USER }} - obs-password: ${{ secrets.OBS_PASSWORD }} + obs-apiurl: ${{ matrix.instance_apiurl }} + obs-user: ${{ matrix.instance_user || vars.OBS_USER }} + obs-password: ${{ secrets[format('OBS_PASSWORD_{0}', matrix.instance_name)] }} - # Same CI profile as the detect job so `qa run` re-resolves project.yaml - # parameters identically (same OBS_ROOTPRJ → same ${OBS_CONTAINER_REGISTRY_ROOTPRJ}). + # The instance's CI profile, as obs-qa-run.yml writes it, so `qa run` + # resolves project.yaml parameters (registry, root, env) for that instance. - name: Write CI profile + shell: bash env: - OBS_APIURL: ${{ vars.OBS_APIURL }} - OBS_ROOTPRJ: ${{ vars.OBS_ROOTPRJ }} + OBS_APIURL: ${{ matrix.instance_apiurl }} + OBS_ROOTPRJ: ${{ matrix.instance_rootprj }} + OBS_REGISTRY: ${{ matrix.instance_registry }} + INSTANCE_ENV: ${{ matrix.instance_env }} JENKINS_URL: ${{ vars.JENKINS_URL }} JENKINS_USER: ${{ vars.JENKINS_USER }} run: | @@ -176,13 +184,18 @@ jobs: cat > .profile/ci.yaml <> .profile/ci.yaml + fi # `name_filter` / `package_filter` / `axis_filters` are tool-generated # flag strings from `qa show --json` (normalised by list_qa_matrix.py), diff --git a/tests/test_list_qa_matrix.py b/tests/test_list_qa_matrix.py index 94fa77b23..836cecfe2 100644 --- a/tests/test_list_qa_matrix.py +++ b/tests/test_list_qa_matrix.py @@ -289,6 +289,9 @@ def test_is_release_tier(): ("ppg:staging:common:containers", "ubi9-16-to-17", "16", False), # not from 16 ("ppg:staging:common:extras:containers", "ubi9-17-to-18", "18", True), ("ppg:staging:common:containers", None, "17", False), + # a major's own project belongs to that major only, whatever the name + ("ppg:staging:18", "ubi9-16-to-17", "17", False), + ("ppg:staging:18", "ubi9-16-to-17", "18", True), ], ) def test_belongs_to_major(project, name, major, expected): @@ -406,3 +409,56 @@ def test_scheduled_run_runs_after_changes(tmp_path, monkeypatch): def test_manual_run_never_skips(tmp_path, monkeypatch): assert _main(tmp_path, monkeypatch, "workflow_dispatch", changed=False)["run"] == "true" + + +# --- nightly: each project on the OBS instance that hosts it ------------------ + +INSTANCES = [ + {"name": "boo", "apiurl": "https://api.example", "rootprj": "isv:percona", "qa_types": "packages"}, + {"name": "labs", "apiurl": "https://labs.example", "rootprj": "isv:percona", + "registry": "registry.labs.example", "qa_types": "containers", "env": {"X": "1"}}, +] + + +def _fake_tree(m, monkeypatch, projects): + monkeypatch.setattr(m, "local_qa_projects", lambda root: projects) + monkeypatch.setattr(m, "_qa_show", lambda project, profile: [{"pipeline": "job", "name": n, "project": project} + for n in ("server", "ubi9-16-to-17")]) + + +def test_each_project_runs_on_its_instance(monkeypatch): + m = _load() + _fake_tree(m, monkeypatch, ["ppg:staging:17", "ppg:staging:17:containers", "ppg:staging:17:tarballs"]) + matrix = m.matrix_per_instance(json.dumps(INSTANCES), Path("root"), "ci", "") + where = {(c["project"], c["instance_name"]) for c in matrix} + assert where == {("ppg:staging:17", "boo"), ("ppg:staging:17:tarballs", "boo"), + ("ppg:staging:17:containers", "labs")} + labs = next(c for c in matrix if c["instance_name"] == "labs") + assert (labs["instance_apiurl"], labs["instance_registry"], json.loads(labs["instance_env"])) == \ + ("https://labs.example", "registry.labs.example", {"X": "1"}) + assert all(c.get("name_filter") is not None for c in matrix) # normalised like before + + +def test_release_snapshots_and_unhosted_projects_are_skipped(monkeypatch, capsys): + m = _load() + only_packages = [INSTANCES[0]] + _fake_tree(m, monkeypatch, ["ppg:releases:17", "ppg:staging:17:containers", "ppg:staging:17"]) + matrix = m.matrix_per_instance(json.dumps(only_packages), Path("root"), "ci", "") + assert {c["project"] for c in matrix} == {"ppg:staging:17"} + assert "no OBS instance hosts ppg:staging:17:containers" in capsys.readouterr().err + + +def test_major_filter_applies_per_instance(monkeypatch): + m = _load() + _fake_tree(m, monkeypatch, ["ppg:staging:common:containers", "ppg:staging:18"]) + matrix = m.matrix_per_instance(json.dumps(INSTANCES), Path("root"), "ci", "17") + # only the upgrade into 17 from the shared project; nothing from PG 18 + assert [(c["project"], c["name"]) for c in matrix] == [("ppg:staging:common:containers", "ubi9-16-to-17")] + + +def test_local_qa_projects_finds_the_real_tree(): + found = _load().local_qa_projects(REPO / "root") + for p in ("ppg:staging:18", "ppg:staging:18:containers", "ppg:staging:18:tarballs", + "ppg:staging:common:tools:containers", "ppg:staging:common:containers"): + assert p in found, p + assert not any("_shared" in p for p in found)