The first scheduled night (PG 18) ran 25 of its 45 builds: the 22 of

.github/scripts/list_qa_matrix.py

@@ -20,6 +20,15 @@
                             ``package``) whose package is not in the OBS
                             subproject.  Set by PR runs, where the PR project
                             holds only promoted packages; unset for the nightly.
+  OBS_INSTANCES             JSON list of OBS instances (the repository variable).
+                            When set (the nightly), projects come from this
+                            checkout's qa: blocks instead of one OBS server's
+                            subproject list, each resolved to the instance that
+                            hosts it (as obs-qa-run.yml does), and every combo
+                            carries that instance (instance_name, _apiurl,
+                            _user, _registry, _rootprj, _env). One server
+                            does not host every project: e.g. container
+                            projects live on another instance than tarballs.
   QA_MAJOR                  a PostgreSQL major (e.g. "17"): keep only that
                             major's combos (see ``belongs_to_major``). Set by the
                             nightly's one-major-per-day schedule; unset = all.
@@ -28,6 +37,7 @@
 from __future__ import annotations
 
 import json
+from pathlib import Path
 import os
 import subprocess
 import sys
@@ -106,14 +116,91 @@ def belongs_to_major(project: str, name: str | None, major: str) -> bool:
     ``-pg<major>``; image upgrades belong to the major they upgrade *into*
     (``...-16-to-17`` is PG 17's), so each upgrade runs exactly once a week.
     """
-    if major in project.split(":"):
-        return True
+    project_majors = [part for part in project.split(":") if part.isdigit()]
+    if project_majors:  # a major's own project: that major only, whatever the name
+        return major in project_majors
     name = name or ""
     return name.endswith(f"-pg{major}") or name.endswith(f"-to-{major}")
 
 
 
+def local_qa_projects(root: Path) -> list[str]:
+    """Projects of this checkout (``root/``) with a qa: block, own or a package's."""
+    found = []
+    for y in sorted(root.rglob("project.yaml")):
+        rel = y.parent.relative_to(root)
+        if "_shared" in rel.parts:
+            continue
+        text = y.read_text(encoding="utf-8")
+        pkgs = [d / "package.yaml" for d in y.parent.iterdir() if (d / "package.yaml").is_file()]
+        if "\nqa:" in text or text.startswith("qa:") or any("\nqa:" in q.read_text(encoding="utf-8") for q in pkgs):
+            found.append(":".join(rel.parts))
+    return found
+
+
+def _qa_show(project: str, profile: str) -> list:
+    result = subprocess.run(
+        ["venv/bin/python", "-m", "percona_obs", "-P", profile, "qa", "show", project, "--json"],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        print(f"warning: qa show {project}: {result.stderr.strip()}", file=sys.stderr)
+        return []
+    for line in result.stderr.rstrip().splitlines():
+        print(f"  {project}: {line.strip()}", file=sys.stderr)
+    try:
+        return json.loads(result.stdout or "[]")
+    except json.JSONDecodeError as exc:
+        print(f"warning: qa show {project}: invalid JSON ({exc})", file=sys.stderr)
+        return []
+
+
+def matrix_per_instance(instances_json: str, root: Path, profile: str, major: str) -> list:
+    """Nightly matrix: every qa: project of the checkout, on the instance hosting it."""
+    sys.path.insert(0, str(Path(__file__).resolve().parent))
+    from resolve_qa_instance import _load_instances, resolve_instance
+
+    instances = _load_instances(instances_json)
+    matrix: list = []
+    for project in local_qa_projects(root):
+        if is_release_tier("root:" + project):
+            continue
+        try:
+            inst = resolve_instance(instances, project)
+        except SystemExit as exc:  # no instance (or several): cannot run, say so
+            print(f"warning: {project}: {exc}", file=sys.stderr)
+            continue
+        entries = _qa_show(project, profile)
+        if major:
+            entries = [e for e in entries if belongs_to_major(project, e.get("name"), major)]
+        if not entries:
+            continue
+        print(f"  {project}: {len(entries)} combo(s) on instance {inst.get('name')}", file=sys.stderr)
+        fields = {
+            "instance_name": str(inst.get("name", "")),
+            "instance_apiurl": str(inst.get("apiurl", "")),
+            "instance_user": str(inst.get("user", "") or ""),
+            "instance_registry": str(inst.get("registry", "") or ""),
+            "instance_rootprj": str(inst.get("rootprj", "") or ""),
+            "instance_env": json.dumps(inst.get("env")) if inst.get("env") else "null",
+        }
+        for e in entries:
+            e.setdefault("project", project)
+            matrix.append({**normalize_entry(e), **fields})
+    return matrix
+
+
 def main() -> None:
+    instances_json = os.environ.get("OBS_INSTANCES", "").strip()
+    if instances_json:
+        major = os.environ.get("QA_MAJOR", "").strip()
+        matrix = matrix_per_instance(instances_json, Path("root"),
+                                     os.environ.get("PERCONA_OBS_PROFILE", "ci"), major)
+        print(f"{('PG ' + major) if major else 'all majors'}: {len(matrix)} combo(s) in total", file=sys.stderr)
+        print(json.dumps(matrix))
+        return
+
     import osc.conf
 
     from percona_obs.obs_api import _fetch_obs_subproject_names

.github/workflows/obs-nightly-qa.yml

@@ -105,6 +105,9 @@ jobs:
           OBS_APIURL: ${{ vars.OBS_APIURL }}
           OBS_PROJECT: ${{ vars.OBS_ROOTPRJ }}
           QA_MAJOR: ${{ needs.plan.outputs.major }}
+          # Each project runs on the instance that hosts it (as obs-qa-run.yml):
+          # one server does not host every project.
+          OBS_INSTANCES: ${{ vars.OBS_INSTANCES }}
           PYTHONPATH: ${{ github.workspace }}
         run: |
           venv/bin/python .github/scripts/list_qa_matrix.py > /tmp/qa-matrix.json
@@ -157,32 +160,42 @@ jobs:
         with:
           ref: main
 
+      # Connect to the instance hosting this combo's project (resolved by the
+      # detect job from OBS_INSTANCES), with that instance's credentials.
       - uses: ./.github/actions/obs-setup
         with:
-          obs-apiurl: ${{ vars.OBS_APIURL }}
-          obs-user: ${{ vars.OBS_USER }}
-          obs-password: ${{ secrets.OBS_PASSWORD }}
+          obs-apiurl: ${{ matrix.instance_apiurl }}
+          obs-user: ${{ matrix.instance_user || vars.OBS_USER }}
+          obs-password: ${{ secrets[format('OBS_PASSWORD_{0}', matrix.instance_name)] }}
 
-      # Same CI profile as the detect job so `qa run` re-resolves project.yaml
-      # parameters identically (same OBS_ROOTPRJ → same ${OBS_CONTAINER_REGISTRY_ROOTPRJ}).
+      # The instance's CI profile, as obs-qa-run.yml writes it, so `qa run`
+      # resolves project.yaml parameters (registry, root, env) for that instance.
       - name: Write CI profile
+        shell: bash
         env:
-          OBS_APIURL: ${{ vars.OBS_APIURL }}
-          OBS_ROOTPRJ: ${{ vars.OBS_ROOTPRJ }}
+          OBS_APIURL: ${{ matrix.instance_apiurl }}
+          OBS_ROOTPRJ: ${{ matrix.instance_rootprj }}
+          OBS_REGISTRY: ${{ matrix.instance_registry }}
+          INSTANCE_ENV: ${{ matrix.instance_env }}
           JENKINS_URL: ${{ vars.JENKINS_URL }}
           JENKINS_USER: ${{ vars.JENKINS_USER }}
         run: |
           mkdir -p .profile
           cat > .profile/ci.yaml <<EOF
           apiurl: ${OBS_APIURL}
           rootprj: ${OBS_ROOTPRJ}
+          ${OBS_REGISTRY:+registry: ${OBS_REGISTRY}}
           jenkins:
             url: ${JENKINS_URL}
             user: ${JENKINS_USER}
           env:
             - name: REMOTE_OBS_ORG_INTERCONNECT
-              value: 'openSUSE.org:'
+              value: ''
           EOF
+          if [ -n "${INSTANCE_ENV:-}" ] && [ "$INSTANCE_ENV" != "null" ]; then
+            jq -r 'to_entries[] | "  - name: \(.key)\n    value: '"'"'\(.value)'"'"'"' \
+              <<<"$INSTANCE_ENV" >> .profile/ci.yaml
+          fi
 
       # `name_filter` / `package_filter` / `axis_filters` are tool-generated
       # flag strings from `qa show --json` (normalised by list_qa_matrix.py),

tests/test_list_qa_matrix.py

@@ -289,6 +289,9 @@ def test_is_release_tier():
         ("ppg:staging:common:containers", "ubi9-16-to-17", "16", False),  # not from 16
         ("ppg:staging:common:extras:containers", "ubi9-17-to-18", "18", True),
         ("ppg:staging:common:containers", None, "17", False),
+        # a major's own project belongs to that major only, whatever the name
+        ("ppg:staging:18", "ubi9-16-to-17", "17", False),
+        ("ppg:staging:18", "ubi9-16-to-17", "18", True),
     ],
 )
 def test_belongs_to_major(project, name, major, expected):
@@ -406,3 +409,56 @@ def test_scheduled_run_runs_after_changes(tmp_path, monkeypatch):
 
 def test_manual_run_never_skips(tmp_path, monkeypatch):
     assert _main(tmp_path, monkeypatch, "workflow_dispatch", changed=False)["run"] == "true"
+
+
+# --- nightly: each project on the OBS instance that hosts it ------------------
+
+INSTANCES = [
+    {"name": "boo", "apiurl": "https://api.example", "rootprj": "isv:percona", "qa_types": "packages"},
+    {"name": "labs", "apiurl": "https://labs.example", "rootprj": "isv:percona",
+     "registry": "registry.labs.example", "qa_types": "containers", "env": {"X": "1"}},
+]
+
+
+def _fake_tree(m, monkeypatch, projects):
+    monkeypatch.setattr(m, "local_qa_projects", lambda root: projects)
+    monkeypatch.setattr(m, "_qa_show", lambda project, profile: [{"pipeline": "job", "name": n, "project": project}
+                                                               for n in ("server", "ubi9-16-to-17")])
+
+
+def test_each_project_runs_on_its_instance(monkeypatch):
+    m = _load()
+    _fake_tree(m, monkeypatch, ["ppg:staging:17", "ppg:staging:17:containers", "ppg:staging:17:tarballs"])
+    matrix = m.matrix_per_instance(json.dumps(INSTANCES), Path("root"), "ci", "")
+    where = {(c["project"], c["instance_name"]) for c in matrix}
+    assert where == {("ppg:staging:17", "boo"), ("ppg:staging:17:tarballs", "boo"),
+                     ("ppg:staging:17:containers", "labs")}
+    labs = next(c for c in matrix if c["instance_name"] == "labs")
+    assert (labs["instance_apiurl"], labs["instance_registry"], json.loads(labs["instance_env"])) == \
+        ("https://labs.example", "registry.labs.example", {"X": "1"})
+    assert all(c.get("name_filter") is not None for c in matrix)  # normalised like before
+
+
+def test_release_snapshots_and_unhosted_projects_are_skipped(monkeypatch, capsys):
+    m = _load()
+    only_packages = [INSTANCES[0]]
+    _fake_tree(m, monkeypatch, ["ppg:releases:17", "ppg:staging:17:containers", "ppg:staging:17"])
+    matrix = m.matrix_per_instance(json.dumps(only_packages), Path("root"), "ci", "")
+    assert {c["project"] for c in matrix} == {"ppg:staging:17"}
+    assert "no OBS instance hosts ppg:staging:17:containers" in capsys.readouterr().err
+
+
+def test_major_filter_applies_per_instance(monkeypatch):
+    m = _load()
+    _fake_tree(m, monkeypatch, ["ppg:staging:common:containers", "ppg:staging:18"])
+    matrix = m.matrix_per_instance(json.dumps(INSTANCES), Path("root"), "ci", "17")
+    # only the upgrade into 17 from the shared project; nothing from PG 18
+    assert [(c["project"], c["name"]) for c in matrix] == [("ppg:staging:common:containers", "ubi9-16-to-17")]
+
+
+def test_local_qa_projects_finds_the_real_tree():
+    found = _load().local_qa_projects(REPO / "root")
+    for p in ("ppg:staging:18", "ppg:staging:18:containers", "ppg:staging:18:tarballs",
+              "ppg:staging:common:tools:containers", "ppg:staging:common:containers"):
+        assert p in found, p
+    assert not any("_shared" in p for p in found)