From 09974f91482eaf368046242d0eb42b9f9f4ee55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lipt=C3=A1k=20P=C3=A9ter?= Date: Mon, 7 Sep 2026 23:17:25 +0200 Subject: [PATCH] feat(career): Add validated staging content transfer Transfer career groups, settings and referenced images through explicit mappings and reviewed plans. Bind imports to the target database, verify content in a transaction, and preserve rollback receipts. Package the server runner in the application image and add transfer checks to CI and releases. --- .dockerignore | 2 + .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 3 + .gitignore | 2 + Dockerfile | 28 ++- docs/career-content-transfer.md | 245 +++++++++++++++++++++ eslint.config.mjs | 1 + package.json | 5 +- scripts/build-career-transfer.mjs | 8 + scripts/career-transfer-server.mjs | 10 + scripts/career-transfer.mjs | 37 ++++ scripts/career-transfer/cli.mjs | 79 +++++++ scripts/career-transfer/content.mjs | 58 +++++ scripts/career-transfer/server-cli.mjs | 66 ++++++ scripts/career-transfer/server-options.mjs | 28 +++ scripts/career-transfer/transfer.mjs | 148 +++++++++++++ scripts/career-transfer/verify-guards.mjs | 51 +++++ scripts/prepare-career-bundle.mjs | 52 +++++ tests/career-transfer-server.test.mjs | 28 +++ tests/career-transfer.test.mjs | 43 ++++ 20 files changed, 886 insertions(+), 11 deletions(-) create mode 100644 docs/career-content-transfer.md create mode 100644 scripts/build-career-transfer.mjs create mode 100644 scripts/career-transfer-server.mjs create mode 100644 scripts/career-transfer.mjs create mode 100644 scripts/career-transfer/cli.mjs create mode 100644 scripts/career-transfer/content.mjs create mode 100644 scripts/career-transfer/server-cli.mjs create mode 100644 scripts/career-transfer/server-options.mjs create mode 100644 scripts/career-transfer/transfer.mjs create mode 100644 scripts/career-transfer/verify-guards.mjs create mode 100644 scripts/prepare-career-bundle.mjs create mode 100644 tests/career-transfer-server.test.mjs create mode 100644 tests/career-transfer.test.mjs diff --git a/.dockerignore b/.dockerignore index a4e834b..7bd678a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,5 @@ public/videos/* private .local-verification .career-tests-* +media +scripts/.career-transfer diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cbf591..88f7e7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,9 @@ jobs: - name: Test career applications run: yarn test:career + - name: Test career content transfer + run: yarn test:career-transfer + container: name: Build Docker image needs: quality diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da565b7..feb3a28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,9 @@ jobs: - name: Test career applications run: yarn test:career + - name: Test career content transfer + run: yarn test:career-transfer + build: name: Build and publish image needs: quality diff --git a/.gitignore b/.gitignore index 262dec2..c6aa4d3 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,6 @@ next-env.d.ts public/videos # Private applicant documents and local verification artifacts /private/ +/media /.local-verification/ +/scripts/.career-transfer/ diff --git a/Dockerfile b/Dockerfile index 94796e1..92e1974 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,32 +11,40 @@ RUN --mount=type=cache,target=/usr/local/share/.cache/yarn \ --network-timeout 600000 \ --network-concurrency 4 -FROM base AS builder +FROM base AS source WORKDIR /app COPY --from=deps /app/node_modules ./node_modules COPY . . ENV NEXT_TELEMETRY_DISABLED=1 +RUN yarn run build:career-transfer + +FROM source AS builder RUN yarn run build -FROM base AS runner +FROM base AS career-transfer-runner WORKDIR /app ENV NODE_ENV=production ENV NEXT_TELEMETRY_DISABLED=1 +COPY --from=deps /app/node_modules ./node_modules +COPY --from=source /app/src ./src +COPY --from=source /app/payload.config.ts ./payload.config.ts +COPY --from=source /app/tsconfig.json ./tsconfig.json +COPY --from=source /app/css-loader-register.mjs ./css-loader-register.mjs +COPY --from=source /app/css-loader-hooks.mjs ./css-loader-hooks.mjs +COPY --from=source /app/package.json ./package.json +COPY --from=source /app/scripts/career-transfer-server.mjs ./scripts/career-transfer-server.mjs +COPY --from=source /app/scripts/.career-transfer ./scripts/.career-transfer + +CMD ["node", "scripts/career-transfer-server.mjs"] + +FROM career-transfer-runner AS runner COPY --from=builder /app/public ./public - COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static -COPY --from=deps /app/node_modules ./node_modules -COPY --from=builder /app/src ./src -COPY --from=builder /app/payload.config.ts ./payload.config.ts -COPY --from=builder /app/tsconfig.json ./tsconfig.json -COPY --from=builder /app/css-loader-register.mjs ./css-loader-register.mjs -COPY --from=builder /app/css-loader-hooks.mjs ./css-loader-hooks.mjs -COPY --from=builder /app/package.json ./package.json EXPOSE 3000 diff --git a/docs/career-content-transfer.md b/docs/career-content-transfer.md new file mode 100644 index 0000000..c7a4e2b --- /dev/null +++ b/docs/career-content-transfer.md @@ -0,0 +1,245 @@ +# Karriertartalom átvitele + +A `scripts/career-transfer.mjs` kizárólag elkülönített helyi próbához készült. +A `127.0.0.1:55441` címen futó PostgreSQL-konténert használja; a staging +másolatából olvas, és csak a `frt_transfer_candidate` adatbázisba importál. +Éles adatbázist ez a helyi futtató nem fogad el. A Docker-image külön +`scripts/career-transfer-server.mjs` futtatót tartalmaz a szerveres végrehajtáshoz. + +## Adatmodell és hatókör + +- `recruitment`: meglévő csoportok frissítése explicit staging → production + összerendeléssel, a production csoportazonosítók megtartásával. +- A csoporton belüli pozíciók és szekciók a staging listáját veszik át; + a korábbi production pozíciólista nem adódik hozzá automatikusan. +- `career-settings`: minden tartalmi mező és jelentkezési beállítás a stagingből. +- A hivatkozott képfájlok és médiarekordok. Egyező fájlnév, SHA-256 és alt esetén + meglévő rekordot használ; egyébként tartalomlenyomatot tartalmazó néven tölt fel. +- A jelentkezések, CV-k, felhasználók, más oldaltartalmak és meglévő médiarekordok + nem módosulnak. Nincs teljes adatbáziscsere vagy SQL-adatdump összeolvasztás. + +Az `applicationMode`, `applicationsOpen`, `spreadsheetUrl` és `googleFormUrl` +értékeit is a staging határozza meg, a felhasználó döntése alapján. +A stagingben megadott Google-táblázat lesz az átvett cél; az üres mezők is +felülírják a korábbi célértékeket. Az integráció hitelesítési adatai továbbra +is a célkörnyezet konfigurációjából származnak, az import ezeket nem másolja. + +## A próba környezete + +Az adatokat és naplókat az ignorált, 700-as jogosultságú `.local-verification/` +könyvtárban tároljuk. Ne commitold, és ne tedd publikus helyre a dumpokat, +exportokat, archívumokat vagy hitelesítési adatokat. + +Előfeltétel: a projekt függőségei, Docker, Python 3 és a két ellenőrzött +PostgreSQL 17 dump, valamint a production/staging fájlarchívum. + +A jelenlegi helyi konténer neve `frt-career-transfer-db`, image-e +`postgres:17-alpine`, kötete `frt-career-transfer-pg`, portja `127.0.0.1:55441`. +A `.local-verification/postgres.env` helyben generált hitelesítési adatokat +tartalmaz: `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB`. + +A három adatbázis szerepe: + +| Adatbázis | Szerep | +| --- | --- | +| `frt_transfer_prod` | Production dump változatlan helyreállítása | +| `frt_transfer_staging` | Staging dump helyreállítása, export forrása | +| `frt_transfer_candidate` | Production másolata, migráció és próbaimport célja | + +A restore `pg_restore --no-owner --no-acl --exit-on-error --single-transaction` +opciókkal történt, új, üres adatbázisokba. A candidate a helyreállított +production adatbázisból `createdb --template` segítségével készült. + +## Futtatás + +Mindig a worktree gyökeréből futtasd. A parancsok nem töltik be a projekt +`.env` fájljait: a wrapper csak a helyi PostgreSQL-jelszót és minimális +környezeti változókat ad át. E-mail küldése és külső `fetch` tiltott. + +```bash +node scripts/career-transfer.mjs frt_transfer_staging export +node scripts/career-transfer.mjs frt_transfer_candidate migrate +node scripts/career-transfer.mjs frt_transfer_candidate export +``` + +Ha a dump `dev` migrációjelölőt tartalmaz, a Payload megerősítést kér. +A meglévő próbában a hat hiányzó migráció sikeresen lefutott, a forrásdumpok +változatlanok maradtak. Éles migrációt csak friss mentés és az aktuális +adatbázisséma ellenőrzése után végezz. + +A `prepare-career-bundle` csak a hivatkozott képeket és az egyező nevű +production képeket bontja ki. Videókat és CV-ket nem másol. A fájlok +méretét a dumpban tárolt mérettel is összeveti. Meglévő fájlt nem ír felül; +új forráscsomaghoz új próbaterületet használj. + +```bash +node scripts/prepare-career-bundle.mjs /abs/staging-files.tar.gz /abs/prod-files.tar.gz +``` + +A `.local-verification/group-mapping.json` fájl minden forráscsoporthoz egy +rekordot tartalmaz: `sourceID`, `sourceName`, `targetID`, `targetName`. +Hiányzó, ismétlődő vagy név alapján már nem egyező összerendelés hibát okoz. +A mostani konkrét megfeleltetés és változáslista a helyi `rehearsal-report.md` +fájlban és az `import-plan-first.json` tervben szerepel. + +```bash +node scripts/career-transfer.mjs frt_transfer_candidate audit audit-before.json +node scripts/career-transfer.mjs frt_transfer_candidate plan +# Az import-plan.json ellenőrzése után, kizárólag a helyi másolaton: +node scripts/career-transfer.mjs frt_transfer_candidate apply +node scripts/career-transfer.mjs frt_transfer_candidate audit audit-after.json +``` + +A terv SHA-256-tal rögzíti a forráscsomagot, az összerendelést és a célállapotot. +Az apply egy PostgreSQL-tranzakcióban zárolja az érintett táblákat, újraellenőrzi +a tervet, végrehajtja az importot, és összehasonlítja az eredményt a forrással. +Validációs hibánál visszagörget és eltávolítja az újonnan írt képfájlokat. +Bizonytalan COMMIT-válasznál a fájlokat megőrzi, a napló `commit-uncertain` +állapotot kap: ilyen esetben adatbázis-ellenőrzés szükséges újrafuttatás előtt. + +Ismételt futtatáshoz készíts új tervet. Változatlan forrás és cél esetén ez +nulla módosítást tartalmaz. A régi, import előtti tervet az apply elutasítja. + +```bash +yarn test:career-transfer +node scripts/career-transfer.mjs frt_transfer_candidate verify-guards +``` + +A guard-próba helyi tesztképet hoz létre, majd szándékosan hibás utolsó +csoporttal teszteli a korábbi módosítások visszagörgetését és a fájltakarítást. +A PostgreSQL sequence számlálója visszagörgetés után is előreléphet; +az adatrekordok és fájlok ellenőrzötten változatlanok maradnak. + +## Éles kiadás előtt + +1. Ellenőrizni kell, változott-e a staging tartalma vagy a production adatbázis + a használt snapshotok óta; eltérés esetén friss export és új terv kell. +2. A pozíciólista és a karrierbeállítások staging szerinti átvétele jóváhagyott. +3. Ellenőrizni kell, hogy a production szolgáltatásfiók hozzáfér-e az átvett + Google-táblázathoz. A tartalmi célértékekről nem szükséges újabb döntés. +4. Productionben az új CV-funkcióhoz csatolni kell az `applicant_cvs` kötetet. +5. Rövid írásszünet alatt friss, összehangolt adatbázis- és fájlmentés kell. +6. A jóváhagyott kódot migrációkkal ki kell adni, majd az alábbi szerveres + importtal átvinni ugyanazt az ellenőrzött tartalomcsomagot. +7. Magyar/angol oldal, képek, űrlap és az integráció beállításainak ellenőrzése. + +A kód kiadása és a tartalom importálása külön művelet. Az importáló nem +indít deployt; adatot kizárólag az explicit `apply` parancs módosít. + +## Docker-csomagolás és szerveres futtató + +A Docker build a `build:career-transfer` scripttel előre lefordítja az importot. +A `career-transfer-runner` stage tartalmazza a Linuxos függőségeket, a pontos +Payload-konfigurációt, migrációkat és az importot; a teljes alkalmazás `runner` +stage-e erre épül. A szerveren az import nem telepít függőséget és nem fordít. +Az image nem tartalmaz feltöltött médiát, CV-ket, mentéseket vagy exportcsomagot. + +```bash +docker build --target career-transfer-runner -t frt-career-transfer:local . +docker build -t frt-app:career-transfer-local . +``` + +A szerveres futtató két parancsot fogad: `plan` és `apply`. Nem futtat automatikus +migrációt. Hiányzó migráció esetén leáll, mielőtt importtervet vagy adatot írna. +A konfiguráció környezeti változóiból csak az adatbáziskapcsolatot, a Payload +titkot és az alkalmazás címét adja át. Google- és e-mail-hitelesítési adatok +nem jutnak az importfolyamatba; külső `fetch` és e-mail-küldés tiltott. + +Minden szerveres terv tartalmazza a környezetet, az alkalmazás originjét, az +adatbázis nevét, hostját, szerepkörét és az elért PostgreSQL-szerver címét. +Az apply újra ellenőrzi ezt az azonosítást, a forráscsomagot, a képfájlokat és +a céladatokat. A helyi próbában készült tervet ezért nem lehet productionben +alkalmazni: ott új, csak olvasást végző `plan` futtatás szükséges. + +### Átadandó fájlok + +A megerősített exportcsomagot a szerverre másold egy korlátozott jogosultságú +könyvtárba, például `/home/frt/imports/career-20260907` alá: + +```text +career-20260907/ + bundle/ + bundle.json + media/ # csak a 9 hivatkozott kép + group-mapping.json +``` + +Ne másold ide a teljes média- és videóarchívumokat. A csomag kb. 77 MB képet +tartalmaz; a mentések továbbra is külön visszaállítási tartalékok. +Az importtervek és futási naplók ugyanebbe a könyvtárba kerülhetnek. + +### Terv készítése a production környezetben + +Előfeltétel: az új kód Docker-image-e már ki van adva, a migrációk lefutottak, +a production média- és CV-kötetek csatolva vannak. A használt image-et a +`/home/frt/frt/.env.deploy` rögzíti. A lent megadott könyvtárak a szerveren +ellenőrzött elrendezést használják, nem a régebbi `/srv` példákat. + +Először csak az alkalmazás originjét és az adatbázis nevét ellenőrizd, +hitelesítési adatok kiírása nélkül: + +```bash +docker exec frt-app-1 node -e ' + const db = new URL(process.env.DATABASE_URI); + console.log(JSON.stringify({ + origin: new URL(process.env.NEXT_PUBLIC_SERVER_URL).origin, + database: decodeURIComponent(db.pathname.slice(1)) + })); +' +``` + +Az alábbi parancsokban az `EXPECTED_ORIGIN` értékét erre az ellenőrzött címre +állítsd. A `frt` adatbázisnévnek szintén egyeznie kell a fenti eredménnyel. +Az írások legyenek szüneteltetve a friss mentéstől az import ellenőrzéséig. + +```bash +cd /home/frt/frt +EXPECTED_ORIGIN='https://AZ-ELLENORZOTT-PRODUCTION-DOMAIN' +TRANSFER_DIR=/home/frt/imports/career-20260907 + +docker compose --env-file .env.deploy -p frt -f docker-compose.yml \ + run --rm -T --no-deps --pull never \ + -v "$TRANSFER_DIR:/transfer" --entrypoint node app \ + scripts/career-transfer-server.mjs plan \ + --environment production --expect-database frt \ + --expect-origin "$EXPECTED_ORIGIN" \ + --bundle /transfer/bundle/bundle.json \ + --mapping /transfer/group-mapping.json \ + --out /transfer/production-plan.json +``` + +Ellenőrizd a kiírt célt, változásszámokat és a tervet. A staging tartalom a +mérvadó, de váratlan célállapot esetén ne kerüld meg a terv ellenőrzését. + +### Az ellenőrzött terv alkalmazása + +Az `APPROVED_PLAN_SHA256` változóba az előző parancs által kiírt, ellenőrzött +terv pontos `sha256` értékét másold. Ez a terv JSON-tartalmának lenyomata, +nem a formázott JSON-fájl `sha256sum` eredménye. + +```bash +APPROVED_PLAN_SHA256='AZ-ELLENORZOTT-TERV-64-KARAKTERES-LENYOMATA' + +docker compose --env-file .env.deploy -p frt -f docker-compose.yml \ + run --rm -T --no-deps --pull never \ + -v "$TRANSFER_DIR:/transfer" --entrypoint node app \ + scripts/career-transfer-server.mjs apply \ + --environment production --expect-database frt \ + --expect-origin "$EXPECTED_ORIGIN" \ + --bundle /transfer/bundle/bundle.json \ + --mapping /transfer/group-mapping.json \ + --plan /transfer/production-plan.json \ + --plan-sha256 "$APPROVED_PLAN_SHA256" \ + --receipt /transfer/production-receipt.json +``` + +A sikeres napló státusza `committed`. `prepared`, `pending-commit` vagy +`commit-uncertain` státusz esetén előbb ellenőrizni kell az adatbázis aktuális +állapotát és a folyamatot; ne feltételezz automatikus visszagörgetést. +`failed-before-commit` esetén a futtató a commit előtti hibát jelezte. +A futtató nem ír felül meglévő tervet vagy naplót. Új próbához új fájlnév kell. + +Siker után új `plan` futtatás külön fájlba nulla csoportmódosítást, nulla új +médiát és változatlan globális beállításokat kell mutasson. Ellenőrizd a +magyar és angol oldalt, az űrlapot és a production szolgáltatásfiók táblázat- +hozzáférését, majd oldd fel az írásszünetet. Az import nem küld tesztjelentkezést. diff --git a/eslint.config.mjs b/eslint.config.mjs index 28cec51..0a7b7a1 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -8,6 +8,7 @@ const eslintConfig = [ "coverage/**", ".local-verification/**", ".career-tests-*/**", + "scripts/.career-transfer/**", "private/**", "next-env.d.ts", "node_modules/**", diff --git a/package.json b/package.json index 9503d0f..7581c85 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,15 @@ "scripts": { "dev": "next dev --turbopack", "build": "next build --webpack", + "build:career-transfer": "node scripts/build-career-transfer.mjs", + "career:transfer": "node scripts/career-transfer-server.mjs", "start": "next start", "lint": "eslint .", "payload:migrate": "node --import ./css-loader-register.mjs ./node_modules/.bin/payload migrate", "payload:migrate-status": "node --import ./css-loader-register.mjs ./node_modules/.bin/payload migrate:status", "payload:generate-types": "node --import ./css-loader-register.mjs ./node_modules/.bin/payload generate:types", - "test:career": "node scripts/test-career.mjs" + "test:career": "node scripts/test-career.mjs", + "test:career-transfer": "node --test tests/career-transfer.test.mjs tests/career-transfer-server.test.mjs" }, "dependencies": { "@fullcalendar/core": "^6.1.19", diff --git a/scripts/build-career-transfer.mjs b/scripts/build-career-transfer.mjs new file mode 100644 index 0000000..5684b0a --- /dev/null +++ b/scripts/build-career-transfer.mjs @@ -0,0 +1,8 @@ +import { build } from 'esbuild' + +await build({ + entryPoints: ['scripts/career-transfer/server-cli.mjs'], + outfile: 'scripts/.career-transfer/server.mjs', + bundle: true, platform: 'node', format: 'esm', packages: 'external', + banner: { js: "import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);" }, +}) diff --git a/scripts/career-transfer-server.mjs b/scripts/career-transfer-server.mjs new file mode 100644 index 0000000..b981d27 --- /dev/null +++ b/scripts/career-transfer-server.mjs @@ -0,0 +1,10 @@ +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import path from 'node:path' + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +// Inherit only DB and application identity, never integration credentials or .env loading. +const env = Object.fromEntries(['PATH', 'HOME', 'TMPDIR', 'DATABASE_URI', 'PAYLOAD_SECRET', 'NEXT_PUBLIC_SERVER_URL'].flatMap(key => process.env[key] === undefined ? [] : [[key, process.env[key]]])) +Object.assign(env, { NODE_ENV: 'production', NEXT_TELEMETRY_DISABLED: '1' }) +const result = spawnSync(process.execPath, ['--import', './css-loader-register.mjs', './scripts/.career-transfer/server.mjs', ...process.argv.slice(2)], { cwd: root, env, stdio: 'inherit' }) +process.exitCode = result.status ?? 1 diff --git a/scripts/career-transfer.mjs b/scripts/career-transfer.mjs new file mode 100644 index 0000000..47128f8 --- /dev/null +++ b/scripts/career-transfer.mjs @@ -0,0 +1,37 @@ +import { build } from 'esbuild' +import { spawnSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { mkdir, readFile } from 'node:fs/promises' +import path from 'node:path' + +// This runner deliberately supports isolated local rehearsal databases only. +const database = process.argv[2] +if (!/^frt_transfer_(candidate|staging)$/.test(database ?? '')) { + throw new Error('Usage: node scripts/career-transfer.mjs frt_transfer_{candidate|staging} [audit-filename.json]') +} +const root = path.resolve('.local-verification') +await mkdir(root, { recursive: true, mode: 0o700 }) +const credentials = Object.fromEntries((await readFile(path.join(root, 'postgres.env'), 'utf8')).trim().split('\n').map(line => { + const separator = line.indexOf('=') + return [line.slice(0, separator), line.slice(separator + 1)] +})) +const uri = new URL(`postgresql://127.0.0.1:55441/${database}`) +uri.username = credentials.POSTGRES_USER +uri.password = credentials.POSTGRES_PASSWORD +const outfile = path.join(root, 'career-transfer-runner.mjs') +await build({ + entryPoints: ['scripts/career-transfer/cli.mjs'], outfile, + bundle: true, platform: 'node', format: 'esm', packages: 'external', + banner: { js: "import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);" }, +}) +const result = spawnSync(process.execPath, ['--import', './css-loader-register.mjs', outfile, ...process.argv.slice(3)], { + stdio: 'inherit', + env: { + PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, + NODE_ENV: 'production', NEXT_TELEMETRY_DISABLED: '1', + DATABASE_URI: uri.href, PAYLOAD_SECRET: randomBytes(32).toString('hex'), + FRT_TRANSFER_DATABASE: database, FRT_TRANSFER_ROOT: root, + CV_STORAGE_DIR: path.join(root, database, 'private/cvs'), + }, +}) +process.exitCode = result.status ?? 1 diff --git a/scripts/career-transfer/cli.mjs b/scripts/career-transfer/cli.mjs new file mode 100644 index 0000000..cae2f90 --- /dev/null +++ b/scripts/career-transfer/cli.mjs @@ -0,0 +1,79 @@ +import { postgresAdapter } from '@payloadcms/db-postgres' +import { getPayload } from 'payload' +import path from 'node:path' +import { mkdir, writeFile, readFile } from 'node:fs/promises' +import appConfig from '../../src/payload.config' +import { migrations } from '../../src/migrations' +import { loadBundle, buildPlan, applyPlan } from './transfer.mjs' +import { verifyGuards } from './verify-guards.mjs' + +process.umask(0o077) +const root = process.env.FRT_TRANSFER_ROOT +const database = process.env.FRT_TRANSFER_DATABASE +const uri = new URL(process.env.DATABASE_URI) +if (uri.hostname !== '127.0.0.1' || uri.port !== '55441' || uri.pathname !== `/${database}` || !/^frt_transfer_(candidate|staging)$/.test(database)) { + throw new Error('Only the dedicated local rehearsal databases are allowed') +} +// No environment files are loaded by this runner. Block fetch even if a future hook adds an integration. +globalThis.fetch = async () => { throw new Error('External requests are disabled during content rehearsal') } +const config = await appConfig +config.telemetry = false +config.db = postgresAdapter({ pool: { connectionString: uri.href }, push: false, disableCreateDatabase: true, migrationDir: path.resolve('src/migrations') }) +config.email = () => ({ name: 'disabled', defaultFromAddress: 'disabled@localhost', defaultFromName: 'Disabled', sendEmail: async () => { throw new Error('Email is disabled during rehearsal') } }) +for (const collection of config.collections) { + if (collection.upload) collection.upload.staticDir = path.join(root, database, collection.slug === 'application-cvs' ? 'private/cvs' : collection.slug) +} +await mkdir(path.join(root, database, 'media'), { recursive: true }) +const payload = await getPayload({ config }) +try { + const command = process.argv[2] + if (command === 'migrate') { + if (database !== 'frt_transfer_candidate') throw new Error('Only the candidate may be migrated') + await payload.db.migrate({ migrations }) + console.log('Migrations completed') + } else if (command === 'export') { + const recruitment = await payload.find({ collection: 'recruitment', depth: 0, pagination: false, sort: 'id' }) + const settings = await payload.findGlobal({ slug: 'career-settings', depth: 0 }) + const media = await payload.find({ collection: 'media', depth: 0, pagination: false, sort: 'id' }) + const result = { database, recruitment: recruitment.docs, settings, media: media.docs } + await writeFile(path.join(root, `${database}.json`), JSON.stringify(result, null, 2)) + console.log(JSON.stringify({ database, groups: result.recruitment.map(doc => ({ id: doc.id, name: doc.groupName, positions: doc.positions.length })), faqs: settings.faqs?.length ?? 0, questions: settings.formQuestions?.length ?? 0, media: media.docs.length })) + } else if (['plan', 'apply', 'verify-guards'].includes(command)) { + if (database !== 'frt_transfer_candidate') throw new Error('Only the candidate may receive an import') + const bundlePath = path.join(root, 'bundle/bundle.json') + const bundle = await loadBundle(bundlePath) + const mappings = JSON.parse(await readFile(path.join(root, 'group-mapping.json'))) + const planPath = path.join(root, 'import-plan.json') + if (command === 'verify-guards') { + console.log(JSON.stringify(await verifyGuards(payload, root, bundle, bundlePath, mappings))) + } else if (command === 'plan') { + const plan = await buildPlan(payload, bundle, mappings) + await writeFile(planPath, JSON.stringify(plan, null, 2)) + console.log(JSON.stringify(plan, null, 2)) + } else { + const plan = JSON.parse(await readFile(planPath)) + console.log(JSON.stringify(await applyPlan(payload, bundle, bundlePath, mappings, plan, path.join(root, 'import-receipt.json')), null, 2)) + } + } else if (command === 'audit') { + const tables = await payload.db.pool.query("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename") + const result = {} + for (const { tablename } of tables.rows) { + const table = `"${tablename.replaceAll('"', '""')}"` + const query = await payload.db.pool.query(`SELECT count(*)::int AS count, md5(coalesce(string_agg(to_jsonb(t)::text, E'\\n' ORDER BY to_jsonb(t)::text), '')) AS hash FROM ${table} t`) + result[tablename] = query.rows[0] + } + const filename = process.argv[3] + if (!filename || path.basename(filename) !== filename || !filename.endsWith('.json')) throw new Error('Provide an audit JSON filename') + await writeFile(path.join(root, filename), JSON.stringify(result, null, 2)) + console.log(`Audited ${tables.rows.length} tables -> ${filename}`) + } else { + throw new Error(`Unknown command: ${command}`) + } +} catch (error) { + console.error(error) + process.exitCode = 1 +} finally { + await payload.destroy() +} +// Payload's own CLI also exits explicitly: adapter.destroy does not close its pool. +process.exit(process.exitCode ?? 0) diff --git a/scripts/career-transfer/content.mjs b/scripts/career-transfer/content.mjs new file mode 100644 index 0000000..49fb29b --- /dev/null +++ b/scripts/career-transfer/content.mjs @@ -0,0 +1,58 @@ +import { createHash } from 'node:crypto' + +const groupFields = ['groupName', 'groupNameEng', 'image', 'order', 'description', 'descriptionEng', 'positions'] +const settingsFields = ['intro', 'introEng', 'applicationsClosedText', 'applicationsClosedTextEng', 'faqs', 'formQuestions', 'applicationMode', 'googleFormUrl', 'spreadsheetUrl', 'applicationsOpen'] +const rowArrays = new Set(['positions', 'sections', 'faqs', 'formQuestions', 'options']) + +// Strip Payload row IDs only, never IDs/keys embedded in rich text or question keys. +function clean(value, isRow = false) { + if (Array.isArray(value)) return value.map(item => clean(item)) + if (!value || typeof value !== 'object') return value + return Object.fromEntries(Object.entries(value).filter(([key]) => !(isRow && key === 'id')).map(([key, item]) => [key, + rowArrays.has(key) && Array.isArray(item) ? item.map(row => clean(row, true)) : clean(item), + ])) +} +export function content(doc, kind) { + return clean(Object.fromEntries((kind === 'group' ? groupFields : settingsFields).map(key => [key, doc[key] ?? null]))) +} +export function canonical(value) { + if (Array.isArray(value)) return value.map(canonical) + if (!value || typeof value !== 'object') return value + return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) +} +export const hash = value => createHash('sha256').update(JSON.stringify(canonical(value))).digest('hex') +export const fileHash = buffer => createHash('sha256').update(buffer).digest('hex') + +export function mediaReferences(value, result = new Set()) { + if (!value || typeof value !== 'object') return result + if (value.relationTo) { + if (value.relationTo !== 'media') throw new Error(`Unmapped rich-text relationship: ${value.relationTo}`) + const id = typeof value.value === 'object' ? value.value?.id : value.value + if (!Number.isInteger(id)) throw new Error('Invalid rich-text media reference') + result.add(id) + } + for (const item of Object.values(value)) mediaReferences(item, result) + return result +} +export function remapMedia(value, mapping) { + if (Array.isArray(value)) return value.map(item => remapMedia(item, mapping)) + if (!value || typeof value !== 'object') return value + const result = Object.fromEntries(Object.entries(value).map(([key, item]) => [key, remapMedia(item, mapping)])) + if (result.relationTo) { + if (result.relationTo !== 'media') throw new Error(`Unsupported relationship: ${result.relationTo}`) + const oldID = typeof result.value === 'object' ? result.value.id : result.value + if (!mapping[oldID]) throw new Error(`Missing media mapping: ${oldID}`) + result.value = mapping[oldID] + } + return result +} +export function validateMapping(source, target, mappings) { + if (mappings.length !== source.length || new Set(mappings.map(x => x.sourceID)).size !== source.length) throw new Error('Every source group needs exactly one mapping') + if (new Set(mappings.map(x => x.targetID)).size !== mappings.length) throw new Error('Target group mappings must be unique') + if (target.some(doc => !mappings.some(mapping => mapping.targetID === doc.id))) throw new Error('Unmapped target groups require an explicit removal plan') + for (const mapping of mappings) { + const s = source.find(x => x.id === mapping.sourceID) + const t = target.find(x => x.id === mapping.targetID) + if (!s || !t || s.groupName !== mapping.sourceName || t.groupName !== mapping.targetName) throw new Error(`Group mapping no longer matches: ${mapping.sourceID} -> ${mapping.targetID}`) + } +} diff --git a/scripts/career-transfer/server-cli.mjs b/scripts/career-transfer/server-cli.mjs new file mode 100644 index 0000000..812508c --- /dev/null +++ b/scripts/career-transfer/server-cli.mjs @@ -0,0 +1,66 @@ +import { postgresAdapter } from '@payloadcms/db-postgres' +import { getPayload } from 'payload' +import { readFile, writeFile, stat } from 'node:fs/promises' +import path from 'node:path' +import appConfig from '../../src/payload.config' +import { migrations } from '../../src/migrations' +import { hash } from './content.mjs' +import { parseOptions } from './server-options.mjs' +import { loadBundle, buildPlan, applyPlan } from './transfer.mjs' + +process.umask(0o077) +globalThis.fetch = async () => { throw new Error('External requests are disabled during content import') } +let payload +try { + const options = parseOptions(process.argv.slice(2), process.env) + const bundle = await loadBundle(options.bundle) + const mappings = JSON.parse(await readFile(options.mapping, 'utf8')) + const approvedPlan = options.command === 'apply' ? JSON.parse(await readFile(options.plan, 'utf8')) : null + if (approvedPlan) { + const { sha256, ...data } = approvedPlan + if (sha256 !== options['plan-sha256'] || hash(data) !== sha256 || !approvedPlan.target) throw new Error('Plan hash mismatch or missing server target identity') + } + const output = options.command === 'plan' ? options.out : options.receipt + if (!(await stat(path.dirname(output))).isDirectory()) throw new Error('Output directory does not exist') + try { await stat(output); throw new Error('Output file already exists; preserve earlier plans and receipts') } + catch (error) { if (error.code !== 'ENOENT') throw error } + const mediaDir = path.resolve('media') + if (!(await stat(mediaDir)).isDirectory()) throw new Error('The application media directory must be mounted') + const config = await appConfig + config.telemetry = false + config.db = postgresAdapter({ pool: { connectionString: process.env.DATABASE_URI }, push: false, disableCreateDatabase: true, migrationDir: path.resolve('src/migrations') }) + config.email = () => ({ name: 'disabled', defaultFromAddress: 'disabled@localhost', defaultFromName: 'Disabled', sendEmail: async () => { throw new Error('Email is disabled during content import') } }) + for (const collection of config.collections) { + if (collection.upload) collection.upload.staticDir = collection.slug === 'media' ? mediaDir : path.resolve('private/cvs') + } + payload = await getPayload({ config }) + const applied = (await payload.db.pool.query('SELECT name FROM payload_migrations')).rows.map(row => row.name) + const pending = migrations.filter(migration => !applied.includes(migration.name)).map(migration => migration.name) + if (pending.length) throw new Error(`Database migrations must run before import: ${pending.join(', ')}`) + const uri = new URL(process.env.DATABASE_URI) + const actual = (await payload.db.pool.query('SELECT current_database() AS database, current_user AS role, inet_server_addr()::text AS "serverAddress", inet_server_port() AS "serverPort"')).rows[0] + if (actual.database !== options['expect-database']) throw new Error('Connected database differs from the expected database') + const targetIdentity = { environment: options.environment, origin: new URL(process.env.NEXT_PUBLIC_SERVER_URL).origin, host: uri.hostname, port: uri.port || '5432', ...actual } + if (approvedPlan && hash(approvedPlan.target) !== hash(targetIdentity)) throw new Error('Connected server identity does not match the approved plan') + if (options.command === 'plan') { + const plan = await buildPlan(payload, bundle, mappings, undefined, targetIdentity) + await writeFile(output, JSON.stringify(plan, null, 2), { flag: 'wx', mode: 0o600 }) + console.log(JSON.stringify({ plan: output, sha256: plan.sha256, target: targetIdentity, groupsToUpdate: plan.groups.filter(x => x.action === 'update').length, mediaToCreate: plan.assets.filter(x => x.action === 'create').length, settings: plan.settings }, null, 2)) + } else { + await writeFile(output, JSON.stringify({ status: 'prepared', planHash: approvedPlan.sha256, target: targetIdentity }, null, 2), { flag: 'wx', mode: 0o600 }) + try { + await applyPlan(payload, bundle, options.bundle, mappings, approvedPlan, output, { targetIdentity }) + console.log(JSON.stringify({ status: 'committed', receipt: output, target: targetIdentity })) + } catch (error) { + const receipt = JSON.parse(await readFile(output, 'utf8')) + if (receipt.status === 'prepared') await writeFile(output, JSON.stringify({ ...receipt, status: 'failed-before-commit' }, null, 2)) + throw error + } + } +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + if (payload) await payload.destroy() +} +process.exit(process.exitCode ?? 0) diff --git a/scripts/career-transfer/server-options.mjs b/scripts/career-transfer/server-options.mjs new file mode 100644 index 0000000..6d29e5c --- /dev/null +++ b/scripts/career-transfer/server-options.mjs @@ -0,0 +1,28 @@ +import path from 'node:path' + +export function parseOptions(argv, env) { + const [command, ...args] = argv + if (!['plan', 'apply'].includes(command)) throw new Error('Usage: career-transfer-server.mjs --environment --expect-database --expect-origin --bundle --mapping [--out | --plan --plan-sha256 --receipt ]') + const allowed = new Set(['environment', 'expect-database', 'expect-origin', 'bundle', 'mapping', ...(command === 'plan' ? ['out'] : ['plan', 'plan-sha256', 'receipt'])]) + const options = { command } + for (let i = 0; i < args.length; i += 2) { + const key = args[i]?.slice(2) + if (!args[i]?.startsWith('--') || !allowed.has(key) || Object.hasOwn(options, key) || !args[i + 1] || args[i + 1].startsWith('--')) throw new Error(`Invalid or duplicate option: ${args[i]}`) + options[key] = args[i + 1] + } + for (const key of allowed) if (!options[key]) throw new Error(`Missing --${key}`) + if (!['production', 'staging', 'rehearsal'].includes(options.environment)) throw new Error('Invalid environment') + if (!env.DATABASE_URI || !env.PAYLOAD_SECRET || !env.NEXT_PUBLIC_SERVER_URL) throw new Error('DATABASE_URI, PAYLOAD_SECRET and NEXT_PUBLIC_SERVER_URL are required') + const database = new URL(env.DATABASE_URI) + if (!['postgres:', 'postgresql:'].includes(database.protocol)) throw new Error('PostgreSQL DATABASE_URI required') + if (decodeURIComponent(database.pathname.slice(1)) !== options['expect-database']) throw new Error('Database does not match --expect-database') + const expectedOrigin = new URL(options['expect-origin']) + const origin = new URL(env.NEXT_PUBLIC_SERVER_URL) + if (!['http:', 'https:'].includes(origin.protocol) || expectedOrigin.origin !== origin.origin || expectedOrigin.username || expectedOrigin.password || origin.username || origin.password) throw new Error('Application origin does not match --expect-origin') + if (expectedOrigin.pathname !== '/' || expectedOrigin.search || expectedOrigin.hash) throw new Error('--expect-origin must contain only the origin') + if (command === 'apply' && !/^[a-f0-9]{64}$/.test(options['plan-sha256'])) throw new Error('An exact SHA-256 plan hash is required') + for (const key of ['bundle', 'mapping', 'out', 'plan', 'receipt']) if (options[key]) options[key] = path.resolve(options[key]) + const files = ['bundle', 'mapping', 'out', 'plan', 'receipt'].flatMap(key => options[key] ? [options[key]] : []) + if (new Set(files).size !== files.length) throw new Error('Input and output paths must be distinct') + return options +} diff --git a/scripts/career-transfer/transfer.mjs b/scripts/career-transfer/transfer.mjs new file mode 100644 index 0000000..54861d0 --- /dev/null +++ b/scripts/career-transfer/transfer.mjs @@ -0,0 +1,148 @@ +import { createLocalReq, initTransaction, commitTransaction, killTransaction } from 'payload' +import { sql } from '@payloadcms/db-postgres' +import { readFile, writeFile, rm, access } from 'node:fs/promises' +import path from 'node:path' +import { content, fileHash, hash, remapMedia, validateMapping, mediaReferences } from './content.mjs' + +export async function readTarget(payload, req) { + const recruitment = await payload.find({ collection: 'recruitment', depth: 0, pagination: false, sort: 'id', req }) + const settings = await payload.findGlobal({ slug: 'career-settings', depth: 0, req }) + const media = await payload.find({ collection: 'media', depth: 0, pagination: false, sort: 'id', req }) + return { recruitment: recruitment.docs, settings, media: media.docs } +} + +export function targetFingerprint(target) { + return hash({ + recruitment: target.recruitment.map(doc => ({ id: doc.id, updatedAt: doc.updatedAt, data: content(doc, 'group') })), + settings: { data: content(target.settings, 'settings'), updatedAt: target.settings.updatedAt }, + media: target.media.map(({ id, filename, alt, filesize, updatedAt }) => ({ id, filename, alt, filesize, updatedAt })), + }) +} + +export async function loadBundle(filename) { + const bundle = JSON.parse(await readFile(filename, 'utf8')) + const { sha256, ...data } = bundle + if (bundle.version !== 1 || hash(data) !== sha256) throw new Error('Bundle checksum mismatch') + const refs = mediaReferences([bundle.recruitment, bundle.settings]) + for (const doc of bundle.recruitment) if (doc.image) refs.add(doc.image) + if (new Set(bundle.assets.map(x => x.sourceID)).size !== bundle.assets.length || refs.size !== bundle.assets.length || [...refs].some(id => !bundle.assets.some(x => x.sourceID === id))) throw new Error('Incomplete or duplicate media assets') + for (const asset of bundle.assets) { + if (path.basename(asset.filename) !== asset.filename) throw new Error('Invalid asset filename') + const bytes = await readFile(path.join(path.dirname(filename), 'media', asset.filename)) + if (bytes.length !== asset.size || fileHash(bytes) !== asset.sha256) throw new Error(`Asset checksum mismatch: ${asset.filename}`) + } + return bundle +} + +export async function buildPlan(payload, bundle, mappings, req, targetIdentity) { + const target = await readTarget(payload, req) + // A completed run may have renamed a target group to its source name. + validateMapping(bundle.recruitment, target.recruitment, mappings.map(m => ({ ...m, + targetName: target.recruitment.find(t => t.id === m.targetID)?.groupName === m.sourceName ? m.sourceName : m.targetName, + }))) + const mediaDir = payload.collections.media.config.upload.staticDir + const assets = [] + for (const asset of bundle.assets) { + const importName = `career-${asset.sha256}${path.extname(asset.filename).toLowerCase()}` + const candidates = target.media.filter(doc => [asset.filename, importName].includes(doc.filename)) + const matches = [] + for (const doc of candidates) { + const bytes = await readFile(path.join(mediaDir, doc.filename)) + if (fileHash(bytes) === asset.sha256 && doc.alt === asset.alt) matches.push(doc) + } + if (matches.length > 1) throw new Error(`Ambiguous target image: ${asset.filename}`) + if (!matches.length && candidates.some(x => x.filename === importName)) throw new Error(`Conflicting import image: ${importName}`) + assets.push({ sourceID: asset.sourceID, action: matches.length ? 'reuse' : 'create', targetID: matches[0]?.id ?? null, filename: matches[0]?.filename ?? importName, sha256: asset.sha256 }) + } + const mediaMap = Object.fromEntries(assets.map(asset => [asset.sourceID, asset.targetID ?? `new:${asset.sha256}`])) + const groups = mappings.map(mapping => { + const source = bundle.recruitment.find(doc => doc.id === mapping.sourceID) + const before = content(target.recruitment.find(doc => doc.id === mapping.targetID), 'group') + const after = remapMedia(content(source, 'group'), mediaMap) + after.image = source.image ? mediaMap[source.image] : null + return { sourceID: mapping.sourceID, targetID: mapping.targetID, name: source.groupName, + action: hash(before) === hash(after) ? 'unchanged' : 'update', + changedFields: Object.keys(after).filter(key => hash(before[key]) !== hash(after[key])), + positionsBefore: before.positions.length, positionsAfter: after.positions.length, + oldPositions: before.positions.map(x => x.positionName), newPositions: after.positions.map(x => x.positionName), + } + }) + const beforeSettings = content(target.settings, 'settings') + const afterSettings = remapMedia(content(bundle.settings, 'settings'), mediaMap) + const settings = { action: hash(beforeSettings) === hash(afterSettings) ? 'unchanged' : 'update', + changedFields: Object.keys(afterSettings).filter(key => hash(beforeSettings[key]) !== hash(afterSettings[key])), + authority: 'staging', + sourceIntegrationConfigured: !!bundle.settings.spreadsheetUrl, + targetIntegrationConfigured: !!target.settings.spreadsheetUrl, + } + const data = { version: 1, bundleHash: bundle.sha256, targetHash: targetFingerprint(target), mappingHash: hash(mappings), assets, groups, settings, ...(targetIdentity ? { target: targetIdentity } : {}) } + return { ...data, sha256: hash(data) } +} + +export async function applyPlan(payload, bundle, bundlePath, mappings, approvedPlan, receiptPath, { targetIdentity } = {}) { + const { sha256, ...planData } = approvedPlan + if (hash(planData) !== sha256 || approvedPlan.bundleHash !== bundle.sha256 || approvedPlan.mappingHash !== hash(mappings)) throw new Error('Plan checksum or input mismatch') + const req = await createLocalReq({ context: { skipSheetSync: true } }, payload) + const createdFiles = new Set() + const mediaDir = payload.collections.media.config.upload.staticDir + let committed = false + let commitAttempted = false + const receipt = { planHash: sha256, bundleHash: bundle.sha256, createdMedia: [], updatedGroups: [], updatedSettings: false, ...(targetIdentity ? { target: targetIdentity } : {}) } + try { + await initTransaction(req) + const transaction = payload.db.sessions[await req.transactionID]?.db + if (!transaction) throw new Error('A real PostgreSQL transaction is required') + await transaction.execute(sql`SET LOCAL lock_timeout = '5s'`) + await transaction.execute(sql`LOCK TABLE recruitment, recruitment_positions, recruitment_positions_sections, career_settings, career_settings_faqs, career_settings_form_questions, career_settings_form_questions_options, media IN SHARE ROW EXCLUSIVE MODE`) + const currentPlan = await buildPlan(payload, bundle, mappings, req, targetIdentity) + if (currentPlan.sha256 !== sha256) throw new Error('Target changed since planning; generate and review a new plan') + const mediaMap = {} + for (const action of approvedPlan.assets) { + if (action.action === 'reuse') { mediaMap[action.sourceID] = action.targetID; continue } + const asset = bundle.assets.find(x => x.sourceID === action.sourceID) + const destination = path.join(mediaDir, action.filename) + try { await access(destination); throw new Error(`Untracked target file already exists: ${action.filename}`) } + catch (error) { if (error.code !== 'ENOENT') throw error } + createdFiles.add(destination) + const data = await readFile(path.join(path.dirname(bundlePath), 'media', asset.filename)) + const doc = await payload.create({ collection: 'media', data: { alt: asset.alt }, + file: { data, name: action.filename, mimetype: asset.mimeType, size: data.length }, req }) + createdFiles.add(path.join(mediaDir, doc.filename)) + if (fileHash(await readFile(path.join(mediaDir, doc.filename))) !== asset.sha256) throw new Error(`Upload changed file content: ${asset.filename}`) + mediaMap[action.sourceID] = doc.id + receipt.createdMedia.push({ sourceID: action.sourceID, targetID: doc.id, filename: doc.filename }) + } + for (const action of approvedPlan.groups) { + if (action.action === 'unchanged') continue + const source = bundle.recruitment.find(doc => doc.id === action.sourceID) + const data = remapMedia(content(source, 'group'), mediaMap) + data.image = source.image ? mediaMap[source.image] : null + await payload.update({ collection: 'recruitment', id: action.targetID, data, depth: 0, req }) + receipt.updatedGroups.push(action.targetID) + } + if (approvedPlan.settings.action !== 'unchanged') { + await payload.updateGlobal({ slug: 'career-settings', data: remapMedia(content(bundle.settings, 'settings'), mediaMap), depth: 0, req }) + receipt.updatedSettings = true + } + const afterPlan = await buildPlan(payload, bundle, mappings, req, targetIdentity) + if (afterPlan.groups.some(x => x.action !== 'unchanged') || afterPlan.assets.some(x => x.action !== 'reuse') || afterPlan.settings.action !== 'unchanged') throw new Error('Imported content differs from the bundle') + receipt.afterHash = afterPlan.targetHash + // Record pending outcome before commit; never report committed until PostgreSQL confirms it. + await writeFile(receiptPath, JSON.stringify({ ...receipt, status: 'pending-commit' }, null, 2)) + commitAttempted = true + await commitTransaction(req) + committed = true + await writeFile(receiptPath, JSON.stringify({ ...receipt, status: 'committed' }, null, 2)) + return receipt + } catch (error) { + if (!committed && !commitAttempted) { + await killTransaction(req) + for (const filename of createdFiles) await rm(filename, { force: true }) + } + if (!committed && commitAttempted) { + // A lost COMMIT response does not prove rollback. Keep files for reconciliation. + await writeFile(receiptPath, JSON.stringify({ ...receipt, status: 'commit-uncertain' }, null, 2)).catch(() => {}) + } + throw error + } +} diff --git a/scripts/career-transfer/verify-guards.mjs b/scripts/career-transfer/verify-guards.mjs new file mode 100644 index 0000000..66d7203 --- /dev/null +++ b/scripts/career-transfer/verify-guards.mjs @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict' +import { mkdtemp, mkdir, readFile, readdir, writeFile, symlink, rm } from 'node:fs/promises' +import path from 'node:path' +import { applyPlan, buildPlan, readTarget, targetFingerprint } from './transfer.mjs' +import { fileHash, hash } from './content.mjs' + +export async function verifyGuards(payload, root, bundle, bundlePath, mappings) { + const before = targetFingerprint(await readTarget(payload)) + const mediaDir = payload.collections.media.config.upload.staticDir + const beforeFiles = (await readdir(mediaDir)).sort() + const directory = await mkdtemp(path.join(root, 'guard-test-')) + try { + const stale = await buildPlan(payload, bundle, mappings) + stale.targetHash = 'deliberately-stale' + const { sha256: ignored, ...staleData } = stale + void ignored + stale.sha256 = hash(staleData) + await assert.rejects(() => applyPlan(payload, bundle, bundlePath, mappings, stale, path.join(directory, 'stale-receipt.json')), /Target changed since planning/) + + // Create a valid, distinct JPEG first, then fail validation on the LAST group. + // This verifies rollback of earlier updates and cleanup of newly written files. + const invalid = structuredClone(bundle) + await mkdir(path.join(directory, 'media')) + for (let i = 0; i < invalid.assets.length; i++) { + const asset = invalid.assets[i] + const source = path.join(path.dirname(bundlePath), 'media', asset.filename) + const target = path.join(directory, 'media', asset.filename) + if (i === 0) { + const bytes = Buffer.concat([await readFile(source), Buffer.from('\nlocal rollback verification\n')]) + await writeFile(target, bytes) + asset.sha256 = fileHash(bytes) + asset.size = bytes.length + } else await symlink(source, target) + } + invalid.recruitment[0].order = 98765 + invalid.recruitment.at(-1).description = null + const { sha256: oldHash, ...invalidData } = invalid + void oldHash + invalid.sha256 = hash(invalidData) + const plan = await buildPlan(payload, invalid, mappings) + assert.ok(plan.assets.some(x => x.action === 'create')) + await assert.rejects(() => applyPlan(payload, invalid, path.join(directory, 'bundle.json'), mappings, plan, path.join(directory, 'rollback-receipt.json')), error => error.name === 'ValidationError') + assert.equal(targetFingerprint(await readTarget(payload)), before) + assert.deepEqual((await readdir(mediaDir)).sort(), beforeFiles) + const result = { stalePlanRejected: true, databaseRollback: true, newFileCleanup: true } + await writeFile(path.join(root, 'guard-results.json'), JSON.stringify(result, null, 2)) + return result + } finally { + await rm(directory, { recursive: true, force: true }) + } +} diff --git a/scripts/prepare-career-bundle.mjs b/scripts/prepare-career-bundle.mjs new file mode 100644 index 0000000..6e56eac --- /dev/null +++ b/scripts/prepare-career-bundle.mjs @@ -0,0 +1,52 @@ +import { spawnSync } from 'node:child_process' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { fileHash, hash, mediaReferences } from './career-transfer/content.mjs' + +process.umask(0o077) +const [stagingArchive, productionArchive] = process.argv.slice(2) +if (!stagingArchive || !productionArchive) throw new Error('Pass staging and production file archives') +const root = path.resolve('.local-verification') +const source = JSON.parse(await readFile(path.join(root, 'frt_transfer_staging.json'))) +const target = JSON.parse(await readFile(path.join(root, 'frt_transfer_candidate.json'))) +const ids = mediaReferences([source.recruitment, source.settings]) +source.recruitment.forEach(doc => { if (doc.image) ids.add(doc.image) }) +const media = [...ids].map(id => { + const doc = source.media.find(item => item.id === id) + if (!doc?.filename || path.basename(doc.filename) !== doc.filename) throw new Error(`Invalid media file: ${id}`) + return doc +}) +const bundleDir = path.join(root, 'bundle') +await mkdir(bundleDir, { recursive: true }) + +async function extract(archive, destination, filenames) { + await mkdir(path.join(destination, 'media'), { recursive: true }) + if (!filenames.length) return + // Extract exact regular files ourselves; no paths or links from tar may escape destination. + const python = `import tarfile,sys,os,shutil,json +names=set(json.loads(sys.argv[3])); found=set() +with tarfile.open(sys.argv[1], 'r|gz') as archive: + for member in archive: + if member.name in names: + if not member.isfile() or member.name in found: raise RuntimeError('Invalid archive member') + dest=os.path.join(sys.argv[2], member.name) + with archive.extractfile(member) as src, open(dest, 'xb') as out: shutil.copyfileobj(src,out) + found.add(member.name) +if found != names: raise RuntimeError('Missing archive members: '+str(names-found)) +` + const result = spawnSync('python3', ['-c', python, archive, destination, JSON.stringify(filenames.map(name => `media/${name}`))], { stdio: 'inherit' }) + if (result.status !== 0) throw new Error('Selective file extraction failed') +} +await extract(stagingArchive, bundleDir, media.map(doc => doc.filename)) +const existing = target.media.filter(doc => media.some(s => s.filename === doc.filename)) +await extract(productionArchive, path.join(root, 'frt_transfer_candidate'), existing.map(doc => doc.filename)) +const assets = [] +for (const doc of media) { + const bytes = await readFile(path.join(bundleDir, 'media', doc.filename)) + if (bytes.length !== doc.filesize) throw new Error(`File size differs from DB: ${doc.filename}`) + assets.push({ sourceID: doc.id, filename: doc.filename, alt: doc.alt, mimeType: doc.mimeType, size: bytes.length, sha256: fileHash(bytes) }) +} +const bundle = { version: 1, recruitment: source.recruitment, settings: source.settings, assets } +bundle.sha256 = hash(bundle) +await writeFile(path.join(bundleDir, 'bundle.json'), JSON.stringify(bundle, null, 2)) +console.log(JSON.stringify({ bundle: path.join(bundleDir, 'bundle.json'), files: assets.length, bytes: assets.reduce((n, x) => n + x.size, 0), sha256: bundle.sha256 })) diff --git a/tests/career-transfer-server.test.mjs b/tests/career-transfer-server.test.mjs new file mode 100644 index 0000000..7ef00c0 --- /dev/null +++ b/tests/career-transfer-server.test.mjs @@ -0,0 +1,28 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { parseOptions } from '../scripts/career-transfer/server-options.mjs' + +const env = { DATABASE_URI: 'postgresql://app:local-password@postgres:5432/frt', PAYLOAD_SECRET: 'local-test-only', NEXT_PUBLIC_SERVER_URL: 'https://frtbme.hu' } +const common = ['--environment', 'production', '--expect-database', 'frt', '--expect-origin', 'https://frtbme.hu', '--bundle', '/transfer/bundle/bundle.json', '--mapping', '/transfer/mapping.json'] +const plan = ['plan', ...common, '--out', '/transfer/plan.json'] +const apply = ['apply', ...common, '--plan', '/transfer/plan.json', '--plan-sha256', 'a'.repeat(64), '--receipt', '/transfer/receipt.json'] + +test('server plan requires explicit database and application identity', () => { + assert.equal(parseOptions(plan, env).command, 'plan') + assert.throws(() => parseOptions(plan, { ...env, DATABASE_URI: 'postgresql://app:pass@postgres/other' }), /Database does not match/) + assert.throws(() => parseOptions(plan, { ...env, NEXT_PUBLIC_SERVER_URL: 'https://staging.frtbme.hu' }), /origin does not match/) +}) +test('server apply requires the exact reviewed plan hash', () => { + assert.equal(parseOptions(apply, env)['plan-sha256'], 'a'.repeat(64)) + assert.throws(() => parseOptions(apply.filter((_, index) => ![apply.indexOf('--plan-sha256'), apply.indexOf('--plan-sha256') + 1].includes(index)), env), /Missing --plan-sha256/) + assert.throws(() => parseOptions(apply.map(value => value === 'a'.repeat(64) ? 'yes' : value), env), /exact SHA-256/) +}) +test('server runner rejects unexpected commands, duplicate flags and path collisions', () => { + assert.throws(() => parseOptions(['migrate', ...common], env), /Usage/) + assert.throws(() => parseOptions([...plan, '--environment', 'staging'], env), /duplicate option/) + assert.throws(() => parseOptions(plan.map(value => value === '/transfer/plan.json' ? '/transfer/bundle/bundle.json' : value), env), /must be distinct/) +}) +test('server identity cannot omit credentials or use an authenticated expected URL', () => { + assert.throws(() => parseOptions(plan, { ...env, PAYLOAD_SECRET: '' }), /required/) + assert.throws(() => parseOptions(plan.map(value => value === 'https://frtbme.hu' ? 'https://user:pass@frtbme.hu' : value), env), /origin does not match/) +}) diff --git a/tests/career-transfer.test.mjs b/tests/career-transfer.test.mjs new file mode 100644 index 0000000..d467f2e --- /dev/null +++ b/tests/career-transfer.test.mjs @@ -0,0 +1,43 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { content, hash, mediaReferences, remapMedia, validateMapping } from '../scripts/career-transfer/content.mjs' + +test('row IDs are removed but question keys and rich-text metadata survive', () => { + const doc = { formQuestions: [{ id: 'staging-row', key: 'stable', options: [{ id: 'option-row', key: 'yes' }] }], intro: { root: { id: 'rich-id', children: [] } } } + const result = content(doc, 'settings') + assert.equal(result.formQuestions[0].id, undefined) + assert.equal(result.formQuestions[0].key, 'stable') + assert.equal(result.formQuestions[0].options[0].id, undefined) + assert.equal(result.formQuestions[0].options[0].key, 'yes') + assert.equal(result.intro.root.id, 'rich-id') +}) +test('staging controls application state and integration URLs, including cleared values', () => { + const source = { spreadsheetUrl: 'https://example.test/sheet', applicationsOpen: false, applicationMode: 'googleForm', googleFormUrl: 'https://example.test/form', intro: {} } + const result = content(source, 'settings') + for (const key of ['spreadsheetUrl', 'applicationsOpen', 'applicationMode', 'googleFormUrl']) assert.equal(result[key], source[key]) + const cleared = content({ ...source, spreadsheetUrl: null, googleFormUrl: null }, 'settings') + assert.equal(cleared.spreadsheetUrl, null) + assert.equal(cleared.googleFormUrl, null) + assert.equal(cleared.applicationsOpen, false) +}) +test('cross-database mappings reject reused IDs and stale names', () => { + const source = [{ id: 6, groupName: 'Önvezető' }, { id: 7, groupName: 'Mechanika' }] + const target = [{ id: 12, groupName: 'Driverless' }, { id: 13, groupName: 'Mechanika' }] + const mapping = [{ sourceID: 6, sourceName: 'Önvezető', targetID: 12, targetName: 'Driverless' }, { sourceID: 7, sourceName: 'Mechanika', targetID: 13, targetName: 'Mechanika' }] + assert.doesNotThrow(() => validateMapping(source, target, mapping)) + assert.throws(() => validateMapping(source, target, [{ ...mapping[0], targetID: 6 }, mapping[1]])) + assert.throws(() => validateMapping(source, target, [mapping[0], { ...mapping[1], targetID: 12 }])) + assert.throws(() => validateMapping(source, target, [mapping[0]])) + assert.throws(() => validateMapping(source, [...target, { id: 14, groupName: 'Extra' }], mapping), /Unmapped target groups/) +}) +test('rich-text relationships are mapped explicitly and unknown collections fail', () => { + const doc = { root: { children: [{ type: 'upload', relationTo: 'media', value: 985 }] } } + assert.deepEqual([...mediaReferences(doc)], [985]) + assert.equal(remapMedia(doc, { 985: 992 }).root.children[0].value, 992) + assert.throws(() => remapMedia(doc, {})) + assert.throws(() => mediaReferences({ relationTo: 'users', value: 1 })) +}) +test('fingerprints ignore object order but retain array order', () => { + assert.equal(hash({ a: 1, b: 2 }), hash({ b: 2, a: 1 })) + assert.notEqual(hash([1, 2]), hash([2, 1])) +})