diff --git a/.gitattributes b/.gitattributes index 2d973c9..f1a2d30 100644 --- a/.gitattributes +++ b/.gitattributes @@ -54,3 +54,5 @@ docs/bchc/** merge=ours .github/ISSUE_TEMPLATE/new-entry.yml merge=ours .github/ISSUE_TEMPLATE/config.yml merge=ours assets/js/configurator/defaults.generated.js merge=ours +submit/answers.schema.json merge=ours +submit/agent-instructions.txt merge=ours diff --git a/.github/workflows/new-entry.yml b/.github/workflows/new-entry.yml index 95d321d..5d47fc0 100644 --- a/.github/workflows/new-entry.yml +++ b/.github/workflows/new-entry.yml @@ -156,6 +156,45 @@ jobs: const notify = await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/lib/notify_github.mjs`).href); await notify.notifyIssue({ github, context, core, status: notify.STATUS.received }); + # Optional: files attached on /submit/ live in a PRIVATE uploads repository + # (docs/submit-uploads-plan.md), and the issue body names them with + # `upload:` lines. Reading them needs a credential for that repository, so + # this mints one — and only when the repository variable UPLOADS_REPO + # (`owner/name`) is set; unset, neither step runs and nothing changes. + # The token is a GitHub App installation token that can READ CONTENTS of + # that one repository and nothing else (`permission-contents: read`), it + # expires within the hour, and it is handed to the scaffold step below and + # to no other step. Deleting uploads is a separate, write-capable token + # minted in uploads-cleanup.yml, never here. The variable is split in a + # step of its own because the action takes the owner and the repository + # name separately and expressions cannot split a string. + # + # Both steps are non-fatal: a misconfigured or unreachable App must not + # cost a submitter their draft. A failed mint leaves the token empty, and + # the scaffolder drafts the entry anyway with one warning per `upload:` + # answer it could not collect (scripts/lib/uploads.mjs). + - name: Split the uploads repository name + id: uploads-repo + if: vars.UPLOADS_REPO != '' + continue-on-error: true + env: + UPLOADS_REPO: ${{ vars.UPLOADS_REPO }} + run: | + echo "owner=${UPLOADS_REPO%%/*}" >> "$GITHUB_OUTPUT" + echo "name=${UPLOADS_REPO#*/}" >> "$GITHUB_OUTPUT" + + - name: Mint a read-only token for the uploads repository + id: uploads-token + if: vars.UPLOADS_REPO != '' + continue-on-error: true + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.UPLOADS_APP_ID }} + private-key: ${{ secrets.UPLOADS_APP_PRIVATE_KEY }} + owner: ${{ steps.uploads-repo.outputs.owner }} + repositories: ${{ steps.uploads-repo.outputs.name }} + permission-contents: read + - name: Scaffold entry files id: scaffold continue-on-error: true @@ -166,6 +205,12 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} # Lets the scaffolder download images attached to a private issue. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Empty when UPLOADS_REPO is unset. The scaffolder resolves `upload:` + # lines against the ref built from the ISSUE AUTHOR's id, so a + # reference copied into someone else's issue finds nothing. + UPLOADS_TOKEN: ${{ steps.uploads-token.outputs.token }} + UPLOADS_REPO: ${{ vars.UPLOADS_REPO }} + ISSUE_AUTHOR_ID: ${{ github.event.issue.user.id }} - name: Report the failure on the issue if: steps.scaffold.outcome == 'failure' diff --git a/.github/workflows/uploads-cleanup.yml b/.github/workflows/uploads-cleanup.yml new file mode 100644 index 0000000..2889c85 --- /dev/null +++ b/.github/workflows/uploads-cleanup.yml @@ -0,0 +1,134 @@ +name: Uploads cleanup + +# Files attached on /submit/ are stored by the optional upload service as one +# branch each (`u///`) in a PRIVATE uploads repository +# (docs/submit-uploads-plan.md). This deletes them once they are not needed: +# +# * when a `content:new-entry` issue closes, or its draft pull request +# (branch `entry/-`) closes merged or declined, the author's +# sets named in the issue body. The pull request trigger exists because a +# close made with the built-in GITHUB_TOKEN (a decline, a "Closes #N" on +# merge) starts no `issues: closed` run; +# * every day, any set older than 14 days that no open `content:new-entry` +# issue by the same author still references. +# +# Deletion targets always come from the uploads repository's own ref listing; +# issue text can only cause a set to be kept (scripts/uploads_cleanup.mjs). +# +# Off until the repository variable UPLOADS_REPO (`owner/name`) is set, with the +# secrets UPLOADS_APP_ID and UPLOADS_APP_PRIVATE_KEY of a GitHub App installed +# on that repository. A manual run from the Actions tab sweeps; its "Preview +# only" box lists what would go without deleting anything. + +on: + issues: + types: + - closed + # Same-repository drafts only (the job's `if`); the branch name is read from + # the event and used through `env:` only. The checkout below names the default + # branch because a declined, conflicting draft has no merge ref to check out. + pull_request: + types: + - closed + schedule: + # 04:47 UTC daily, off the hour-zero peak where scheduled runs queue for ages. + - cron: "47 4 * * *" + workflow_dispatch: + inputs: + dry_run: + description: "Preview only — list what would be deleted, delete nothing" + type: boolean + default: false + +permissions: {} + +# GitHub keeps only ONE pending run per concurrency group and cancels any +# older pending one, so a single shared group would drop queued close runs. +# Each closed issue or pull request gets a group of its own (a re-close of the +# same one waits behind the first, and a dropped pending duplicate would have +# done the same work); the daily and manual sweeps share the `sweep` group. +# Runs in different groups may overlap: every deletion target comes from a +# fresh ref listing and a ref that is already gone (404/422) counts as "already +# gone", not a failure, so two runs deleting the same files is harmless. +concurrency: + group: uploads-cleanup-${{ github.event.issue.number || github.event.pull_request.number || 'sweep' }} + cancel-in-progress: false + +jobs: + # A job whose `if` is false shows up as a bare grey "skipped" with no reason + # (see metrics.yml). This one runs only when the feature is off and says so. + turned-off: + if: >- + vars.UPLOADS_REPO == '' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'content:new-entry') && github.event.pull_request.head.repo.full_name == github.repository && startsWith(github.event.pull_request.head.ref, 'entry/'))) + && (github.event_name != 'issues' || contains(github.event.issue.labels.*.name, 'content:new-entry')) + runs-on: ubuntu-latest + steps: + - name: Say why nothing ran + run: | + echo "Uploads cleanup is turned off: the repository variable UPLOADS_REPO is not set, so there is no uploads repository to clean. Set it in Settings -> Secrets and variables -> Actions -> Variables to turn uploads on." >> "$GITHUB_STEP_SUMMARY" + + cleanup: + if: >- + vars.UPLOADS_REPO != '' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'content:new-entry') && github.event.pull_request.head.repo.full_name == github.repository && startsWith(github.event.pull_request.head.ref, 'entry/'))) + && (github.event_name != 'issues' || contains(github.event.issue.labels.*.name, 'content:new-entry')) + runs-on: ubuntu-latest + permissions: + # Checkout, and listing this repository's open issues for the sweep. + contents: read + issues: read + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: ".node-version" + cache: npm + + # The script uses Node built-ins and scripts/lib only; no npm install. + - name: Select exact npm + run: node scripts/install_exact_npm.mjs + + - name: Split the uploads repository name + id: uploads-repo + env: + UPLOADS_REPO: ${{ vars.UPLOADS_REPO }} + run: | + echo "owner=${UPLOADS_REPO%%/*}" >> "$GITHUB_OUTPUT" + echo "name=${UPLOADS_REPO#*/}" >> "$GITHUB_OUTPUT" + + # The only place a write-capable token for the uploads repository is + # minted: contents write on that one repository (deleting a branch is a + # contents write), valid for under an hour. new-entry.yml mints a + # read-only one for scaffolding. + - name: Mint a token for the uploads repository + id: uploads-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.UPLOADS_APP_ID }} + private-key: ${{ secrets.UPLOADS_APP_PRIVATE_KEY }} + owner: ${{ steps.uploads-repo.outputs.owner }} + repositories: ${{ steps.uploads-repo.outputs.name }} + permission-contents: write + + # Issue text reaches the script through `env:` only, never a `run:` line. + - name: Delete unneeded uploads + run: node scripts/uploads_cleanup.mjs + env: + MODE: ${{ github.event_name == 'issues' && 'closed' || github.event_name == 'pull_request' && 'pr-closed' || 'sweep' }} + # pr-closed: the issue is found from the draft branch name, which the + # script re-checks; the fork check is in the job's `if`. + PR_BRANCH: ${{ github.event.pull_request.head.ref }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + UPLOADS_TOKEN: ${{ steps.uploads-token.outputs.token }} + UPLOADS_REPO: ${{ vars.UPLOADS_REPO }} + CATALOG_REPO: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_BODY: ${{ github.event.issue.body }} + ISSUE_AUTHOR_ID: ${{ github.event.issue.user.id }} + DRY_RUN: ${{ inputs.dry_run }} diff --git a/.phct/ownership.yml b/.phct/ownership.yml index f2bb421..65cab77 100644 --- a/.phct/ownership.yml +++ b/.phct/ownership.yml @@ -39,6 +39,8 @@ ownership: - .github/ISSUE_TEMPLATE/new-entry.yml - .github/ISSUE_TEMPLATE/config.yml - assets/js/configurator/defaults.generated.js + - submit/answers.schema.json + - submit/agent-instructions.txt # Protected from the parent merge, then deliberately updated by the updater. # It is excluded from byte-for-byte deployment content checks. diff --git a/.prettierignore b/.prettierignore index 0366379..da06cca 100644 --- a/.prettierignore +++ b/.prettierignore @@ -21,5 +21,9 @@ assets/css/ # Liquid template that emits JSON — it is not parseable JSON at rest. entries.json +# Generated by `npm run generate`: Liquid inside JSON, and plain text; neither is formatted. +submit/answers.schema.json +submit/agent-instructions.txt + # Written by `vercel link` when the sign-in service is deployed; gitignored there. .vercel/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d40127a..2470dc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,45 @@ major version, and each entry says so when it happens. ## [Unreleased] +### Added + +- Optional **attach files on the submit page**. Set `submit.uploads_url` in + `_data/site.yml` (it needs `submit.signin_url`; `npm run validate` checks + both) and a signed-in submitter gets a file picker on the screenshots question, + with alt text beside each picture, and on each file question (the slide deck). + Each file uploads as soon as it is picked, pictures are re-encoded in the browser + first (EXIF and location data stripped, longest side 2,400 px), the draft remembers + the files, and the issue carries `upload:` references instead of asking the + submitter to drag files onto GitHub afterwards. Off by default; blank keeps the page + exactly as it was. Files are limited to 4 MB each, 3 drafts and 9 files per draft per + person. +- `services/uploads/`, the **upload service** behind it: a stateless Vercel project with + no runtime dependencies. It checks the submitter's GitHub token, the origin, the + file's real type and size and the per-person caps, then writes each file as its own + parentless commit and branch in a private uploads repository, with a second GitHub App + of its own. Setup is in `services/uploads/README.md`. +- The scaffolder resolves `upload:` references. `new-entry.yml` mints a read-only token for + the uploads repository (repository variable `UPLOADS_REPO`, secrets `UPLOADS_APP_ID` and + `UPLOADS_APP_PRIVATE_KEY`) and hands it to the scaffold step only; the files are read as + the issue's author, re-hashed, checked like any attachment and committed into the entry. + An unset or broken setup, or a file that has expired, becomes a warning on the draft, + and dragging files onto the issue keeps working. +- **Uploads cleanup** workflow (`uploads-cleanup.yml`): deletes a submission's files when + its draft pull request is merged or closed (or its issue is closed by hand), and + daily sweeps any file older than 14 days that no open submission names. A manual run with + **Preview only** lists what would go. Admin guide: *Files attached on the submit page*, + including how to purge something sensitive. +- "Using an AI agent?" on `/submit/`: an instructions file and a JSON Schema for answers, + both generated from `_data/schema.yml` by `npm run generate`, and an **Import answers** + box that loads an agent's JSON into the form and runs the form's own checks. + +### Security + +- `SECURITY.md` and the sign-in README now say that, on a deployment that also runs the + upload service, the submitter's sign-in token is sent to that service as well as to + `api.github.com` (one `GET /user` call), and that the private uploads repository does + not keep screenshots private once a submission is scaffolded. + ## [1.11.0-rc.10] — 2026-10-13 Tenth candidate. rc.9 failed closed in BCHC on three new tests that read the diff --git a/SECURITY.md b/SECURITY.md index 10152e6..eecd55b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -108,6 +108,42 @@ See [docs/admin-guide.md](docs/admin-guide.md#checks-on-a-generated-pull-request Every `uses:` in `.github/workflows/` names a full commit SHA with the release tag in a trailing comment. Dependabot proposes updates weekly. +## File uploads on the submit page (optional) + +Off unless a deployment sets `submit.uploads_url`. Then a signed-in submitter can attach screenshots +and a PDF deck on `/submit/` through a small service the deployment runs +([`services/uploads/`](services/uploads/README.md)). What that changes: + +- **The service stores nothing.** No database, no file storage, nothing on disk. It checks each file + (origin, a real PNG, JPEG, WebP or PDF judged by its bytes, at most 4 MB, per-person caps) and writes + it to a **private** GitHub repository in the same request. It has no runtime dependencies. +- **A private repository does not keep screenshots private.** It protects drafts that were never + submitted and stops the service being an anonymous file host. Once a submission is scaffolded, its + files are committed to a public `entry/*` draft branch and shown in the draft pull request before + anyone has reviewed them, just as files dragged into an issue are public the moment they are attached. + The page tells submitters not to show personal information; reviewers still open every picture at + full size ([docs/incidents.md](docs/incidents.md)). +- **The submitter's sign-in token goes to the upload service too.** Without uploads it is sent only to + `api.github.com`. With uploads the page also sends it to the service the deployment runs, which uses + it for one `GET /user` call to learn who is uploading and never logs, stores or forwards it. It can + do only what the sign-in App allows, and cannot touch the uploads repository. +- **A reference cannot name someone else's files.** The scaffolder builds the branch name itself from + the issue author's numeric id, so a reference copied into another person's issue finds nothing, and + no reference can name another repository. It re-hashes every file and rejects a mismatch. +- **The uploads token is read-only and reaches one step.** The scaffold job mints a token that can + read contents of the uploads repository only, hands it to the scaffold step alone, and the generic + image and attachment fetchers (which send their token to any GitHub host a submitter names) never + see it. Deleting uses a separate write token minted only in `uploads-cleanup.yml`, whose deletion + targets always come from the repository's own branch listing; issue text can only cause a set to be + kept. +- **Deleted is not erased.** One parentless commit per file means deleting a branch leaves the file + unreachable, but git keeps it until GitHub garbage-collects. To purge something sensitive at once, + contact GitHub Support. The procedure is in + [docs/admin-guide.md](docs/admin-guide.md#files-attached-on-the-submit-page). + +Report a problem with the upload service the same way as any other: see +[Reporting a vulnerability](#reporting-a-vulnerability) below. + ## Who can submit Open by default: the catalog is meant to collect work from people without diff --git a/_data/site.yml b/_data/site.yml index 03a857e..978d281 100644 --- a/_data/site.yml +++ b/_data/site.yml @@ -123,6 +123,12 @@ submit: # answers as an issue straight from /submit/, however long. Blank keeps the # link hand-off only. See docs/configuration.md. signin_url: "" + # Optional file uploads on /submit/: the address of the upload service you + # deploy from services/uploads/ (e.g. https://catalog-uploads.vercel.app). + # Signed-in submitters can then attach screenshots and a PDF deck right on + # the page. Needs signin_url, because uploads work only for signed-in + # submitters. Blank turns it off. See docs/configuration.md. + uploads_url: "" # "Check your submission" page (module: status). A submitter types the number # GitHub gave their submission and sees its title, date and stage, read from diff --git a/assets/js/configurator/defaults.generated.js b/assets/js/configurator/defaults.generated.js index 2b5335a..e20afbc 100644 --- a/assets/js/configurator/defaults.generated.js +++ b/assets/js/configurator/defaults.generated.js @@ -96,7 +96,8 @@ export const SITE = { "turnaround": "Intake checks it within about five business days and the Governance Committee reviews it within about ten more; you keep ownership of anything you share.", "review_note": "Please do not include protected health information, credentials or non-public data. Link out to repositories and documents rather than pasting sensitive content.", "fallback_email": "catalog@example.org", - "signin_url": "" + "signin_url": "", + "uploads_url": "" }, "status": { "heading": "Check your submission", diff --git a/assets/js/submit.js b/assets/js/submit.js index a1ce4f0..cc8a834 100644 --- a/assets/js/submit.js +++ b/assets/js/submit.js @@ -1,7 +1,8 @@ /** * Submit form — entry point. Wires the pieces in assets/js/submit/*.js: * fields (registry) → validate → repeatable → preview → draft → handoff → - * review, and direct (Sign in with GitHub) when the site has it on. + * review, import (an AI agent's JSON), and direct (Sign in with GitHub) and uploads (file pickers for + * signed-in submitters) when the site has them on. * * Loaded last by the `scripts:` list in submit/index.md front matter, so every * window.SubmitForm helper it calls is already defined. @@ -241,10 +242,15 @@ draft.save(); } + // File uploads (assets/js/submit/uploads.js), set up below beside the + // sign-in they depend on; a restored draft hands its record of them over. + let uploads = null; + draft = ns.initDraft( form, fields, - function afterRestore(ui) { + function afterRestore(ui, uploaded) { + if (uploads) uploads.restore(uploaded); fields.filter((field) => field.type === 'images').forEach(ns.renderImagePreviews); // Visibility before position: the short-form recompute decides which // sections still have anything to show before the stepper lands on one. @@ -349,6 +355,16 @@ true ); + /** @returns {string} the issue form with nothing prefilled */ + function blankFormUrl() { + return ( + 'https://github.com/' + + form.dataset.repo + + '/issues/new?template=' + + (form.dataset.template || 'new-entry.yml') + ); + } + /** * Hand the answers to GitHub, from the review panel. * @@ -379,10 +395,7 @@ // not: the chooser has blank issues off, and submitters cannot label. showFallback( 'Your answers are too long to carry across in a link. Open the issue form with the link below, then copy each answer from the box into the question with the same heading.', - 'https://github.com/' + - form.dataset.repo + - '/issues/new?template=' + - (form.dataset.template || 'new-entry.yml') + blankFormUrl() ); return; } @@ -426,6 +439,28 @@ // it can — the link hand-off is the way. const direct = titleField && typeof ns.initDirect === 'function' ? ns.initDirect(form) : null; + // Only with sign-in, and the service's address: submit/index.md renders + // data-uploads-url and uploads.js together, and only with signin_url set. + uploads = + direct && form.dataset.uploadsUrl && typeof ns.initUploads === 'function' + ? ns.initUploads({ form: form, fields: fields, direct: direct, draft: draft, onChange: refresh }) + : null; + + // "Using an AI agent?" (assets/js/submit/import.js): the pasted JSON fills + // the form, then the form's own validation reports on it. + if (typeof ns.initImport === 'function') { + ns.initImport({ + root: root, + fields: fields, + summary: summary, + afterFill: function (problems) { + progress.errors(problems); + if (shortform) shortform.refresh(); + refresh(); + }, + }); + } + /** * Send the answers straight to GitHub as the signed-in submitter. The * draft is deleted only once GitHub answers 201 — every other outcome @@ -433,6 +468,12 @@ * @param {HTMLButtonElement} button held while the request is out */ function sendDirect(button) { + // Counted before the draft is cleared: the panel says where to drag the + // file for each file question that has no upload. + const uploaded = uploads ? uploads.count() : 0; + const fileLabels = fields + .filter((field) => field.type === 'file' && !(uploads && uploads.hasFiles(field))) + .map((field) => field.label); direct.submit({ button: button, status: form.querySelector('[data-review-status]'), @@ -442,16 +483,60 @@ draft.clear(); countEvent('Send to GitHub'); say(status, ''); - ns.renderSubmitted( - form, - issue, - fields.filter((field) => field.type === 'file').map((field) => field.label) - ); + if (uploads) uploads.reset(); + ns.renderSubmitted(form, issue, fileLabels, uploaded); }, - onFallback: send, + onFallback: fallbackWithFiles, }); } + /** + * "Use GitHub's form instead" after a direct send failed. The prefilled + * link never carries `upload:` lines (handoff.js), so with files attached + * it would drop them unannounced: hand over the copy-out box instead, + * which carries them, and say why. Files held by an expired sign-in: say + * that only signing in again keeps them. No files, or no repository: the + * ordinary link hand-off. + */ + function fallbackWithFiles() { + const files = uploads ? uploads.count() : 0; + const held = uploads ? uploads.held() : 0; + if (files === 0 && held > 0 && form.dataset.repo) { + // The sign-in expired (a 401 signs the page out): the files are still + // stored, but nothing sent from a signed-out page can name them. + ns.exitReview(form); + const url = ns.issueUrl(form, fields, entryTitle()); + showFallback( + 'Your GitHub sign-in has expired, so ' + + (held === 1 ? 'the file you uploaded is' : 'the ' + held + ' files you uploaded are') + + ' not in the box or the link below. To keep ' + + (held === 1 ? 'it' : 'them') + + ', sign in again and send from this page. To go on without ' + + (held === 1 ? 'it' : 'them') + + ', use GitHub’s form.', + url.length <= MAX_URL ? url : blankFormUrl() + ); + return; + } + if (files === 0 || !form.dataset.repo) { + send(); + return; + } + const url = ns.issueUrl(form, fields, entryTitle()); + const fits = url.length <= MAX_URL; + ns.exitReview(form); + showFallback( + (files === 1 ? 'The file you uploaded does' : 'The ' + files + ' files you uploaded do') + + ' not travel in a link to GitHub’s form, but the box below carries ' + + (files === 1 ? 'it' : 'them') + + ' with your answers. ' + + (fits + ? 'Open the prefilled issue with the link below, then copy the upload: lines from the box into the questions with the same headings.' + : 'Open the issue form with the link below, then copy each answer from the box into the question with the same heading.'), + fits ? url : blankFormUrl() + ); + } + form.addEventListener('submit', (event) => { event.preventDefault(); // Implicit submission (Enter in a text input) before the last step is a diff --git a/assets/js/submit/direct.js b/assets/js/submit/direct.js index 68b8acd..ebfb5cd 100644 --- a/assets/js/submit/direct.js +++ b/assets/js/submit/direct.js @@ -29,7 +29,13 @@ * [data-direct-copy] (the intro's two versions of the last step, one shown); * [data-review-status] (rendered by review.js when `handlers.direct` is set). * - * Exposes: window.SubmitForm.initDirect + * The object initDirect returns also lets assets/js/submit/uploads.js (file + * uploads, only when `submit.uploads_url` is set) use the sign-in: `token()` + * hands out the bearer token for a request to the upload service (never to be + * stored or logged), `userId()` / `login()` say whose it is, `onChange(fn)` + * reports every sign-in state change, and `expire()` treats the token as dead. + * + * Exposes: window.SubmitForm.initDirect, .serviceOrigin */ (function (ns) { 'use strict'; @@ -65,10 +71,14 @@ return url.protocol === 'https:' || local ? url.origin : ''; } + ns.serviceOrigin = serviceOrigin; + /** * Wire "Sign in with GitHub" into the page. * @param {HTMLFormElement} form - * @returns {null|{signedIn: () => boolean, login: () => string, submit: (options: object) => void}} + * @returns {null|{signedIn: () => boolean, login: () => string, userId: () => number|null, + * token: () => string|null, onChange: (fn: () => void) => void, expire: () => void, + * submit: (options: object) => void}} * null when the site has no sign-in service (or no repository to send to) */ ns.initDirect = function initDirect(form) { @@ -86,6 +96,10 @@ /** Fallback when sessionStorage is refused: the token lives as long as the page. */ let memory = null; let login = ''; + /** GitHub's numeric id for the signed-in account, once GitHub has been asked. */ + let userId = null; + /** Callbacks told whenever the sign-in state is repainted. */ + const listeners = []; /** The sign-in window in progress: {popup, resolve, timer}. */ let pending = null; let inFlight = false; @@ -123,6 +137,7 @@ function clearToken() { memory = null; login = ''; + userId = null; try { window.sessionStorage.removeItem(storeKey); } catch { @@ -160,6 +175,18 @@ /** @param {'out'|'checking'|'in'} state */ function paint(state) { + paintControls(state); + listeners.forEach((listener) => { + try { + listener(); + } catch { + // A listener's trouble is not the sign-in's. + } + }); + } + + /** @param {'out'|'checking'|'in'} state */ + function paintControls(state) { // The intro's "what happens" steps, outside any template: say which // route a press of the last button will take. const signed = state !== 'out'; @@ -229,12 +256,14 @@ } return response.json().then((user) => { login = user && LOGIN_PATTERN.test(String(user.login)) ? String(user.login) : ''; + userId = user && Number.isInteger(user.id) && user.id > 0 ? user.id : null; paint('in'); }); }) .catch(() => { // GitHub out of reach: keep the token, and say only that it is there. login = ''; + userId = null; paint(readToken() ? 'in' : 'out'); }); } @@ -561,6 +590,20 @@ return { signedIn: () => readToken() !== null, login: () => login, + userId: () => (readToken() !== null ? userId : null), + token: () => { + const saved = readToken(); + return saved ? saved.token : null; + }, + onChange: (listener) => { + listeners.push(listener); + }, + // A service that took the token said it is no longer good. + expire: () => { + clearToken(); + paint('out'); + sayNote('Your GitHub sign-in has expired. Sign in again to submit from this page.'); + }, submit: submit, }; }; diff --git a/assets/js/submit/draft.js b/assets/js/submit/draft.js index f47a048..1ba2181 100644 --- a/assets/js/submit/draft.js +++ b/assets/js/submit/draft.js @@ -27,7 +27,10 @@ * Bump the version segment when the stored shape changes *incompatibly*. * The optional `ui` key (stepper position, short-form mode) rode in without * a bump on purpose: readers on either side ignore keys they don't know, so - * a bump would only have thrown away everyone's saved answers. + * a bump would only have thrown away everyone's saved answers. So did the + * optional `uploads` key (files attached through assets/js/submit/uploads.js): + * `{set, user: {id, login}, files: {: [{ref, name, alt, bytes, kind}]}}`. + * It holds references only — never a sign-in token, never file contents. */ const KEY_PREFIX = 'catalog-template:submit-draft:v2'; const DEBOUNCE_MS = 500; @@ -87,11 +90,16 @@ * typed answers are still not in storage — the one case worth a warning. * @param {HTMLFormElement} form * @param {object[]} fields descriptors from readFields - * @param {(ui: object|null) => void} onRestore called after a draft is - * written into the form, with the draft's saved UI state (or null) + * @param {(ui: object|null, uploads: object|null) => void} onRestore called + * after a draft is written into the form, with the draft's saved UI state + * and uploaded-files record (or null) * @param {(() => object|null)|undefined} uiState reads the UI state (stepper * position, short-form mode) to store beside the answers - * @returns {{save: Function, clear: Function, flush: Function, unsaved: () => boolean}} + * @returns {{save: Function, clear: Function, flush: Function, unsaved: () => boolean, + * trackUploads: (read: () => object|null) => void}} + * `trackUploads` registers the reader of the uploaded-files record stored + * beside the answers (null for none); a draft restored later hands the + * record it holds to `onRestore` as its second argument. */ ns.initDraft = function initDraft(form, fields, onRestore, uiState) { const key = storageKey(form); @@ -124,6 +132,16 @@ // True after the kept draft is restored and until its answers are saved // under `key`: until then `:previous` is still the only stored copy. let dropKept = false; + // Reads the uploaded-files record to store beside the answers; see trackUploads. + let uploadsState = null; + + /** @returns {object|null} the uploaded-files record, when it names any file */ + function uploadsRecord() { + const record = uploadsState ? uploadsState() : null; + if (!record || !record.files) return null; + const any = Object.keys(record.files).some((name) => record.files[name].length > 0); + return any ? record : null; + } /** @returns {object} the current answers, keyed by schema key */ function snapshot() { @@ -183,7 +201,8 @@ timer = null; if (held) return; const data = snapshot(); - const empty = Object.keys(data).length === 0; + const uploads = uploadsRecord(); + const empty = Object.keys(data).length === 0 && !uploads; if (!memory) { dirty = !empty; return; @@ -206,6 +225,7 @@ const payload = { saved: stamp, fields: data }; const ui = uiState ? uiState() : null; if (ui) payload.ui = ui; + if (uploads) payload.uploads = uploads; // A failed setItem leaves the key as it was, so the offered draft stays // under `key` until the new answers have actually replaced it; the copy // under `:previous` is undone if they have not. @@ -282,7 +302,7 @@ /** * @param {string} from storage key - * @returns {{saved: string, fields: object, ui: object|null}|null} the stored draft, if any + * @returns {{saved: string, fields: object, ui: object|null, uploads: object|null}|null} the stored draft, if any */ function read(from) { if (!memory) return null; @@ -293,6 +313,7 @@ saved: String(parsed.saved || ''), fields: parsed.fields, ui: parsed.ui && typeof parsed.ui === 'object' ? parsed.ui : null, + uploads: parsed.uploads && typeof parsed.uploads === 'object' ? parsed.uploads : null, }; } catch (error) { return null; @@ -326,7 +347,7 @@ offered = ''; if (bar) bar.hidden = true; setStatus(message); - onRestore(draft.ui); + onRestore(draft.ui, draft.uploads); savedAt = ''; save(); } @@ -398,7 +419,7 @@ // After the caller has finished starting up: the answers are in place // for its controllers to read, but the stepper and short-form mode the // saved UI state moves do not exist until initDraft has returned. - Promise.resolve().then(() => onRestore(existing.ui)); + Promise.resolve().then(() => onRestore(existing.ui, existing.uploads)); } else if (existing) offerRestore(existing, key); else if (kept) offerRestore(kept, previousKey); } @@ -475,6 +496,14 @@ } }); - return { save: save, clear: clear, flush: flush, unsaved: unsaved }; + return { + save: save, + clear: clear, + flush: flush, + unsaved: unsaved, + trackUploads: function trackUploads(read) { + uploadsState = read; + }, + }; }; })((window.SubmitForm = window.SubmitForm || {})); diff --git a/assets/js/submit/handoff.js b/assets/js/submit/handoff.js index 6188a73..9528e7a 100644 --- a/assets/js/submit/handoff.js +++ b/assets/js/submit/handoff.js @@ -17,6 +17,13 @@ * assets/js/configurator/issue-form-ids.js, the same helper the generator * names the form's elements with. * + * Files attached through assets/js/submit/uploads.js (a signed-in submitter, + * `submit.uploads_url` set) are written into the two bodies — never into the + * prefilled URL, which is the route for submitters who are not signed in — + * as `upload:/. | ` lines in a screenshots + * question and a bare `upload:…` reference in a `file` question, where they + * replace any typed link. The scaffolder resolves them (scripts/lib/upload_refs.mjs). + * * Exposes: window.SubmitForm.issueUrl, .markdownBody, .issueFormBody, .yamlFrontMatter, * .mailtoUrl, .unprefillable */ @@ -34,6 +41,20 @@ .filter((entry) => entry.text !== ''); } + /** + * A field's answer as the issue body carries it: its attached files first + * (see the header), then what was typed. An attached file replaces a typed + * answer in a `file` question. + * @param {object} field + * @returns {string} '' when there is nothing to say + */ + function bodyText(field) { + const lines = typeof ns.uploadLines === 'function' ? ns.uploadLines(field) : []; + if (lines.length === 0) return ns.serialize(field); + if (field.type === 'file') return lines[0]; + return lines.concat(ns.serialize(field) === '' ? [] : [ns.serialize(field)]).join('\n'); + } + /** * Fields the submitter will have to re-answer on GitHub. * @param {object[]} fields @@ -46,7 +67,9 @@ }; /** - * The prefilled "new issue" URL. + * The prefilled "new issue" URL. Never carries `upload:` lines (see the + * header): with files attached, assets/js/submit.js offers the copy-out box + * instead, which does. * @param {HTMLFormElement} form carries data-repo / data-template / data-title-prefix * @param {object[]} fields * @param {string} title the entry title, used for the issue title @@ -69,7 +92,9 @@ * @returns {string} */ ns.markdownBody = function markdownBody(fields) { - return answered(fields) + return fields + .map((field) => ({ field: field, text: bodyText(field) })) + .filter((entry) => entry.text !== '') .map((entry) => '### ' + entry.field.label + '\n\n' + entry.text + '\n') .join('\n'); }; @@ -87,7 +112,7 @@ */ ns.issueFormBody = function issueFormBody(fields) { return fields - .map((field) => '### ' + field.label + '\n\n' + (ns.serialize(field) || '_No response_')) + .map((field) => '### ' + field.label + '\n\n' + (bodyText(field) || '_No response_')) .join('\n\n'); }; diff --git a/assets/js/submit/import.js b/assets/js/submit/import.js new file mode 100644 index 0000000..b630332 --- /dev/null +++ b/assets/js/submit/import.js @@ -0,0 +1,405 @@ +/** + * Submit form — "Import answers": paste the JSON an AI agent prepared and have + * it fill the form. + * + * The JSON follows submit/answers.schema.json (generated from _data/schema.yml + * by scripts/lib/agent_files.mjs). Nothing here names a field: values are matched + * to the form by `data-field` key, and each is checked against the controls the + * page actually rendered — its radios, boxes and options. Filling goes through + * the form's own `setValue`, and the result through its own `validateAll`, so + * an imported answer is held to exactly what a typed one is. File inputs and + * uploads are never touched: files are not part of the JSON. + * + * DOM contract (submit/index.md, inside the intro, outside the form): + * [data-import] the container + * [data-import-input] the +
+ +
+ +

+ + + + {% if sub_art != '' %}{% endif %} @@ -306,7 +353,8 @@ form says the same (assets/js/configurator/issue-template.js). {%- endcomment -% data-entry-path="{{ schema.entry.path | default: 'catalog' }}" data-draft-key="{{ cfg.name | default: 'catalog' | slugify }}" data-section-count="{{ form_groups.size }}"{% if signin_url != '' %} - data-signin-url="{{ signin_url | escape }}"{% endif %}> + data-signin-url="{{ signin_url | escape }}"{% endif %}{% if uploads_url != '' %} + data-uploads-url="{{ uploads_url | escape }}"{% endif %}> @@ -648,5 +696,11 @@ form says the same (assets/js/configurator/issue-template.js). {%- endcomment -% {%- if signin_url != '' %} {%- comment -%} Before the front-matter scripts, which run in document order after it: it only defines window.SubmitForm.initDirect, which assets/js/submit.js calls. {%- endcomment %} +{%- if uploads_url != '' %} +{%- comment -%} Also before the front-matter scripts: it only defines window.SubmitForm.initUploads, which assets/js/submit.js calls once sign-in is wired. {%- endcomment %} + + + +{%- endif -%} {%- endif -%} {%- endif -%} diff --git a/test/configurator/issue-form-limits.test.mjs b/test/configurator/issue-form-limits.test.mjs index b83c86a..de98379 100644 --- a/test/configurator/issue-form-limits.test.mjs +++ b/test/configurator/issue-form-limits.test.mjs @@ -244,7 +244,14 @@ test('issueFormProblems names each broken rule and where it is', () => { /** A throwaway copy of what generate.mjs reads, sharing this checkout's node_modules. */ function scratchRepository() { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'issue-form-limits-')); - for (const dir of ['_data', '_includes', 'scripts', 'assets/js/configurator', '.github/ISSUE_TEMPLATE']) { + for (const dir of [ + '_data', + '_includes', + 'scripts', + 'submit', + 'assets/js/configurator', + '.github/ISSUE_TEMPLATE', + ]) { fs.cpSync(path.join(ROOT, dir), path.join(root, dir), { recursive: true }); } fs.copyFileSync(path.join(ROOT, '_config.yml'), path.join(root, '_config.yml')); diff --git a/test/fixtures/scaffold-fetch-stub.mjs b/test/fixtures/scaffold-fetch-stub.mjs new file mode 100644 index 0000000..4663f07 --- /dev/null +++ b/test/fixtures/scaffold-fetch-stub.mjs @@ -0,0 +1,93 @@ +/** + * Preloaded (`node --import`) into a real run of scripts/new_entry_from_issue.mjs + * by test/scripts/new_entry_scaffold.test.mjs, so the scaffolder can be run end + * to end without a network: every hostname resolves to a public address, and + * `fetch` answers from the plan named by `FETCH_STUB_PLAN` (JSON): + * + * { + * "log": "", + * "urls": { "": { "contentType": "image/png", "base64": "…" } }, + * "uploads": { "repo": "owner/name", "branches": { "u/1//": { "path": ".png", "base64": "…" } } } + * } + * + * The uploads repository is served the way the GitHub REST API serves a git + * database: ref → commit → tree → blob. Anything not in the plan is a 404. + * + * Without `FETCH_STUB_PLAN` it does nothing (`npm test` runs every .mjs under + * test/ as a test file, this one included). + */ + +import crypto from 'node:crypto'; +import dns from 'node:dns/promises'; +import fs from 'node:fs'; +import process from 'node:process'; + +if (process.env.FETCH_STUB_PLAN) install(); + +function install() { + const plan = JSON.parse(fs.readFileSync(process.env.FETCH_STUB_PLAN, 'utf8')); + + dns.lookup = async () => [{ address: '93.184.216.34', family: 4 }]; + + const objectId = (text) => crypto.createHash('sha1').update(text).digest('hex'); + + /** Fake git objects for every branch in the plan, keyed by their ids. */ + const commits = new Map(); + const trees = new Map(); + const blobs = new Map(); + const refs = new Map(); + for (const [branch, file] of Object.entries(plan.uploads?.branches ?? {})) { + const bytes = Buffer.from(file.base64, 'base64'); + const blob = objectId(`blob ${branch}`); + const tree = objectId(`tree ${branch}`); + const commit = objectId(`commit ${branch}`); + blobs.set(blob, bytes); + trees.set(tree, [{ path: file.path, mode: '100644', type: 'blob', sha: blob, size: bytes.length }]); + commits.set(commit, tree); + refs.set(branch, commit); + } + + const json = (body) => + new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }); + const notFound = () => new Response('{"message":"Not Found"}', { status: 404 }); + + globalThis.fetch = async (input, init = {}) => { + const url = String(input); + const headers = Object.fromEntries( + Object.entries(init.headers ?? {}).map(([k, v]) => [k.toLowerCase(), v]) + ); + fs.appendFileSync( + plan.log, + `${JSON.stringify({ url, method: init.method ?? 'GET', authorization: headers.authorization ?? '' })}\n` + ); + + const api = `https://api.github.com/repos/${plan.uploads?.repo}/git/`; + if (plan.uploads && url.startsWith(api)) { + const route = url.slice(api.length); + let match; + if ((match = /^ref\/heads\/(.+)$/.exec(route)) && refs.has(match[1])) { + return json({ ref: `refs/heads/${match[1]}`, object: { type: 'commit', sha: refs.get(match[1]) } }); + } + if ((match = /^commits\/([0-9a-f]+)$/.exec(route)) && commits.has(match[1])) { + return json({ sha: match[1], tree: { sha: commits.get(match[1]) } }); + } + if ((match = /^trees\/([0-9a-f]+)$/.exec(route)) && trees.has(match[1])) { + return json({ sha: match[1], tree: trees.get(match[1]), truncated: false }); + } + if ((match = /^blobs\/([0-9a-f]+)$/.exec(route)) && blobs.has(match[1])) { + return new Response(blobs.get(match[1]), { + status: 200, + headers: { 'content-type': 'application/octet-stream' }, + }); + } + return notFound(); + } + + const answer = plan.urls?.[url]; + if (!answer) return notFound(); + return new Response(Buffer.from(answer.base64, 'base64'), { + status: 200, + headers: { 'content-type': answer.contentType }, + }); + }; +} diff --git a/test/scripts/agent_files.test.mjs b/test/scripts/agent_files.test.mjs new file mode 100644 index 0000000..f5a8343 --- /dev/null +++ b/test/scripts/agent_files.test.mjs @@ -0,0 +1,287 @@ +/** + * submit/answers.schema.json and submit/agent-instructions.txt + * (scripts/lib/agent_files.mjs), generated from the content model. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import * as jsYaml from 'js-yaml'; + +import { presets } from '../../assets/js/configurator/presets.js'; +import { groupedFormFields } from '../../assets/js/configurator/issue-template.js'; +import { + agentInstructions, + agentInstructionsFile, + answerFields, + answersSchema, + answersSchemaFile, +} from '../../scripts/lib/agent_files.mjs'; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const read = (relative) => fs.readFileSync(path.join(ROOT, relative), 'utf8'); +const shippedSchema = jsYaml.load(read('_data/schema.yml')); +const shippedSite = jsYaml.load(read('_data/site.yml')); + +/** Every configuration the template starts from: the shipped data and each preset. */ +const CONFIGS = [ + { id: 'shipped _data', schema: shippedSchema, site: shippedSite }, + ...presets.map((preset) => ({ + id: `preset ${preset.id}`, + schema: preset.config.schema, + site: preset.config.site, + })), +]; + +/** Drop the front matter block. */ +const body = (file) => file.replace(/^---\n[\s\S]*?\n---\n/, ''); + +const draft = 'https://json-schema.org/draft/2020-12/schema'; + +for (const { id, schema, site } of CONFIGS) { + test(`${id}: one property per form question, in order, with the right types`, () => { + const result = answersSchema(schema, site); + const fields = groupedFormFields(schema).flatMap((section) => section.fields); + assert.deepEqual( + Object.keys(result.properties), + answerFields(schema).map((f) => f.key) + ); + assert.ok( + fields.every((field) => has(result.properties, field.key)), + 'every form field has a property' + ); + assert.ok( + fields.filter((field) => field.form === false).every((field) => !has(result.properties, field.key)), + 'form: false fields are not asked' + ); + for (const field of fields) { + const prop = result.properties[field.key]; + assert.equal(prop.title, field.label, `${field.key} title`); + switch (field.type) { + case 'select': + assert.equal(prop.type, 'string'); + assert.deepEqual(prop.enum, field.options.map(String), `${field.key} enum is verbatim`); + break; + case 'multiselect': + assert.equal(prop.type, 'array'); + assert.equal(prop.uniqueItems, true); + assert.deepEqual(prop.items.enum, field.options.map(String)); + break; + case 'list': + assert.equal(prop.type, 'array'); + assert.equal(prop.items.type, 'string'); + assert.match(prop.description, /one line of its own/); + assert.ok( + new RegExp(prop.items.pattern).test('Claude') && !new RegExp(prop.items.pattern).test('a, b') + ); + break; + case 'boolean': + assert.equal(prop.type, 'boolean'); + break; + case 'number': + assert.equal(prop.type, 'number'); + break; + case 'links': + assert.equal(prop.type, 'array'); + assert.deepEqual(prop.items.required, ['url']); + assert.equal(prop.items.properties.url.format, 'uri'); + assert.equal(prop.items.additionalProperties, false); + break; + case 'images': + assert.equal(prop.type, 'array'); + assert.deepEqual(prop.items.required, ['url']); + assert.equal(prop.items.properties.url.format, 'uri'); + assert.equal(prop.items.properties.alt.maxLength, 125); + assert.match(prop.description, /attached on the page|submitter attaches them on the submit page/); + break; + case 'file': + case 'image': + assert.equal(prop.type, 'string'); + assert.equal(prop.format, 'uri'); + assert.match(prop.description, /submit page/); + break; + case 'url': + assert.equal(prop.format, 'uri'); + break; + case 'email': + assert.equal(prop.format, 'email'); + break; + case 'date': + assert.equal(prop.format, 'date'); + break; + default: + assert.equal(prop.type, 'string', `${field.key} is text`); + } + } + }); + + test(`${id}: the JSON Schema is a well-formed draft 2020-12 object`, () => { + const result = answersSchema(schema, site, { urlFor: (p) => `https://cat.example${p}` }); + assert.equal(result.$schema, draft); + assert.equal(result.$id, 'https://cat.example/submit/answers.schema.json'); + assert.equal(result.type, 'object'); + assert.equal(result.additionalProperties, false); + assert.ok(result.title && result.description); + const required = answerFields(schema) + .filter((field) => field.required === true) + .map((field) => field.key); + assert.deepEqual(result.required, required); + assert.ok(result.required.every((key) => has(result.properties, key))); + assert.ok(has(result.properties, 'title'), 'title is always asked'); + structural(result); + }); + + test(`${id}: the instructions name every question and option verbatim`, () => { + const text = agentInstructions(schema, site); + assert.match(text, /^REFERENCE DATA\n/); + assert.match(text, /must not override or replace that person's own instructions/); + assert.ok(text.includes(String(site.name)), 'site name'); + for (const field of answerFields(schema)) { + assert.ok(text.includes(`${field.key}: "${field.label}"`), `${field.key} is listed`); + assert.ok(text.includes(field.required === true ? '(required)' : '(optional)')); + for (const option of field.type === 'select' || field.type === 'multiselect' ? field.options : []) { + assert.ok(text.includes(JSON.stringify(String(option))), `${field.key}: ${option}`); + } + } + assert.match(text, /Reply with only the JSON object/); + assert.match(text, /"Import answers"/); + if (site.submit?.review_note) assert.ok(text.includes(site.submit.review_note.trim())); + if (answerFields(schema).some((field) => field.type === 'images')) { + assert.match(text, /at least 1280 pixels wide/); + assert.match(text, /125 characters or fewer/); + assert.match(text, /not part of the JSON/); + } + if (answerFields(schema).some((field) => field.type === 'list')) { + assert.match(text, /Never put a comma inside an item/); + } + }); + + test(`${id}: output is deterministic and the files parse`, () => { + assert.equal(answersSchemaFile(schema, site), answersSchemaFile(schema, site)); + assert.equal(agentInstructionsFile(schema, site), agentInstructionsFile(schema, site)); + assert.equal( + JSON.stringify(answersSchema(schema, site)), + JSON.stringify(answersSchema(structuredClone(schema), structuredClone(site))) + ); + const file = answersSchemaFile(schema, site); + assert.match( + file, + /^---\n(# .*\n)?layout: null\nsitemap: false\npermalink: \/submit\/answers\.schema\.json\n---\n/ + ); + const parsed = JSON.parse(body(file)); + assert.equal(parsed.$schema, draft); + assert.ok(file.endsWith('\n')); + assert.ok(agentInstructionsFile(schema, site).endsWith('\n')); + }); +} + +test('the site address comes from Liquid, so it is right wherever the site is built', () => { + const file = answersSchemaFile(shippedSchema, shippedSite); + assert.match(body(file), /"\$id": "\{\{ '\/submit\/answers\.schema\.json' \| absolute_url \}\}"/); + const text = agentInstructionsFile(shippedSchema, shippedSite); + assert.match(text, /Submission page: \{\{ '\/submit\/' \| absolute_url \}\}/); + assert.match(text, /JSON Schema at \{\{ '\/submit\/answers\.schema\.json' \| absolute_url \}\}/); +}); + +test('copy that looks like Liquid cannot break the files', () => { + const schema = { + entry: { singular: 'thing' }, + fields: [ + { + key: 'title', + label: 'Title {{ x }}', + type: 'text', + required: true, + description: 'Use {% raw %} never', + }, + { key: 'kind', label: 'Kind', type: 'select', options: ['{{ a }}', 'b'] }, + ], + }; + for (const file of [ + answersSchemaFile(schema, { name: '{{ site }}' }), + agentInstructionsFile(schema, { name: '{{ site }}' }), + ]) { + // Every `{{` or `{%` left in the file opens one of our own, known tags. + const rest = body(file) + .replace(/\{\{ '[^']*' \| absolute_url \}\}/g, '') + .replace(/\{\{ '(\{\{|\{%)' \}\}/g, ''); + assert.doesNotMatch(rest, /\{\{|\{%/); + } + assert.doesNotThrow(() => JSON.parse(body(answersSchemaFile(schema, {})))); +}); + +test('a schema without a title question still gets one', () => { + const result = answersSchema( + { entry: { singular: 'Thing' }, fields: [{ key: 'x', label: 'X', type: 'text' }] }, + {} + ); + assert.deepEqual(Object.keys(result.properties), ['title', 'x']); + assert.deepEqual(result.required, ['title']); +}); + +test('the committed files equal freshly generated output', () => { + assert.equal(read('submit/answers.schema.json'), answersSchemaFile(shippedSchema, shippedSite)); + assert.equal(read('submit/agent-instructions.txt'), agentInstructionsFile(shippedSchema, shippedSite)); +}); + +test('both files are listed as generator outputs, so updates keep them regenerated', () => { + const generated = jsYaml.load(read('.phct/ownership.yml')); + const text = JSON.stringify(generated); + for (const file of ['submit/answers.schema.json', 'submit/agent-instructions.txt']) { + assert.ok(text.includes(file), `${file} in .phct/ownership.yml`); + const lines = read('.gitattributes') + .split('\n') + .map((line) => line.trim().split(/\s+/)); + assert.ok( + lines.some(([path, attr]) => path === file && attr === 'merge=ours'), + `${file} merge=ours in .gitattributes` + ); + } +}); + +function has(object, key) { + return Object.prototype.hasOwnProperty.call(object, key); +} + +/** Structural checks for the subset of draft 2020-12 these schemas use. */ +function structural(node, where = '#') { + const types = ['string', 'number', 'integer', 'boolean', 'array', 'object', 'null']; + const allowed = new Set([ + '$schema', + '$id', + 'title', + 'description', + 'type', + 'properties', + 'required', + 'additionalProperties', + 'items', + 'enum', + 'uniqueItems', + 'minItems', + 'minLength', + 'maxLength', + 'pattern', + 'format', + ]); + for (const key of Object.keys(node)) assert.ok(allowed.has(key), `${where}: unexpected keyword ${key}`); + if (node.type !== undefined) assert.ok(types.includes(node.type), `${where}: type`); + if (node.enum !== undefined) { + assert.ok(Array.isArray(node.enum) && node.enum.length > 0, `${where}: enum is non-empty`); + assert.equal(new Set(node.enum).size, node.enum.length, `${where}: enum values are unique`); + } + if (node.pattern !== undefined) assert.doesNotThrow(() => new RegExp(node.pattern), `${where}: pattern`); + if (node.type === 'array') assert.ok(node.items && typeof node.items === 'object', `${where}: items`); + if (node.items) structural(node.items, `${where}/items`); + if (node.properties) { + assert.equal(node.type, 'object'); + for (const [key, child] of Object.entries(node.properties)) structural(child, `${where}/${key}`); + } + if (node.required) { + assert.ok(Array.isArray(node.required)); + for (const key of node.required) + assert.ok(has(node.properties || {}, key), `${where}: required ${key} is defined`); + } +} diff --git a/test/scripts/attachments.test.mjs b/test/scripts/attachments.test.mjs index 8fd3207..554f9c4 100644 --- a/test/scripts/attachments.test.mjs +++ b/test/scripts/attachments.test.mjs @@ -10,6 +10,7 @@ import { isGitHubAttachmentUrl, matchesExtension, shouldDownload, + storeAttachment, } from '../../scripts/lib/attachments.mjs'; const PDF = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x37]); @@ -252,3 +253,75 @@ test('attachmentValue keeps the link when a file could not be fetched', () => { assert.equal(attachmentValue('image', { saved: true, publicPath, url }), publicPath); assert.equal(attachmentValue('image', { saved: false, publicPath, url }), ''); }); + +// The upload path (scripts/lib/uploads.mjs) and the URL path share one copy of +// the checks (`storeAttachment`). + +test('bytes from an upload meet exactly the checks bytes from a URL do', async () => { + // [case, filename, body, maxBytes, expected warning] + const cases = [ + ['HTML named as a PDF', 'deck.pdf', HTML, undefined, /does not look like PDF/], + ['a PNG named as a PDF', 'deck.pdf', PNG, undefined, /does not look like PDF/], + ['a PNG named as a JPEG', 'photo.jpg', PNG, undefined, /does not look like JPG/], + ['an extension nobody can verify', 'deck.exe', PDF, undefined, /`deck\.exe` is not one of/], + ['a file over the cap', 'deck.pdf', PDF, PDF.length - 1, /larger than the .* size cap/], + ]; + for (const [name, filename, body, maxBytes, expected] of cases) { + const viaUrl = stubFs(); + const fromUrl = await downloadAttachment('https://example.org/x', { + ...base, + filename, + maxBytes, + fsImpl: viaUrl, + fetchImpl: async () => stubResponse(body), + }); + const viaUpload = stubFs(); + const fromUpload = await storeAttachment('the uploaded file `upload:x`', async () => body, { + destDir: base.destDir, + filename, + maxBytes, + fsImpl: viaUpload, + }); + for (const [result, files] of [ + [fromUrl, viaUrl], + [fromUpload, viaUpload], + ]) { + assert.equal(result.saved, false, name); + assert.match(result.warning, expected, name); + assert.equal(files.written.size, 0, name); + } + assert.match(fromUpload.warning, /^Could not attach the uploaded file `upload:x` \(/, name); + } +}); + +test('storeAttachment writes good bytes under the schema’s name and passes the cap to the loader', async () => { + const fsImpl = stubFs(); + const caps = []; + const result = await storeAttachment( + 'the uploaded file', + async (maxBytes) => { + caps.push(maxBytes); + return PDF; + }, + { destDir: base.destDir, filename: 'deck.pdf', fsImpl } + ); + assert.deepEqual(result, { saved: true, bytes: PDF.length, warning: '' }); + assert.deepEqual(caps, [MAX_ATTACHMENT_BYTES]); + assert.deepEqual([...fsImpl.written.keys()], [path.join('/repo/catalog/brief', 'deck.pdf')]); +}); + +test('storeAttachment never calls the loader for a name it cannot verify, and keeps a loader’s own warning', async () => { + const refused = await storeAttachment('x', () => assert.fail('no load'), { + destDir: base.destDir, + filename: 'deck.docx', + fsImpl: stubFs(), + }); + assert.equal(refused.saved, false); + + const missing = await storeAttachment('x', async () => ({ warning: 'It may have expired.' }), { + destDir: base.destDir, + filename: 'deck.pdf', + fsImpl: stubFs(), + }); + assert.deepEqual(missing, { saved: false, bytes: 0, warning: 'It may have expired.' }); +}); diff --git a/test/scripts/coverage.test.mjs b/test/scripts/coverage.test.mjs index 3bbf32b..79c7170 100644 --- a/test/scripts/coverage.test.mjs +++ b/test/scripts/coverage.test.mjs @@ -6,7 +6,7 @@ import { COVERAGE_GROUPS, coverageArguments, parseCoverageSummary } from '../../ test('coverage groups include explicit security-parser and updater expectations', () => { assert.deepEqual( COVERAGE_GROUPS.map((group) => group.id), - ['full', 'signin-service', 'security-parsers', 'updater-workflows'] + ['full', 'signin-service', 'uploads-service', 'security-parsers', 'updater-workflows'] ); for (const group of COVERAGE_GROUPS) { assert.ok(group.thresholds.lines > 0); diff --git a/test/scripts/images.test.mjs b/test/scripts/images.test.mjs index 013894c..cb58e7d 100644 --- a/test/scripts/images.test.mjs +++ b/test/scripts/images.test.mjs @@ -7,6 +7,7 @@ import { extensionForContentType, ipv6Groups, isBlockedAddress, + saveImages, sniffImageType, } from '../../scripts/lib/images.mjs'; @@ -465,3 +466,154 @@ test('a streamed body under the cap is reassembled byte for byte', async () => { assert.deepEqual(result.warnings, []); assert.deepEqual(new Uint8Array([...files.written.values()][0]), PNG); }); + +// The upload path (scripts/lib/uploads.mjs, through `resolveUpload`) and the +// URL path share one copy of the checks (`saveImages`). + +const SET = 'abcdefghijklmnopqrstuvwxyz'; +const upload = (char, ext) => ({ set: SET, sha256: char.repeat(64), ext }); + +test('bytes from an upload meet exactly the checks bytes from a URL do', async () => { + // [case, URL body + content type, upload body + reference extension, expected warning] + const cases = [ + [ + 'bytes that are not an image', + [TEXT, 'image/png'], + [TEXT, 'png'], + /does not look like a PNG, JPEG, GIF or WebP/, + ], + [ + 'a type that is not an image', + [PNG, 'image/svg+xml'], + [PNG, 'pdf'], + /content type is not PNG, JPEG, GIF or WebP/, + ], + ['an HTML page', [TEXT, 'text/html'], [TEXT, 'pdf'], /content type is not/], + ]; + for (const [name, [urlBytes, contentType], [uploadBytes, ext], expected] of cases) { + const viaUrl = stubFs(); + const fromUrl = await downloadImages([{ url: 'https://e.org/x' }], { + destDir: '/d', + publicPrefix: '/p', + dnsImpl: PUBLIC_DNS, + fsImpl: viaUrl, + fetchImpl: async () => stubResponse(urlBytes, contentType), + }); + const viaUpload = stubFs(); + const fromUpload = await downloadImages([{ upload: upload('a', ext) }], { + destDir: '/d', + publicPrefix: '/p', + fsImpl: viaUpload, + resolveUpload: async () => uploadBytes, + fetchImpl: () => assert.fail('an upload never goes through the URL fetcher'), + }); + for (const [result, files] of [ + [fromUrl, viaUrl], + [fromUpload, viaUpload], + ]) { + assert.deepEqual(result.items, [], name); + assert.equal(result.warnings.length, 1, name); + assert.match(result.warnings[0], expected, name); + assert.equal(files.written.size, 0, name); + } + assert.match(fromUpload.warnings[0], /^Skipped the uploaded file `upload:/, name); + } +}); + +test('an upload is held to the shared size budget, before and after it is read', async () => { + const budgets = []; + const files = stubFs(); + const result = await downloadImages( + [{ url: 'https://e.org/a.png' }, { upload: upload('a', 'png') }, { upload: upload('b', 'png') }], + { + destDir: '/d', + publicPrefix: '/p', + maxTotalBytes: PNG.length * 2 - 1, + dnsImpl: PUBLIC_DNS, + fsImpl: files, + fetchImpl: async () => stubResponse(PNG, 'image/png'), + // Misbehaves: ignores the cap it is given, so the shared total check has to catch it. + resolveUpload: async (_ref, { maxBytes }) => { + budgets.push(maxBytes); + return PNG; + }, + } + ); + assert.deepEqual(budgets, [PNG.length - 1, PNG.length - 1], 'each read is capped at what is left'); + assert.equal(result.items.length, 1); + assert.equal(result.warnings.length, 2); + for (const warning of result.warnings) assert.match(warning, /the images add up to more than/); + assert.equal(files.written.size, 1); +}); + +test('uploads and URLs are numbered in one sequence, and only URLs reach the fetcher', async () => { + const fetched = []; + const resolved = []; + const files = stubFs(); + const result = await downloadImages( + [ + { upload: upload('a', 'png'), alt: 'Landing page' }, + { url: 'https://github.com/user-attachments/assets/b.jpg', alt: 'Queue' }, + { upload: upload('c', 'webp') }, + ], + { + destDir: '/d', + publicPrefix: '/p', + altFallback: 'X — screenshot', + token: 'job-token', + dnsImpl: PUBLIC_DNS, + fsImpl: files, + fetchImpl: async (url, init) => { + fetched.push([url, init.headers.authorization]); + return stubResponse(JPG, 'image/jpeg'); + }, + resolveUpload: async (...args) => { + resolved.push(args); + return args[0].ext === 'png' ? PNG : WEBP; + }, + } + ); + assert.deepEqual(result.items, [ + { src: '/p/01.png', alt: 'Landing page' }, + { src: '/p/02.jpg', alt: 'Queue' }, + { src: '/p/03.webp', alt: 'X — screenshot' }, + ]); + assert.deepEqual(fetched, [['https://github.com/user-attachments/assets/b.jpg', 'Bearer job-token']]); + // The resolver is handed the reference and a cap, and nothing else: no token. + assert.deepEqual( + resolved.map(([ref, opts]) => [ref.sha256[0], Object.keys(opts)]), + [ + ['a', ['maxBytes']], + ['c', ['maxBytes']], + ] + ); +}); + +test('an upload the resolver cannot find is left out with the resolver’s own warning', async () => { + const result = await downloadImages([{ upload: upload('a', 'png') }, { upload: upload('b', 'png') }], { + destDir: '/d', + publicPrefix: '/p', + fsImpl: stubFs(), + resolveUpload: async (ref) => { + if (ref.sha256[0] === 'a') return { warning: 'It may have expired.' }; + throw new Error('the file does not match its upload reference'); + }, + }); + assert.deepEqual(result.items, []); + assert.equal(result.warnings[0], 'It may have expired.'); + assert.match( + result.warnings[1], + /^Could not download the uploaded file `upload:abcdefghijklmnopqrstuvwxyz\/b{64}\.png` \(the file does not match its upload reference\)\. Left it out/ + ); +}); + +test('without a resolver an upload is skipped, not guessed at', async () => { + const result = await downloadImages([{ upload: upload('a', 'png') }], { + destDir: '/d', + publicPrefix: '/p', + fsImpl: stubFs(), + }); + assert.deepEqual(result.items, []); + assert.match(result.warnings[0], /cannot be collected here/); + assert.deepEqual(await saveImages([], { destDir: '/d', publicPrefix: '/p' }), { items: [], warnings: [] }); +}); diff --git a/test/scripts/issue_body.test.mjs b/test/scripts/issue_body.test.mjs index a794523..da74db9 100644 --- a/test/scripts/issue_body.test.mjs +++ b/test/scripts/issue_body.test.mjs @@ -470,3 +470,62 @@ test('coerceChoice strips wrapping quotes, and a long run of quotes stays fast', coerceChoice(field, `a${'"'.repeat(65000)}a`); assert.ok(Date.now() - started < 500, 'quote stripping is linear'); }); + +// `upload:` references: files attached on /submit/ through the optional upload +// service (scripts/lib/upload_refs.mjs). +const UPLOAD_SET = 'abcdefghijklmnopqrstuvwxyz'; +const shaOf = (char) => char.repeat(64); +const uploadLine = (char, ext) => `upload:${UPLOAD_SET}/${shaOf(char)}.${ext}`; +const uploadRef = (char, ext) => ({ set: UPLOAD_SET, sha256: shaOf(char), ext }); + +test('parseImageRefs reads upload: lines with and without alt text, in order, among URLs', () => { + const raw = [ + `${uploadLine('a', 'png')} | Landing page with a Get Started button`, + 'https://example.org/shot2.jpg | The queue view', + `- ${uploadLine('b', 'webp')}`, + `${uploadLine('c', 'jpg')}|Results`, + ].join('\n'); + assert.deepEqual(parseImageRefs(raw), [ + { upload: uploadRef('a', 'png'), alt: 'Landing page with a Get Started button' }, + { url: 'https://example.org/shot2.jpg', alt: 'The queue view' }, + { upload: uploadRef('b', 'webp'), alt: '' }, + { upload: uploadRef('c', 'jpg'), alt: 'Results' }, + ]); +}); + +test('parseImageRefs collapses a repeated reference, keeping the first alt text', () => { + const raw = `${uploadLine('a', 'png')} | First\n${uploadLine('a', 'png')} | Second\n${uploadLine('a', 'png')}`; + assert.deepEqual(parseImageRefs(raw), [{ upload: uploadRef('a', 'png'), alt: 'First' }]); +}); + +test('parseImageRefs drops malformed upload: lines the way it drops malformed URLs', () => { + const malformed = [ + 'upload:', + `upload:${UPLOAD_SET}/${shaOf('a')}`, // no extension + `upload:${UPLOAD_SET}/${shaOf('a')}.gif`, // not an allowed extension + `upload:${UPLOAD_SET}/${'a'.repeat(63)}.png | short`, // truncated hash + `upload:${UPLOAD_SET.toUpperCase()}/${shaOf('a')}.png`, // set id is lower-case base32 + `upload:${UPLOAD_SET}/${shaOf('A')}.png`, // hash is lower-case hex + `upload:../${shaOf('a')}.png`, + `upload:${UPLOAD_SET}/${shaOf('a')}.png/../x.png`, + `see upload:${UPLOAD_SET}/${shaOf('a')}.png`, + 'ftp://example.org/a.png', + ].join('\n'); + assert.deepEqual(parseImageRefs(malformed), []); +}); + +test('parseAttachmentRef reads a bare upload: reference', () => { + assert.deepEqual(parseAttachmentRef(uploadLine('f', 'pdf')), { upload: uploadRef('f', 'pdf'), name: '' }); + assert.deepEqual(parseAttachmentRef(`\n ${uploadLine('f', 'pdf')} \n`), { + upload: uploadRef('f', 'pdf'), + name: '', + }); + // A malformed reference is no attachment at all, like text with no link. + assert.equal(parseAttachmentRef(`upload:${UPLOAD_SET}/${shaOf('f')}.exe`), null); + assert.equal(parseAttachmentRef('upload:nothing'), null); + // A URL answer is unchanged. + assert.deepEqual(parseAttachmentRef('https://example.org/deck.pdf'), { + url: 'https://example.org/deck.pdf', + name: '', + }); +}); diff --git a/test/scripts/new_entry_scaffold.test.mjs b/test/scripts/new_entry_scaffold.test.mjs index 4481cae..a1e2902 100644 --- a/test/scripts/new_entry_scaffold.test.mjs +++ b/test/scripts/new_entry_scaffold.test.mjs @@ -10,15 +10,19 @@ */ import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; import { spawnSync } from 'node:child_process'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import * as yaml from 'js-yaml'; +import { formatUploadRef, uploadBranch, uploadFilename } from '../../scripts/lib/upload_refs.mjs'; +import { notConfiguredWarning, notFoundWarning } from '../../scripts/lib/uploads.mjs'; + const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const SCRIPT = path.join(ROOT, 'scripts', 'new_entry_from_issue.mjs'); const ISSUE_BODY = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'issue-basic.md'), 'utf8'); @@ -312,3 +316,314 @@ test('a file or image question left blank is stored blank, not as a path no file ); assert.equal(dryRun(ROOT, attached)[file.key], `/${schema.entry.path}/${slug}/${file.filename}`); }); + +// --- files attached on /submit/ (`upload:` references) ---------------------- +// The real script, run end to end (not a dry run) in a throwaway checkout, with +// `fetch` and DNS answered by test/fixtures/scaffold-fetch-stub.mjs: GitHub +// attachment URLs, and a private uploads repository served the way the REST API +// serves a git database. Every request is logged, so the tests can see which +// credential went where. + +const STUB = pathToFileURL(path.join(ROOT, 'test', 'fixtures', 'scaffold-fetch-stub.mjs')).href; +const UPLOADS_TOKEN = 'ghs_uploadsTokenThatMustNeverLeak0123456789'; +const JOB_TOKEN = 'ghs_jobToken'; +const UPLOADS_REPO = 'example-org/catalog-uploads'; +const AUTHOR_ID = '4242'; +const UPLOAD_SET = 'abcdefghijklmnopqrstuvwxyz'; +const UPLOADS_ON = { UPLOADS_TOKEN, UPLOADS_REPO, ISSUE_AUTHOR_ID: AUTHOR_ID }; +// What the workflow passes when `vars.UPLOADS_REPO` is unset: the token step is +// skipped, so its output is empty, and the author id is always there. +const UPLOADS_OFF = { UPLOADS_TOKEN: '', UPLOADS_REPO: '', ISSUE_AUTHOR_ID: AUTHOR_ID }; + +const PNG_A = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0a]); +const PNG_URL = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0b]); +const JPG_B = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0x0b]); +const JPG_URL = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0x0c]); +const PDF = Buffer.from('%PDF-1.7 uploaded deck'); +const PDF_URL = Buffer.from('%PDF-1.7 attached deck'); + +const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); +const uploadOf = (bytes, ext) => ({ set: UPLOAD_SET, sha256: sha256(bytes), ext }); +const UP_A = uploadOf(PNG_A, 'png'); +const UP_B = uploadOf(JPG_B, 'jpg'); +const UP_DECK = uploadOf(PDF, 'pdf'); + +/** The uploads repository and the attachment URLs every run can reach. */ +const FETCH_PLAN = { + urls: { + 'https://github.com/user-attachments/assets/aaa.png': { + contentType: 'image/png', + base64: PNG_URL.toString('base64'), + }, + 'https://example.org/shot2.jpg': { contentType: 'image/jpeg', base64: JPG_URL.toString('base64') }, + 'https://github.com/user-attachments/files/12345678/deck.pdf': { + contentType: 'application/pdf', + base64: PDF_URL.toString('base64'), + }, + }, + uploads: { + repo: UPLOADS_REPO, + branches: Object.fromEntries( + [ + [UP_A, PNG_A], + [UP_B, JPG_B], + [UP_DECK, PDF], + ].map(([ref, bytes]) => [ + uploadBranch(AUTHOR_ID, ref), + { path: uploadFilename(ref), base64: bytes.toString('base64') }, + ]) + ), + }, +}; + +/** issue-basic.md with its screenshots and deck attached on /submit/ instead. */ +function uploadBody() { + const screenshots = [ + '![Daily brief](https://github.com/user-attachments/assets/aaa.png)\nhttps://example.org/shot2.jpg | The queue view', + [ + `${formatUploadRef(UP_A)} | Landing page with a Get Started button`, + 'https://github.com/user-attachments/assets/aaa.png | Daily brief', + `${formatUploadRef(UP_B)} | Results table`, + ].join('\n'), + ]; + const deck = [ + '[deck.pdf](https://github.com/user-attachments/files/12345678/deck.pdf)', + formatUploadRef(UP_DECK), + ]; + assert.ok( + ISSUE_BODY.includes(screenshots[0]) && ISSUE_BODY.includes(deck[0]), + 'the fixture still has both answers' + ); + return ISSUE_BODY.replace(screenshots[0], screenshots[1]).replace(deck[0], deck[1]); +} + +/** `$GITHUB_OUTPUT`'s heredoc pairs, as a plain object. */ +function readOutputs(file) { + const outputs = {}; + const text = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : ''; + for (const match of text.matchAll(/^([a-z_]+)<<(GHEOF_\S+)\n([\s\S]*?)\n\2$/gm)) + outputs[match[1]] = match[3]; + return outputs; +} + +/** + * Scaffold for real in a fresh checkout holding this repository's _data. + * @param {string} body issue body + * @param {Record} uploadsEnv + * @param {string} log request log shared across runs (each line one request) + */ +function scaffold( + body, + uploadsEnv, + log = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'scaffold-log-')), 'requests.jsonl') +) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'scaffold-uploads-')); + fs.mkdirSync(path.join(root, '_data')); + for (const name of ['schema.yml', 'governance.yml']) { + fs.copyFileSync(path.join(ROOT, '_data', name), path.join(root, '_data', name)); + } + const planFile = path.join(root, 'fetch-plan.json'); + fs.writeFileSync(planFile, JSON.stringify({ ...FETCH_PLAN, log })); + const outputFile = path.join(root, 'github-output'); + const env = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !/^(GITHUB_TOKEN|GITHUB_OUTPUT|GITHUB_STEP_SUMMARY|UPLOADS_.*|ISSUE_AUTHOR_ID)$/.test(key) + ) + ); + const result = spawnSync(process.execPath, ['--import', STUB, SCRIPT], { + cwd: root, + env: { + ...env, + ...uploadsEnv, + ISSUE_BODY: body, + ISSUE_TITLE: '[Use case] Service request routing', + ISSUE_NUMBER: '7', + GITHUB_TOKEN: JOB_TOKEN, + GITHUB_OUTPUT: outputFile, + FETCH_STUB_PLAN: planFile, + }, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + const outputs = readOutputs(outputFile); + const entryDir = path.join(root, outputs.entry_dir); + const page = fs.readFileSync(path.join(entryDir, 'index.md'), 'utf8'); + const files = Object.fromEntries( + fs + .readdirSync(entryDir, { recursive: true }) + .filter((name) => fs.statSync(path.join(entryDir, name)).isFile()) + .sort() + .map((name) => [name, fs.readFileSync(path.join(entryDir, name))]) + ); + const calls = fs.existsSync(log) + ? fs + .readFileSync(log, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) + : []; + fs.rmSync(root, { recursive: true, force: true }); + return { + ...result, + outputs, + page, + files, + calls, + frontMatter: yaml.load(/^---\n([\s\S]*?)\n---\n/.exec(page)[1]), + }; +} + +const imagesField = () => repoSchema().fields.find((f) => f.type === 'images'); +const fileField = () => repoSchema().fields.find((f) => f.type === 'file'); +const isUploadsApi = (call) => call.url.startsWith(`https://api.github.com/repos/${UPLOADS_REPO}/git/`); + +test('upload: screenshots and deck are saved into the entry exactly as attachments are', () => { + const images = imagesField(); + const file = fileField(); + const attached = scaffold(ISSUE_BODY, UPLOADS_ON); + const uploaded = scaffold(uploadBody(), UPLOADS_ON); + const slug = uploaded.frontMatter.slug; + const shots = `/${repoSchema().entry.path}/${slug}/screenshots`; + + assert.deepEqual(uploaded.frontMatter[images.key], [ + { src: `${shots}/01.png`, alt: 'Landing page with a Get Started button' }, + { src: `${shots}/02.png`, alt: 'Daily brief' }, + { src: `${shots}/03.jpg`, alt: 'Results table' }, + ]); + assert.deepEqual( + Object.keys(uploaded.frontMatter[images.key][0]), + Object.keys(attached.frontMatter[images.key][0]) + ); + // The deck names the same entry path a downloaded GitHub attachment does. + assert.equal(uploaded.frontMatter[file.key], attached.frontMatter[file.key]); + assert.equal(uploaded.frontMatter[file.key], `/${repoSchema().entry.path}/${slug}/${file.filename}`); + + assert.deepEqual(Object.keys(uploaded.files), [ + file.filename, + 'index.md', + 'screenshots/01.png', + 'screenshots/02.png', + 'screenshots/03.jpg', + ]); + // The same layout the attachment run produced: the deck beside the page, screenshots in their folder. + assert.deepEqual(Object.keys(attached.files), [ + file.filename, + 'index.md', + 'screenshots/01.png', + 'screenshots/02.jpg', + ]); + assert.ok(uploaded.files['screenshots/01.png'].equals(PNG_A)); + assert.ok(uploaded.files['screenshots/02.png'].equals(PNG_URL)); + assert.ok(uploaded.files['screenshots/03.jpg'].equals(JPG_B)); + assert.ok(uploaded.files[file.filename].equals(PDF)); + assert.equal(uploaded.outputs.warnings, ''); + assert.match(uploaded.stdout, /Files written: 5 \(1 page, 3 screenshots, 1 attachment\)/); +}); + +test('UPLOADS_TOKEN reaches the uploads API only, never the generic fetchers or any output', () => { + const run = scaffold(uploadBody(), UPLOADS_ON); + const api = run.calls.filter(isUploadsApi); + const generic = run.calls.filter((call) => !isUploadsApi(call)); + assert.equal(api.length, 12, 'ref, commit, tree and blob for each of the three references'); + for (const call of api) assert.equal(call.authorization, `Bearer ${UPLOADS_TOKEN}`, call.url); + // The generic fetchers attach their token to any GitHub host a submitter names. + assert.deepEqual(generic, [ + { + url: 'https://github.com/user-attachments/assets/aaa.png', + method: 'GET', + authorization: `Bearer ${JOB_TOKEN}`, + }, + ]); + for (const text of [run.stdout, run.stderr, run.page, JSON.stringify(run.outputs)]) { + assert.ok(!text.includes(UPLOADS_TOKEN)); + } +}); + +test('an issue edit scaffolds again: every reference is resolved anew, with reads only', () => { + const log = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'scaffold-log-')), 'requests.jsonl'); + const first = scaffold(uploadBody(), UPLOADS_ON, log); + const firstCount = first.calls.filter(isUploadsApi).length; + // The draft branch is rebuilt from main, so the second run starts from an empty folder. + const second = scaffold(uploadBody(), UPLOADS_ON, log); + assert.equal(firstCount, 12); + assert.equal(second.calls.filter(isUploadsApi).length, 24, 'the second run resolved all three again'); + assert.deepEqual(second.files, first.files); + // Nothing is ever written to or deleted from the uploads repository. + assert.deepEqual([...new Set(second.calls.map((call) => call.method))], ['GET']); +}); + +test('uploads not set up: upload: lines become a warning, and everything else still scaffolds', () => { + const images = imagesField(); + const file = fileField(); + const run = scaffold(uploadBody(), UPLOADS_OFF); + const slug = run.frontMatter.slug; + const entryPath = `/${repoSchema().entry.path}/${slug}`; + + assert.equal(run.calls.filter((call) => call.url.startsWith('https://api.github.com/')).length, 0); + assert.deepEqual(run.frontMatter[images.key], [ + { src: `${entryPath}/screenshots/01.png`, alt: 'Daily brief' }, + ]); + assert.equal(run.frontMatter.title, 'Service request routing assistant'); + // The deck gets what a GitHub attachment whose link cannot be kept gets. + assert.equal(run.frontMatter[file.key], `${entryPath}/${file.filename}`); + assert.deepEqual(Object.keys(run.files), ['index.md', 'screenshots/01.png']); + + const warnings = run.outputs.warnings; + assert.ok(warnings.includes(`- ${notConfiguredWarning(images.label, 2)}`), warnings); + assert.ok( + warnings.includes( + `- ${notConfiguredWarning(file.label, 1)} Commit the file as \`${entryPath}/${file.filename}\` in this pull request if it should be included.` + ), + warnings + ); + assert.match( + run.stderr, + /upload: references skipped: uploads are not configured \(UPLOADS_TOKEN, UPLOADS_REPO not set\)/ + ); +}); + +test('an upload that is gone is a not-found warning, and the rest still scaffolds', () => { + const images = imagesField(); + // A reference the uploads repository does not hold under this author. + const gone = { set: UPLOAD_SET, sha256: 'f'.repeat(64), ext: 'png' }; + const body = uploadBody().replace(formatUploadRef(UP_B), formatUploadRef(gone)); + const run = scaffold(body, UPLOADS_ON); + assert.equal(run.frontMatter[images.key].length, 2); + assert.ok(run.outputs.warnings.includes(`- ${notFoundWarning(images.label, gone)}`), run.outputs.warnings); +}); + +test('no upload: lines: the scaffold is identical whether uploads are set up or not', () => { + const off = scaffold(ISSUE_BODY, UPLOADS_OFF); + const on = scaffold(ISSUE_BODY, UPLOADS_ON); + const none = scaffold(ISSUE_BODY, {}); + for (const run of [off, on, none]) { + assert.equal(run.calls.filter((call) => call.url.startsWith('https://api.github.com/')).length, 0); + } + assert.equal(on.page, off.page); + assert.equal(none.page, off.page); + assert.deepEqual(on.files, off.files); + assert.deepEqual(none.outputs, off.outputs); + assert.deepEqual(on.outputs, off.outputs); + assert.equal(on.stdout, off.stdout); + assert.equal(off.stderr, ''); +}); + +test('a dry run lists upload: references and fetches nothing', () => { + const file = fileField(); + const stdout = dryRunOutput(ROOT, uploadBody()); + const fm = dryRun(ROOT, uploadBody()); + assert.equal(fm[imagesField().key].length, 3); + assert.equal(fm[file.key], `/${repoSchema().entry.path}/${fm.slug}/${file.filename}`); + const warnings = stdout.split('# warnings')[1]; + assert.ok(warnings.includes('- Dry run: 1 image URL(s) were parsed but not downloaded.'), warnings); + assert.ok( + warnings.includes( + `- Dry run: \`${formatUploadRef(UP_A)}\`, \`${formatUploadRef(UP_B)}\` parsed but not fetched.` + ), + warnings + ); + assert.ok( + warnings.includes(`- Dry run: \`${formatUploadRef(UP_DECK)}\` was parsed but not fetched.`), + warnings + ); +}); diff --git a/test/scripts/notify_workflows.test.mjs b/test/scripts/notify_workflows.test.mjs index 9f8fe66..0b647f0 100644 --- a/test/scripts/notify_workflows.test.mjs +++ b/test/scripts/notify_workflows.test.mjs @@ -355,8 +355,12 @@ test('create-pull-request only builds branches that are new on every run', () => }); test('an "also deployed by" draft is declined by closing the pull request, not the issue, as the admin guide says', () => { - // Nothing runs when an issue is closed, so closing one moves no status. - for (const name of fs.readdirSync(DIR).filter((file) => file.endsWith('.yml'))) { + // Nothing that moves a status runs when an issue is closed, so closing one + // moves no status. uploads-cleanup.yml does run then, but only deletes the + // files attached on the submit page; it posts and labels nothing. + for (const name of fs + .readdirSync(DIR) + .filter((file) => file.endsWith('.yml') && file !== 'uploads-cleanup.yml')) { const types = parse(name).on?.issues?.types ?? []; assert.ok(!types.includes('closed'), `${name} runs when an issue is closed`); } diff --git a/test/scripts/submit_import.test.mjs b/test/scripts/submit_import.test.mjs new file mode 100644 index 0000000..aa5db69 --- /dev/null +++ b/test/scripts/submit_import.test.mjs @@ -0,0 +1,357 @@ +/** + * "Import answers" on the submit page (assets/js/submit/import.js), driven + * through a real DOM. + * + * The page is the Liquid snapshot submit_form.test.mjs uses, with the + * "Using an AI agent?" block laid over it. That block lives in the page intro, + * which the snapshot leaves out, so it is lifted from submit/index.md itself + * (Liquid links resolved) rather than copied here. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { JSDOM } from 'jsdom'; +import * as jsYaml from 'js-yaml'; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const FIXTURE = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'submit-form.html'), 'utf8'); +const INDEX = fs.readFileSync(path.join(ROOT, 'submit', 'index.md'), 'utf8'); +const SCRIPTS = jsYaml.load(INDEX.split(/^---$/m)[1]).scripts.map((src) => src.replace(/^\//, '')); + +/** The import block as submit/index.md writes it, with its Liquid links resolved. */ +const IMPORT_BLOCK = (() => { + const match = /]*data-import>[\s\S]*?<\/details>/.exec(INDEX); + assert.ok(match, 'submit/index.md has no [data-import] block'); + return match[0].replace(/\{\{\s*'([^']+)'\s*\|\s*relative_url\s*\}\}/g, '$1'); +})(); + +const booted = []; +test.after(() => booted.forEach((dom) => dom.window.close())); + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** + * A booted submit page. + * @param {{draftSeed?: boolean}} [options] + */ +async function boot() { + const dom = new JSDOM('' + IMPORT_BLOCK + FIXTURE + '', { + url: 'https://example.org/submit/', + runScripts: 'outside-only', + }); + booted.push(dom); + const { window } = dom; + window.open = () => null; + await new Promise((resolve) => { + if (window.document.readyState === 'complete') resolve(); + else window.addEventListener('load', resolve); + }); + const { document } = window; + + // A file question as the page renders it when uploads are on: a real file + // input and the picker's list, which import must leave exactly as they are. + const deck = document.querySelector('[data-field="deck_pdf"]'); + deck.insertAdjacentHTML( + 'beforeend', + '
  • deck.pdf
' + ); + const shots = document.querySelector('[data-field="screenshots"]'); + shots.insertAdjacentHTML( + 'beforeend', + '
  • a.png
' + ); + + SCRIPTS.forEach((rel) => window.eval(fs.readFileSync(path.join(ROOT, rel), 'utf8'))); + await flush(); + + const ctx = { window, document, form: document.querySelector('[data-submit-form]') }; + ctx.box = document.querySelector('[data-import]'); + ctx.input = ctx.box.querySelector('[data-import-input]'); + ctx.button = ctx.box.querySelector('[data-import-button]'); + ctx.status = ctx.box.querySelector('[data-import-status]'); + ctx.report = ctx.box.querySelector('[data-import-report]'); + ctx.confirm = ctx.box.querySelector('[data-import-confirm]'); + ctx.summary = ctx.form.querySelector('[data-error-summary]'); + ctx.field = (key) => ctx.form.querySelector('[data-field="' + key + '"]'); + ctx.fields = window.SubmitForm.readFields(ctx.form); + // Through JSON: arrays built inside the jsdom window are a different realm's. + ctx.read = (key) => + JSON.parse(JSON.stringify(window.SubmitForm.readValue(ctx.fields.find((f) => f.key === key)))); + /** The first option of a select/multiselect, as the page offers it. */ + ctx.option = (key, index = 0) => + Array.from(ctx.field(key).querySelectorAll('input[value]:not([data-clear]), select option[value]')) + .map((el) => el.value) + .filter(Boolean)[index]; + ctx.importJson = (value) => { + ctx.input.value = typeof value === 'string' ? value : JSON.stringify(value); + ctx.button.click(); + }; + return ctx; +} + +test('the page ships the import block, hooks and script', () => { + assert.ok(SCRIPTS.includes('assets/js/submit/import.js'), 'import.js is in the page scripts'); + assert.ok( + SCRIPTS.indexOf('assets/js/submit/import.js') < SCRIPTS.indexOf('assets/js/submit.js'), + 'import.js loads before submit.js' + ); + for (const hook of [ + 'data-import-input', + 'data-import-button', + 'data-import-status', + 'data-import-report', + 'data-import-confirm-yes', + 'data-import-confirm-no', + ]) { + assert.ok(IMPORT_BLOCK.includes(hook), `${hook} is in the markup`); + } + assert.match(IMPORT_BLOCK, /]*>Using an AI agent\?<\/summary>/); + assert.match(IMPORT_BLOCK, /href="\/submit\/agent-instructions\.txt"/); + assert.match(IMPORT_BLOCK, /href="\/submit\/answers\.schema\.json"/); + assert.match(IMPORT_BLOCK, /]*for="import-json"/); +}); + +test('valid JSON fills the form and runs the form’s own validation', async () => { + const ctx = await boot(); + const type = ctx.option('solution_type', 1); + const areas = [ctx.option('area', 0), ctx.option('area', 1)]; + ctx.importJson({ + title: 'Hotline triage', + solution_type: type, + area: areas, + summary: 'Sorts calls by urgency.', + ai_tools: ['Claude', 'Copilot'], + repo_url: 'https://example.org/repo', + resources: [{ label: 'Report', url: 'https://example.org/r' }, { url: 'https://www.example.org/x' }], + screenshots: [{ url: 'https://example.org/a.png', alt: 'The triage queue' }], + }); + + assert.equal(ctx.read('title'), 'Hotline triage'); + assert.equal(ctx.read('solution_type'), type); + assert.deepEqual(ctx.read('area'), areas); + assert.deepEqual(ctx.read('ai_tools'), ['Claude', 'Copilot']); + assert.equal(ctx.field('ai_tools').querySelector('textarea').value, 'Claude\nCopilot'); + assert.deepEqual(ctx.read('resources'), [ + { label: 'Report', url: 'https://example.org/r' }, + { label: 'example.org', url: 'https://www.example.org/x' }, + ]); + assert.deepEqual(ctx.read('screenshots'), [{ url: 'https://example.org/a.png', alt: 'The triage queue' }]); + + // validateAll ran: other required questions are still empty, so the form's + // error summary lists them as links to their fields. + assert.equal(ctx.summary.hidden, false); + const links = Array.from(ctx.summary.querySelectorAll('[data-error-summary-list] a')); + assert.ok(links.length > 0, 'the summary lists what is left'); + assert.ok(links.every((a) => a.getAttribute('href').startsWith('#'))); + assert.equal(ctx.field('organization').querySelector('[aria-invalid="true"]') !== null, true); + assert.equal(ctx.field('title').querySelector('[aria-invalid="true"]'), null); + assert.match(ctx.status.textContent, /Filled in 8 answers/); + assert.match(ctx.status.textContent, /still need attention/); +}); + +test('a complete answer set passes validation and closes the summary', async () => { + const ctx = await boot(); + const answers = {}; + for (const field of ctx.fields) { + if (!field.required) continue; + if (field.type === 'select') answers[field.key] = ctx.option(field.key); + else if (field.type === 'multiselect') answers[field.key] = [ctx.option(field.key)]; + else if (field.type === 'boolean') answers[field.key] = true; + else if (field.type === 'email') answers[field.key] = 'a@example.org'; + else if (field.type === 'url') answers[field.key] = 'https://example.org'; + else answers[field.key] = 'Text for ' + field.key; + } + ctx.summary.hidden = false; + ctx.importJson(answers); + assert.equal(ctx.summary.hidden, true); + assert.match(ctx.status.textContent, /passes the form’s checks/); +}); + +test('a syntax error gets a friendly message and changes nothing', async () => { + const ctx = await boot(); + ctx.importJson('{ "title": "Oops",'); + assert.match(ctx.status.textContent, /not valid JSON/); + assert.match(ctx.status.textContent, /only the JSON object/); + assert.equal(ctx.input.getAttribute('aria-invalid'), 'true'); + assert.equal(ctx.read('title'), ''); + assert.equal(ctx.window.document.activeElement, ctx.input); +}); + +test('empty input and non-objects are refused', async () => { + const ctx = await boot(); + ctx.importJson(' '); + assert.match(ctx.status.textContent, /Paste the JSON/); + for (const bad of ['[1,2]', '"text"', '42', 'null']) { + ctx.importJson(bad); + assert.match(ctx.status.textContent, /Expected one JSON object/, bad); + } + assert.equal(ctx.read('title'), ''); +}); + +test('unknown keys are ignored and reported, the rest still imports', async () => { + const ctx = await boot(); + ctx.importJson({ title: 'Known', not_a_question: 'x', also_missing: 1 }); + assert.equal(ctx.read('title'), 'Known'); + assert.match(ctx.report.textContent, /not a question on this form: not_a_question, also_missing/); + assert.equal(ctx.report.hidden, false); + assert.equal(ctx.field('not_a_question'), null); +}); + +test('a value outside the options is reported against its own question and left alone', async () => { + const ctx = await boot(); + ctx.importJson({ + title: 'T', + solution_type: 'Not an option', + area: [ctx.option('area'), 'Imaginary area'], + }); + assert.equal(ctx.read('solution_type'), ''); + assert.match(ctx.report.textContent, /“What are you sharing\?”: “Not an option” is not one of its options/); + // One good multiselect option survives; the bad one is named. + assert.deepEqual(ctx.read('area'), [ctx.option('area')]); + assert.match(ctx.report.textContent, /“Imaginary area”/); + // Option text matches ignoring case, and is stored as the page spells it. + ctx.importJson({ solution_type: ctx.option('solution_type', 2).toUpperCase() }); + assert.equal(ctx.read('solution_type'), ctx.option('solution_type', 2)); +}); + +test('wrong types are rejected without touching the field', async () => { + const ctx = await boot(); + ctx.importJson({ + title: { nested: true }, + ai_tools: 'one', + no_pii_attestation: 'maybe', + resources: 'https://x.org', + }); + assert.equal(ctx.read('title'), ''); + assert.match(ctx.report.textContent, /expects text/); + assert.match(ctx.report.textContent, /true or false/); + assert.match(ctx.report.textContent, /a list of links/); + // A single string is a one-item list, not an error. + assert.deepEqual(ctx.read('ai_tools'), ['one']); +}); + +test('list items go one per line, newlines split and commas are flagged', async () => { + const ctx = await boot(); + ctx.importJson({ ai_tools: ['Claude', 'GPT\nGemini', 'Acme, Inc.'] }); + assert.equal(ctx.field('ai_tools').querySelector('textarea').value, 'Claude\nGPT\nGemini\nAcme, Inc.'); + assert.match(ctx.report.textContent, /“Acme, Inc\.” contains a comma/); +}); + +test('booleans take true or false; null and empty values are skipped', async () => { + const ctx = await boot(); + ctx.importJson({ no_pii_attestation: true, vendor: null, impact: '', area: [] }); + assert.equal(ctx.read('no_pii_attestation'), 'true'); + assert.equal(ctx.read('vendor'), ''); + ctx.importJson({ no_pii_attestation: false }); + ctx.box.querySelector('[data-import-confirm-yes]').click(); // it was ticked, so replacing it asks first + assert.equal(ctx.read('no_pii_attestation'), ''); +}); + +test('an answer the form rejects shows up in the summary (validation is the form’s own)', async () => { + const ctx = await boot(); + ctx.importJson({ repo_url: 'not a url' }); + assert.equal(ctx.read('repo_url'), 'not a url'); + assert.match(ctx.summary.textContent, /web address starting with https/); +}); + +test('answers the submitter already typed ask before being replaced', async () => { + const ctx = await boot(); + const title = ctx.field('title').querySelector('input'); + title.value = 'My own title'; + title.dispatchEvent(new ctx.window.Event('input', { bubbles: true })); + + ctx.importJson({ title: 'Agent title', vendor: 'Acme' }); + assert.equal(ctx.confirm.hidden, false, 'an inline confirmation is shown'); + assert.match(ctx.confirm.textContent, /already answered .*Title|What is it called/); + assert.equal(ctx.read('title'), 'My own title', 'nothing changes while it waits'); + assert.equal(ctx.read('vendor'), '', 'not even the fields that would not clash'); + assert.equal(ctx.window.document.activeElement, ctx.box.querySelector('[data-import-confirm-yes]')); + assert.match(ctx.status.textContent, /replace 1 answer/); + + // Keep mine: nothing changes, focus returns to the Import button. + ctx.box.querySelector('[data-import-confirm-no]').click(); + assert.equal(ctx.confirm.hidden, true); + assert.equal(ctx.read('title'), 'My own title'); + assert.equal(ctx.window.document.activeElement, ctx.button); + + // Replace them: everything lands. + ctx.button.click(); + ctx.box.querySelector('[data-import-confirm-yes]').click(); + assert.equal(ctx.confirm.hidden, true); + assert.equal(ctx.read('title'), 'Agent title'); + assert.equal(ctx.read('vendor'), 'Acme'); +}); + +test('no confirmation when nothing the JSON names was answered', async () => { + const ctx = await boot(); + const vendor = ctx.field('vendor').querySelector('input'); + vendor.value = 'Typed'; + vendor.dispatchEvent(new ctx.window.Event('input', { bubbles: true })); + ctx.importJson({ title: 'Fresh' }); + assert.equal(ctx.confirm.hidden, true); + assert.equal(ctx.read('title'), 'Fresh'); + assert.equal(ctx.read('vendor'), 'Typed', 'questions the JSON does not name are kept'); +}); + +test('file inputs and uploads are never touched; a file answer is reported', async () => { + const ctx = await boot(); + const before = ctx.form.querySelectorAll('[data-uploads]'); + const snapshot = Array.from(before).map((node) => node.outerHTML); + ctx.importJson({ + title: 'T', + deck_pdf: 'https://example.org/deck.pdf', + screenshots: [{ url: 'https://example.org/a.png', alt: 'x' }], + }); + assert.deepEqual( + Array.from(ctx.form.querySelectorAll('[data-uploads]')).map((node) => node.outerHTML), + snapshot + ); + ctx.form.querySelectorAll('input[type=file]').forEach((input) => assert.equal(input.value, '')); + assert.match(ctx.report.textContent, /is a file, so it was skipped/); + // The typed-address box of the images question is separate from the picker. + assert.equal(ctx.read('screenshots').length, 1); +}); + +test('over-long alt text is flagged but kept', async () => { + const ctx = await boot(); + ctx.importJson({ screenshots: [{ url: 'https://example.org/a.png', alt: 'x'.repeat(126) }] }); + assert.match(ctx.report.textContent, /over 125 characters/); + assert.equal(ctx.read('screenshots')[0].alt.length, 126); +}); + +test('the box is keyboard operable and announces itself', async () => { + const ctx = await boot(); + assert.equal(ctx.button.tagName, 'BUTTON'); + assert.equal(ctx.button.type, 'button'); + assert.equal(ctx.input.tagName, 'TEXTAREA'); + assert.equal(ctx.box.tagName, 'DETAILS'); + assert.equal(ctx.box.querySelector('summary').tabIndex, 0); + for (const node of [ctx.button, ctx.input]) assert.notEqual(node.getAttribute('tabindex'), '-1'); + assert.ok( + ctx.input.getAttribute('aria-describedby').split(' ').includes(ctx.status.id), + 'the live region describes the textarea' + ); + assert.equal(ctx.status.getAttribute('role'), 'status'); + assert.equal(ctx.status.getAttribute('aria-live'), 'polite'); + + // The js-only part is revealed by the page's boot, not left hidden. + assert.equal(ctx.input.closest('[data-js-only]').hidden, false); + + assert.equal(ctx.status.textContent, ''); + ctx.importJson({ title: 'Hello' }); + assert.match(ctx.status.textContent, /Filled in 1 answer\./); + ctx.importJson('{'); + assert.match(ctx.status.textContent, /not valid JSON/); +}); + +test('importing saves a draft like typing does', async () => { + const ctx = await boot(); + ctx.importJson({ title: 'Drafted' }); + await new Promise((resolve) => setTimeout(resolve, 700)); + const key = Object.keys(ctx.window.localStorage).find((k) => k.includes('submit-draft')); + assert.ok(key, 'a draft was stored'); + assert.equal(JSON.parse(ctx.window.localStorage.getItem(key)).fields.title, 'Drafted'); +}); diff --git a/test/scripts/submit_uploads.test.mjs b/test/scripts/submit_uploads.test.mjs new file mode 100644 index 0000000..a84ab09 --- /dev/null +++ b/test/scripts/submit_uploads.test.mjs @@ -0,0 +1,1159 @@ +/** + * File uploads on the submit page (assets/js/submit/uploads.js, + * uploads-view.js and uploads-core.js), driven through a real DOM. + * + * Same harness as submit_direct.test.mjs: the Liquid snapshot of /submit/ with + * the markup submit/index.md adds when `submit.signin_url` and + * `submit.uploads_url` are set laid over it (`data-signin-url`, + * `data-uploads-url`), and the shipped scripts in the order the page loads + * them. GitHub's API and the upload service are stand-ins; so are the canvas + * re-encode and the sleep between retries, which JSDOM cannot do (injected + * through `window.SubmitForm.uploadHooks`, the seam uploads.js documents). + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { JSDOM } from 'jsdom'; +import * as jsYaml from 'js-yaml'; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const read = (rel) => fs.readFileSync(path.join(ROOT, rel), 'utf8'); +const FIXTURE = read('test/fixtures/submit-form.html'); +const INDEX = read('submit/index.md'); +const SCRIPTS = jsYaml.load(INDEX.split(/^---$/m)[1]).scripts.map((src) => src.replace(/^\//, '')); +const DIRECT = read('assets/js/submit/direct.js'); +const CORE = read('assets/js/submit/uploads-core.js'); +const VIEW = read('assets/js/submit/uploads-view.js'); +const UPLOADS = read('assets/js/submit/uploads.js'); + +const SIGNIN = 'https://signin.example'; +const UPLOADS_URL = 'https://uploads.example'; +const PAGE = 'https://example.org/submit/'; +const REPO = 'crypticpy/phct'; +const TOKEN = 'ghu_' + 'a'.repeat(36); +const STORE = 'phct:github-signin:v1:' + REPO + '@' + SIGNIN; +const ISSUES_URL = 'https://api.github.com/repos/' + REPO + '/issues'; +const DRAFT_KEY = 'catalog-template:submit-draft:v2:' + /data-draft-key="([^"]+)"/.exec(FIXTURE)[1]; +const SET_RE = /^[a-z2-7]{26}$/; +const REF_RE = /^upload:[a-z2-7]{26}\/[0-9a-f]{64}\.(png|jpg|webp|pdf)$/; + +const NOTICE = + '

Sign in first.

' + + '
  1. Press Submit new issue on GitHub
'; +const HANDOFF_STEP = + '
  • 1. Sending opens a GitHub issue with your answers already filled in. Nothing is submitted until you press Submit new issue there.
    • '; + +/** The fixture as a site with the given switches renders it. */ +function page({ signin = true, uploads = true } = {}) { + assert.ok(FIXTURE.includes(HANDOFF_STEP), 'the fixture no longer has the hand-off step'); + const attrs = + (signin ? ' data-signin-url="' + SIGNIN + '"' : '') + + (uploads ? ' data-uploads-url="' + UPLOADS_URL + '"' : ''); + return ( + NOTICE + + FIXTURE.replace(' data-submit-form=""', ' data-submit-form=""' + attrs).replace( + HANDOFF_STEP, + '
  • 1. Sending opens a GitHub issue.
    • ' + ) + ); +} + +const booted = []; +test.after(() => booted.forEach((dom) => dom.window.close())); + +const flush = () => + new Promise((resolve) => setTimeout(resolve, 0)).then(() => new Promise((r) => setTimeout(r, 0))); +/** Let chains of awaited stand-ins (re-encode, send, sleep) run to the end. */ +async function settle() { + for (let i = 0; i < 8; i += 1) await flush(); +} +const settleDraft = () => new Promise((resolve) => setTimeout(resolve, 700)); + +function reply(status, data, retryAfter = null) { + return { status, retryAfter, data }; +} + +/** A fixed 64-hex digest per upload, so refs are predictable. */ +const digest = (n) => + String(n) + .padStart(64, '0') + .replace(/0/g, (_, i) => (i % 2 ? 'a' : 'b')); + +/** + * A booted submit page. + * @param {object} [options] + * @param {boolean} [options.signin=true] data-signin-url on the form + * @param {boolean} [options.uploads=true] data-uploads-url and the upload scripts + * @param {boolean} [options.signedIn=true] a token in sessionStorage + * @param {object} [options.user] what GET /user answers + * @param {string} [options.draft] raw localStorage draft to start from + * @param {(request: object, ctx: object) => any} [options.send] the upload service + * @param {(file: File) => any} [options.reencode] the canvas + * @param {number} [options.issueStatus=201] what POST /issues answers + */ +async function boot(options = {}) { + const dom = new JSDOM('' + page(options) + '', { + url: PAGE, + runScripts: 'outside-only', + }); + booted.push(dom); + const { window } = dom; + const ctx = { + window, + document: window.document, + requests: [], + uploadsSent: [], + sleeps: [], + reencoded: [], + opened: [], + count: 0, + }; + + const send = async (request) => { + ctx.uploadsSent.push(request); + if (options.send) return options.send(request, ctx); + if (request.method === 'DELETE') return reply(204, null); + ctx.count += 1; + const kind = request.headers['X-Upload-Kind']; + const ext = kind === 'pdf' ? 'pdf' : 'png'; + const sha = digest(ctx.count); + return reply(201, { + ref: 'upload:' + request.headers['X-Upload-Set'] + '/' + sha + '.' + ext, + sha256: sha, + bytes: request.body.size, + ext, + }); + }; + window.SubmitForm = { + uploadHooks: { + send, + sleep: async (ms) => { + ctx.sleeps.push(ms); + }, + reencode: async (file) => { + if (options.reencode) return options.reencode(file); + ctx.reencoded.push(file); + return { blob: new window.Blob(['re-encoded ' + file.name], { type: file.type }), type: file.type }; + }, + }, + }; + + window.open = (url) => { + ctx.opened.push(url); + return {}; + }; + window.fetch = (url, init = {}) => { + ctx.requests.push({ url, init }); + if (url === 'https://api.github.com/user') + return Promise.resolve({ + status: 200, + ok: true, + json: () => Promise.resolve(options.user || { login: 'octo-cat', id: 1001 }), + }); + if (url === ISSUES_URL && init.method === 'POST' && options.issueStatus) + return Promise.resolve({ + status: options.issueStatus, + ok: false, + json: () => Promise.resolve({ message: 'Validation Failed' }), + }); + if (url === ISSUES_URL && init.method === 'POST') + return Promise.resolve({ + status: 201, + ok: true, + json: () => Promise.resolve({ number: 42, html_url: 'https://github.com/' + REPO + '/issues/42' }), + }); + return Promise.reject(new TypeError('unexpected request to ' + url)); + }; + if (options.signedIn !== false) + window.sessionStorage.setItem(STORE, JSON.stringify({ token: TOKEN, expiresAt: Date.now() + 3600000 })); + if (options.draft) window.localStorage.setItem(DRAFT_KEY, options.draft); + + await new Promise((resolve) => { + if (window.document.readyState === 'complete') resolve(); + else window.addEventListener('load', resolve); + }); + ctx.form = window.document.querySelector('[data-submit-form]'); + ctx.rawDraft = () => window.localStorage.getItem(DRAFT_KEY); + ctx.storedDraft = () => JSON.parse(ctx.rawDraft() || 'null'); + ctx.live = () => ctx.form.querySelector('[data-uploads-live]'); + + if (options.signin !== false) window.eval(DIRECT); + if (options.uploads !== false) { + window.eval(CORE); + window.eval(VIEW); + window.eval(UPLOADS); + } + SCRIPTS.forEach((rel) => window.eval(read(rel))); + await settle(); + ctx.fields = window.SubmitForm.readFields(ctx.form); + return ctx; +} + +const wrapOf = (ctx, key) => ctx.form.querySelector('[data-field="' + key + '"]'); +const pickerOf = (ctx, key) => wrapOf(ctx, key).querySelector('input[type="file"]'); +const rowsOf = (ctx, key) => Array.from(wrapOf(ctx, key).querySelectorAll('[data-upload-item]')); + +/** Pick files the way the browser reports them, then let the uploads run. */ +async function pick(ctx, key, files) { + const input = pickerOf(ctx, key); + assert.ok(input, 'no file picker for ' + key); + Object.defineProperty(input, 'files', { value: files, configurable: true }); + input.dispatchEvent(new ctx.window.Event('change', { bubbles: true })); + await settle(); +} + +const file = (ctx, name, type, size = 10) => new ctx.window.File([new Uint8Array(size)], name, { type }); +const png = (ctx, name = 'shot.png') => file(ctx, name, 'image/png'); +const pdf = (ctx, name = 'deck.pdf', size = 10) => file(ctx, name, 'application/pdf', size); + +function describe(ctx, key, index, alt) { + const input = rowsOf(ctx, key)[index].querySelector('[data-upload-alt]'); + input.value = alt; + input.dispatchEvent(new ctx.window.Event('input', { bubbles: true })); +} + +function answer(ctx, key, value) { + // The screenshots question’s typed addresses live in its textarea; its file + // picker is an input too. + const control = wrapOf(ctx, key).querySelector(key === 'screenshots' ? 'textarea' : 'input, textarea'); + control.value = value; + control.dispatchEvent(new ctx.window.Event('input', { bubbles: true })); +} + +function fillRequired(ctx) { + ctx.form.querySelectorAll('[data-required="true"]').forEach((wrap) => { + const type = wrap.dataset.type; + const key = wrap.dataset.field; + if (type === 'select' || type === 'multiselect') { + const input = wrap.querySelector('input[value]:not([data-clear])'); + if (input) { + input.checked = true; + input.dispatchEvent(new ctx.window.Event('change', { bubbles: true })); + } else { + const select = wrap.querySelector('select'); + select.value = Array.from(select.options).find((o) => o.value !== '').value; + select.dispatchEvent(new ctx.window.Event('change', { bubbles: true })); + } + } else if (type === 'boolean') { + const box = wrap.querySelector('input[type="checkbox"]'); + box.checked = true; + box.dispatchEvent(new ctx.window.Event('change', { bubbles: true })); + } else if (type === 'email') answer(ctx, key, 'someone@example.org'); + else if (type === 'url') answer(ctx, key, 'https://example.org'); + else if (type !== 'file' && type !== 'images') answer(ctx, key, 'Value for ' + key); + }); + answer(ctx, 'title', 'Water routing'); +} + +/** Press through the steps; true when the review panel opened. */ +function tryReview(ctx) { + const cap = ctx.form.querySelectorAll('[data-section]').length + 1; + for (let i = 0; i < cap; i += 1) { + ctx.form.dispatchEvent(new ctx.window.Event('submit', { bubbles: true, cancelable: true })); + if (!ctx.form.querySelector('[data-review]').hidden) return true; + } + return false; +} + +const summaryText = (ctx) => ctx.form.querySelector('[data-error-summary]').textContent; + +/** Drive the review panel's send button. */ +async function submitToGitHub(ctx) { + const button = Array.from(ctx.form.querySelectorAll('[data-review] button')).find( + (b) => b.textContent.trim() === 'Submit to GitHub' + ); + assert.ok(button, 'no Submit to GitHub button'); + button.click(); + await settle(); +} + +/* ------------------------------------------------------------- off */ + +test('feature off: no pickers, no script, no data attribute — in the markup and at runtime', async () => { + // The Liquid: the attribute and both scripts exist only inside the guard, + // and the guard needs a sign-in service first (the signin_url pattern). + assert.match( + INDEX, + /\{%- assign uploads_url = '' -%\}\n\{%- if signin_url != '' -%\}\{%- assign uploads_url = cfg\.submit\.uploads_url \| default: '' \| strip -%\}\{%- endif -%\}/ + ); + assert.match( + INDEX, + /\{% if uploads_url != '' %\}\s*data-uploads-url="\{\{ uploads_url \| escape \}\}"\{% endif %\}/ + ); + const scripts = INDEX.split('\n').filter( + (line) => /uploads(-core|-view)?\.js/.test(line) && line.includes(' block.includes(line)), + 'an uploads script sits outside the uploads_url guard' + ); + assert.ok( + !SCRIPTS.some((src) => /uploads/.test(src)), + 'the uploads scripts must not be in the always-on list' + ); + assert.match(read('_data/site.yml'), /^ {2}uploads_url: ""/m, 'uploads must be off by default'); + assert.doesNotMatch(FIXTURE, /uploads/i, 'the off-by-default snapshot mentions uploads'); + + // At runtime: signed in, but no uploads script and no attribute. + const ctx = await boot({ uploads: false }); + assert.equal(ctx.form.dataset.uploadsUrl, undefined); + assert.equal(typeof ctx.window.SubmitForm.initUploads, 'undefined'); + assert.equal(ctx.form.querySelector('input[type="file"]'), null); + assert.equal(ctx.form.querySelector('[data-uploads]'), null); + assert.equal(ctx.form.querySelector('[data-uploads-live]'), null); +}); + +test('data-uploads-url without a sign-in service does nothing (initUploads is only called with direct)', async () => { + const ctx = await boot({ signin: false }); + assert.equal(ctx.form.querySelector('input[type="file"]'), null); +}); + +test('signed out: no pickers at all', async () => { + const ctx = await boot({ signedIn: false }); + assert.equal(ctx.form.querySelector('input[type="file"]'), null); + assert.equal(ctx.form.querySelector('[data-uploads]:not([hidden])'), null); + assert.equal(ctx.uploadsSent.length, 0); +}); + +test('signed in: a picker for every screenshots question and every file question', async () => { + const ctx = await boot(); + const images = ctx.fields.filter((f) => f.type === 'images'); + const files = ctx.fields.filter((f) => f.type === 'file'); + assert.ok(images.length > 0 && files.length > 0); + assert.equal(ctx.form.querySelectorAll('input[type="file"]').length, images.length + files.length); + images.forEach((f) => { + const input = pickerOf(ctx, f.key); + assert.equal(input.accept, 'image/png,image/jpeg,image/webp'); + assert.equal(input.multiple, true); + assert.doesNotMatch(input.accept, /gif/); + }); + files.forEach((f) => { + const input = pickerOf(ctx, f.key); + assert.equal(input.accept, 'application/pdf'); + assert.equal(input.multiple, false); + // Native, labelled, and the page’s "drag it into GitHub" note is hidden. + assert.ok(wrapOf(ctx, f.key).querySelector('label[for="' + input.id + '"]')); + assert.equal(wrapOf(ctx, f.key).querySelector('.field-note:not([data-upload-signed-out])').hidden, true); + }); + assert.match(wrapOf(ctx, 'screenshots').textContent, /A reviewer checks every file/); + assert.match(wrapOf(ctx, 'screenshots').textContent, /do not show personal information/); + assert.match(wrapOf(ctx, 'deck_pdf').textContent, /used instead of any link/); + const live = ctx.live(); + assert.equal(live.getAttribute('aria-live'), 'polite'); + assert.equal(live.getAttribute('role'), 'status'); +}); + +test('signing out hides the pickers; signing back in shows them', async () => { + const ctx = await boot(); + assert.ok(pickerOf(ctx, 'screenshots')); + ctx.document.querySelector('[data-direct-action="signout"]').click(); + assert.equal(ctx.form.querySelector('input[type="file"]'), null); + assert.equal( + wrapOf(ctx, 'deck_pdf').querySelector('.field-note:not([data-upload-signed-out])').hidden, + false + ); +}); + +/* ---------------------------------------------------------- uploading */ + +test('an upload request carries exactly the contract headers, the re-encoded bytes, and never reaches the draft', async () => { + const ctx = await boot(); + await pick(ctx, 'screenshots', [png(ctx, 'My Screenshot é.png')]); + assert.equal(ctx.uploadsSent.length, 1); + const request = ctx.uploadsSent[0]; + assert.equal(request.method, 'POST'); + assert.equal(request.url, UPLOADS_URL + '/api/upload'); + assert.deepEqual(Object.keys(request.headers).sort(), [ + 'Authorization', + 'Content-Type', + 'X-Upload-Kind', + 'X-Upload-Name', + 'X-Upload-Set', + ]); + assert.equal(request.headers.Authorization, 'Bearer ' + TOKEN); + assert.equal(request.headers['Content-Type'], 'application/octet-stream'); + assert.equal(request.headers['X-Upload-Kind'], 'image'); + assert.match(request.headers['X-Upload-Set'], SET_RE); + assert.equal(request.headers['X-Upload-Name'], 'My Screenshot _.png'); + assert.equal(await request.body.text(), 're-encoded My Screenshot é.png'); + assert.equal(ctx.reencoded.length, 1); + + describe(ctx, 'screenshots', 0, 'The queue'); + await settleDraft(); + const raw = ctx.rawDraft(); + assert.doesNotMatch(raw, new RegExp(TOKEN)); + assert.doesNotMatch(raw, /Bearer|Authorization/); + assert.doesNotMatch(ctx.window.localStorage.getItem(STORE) || '', /./); + const stored = ctx.storedDraft().uploads; + assert.equal(stored.set, request.headers['X-Upload-Set']); + assert.deepEqual(stored.user, { id: 1001, login: 'octo-cat' }); + assert.equal(stored.files.screenshots.length, 1); + assert.deepEqual(Object.keys(stored.files.screenshots[0]).sort(), ['alt', 'bytes', 'kind', 'name', 'ref']); + assert.match(stored.files.screenshots[0].ref, REF_RE); + assert.equal(stored.files.screenshots[0].alt, 'The queue'); + assert.equal(stored.files.screenshots[0].kind, 'image'); +}); + +test('a PDF is sent as it is, as kind pdf; a second one waits for the first to be removed', async () => { + const ctx = await boot(); + const deck = pdf(ctx); + await pick(ctx, 'deck_pdf', [deck]); + assert.equal(ctx.reencoded.length, 0, 'a PDF must not be re-encoded'); + const request = ctx.uploadsSent[0]; + assert.equal(request.headers['X-Upload-Kind'], 'pdf'); + assert.equal(request.body, deck); + assert.equal(pickerOf(ctx, 'deck_pdf'), null, 'one deck: the picker steps aside'); + assert.equal(rowsOf(ctx, 'deck_pdf').length, 1); +}); + +test('uploads go one at a time, in order, and share one set', async () => { + let active = 0; + let overlap = 0; + const ctx = await boot({ + send: async (request, c) => { + active += 1; + overlap = Math.max(overlap, active); + await flush(); + active -= 1; + c.count += 1; + const sha = digest(c.count); + return reply(201, { + ref: 'upload:' + request.headers['X-Upload-Set'] + '/' + sha + '.png', + sha256: sha, + bytes: 1, + ext: 'png', + }); + }, + }); + await pick(ctx, 'screenshots', [png(ctx, 'a.png'), png(ctx, 'b.png'), png(ctx, 'c.png')]); + assert.equal(overlap, 1); + const sets = new Set(ctx.uploadsSent.map((r) => r.headers['X-Upload-Set'])); + assert.equal(sets.size, 1); + assert.deepEqual( + ctx.uploadsSent.map((r) => r.headers['X-Upload-Name']), + ['a.png', 'b.png', 'c.png'] + ); + assert.equal(rowsOf(ctx, 'screenshots').length, 3); +}); + +test('the set id is made once per draft and survives a reload', async () => { + const first = await boot(); + await pick(first, 'screenshots', [png(first, 'a.png')]); + describe(first, 'screenshots', 0, 'First'); + await settleDraft(); + const set = first.storedDraft().uploads.set; + assert.match(set, SET_RE); + + const second = await boot({ draft: first.rawDraft() }); + second.document.querySelector('[data-draft-action="restore"]').click(); + await settle(); + assert.equal(rowsOf(second, 'screenshots').length, 1); + await pick(second, 'screenshots', [png(second, 'b.png')]); + assert.equal(second.uploadsSent[0].headers['X-Upload-Set'], set, 'a restored draft kept its set'); +}); + +test('a 401 means the sign-in is over: signed-out state, pickers gone, files kept as failed', async () => { + const ctx = await boot({ + send: async () => reply(401, { error: 'unauthorized', message: 'Token expired' }), + }); + await pick(ctx, 'screenshots', [png(ctx)]); + assert.equal(ctx.window.sessionStorage.getItem(STORE), null, 'the dead token was kept'); + assert.equal(ctx.form.querySelector('input[type="file"]'), null); + assert.ok(ctx.document.querySelector('[data-direct-action="signin"]'), 'the sign-in button is back'); + assert.match(ctx.document.querySelector('[data-signin-notice]').textContent, /sign-in has expired/); + assert.match(ctx.live().textContent, /did not upload/); + assert.equal(ctx.uploadsSent.length, 1, 'a 401 must not be retried'); +}); + +test('a 429 waits as long as Retry-After says, then retries, up to three times', async () => { + const answers = [ + reply(429, { error: 'rate_limited', message: 'Busy' }, '3'), + reply(429, { error: 'rate_limited', message: 'Busy' }, '7'), + ]; + const ctx = await boot({ + send: async (request, c) => { + const next = answers.shift(); + if (next) return next; + c.count += 1; + const sha = digest(c.count); + return reply(201, { + ref: 'upload:' + request.headers['X-Upload-Set'] + '/' + sha + '.png', + sha256: sha, + bytes: 1, + ext: 'png', + }); + }, + }); + await pick(ctx, 'screenshots', [png(ctx)]); + assert.deepEqual(ctx.sleeps, [3000, 7000]); + assert.equal(ctx.uploadsSent.length, 3); + assert.match(rowsOf(ctx, 'screenshots')[0].textContent, /Uploaded\./); + + const never = await boot({ + send: async () => + reply(429, { error: 'rate_limited', message: 'Too many uploads. Try again in a minute.' }, '1'), + }); + await pick(never, 'screenshots', [png(never)]); + assert.equal(never.uploadsSent.length, 4, 'the first try and three retries'); + assert.deepEqual(never.sleeps, [1000, 1000, 1000]); + const row = rowsOf(never, 'screenshots')[0]; + assert.match(row.textContent, /Not uploaded: Too many uploads\. Try again in a minute\./); + assert.equal(row.querySelector('[data-upload-action="retry"]').hidden, false); +}); + +test('a 429 for a per-user cap (too_many_files, too_many_drafts) is not retried; its message is shown', async () => { + for (const code of ['too_many_files', 'too_many_drafts']) { + const ctx = await boot({ + send: async () => reply(429, { error: code, message: 'Cap reached: ' + code }, '5'), + }); + await pick(ctx, 'screenshots', [png(ctx)]); + assert.equal(ctx.uploadsSent.length, 1, code + ' was retried'); + assert.deepEqual(ctx.sleeps, []); + assert.match(rowsOf(ctx, 'screenshots')[0].textContent, new RegExp('Not uploaded: Cap reached: ' + code)); + } +}); + +test('a refusal shows the service’s message; a network failure offers Try again, which uploads', async () => { + let fail = true; + const ctx = await boot({ + send: async (request, c) => { + if (fail) throw new Error('network'); + c.count += 1; + const sha = digest(c.count); + return reply(201, { + ref: 'upload:' + request.headers['X-Upload-Set'] + '/' + sha + '.png', + sha256: sha, + bytes: 1, + ext: 'png', + }); + }, + }); + await pick(ctx, 'screenshots', [png(ctx)]); + const row = () => rowsOf(ctx, 'screenshots')[0]; + assert.match(row().textContent, /could not reach the upload service/); + fail = false; + row().querySelector('[data-upload-action="retry"]').click(); + await settle(); + assert.match(row().textContent, /Uploaded\./); + assert.equal(ctx.reencoded.length, 1, 'a retry must not re-encode'); + + const refused = await boot({ + send: async () => reply(413, { error: 'too_large', message: 'That file is over 4 MB.' }), + }); + await pick(refused, 'screenshots', [png(refused)]); + assert.match(rowsOf(refused, 'screenshots')[0].textContent, /That file is over 4 MB\./); +}); + +test('remove: DELETE with the ref, then the row is gone, focus moves on, the draft forgets it', async () => { + const ctx = await boot(); + await pick(ctx, 'screenshots', [png(ctx, 'a.png'), png(ctx, 'b.png')]); + const ref = rowsOf(ctx, 'screenshots').length && ctx.uploadsSent[0].headers['X-Upload-Set']; + assert.ok(ref); + const first = rowsOf(ctx, 'screenshots')[0]; + const button = first.querySelector('[data-upload-action="remove"]'); + assert.match(button.getAttribute('aria-label'), /a\.png/); + button.click(); + await settle(); + const del = ctx.uploadsSent.find((r) => r.method === 'DELETE'); + assert.ok(del, 'no DELETE was sent'); + assert.match( + del.url, + /^https:\/\/uploads\.example\/api\/upload\?ref=upload%3A[a-z2-7]{26}%2F[0-9a-f]{64}\.png$/ + ); + assert.deepEqual(Object.keys(del.headers), ['Authorization']); + assert.equal(del.headers.Authorization, 'Bearer ' + TOKEN); + assert.equal(rowsOf(ctx, 'screenshots').length, 1); + assert.match(rowsOf(ctx, 'screenshots')[0].textContent, /b\.png/); + assert.equal( + ctx.document.activeElement, + rowsOf(ctx, 'screenshots')[0].querySelector('[data-upload-action="remove"]') + ); + assert.match(ctx.live().textContent, /Removed a\.png/); + + await settleDraft(); + assert.equal(ctx.storedDraft().uploads.files.screenshots.length, 1); + + // Last one out: focus lands on the picker. + rowsOf(ctx, 'screenshots')[0].querySelector('[data-upload-action="remove"]').click(); + await settle(); + assert.equal(rowsOf(ctx, 'screenshots').length, 0); + assert.equal(ctx.document.activeElement, pickerOf(ctx, 'screenshots')); +}); + +test('two quick removes take out the two files pressed, whichever DELETE answers first', async () => { + // b's DELETE is held until a's has come back: the list shifts under it. + let release; + const held = new Promise((resolve) => { + release = resolve; + }); + const ctx = await boot({ + send: async (request, c) => { + if (request.method !== 'DELETE') return defaultCreated(request, c); + if (request.url.includes(digest(2))) await held; + return reply(204, null); + }, + }); + await pick(ctx, 'screenshots', [png(ctx, 'a.png'), png(ctx, 'b.png'), png(ctx, 'c.png')]); + ['A', 'B', 'C'].forEach((alt, i) => describe(ctx, 'screenshots', i, 'Alt ' + alt)); + const set = ctx.uploadsSent[0].headers['X-Upload-Set']; + const refC = 'upload:' + set + '/' + digest(3) + '.png'; + + rowsOf(ctx, 'screenshots')[1].querySelector('[data-upload-action="remove"]').click(); + await settle(); + rowsOf(ctx, 'screenshots')[0].querySelector('[data-upload-action="remove"]').click(); + await settle(); + release(); + await settle(); + + assert.deepEqual( + rowsOf(ctx, 'screenshots').map((row) => /([abc])\.png/.exec(row.textContent)[1]), + ['c'] + ); + const { issueFormBody, markdownBody } = ctx.window.SubmitForm; + for (const body of [issueFormBody(ctx.fields), markdownBody(ctx.fields)]) + assert.deepEqual( + body.split('\n').filter((line) => line.startsWith('upload:')), + [refC + ' | Alt C'] + ); + await settleDraft(); + assert.deepEqual( + ctx.storedDraft().uploads.files.screenshots.map((entry) => [entry.name, entry.ref]), + [['c.png', refC]] + ); +}); + +test('a deck removed brings its picker back; a failed DELETE keeps the row and says so', async () => { + const ctx = await boot({ + send: async (request, c) => + request.method === 'DELETE' + ? reply(500, { error: 'x', message: 'GitHub is down.' }) + : defaultCreated(request, c), + }); + await pick(ctx, 'deck_pdf', [pdf(ctx)]); + rowsOf(ctx, 'deck_pdf')[0].querySelector('[data-upload-action="remove"]').click(); + await settle(); + assert.equal(rowsOf(ctx, 'deck_pdf').length, 1); + assert.match( + wrapOf(ctx, 'deck_pdf').querySelector('[data-upload-message]').textContent, + /could not be removed.*GitHub is down\./ + ); + + const ok = await boot(); + await pick(ok, 'deck_pdf', [pdf(ok)]); + rowsOf(ok, 'deck_pdf')[0].querySelector('[data-upload-action="remove"]').click(); + await settle(); + assert.equal(rowsOf(ok, 'deck_pdf').length, 0); + assert.ok(pickerOf(ok, 'deck_pdf')); + assert.equal(ok.document.activeElement, pickerOf(ok, 'deck_pdf')); +}); + +function defaultCreated(request, c) { + c.count += 1; + const sha = digest(c.count); + const ext = request.headers['X-Upload-Kind'] === 'pdf' ? 'pdf' : 'png'; + return reply(201, { + ref: 'upload:' + request.headers['X-Upload-Set'] + '/' + sha + '.' + ext, + sha256: sha, + bytes: 1, + ext, + }); +} + +/* ------------------------------------------------------- refusals */ + +test('refusals before any upload: a big PDF, a GIF, a non-PDF, too many pictures', async () => { + const ctx = await boot(); + await pick(ctx, 'deck_pdf', [pdf(ctx, 'big.pdf', 4 * 1024 * 1024 + 1)]); + const message = wrapOf(ctx, 'deck_pdf').querySelector('[data-upload-message]'); + assert.equal(message.hidden, false); + assert.match(message.textContent, /big\.pdf.*over the 4 MB limit/); + assert.match(message.textContent, /export it again as PDF, minimum size/); + assert.match(message.textContent, /paste a link/); + assert.equal(ctx.uploadsSent.length, 0); + assert.match(ctx.live().textContent, /big\.pdf/); + + await pick(ctx, 'deck_pdf', [file(ctx, 'notes.txt', 'text/plain')]); + assert.match(message.textContent, /is not a PDF/); + await pick(ctx, 'screenshots', [file(ctx, 'move.gif', 'image/gif')]); + assert.match( + wrapOf(ctx, 'screenshots').querySelector('[data-upload-message]').textContent, + /not a PNG, JPEG or WebP/ + ); + assert.equal(ctx.uploadsSent.length, 0); + + await pick( + ctx, + 'screenshots', + Array.from({ length: 9 }, (_, i) => png(ctx, 'p' + i + '.png')) + ); + assert.equal(rowsOf(ctx, 'screenshots').length, 8); + assert.match( + wrapOf(ctx, 'screenshots').querySelector('[data-upload-message]').textContent, + /Up to 8 screenshots/ + ); +}); + +test('a picture that cannot be shrunk under 4 MB, or read, is reported with a way to remove it', async () => { + const fails = (code) => async () => { + const error = new Error(code); + error.code = code; + throw error; + }; + const ctx = await boot({ reencode: fails('toolarge') }); + await pick(ctx, 'screenshots', [png(ctx)]); + const row = rowsOf(ctx, 'screenshots')[0]; + assert.match(row.textContent, /over 4 MB/); + assert.equal(row.querySelector('[data-upload-action="remove"]').hidden, false); + assert.equal(ctx.uploadsSent.length, 0); + + const unreadable = await boot({ reencode: fails('decode') }); + await pick(unreadable, 'screenshots', [png(unreadable)]); + assert.match(rowsOf(unreadable, 'screenshots')[0].textContent, /could not be read as a picture/); +}); + +/* ----------------------------------------------------------- the body */ + +test('the issue body carries upload lines for screenshots (with alt, then typed URLs) and a bare ref for the deck', async () => { + const ctx = await boot(); + await pick(ctx, 'screenshots', [png(ctx, 'a.png'), png(ctx, 'b.png')]); + await pick(ctx, 'deck_pdf', [pdf(ctx)]); + describe(ctx, 'screenshots', 0, 'The intake queue'); + describe(ctx, 'screenshots', 1, 'Routing result'); + answer(ctx, 'screenshots', 'https://example.org/extra.png | Extra view'); + const set = ctx.uploadsSent[0].headers['X-Upload-Set']; + const refs = ctx.uploadsSent.map( + (r, i) => 'upload:' + set + '/' + digest(i + 1) + (r.headers['X-Upload-Kind'] === 'pdf' ? '.pdf' : '.png') + ); + const { issueFormBody, markdownBody, issueUrl } = ctx.window.SubmitForm; + const expectedShots = + refs[0] + + ' | The intake queue\n' + + refs[1] + + ' | Routing result\nhttps://example.org/extra.png | Extra view'; + + const body = issueFormBody(ctx.fields); + assert.ok(body.includes('### Screenshots\n\n' + expectedShots), body); + assert.ok( + body.includes('### Slide deck or one-pager (PDF)\n\n' + refs[2] + '\n\n') || + body.endsWith('### Slide deck or one-pager (PDF)\n\n' + refs[2]), + body + ); + const markdown = markdownBody(ctx.fields); + assert.ok(markdown.includes('### Screenshots\n\n' + expectedShots + '\n')); + assert.ok(markdown.includes('### Slide deck or one-pager (PDF)\n\n' + refs[2] + '\n')); + for (const line of body.split('\n').filter((l) => l.startsWith('upload:'))) + assert.match(line.split(' | ')[0], REF_RE); + + // The prefilled link is for signed-out submitters: never an upload in it. + const url = issueUrl(ctx.form, ctx.fields, 'Water routing'); + assert.doesNotMatch(decodeURIComponent(url), /upload:/); + assert.match(decodeURIComponent(url), /extra\.png/); +}); + +test('with no uploads the body is exactly what it was', async () => { + const ctx = await boot(); + answer(ctx, 'screenshots', 'https://example.org/a.png | A'); + const plain = await boot({ uploads: false }); + answer(plain, 'screenshots', 'https://example.org/a.png | A'); + assert.equal( + ctx.window.SubmitForm.issueFormBody(ctx.fields), + plain.window.SubmitForm.issueFormBody(plain.fields) + ); + assert.equal( + ctx.window.SubmitForm.markdownBody(ctx.fields), + plain.window.SubmitForm.markdownBody(plain.fields) + ); +}); + +test('signed out, hidden uploads are not written into the body', async () => { + const ctx = await boot(); + await pick(ctx, 'screenshots', [png(ctx)]); + describe(ctx, 'screenshots', 0, 'Alt'); + ctx.document.querySelector('[data-direct-action="signout"]').click(); + assert.doesNotMatch(ctx.window.SubmitForm.issueFormBody(ctx.fields), /upload:/); + assert.doesNotMatch(ctx.window.SubmitForm.markdownBody(ctx.fields), /upload:/); +}); + +/* ------------------------------------------------------ draft restore */ + +/** A draft with one uploaded screenshot, made by `user`. */ +async function draftWithUpload(user = { login: 'octo-cat', id: 1001 }) { + const ctx = await boot({ user }); + await pick(ctx, 'screenshots', [png(ctx, 'kept.png')]); + describe(ctx, 'screenshots', 0, 'Kept alt'); + await pick(ctx, 'deck_pdf', [pdf(ctx, 'kept.pdf')]); + await settleDraft(); + return ctx.rawDraft(); +} + +test('restoring a draft as the same user shows the files again, with their descriptions', async () => { + const draft = await draftWithUpload(); + const ctx = await boot({ draft }); + assert.equal(rowsOf(ctx, 'screenshots').length, 0, 'files must wait for Restore, like the answers'); + ctx.document.querySelector('[data-draft-action="restore"]').click(); + await settle(); + const shots = rowsOf(ctx, 'screenshots'); + assert.equal(shots.length, 1); + assert.match(shots[0].textContent, /kept\.png/); + assert.equal(shots[0].querySelector('[data-upload-alt]').value, 'Kept alt'); + assert.equal(rowsOf(ctx, 'deck_pdf').length, 1); + assert.match( + ctx.window.SubmitForm.issueFormBody(ctx.fields), + /upload:[a-z2-7]{26}\/[0-9a-f]{64}\.png \| Kept alt/ + ); + assert.equal(ctx.uploadsSent.length, 0, 'restoring must not upload anything'); +}); + +test('restoring as a different user drops the files with a notice', async () => { + const draft = await draftWithUpload(); + const ctx = await boot({ draft, user: { login: 'someone-else', id: 2002 } }); + ctx.document.querySelector('[data-draft-action="restore"]').click(); + await settle(); + assert.equal(rowsOf(ctx, 'screenshots').length, 0); + assert.equal(rowsOf(ctx, 'deck_pdf').length, 0); + const note = wrapOf(ctx, 'screenshots').querySelector('[data-upload-message]'); + assert.equal(note.hidden, false); + assert.match(note.textContent, /uploaded by @octo-cat.*signed in as @someone-else.*removed from the draft/); + assert.match(ctx.live().textContent, /removed from the draft/); + assert.doesNotMatch(ctx.window.SubmitForm.issueFormBody(ctx.fields), /upload:/); + await settleDraft(); + assert.equal((ctx.storedDraft() || {}).uploads, undefined, 'the other user’s refs stayed in the draft'); +}); + +test('restoring while signed out keeps the files in the draft, hides the pickers, and says to sign in', async () => { + const draft = await draftWithUpload(); + const ctx = await boot({ draft, signedIn: false }); + ctx.document.querySelector('[data-draft-action="restore"]').click(); + await settle(); + assert.equal(ctx.form.querySelector('input[type="file"]'), null); + const note = wrapOf(ctx, 'screenshots').querySelector('[data-upload-signed-out]'); + assert.equal(note.hidden, false); + assert.match(note.textContent, /Sign in with GitHub again/); + assert.doesNotMatch(ctx.window.SubmitForm.issueFormBody(ctx.fields), /upload:/); + await settleDraft(); + assert.equal(ctx.storedDraft().uploads.files.screenshots.length, 1, 'the files were lost from the draft'); + assert.equal(ctx.storedDraft().uploads.user.id, 1001); + + // Signing in as the same user brings them back. + ctx.window.sessionStorage.setItem(STORE, JSON.stringify({ token: TOKEN, expiresAt: null })); + ctx.window.dispatchEvent( + Object.assign( + new ctx.window.MessageEvent('message', { + data: { type: 'phct-github-token', token: TOKEN }, + origin: SIGNIN, + }) + ) + ); +}); + +test('an old draft, from before uploads, loads exactly as it did', async () => { + const draft = JSON.stringify({ saved: new Date().toISOString(), fields: { title: 'Old draft' } }); + const ctx = await boot({ draft }); + ctx.document.querySelector('[data-draft-action="restore"]').click(); + await settle(); + assert.equal(wrapOf(ctx, 'title').querySelector('input').value, 'Old draft'); + assert.equal(rowsOf(ctx, 'screenshots').length, 0); +}); + +test('a hand-edited uploads record is checked: wrong set, wrong ref, wrong kind are dropped', async () => { + const set = 'a'.repeat(26); + const good = 'upload:' + set + '/' + 'b'.repeat(64) + '.png'; + const draft = JSON.stringify({ + saved: new Date().toISOString(), + fields: {}, + uploads: { + set, + user: { id: 1001, login: 'octo-cat' }, + files: { + screenshots: [ + { ref: good, name: 'ok.png', alt: 'ok', bytes: 5, kind: 'image' }, + { + ref: 'upload:' + 'c'.repeat(26) + '/' + 'b'.repeat(64) + '.png', + name: 'other-set.png', + alt: '', + bytes: 5, + kind: 'image', + }, + { ref: 'https://evil.example/x.png', name: 'url.png', alt: '', bytes: 5, kind: 'image' }, + { + ref: 'upload:' + set + '/' + 'd'.repeat(64) + '.pdf', + name: 'pdf-in-images.pdf', + alt: '', + bytes: 5, + kind: 'pdf', + }, + ], + }, + }, + }); + const ctx = await boot({ draft }); + ctx.document.querySelector('[data-draft-action="restore"]').click(); + await settle(); + assert.deepEqual( + rowsOf(ctx, 'screenshots').map( + (row) => /ok\.png|other-set|url\.png|pdf-in-images/.exec(row.textContent)[0] + ), + ['ok.png'] + ); +}); + +/* -------------------------------------------------- blocking submit */ + +test('Submit is blocked while a file uploads, after one fails, and while a description is missing', async () => { + let release; + const ctx = await boot({ + send: (request, c) => + new Promise((resolve) => { + release = () => resolve(defaultCreated(request, c)); + }), + }); + fillRequired(ctx); + await pick(ctx, 'screenshots', [png(ctx)]); + // Still uploading. + assert.equal(tryReview(ctx), false); + assert.match(summaryText(ctx), /still uploading/); + assert.equal(rowsOf(ctx, 'screenshots')[0].querySelector('progress').hidden, false); + + release(); + await settle(); + // Uploaded, but no description yet. + assert.equal(tryReview(ctx), false); + assert.match(summaryText(ctx), /Describe every screenshot/); + describe(ctx, 'screenshots', 0, 'The queue'); + assert.equal(tryReview(ctx), true, 'a described, uploaded screenshot should let the form through'); + + // A failed upload blocks until it is dealt with. + const failed = await boot({ send: async () => reply(500, { error: 'x', message: 'Broke.' }) }); + fillRequired(failed); + await pick(failed, 'screenshots', [png(failed)]); + assert.equal(tryReview(failed), false); + assert.match(summaryText(failed), /did not upload/); + rowsOf(failed, 'screenshots')[0].querySelector('[data-upload-action="remove"]').click(); + await settle(); + assert.equal(tryReview(failed), true); +}); + +test('a required question is satisfied by an upload alone', async () => { + const ctx = await boot(); + const field = ctx.fields.find((f) => f.key === 'deck_pdf'); + field.required = true; + assert.match(ctx.window.SubmitForm.checkField(field), /Enter an answer/); + await pick(ctx, 'deck_pdf', [pdf(ctx)]); + assert.equal(ctx.window.SubmitForm.checkField(field), ''); +}); + +test('the review reads the files back by name and description', async () => { + const ctx = await boot(); + fillRequired(ctx); + await pick(ctx, 'screenshots', [png(ctx, 'queue.png')]); + describe(ctx, 'screenshots', 0, 'The queue'); + assert.equal(tryReview(ctx), true); + assert.match(ctx.form.querySelector('[data-review]').textContent, /queue\.png — The queue/); +}); + +/* ------------------------------------------------------ sending, copy */ + +test('Submit to GitHub sends the upload lines; the confirmation no longer says to drag the deck in', async () => { + const ctx = await boot(); + fillRequired(ctx); + await pick(ctx, 'screenshots', [png(ctx)]); + describe(ctx, 'screenshots', 0, 'The queue'); + await pick(ctx, 'deck_pdf', [pdf(ctx)]); + assert.equal(tryReview(ctx), true); + await submitToGitHub(ctx); + const post = ctx.requests.find((r) => r.url === ISSUES_URL); + assert.ok(post, 'no issue was created'); + const body = JSON.parse(post.init.body).body; + assert.match(body, /### Screenshots\n\nupload:[a-z2-7]{26}\/[0-9a-f]{64}\.png \| The queue/); + assert.match(body, /### Slide deck or one-pager \(PDF\)\n\nupload:[a-z2-7]{26}\/[0-9a-f]{64}\.pdf/); + const done = ctx.form.querySelector('[data-review]').textContent; + assert.match(done, /Submitted — thank you/); + assert.doesNotMatch(done, /Files cannot be sent from this page/); + assert.match(done, /Your 2 uploaded files went with it/); + assert.equal(ctx.rawDraft(), null, 'the draft survived the send'); +}); + +test('without an upload the confirmation still says to drag the deck into the issue', async () => { + const ctx = await boot(); + fillRequired(ctx); + assert.equal(tryReview(ctx), true); + await submitToGitHub(ctx); + const done = ctx.form.querySelector('[data-review]').textContent; + assert.match(done, /Files cannot be sent from this page.*Slide deck or one-pager \(PDF\)/); + assert.doesNotMatch(done, /uploaded file/); +}); + +test('a failed send with files: GitHub’s form is offered as the copy-out box, which keeps them', async () => { + const ctx = await boot({ issueStatus: 422 }); + fillRequired(ctx); + await pick(ctx, 'screenshots', [png(ctx)]); + describe(ctx, 'screenshots', 0, 'The queue'); + await pick(ctx, 'deck_pdf', [pdf(ctx)]); + assert.equal(tryReview(ctx), true); + await submitToGitHub(ctx); + ctx.form.querySelector('[data-direct-action="fallback"]').click(); + await settle(); + + assert.deepEqual(ctx.opened, [], 'the prefilled link was opened, dropping the files'); + assert.match( + ctx.form.querySelector('[data-submit-status]').textContent, + /The 2 files you uploaded do not travel in a link to GitHub’s form, but the box below carries them/ + ); + assert.equal(ctx.form.querySelector('[data-fallback]').hidden, false); + const box = ctx.form.querySelector('[data-fallback-body]').value; + assert.equal(box, ctx.window.SubmitForm.markdownBody(ctx.fields)); + assert.match(box, /### Screenshots\n\nupload:[a-z2-7]{26}\/[0-9a-f]{64}\.png \| The queue/); + assert.match(box, /### Slide deck or one-pager \(PDF\)\n\nupload:[a-z2-7]{26}\/[0-9a-f]{64}\.pdf/); + const link = ctx.form.querySelector('[data-fallback-link]'); + assert.equal(link.hidden, false); + assert.equal(link.href, ctx.window.SubmitForm.issueUrl(ctx.form, ctx.fields, 'Water routing')); + await settleDraft(); + assert.equal(ctx.storedDraft().uploads.files.screenshots.length, 1, 'a failed send lost the draft'); +}); + +test('a send that finds the sign-in expired says only a fresh sign-in keeps the files', async () => { + const ctx = await boot({ issueStatus: 401 }); + fillRequired(ctx); + await pick(ctx, 'screenshots', [png(ctx)]); + describe(ctx, 'screenshots', 0, 'The queue'); + assert.equal(tryReview(ctx), true); + await submitToGitHub(ctx); + ctx.form.querySelector('[data-direct-action="fallback"]').click(); + await settle(); + + assert.deepEqual(ctx.opened, [], 'GitHub’s form opened without saying the files were left behind'); + assert.match( + ctx.form.querySelector('[data-submit-status]').textContent, + /Your GitHub sign-in has expired, so the file you uploaded is not in the box or the link below\. To keep it, sign in again/ + ); + const link = ctx.form.querySelector('[data-fallback-link]'); + assert.equal(link.hidden, false); + assert.doesNotMatch(ctx.form.querySelector('[data-fallback-body]').value, /upload:/); + await settleDraft(); + assert.equal( + ctx.storedDraft().uploads.files.screenshots.length, + 1, + 'the expired sign-in lost the draft’s files' + ); +}); + +test('a failed send without files: GitHub’s form opens as before, with no files warning', async () => { + const ctx = await boot({ issueStatus: 422 }); + fillRequired(ctx); + assert.equal(tryReview(ctx), true); + await submitToGitHub(ctx); + ctx.form.querySelector('[data-direct-action="fallback"]').click(); + await settle(); + + assert.equal(ctx.opened.length, 1); + assert.match(ctx.opened[0], /^https:\/\/github\.com\/crypticpy\/phct\/issues\/new\?/); + assert.doesNotMatch(ctx.form.textContent, /files? you uploaded/); + assert.equal(ctx.form.querySelector('[data-fallback]').hidden, true); +}); + +/* ------------------------------------------- the core's decision logic */ + +function core() { + const dom = new JSDOM('', { runScripts: 'outside-only' }); + booted.push(dom); + dom.window.eval(CORE); + return dom.window.SubmitForm.uploadsCore; +} + +test('encodeUnderLimit keeps the format at quality 0.9 when that fits', async () => { + const c = core(); + const calls = []; + const result = await c.encodeUnderLimit(async (type, quality) => { + calls.push([type, quality]); + return { size: 1000, type }; + }, 'image/png'); + assert.deepEqual(calls, [['image/png', 0.9]]); + assert.equal(result.type, 'image/png'); +}); + +test('encodeUnderLimit falls back to JPEG at falling quality until it fits', async () => { + const c = core(); + const limit = 4 * 1024 * 1024; + const calls = []; + const sizes = { + 'image/webp@0.9': limit + 1, + 'image/jpeg@0.85': limit + 1, + 'image/jpeg@0.75': limit + 1, + 'image/jpeg@0.65': limit, + }; + const result = await c.encodeUnderLimit(async (type, quality) => { + calls.push(type + '@' + quality); + return { size: sizes[type + '@' + quality] ?? 1, type }; + }, 'image/webp'); + assert.deepEqual(calls, ['image/webp@0.9', 'image/jpeg@0.85', 'image/jpeg@0.75', 'image/jpeg@0.65']); + assert.equal(result.type, 'image/jpeg'); + assert.equal(result.blob.size, limit, 'exactly 4 MiB is allowed'); +}); + +test('encodeUnderLimit gives up (null) when no quality fits, after trying every step', async () => { + const c = core(); + const calls = []; + const result = await c.encodeUnderLimit(async (type, quality) => { + calls.push(quality); + return { size: 10 * 1024 * 1024, type }; + }, 'image/jpeg'); + assert.equal(result, null); + assert.deepEqual(calls, [0.9, ...c.FALLBACK_QUALITIES]); + assert.ok( + c.FALLBACK_QUALITIES.every((q, i, all) => i === 0 || q < all[i - 1]), + 'qualities must decrease' + ); +}); + +test('fitWithin caps the long edge at 2400 px, keeps the ratio, never scales up', () => { + const c = core(); + const fit = (core, w, h) => ({ ...core.fitWithin(w, h) }); + assert.deepEqual(fit(c, 4800, 2400), { width: 2400, height: 1200 }); + assert.deepEqual(fit(c, 1000, 6000), { width: 400, height: 2400 }); + assert.deepEqual(fit(c, 2400, 2400), { width: 2400, height: 2400 }); + assert.deepEqual(fit(c, 800, 600), { width: 800, height: 600 }); + assert.deepEqual(fit(c, 10000, 1), { width: 2400, height: 1 }); +}); + +test('outputType keeps PNG, JPEG and WebP and refuses GIF; kindOf reads type then extension', () => { + const c = core(); + assert.equal(c.outputType('image/png'), 'image/png'); + assert.equal(c.outputType('image/jpeg'), 'image/jpeg'); + assert.equal(c.outputType('image/webp'), 'image/webp'); + assert.equal(c.outputType('image/gif'), ''); + assert.equal(c.kindOf({ name: 'a.PDF', type: '' }), 'pdf'); + assert.equal(c.kindOf({ name: 'a.jpeg', type: '' }), 'image'); + assert.equal(c.kindOf({ name: 'a.gif', type: 'image/gif' }), ''); + assert.equal(c.kindOf({ name: 'a.png', type: 'application/pdf' }), 'pdf'); +}); + +test('newSetId is 26 base32 characters from the secure source, 5 bits per byte', () => { + const c = core(); + const id = c.newSetId({ getRandomValues: (bytes) => bytes.fill(0xff) }); + assert.equal(id, '7'.repeat(26)); + assert.equal(c.newSetId({ getRandomValues: (bytes) => bytes.fill(0) }), 'a'.repeat(26)); + assert.match(c.newSetId(), SET_RE); + assert.notEqual(c.newSetId(), c.newSetId()); + assert.throws(() => c.newSetId({}), /secure random/); +}); + +test('retryDelay honours Retry-After seconds within reason', () => { + const c = core(); + assert.equal(c.retryDelay('3'), 3000); + assert.equal(c.retryDelay(0), 0); + assert.equal(c.retryDelay('1.5'), 1500); + assert.equal(c.retryDelay('9999'), 60000); + for (const bad of [null, undefined, '', 'soon', '-4']) assert.equal(c.retryDelay(bad), 2000, String(bad)); +}); + +test('headerName is printable ASCII, at most 200 characters', () => { + const c = core(); + assert.equal(c.headerName('Résumé.pdf'), 'R_sum_.pdf'); + assert.equal(c.headerName('a'.repeat(300)).length, 200); + assert.equal(c.headerName(''), 'file'); +}); diff --git a/test/scripts/upload_refs.test.mjs b/test/scripts/upload_refs.test.mjs new file mode 100644 index 0000000..2a55d8d --- /dev/null +++ b/test/scripts/upload_refs.test.mjs @@ -0,0 +1,56 @@ +/** + * The `upload:` reference contract (scripts/lib/upload_refs.mjs) that the + * submit page writes, the upload service stores under, and the scaffolder and + * the cleanup workflow resolve. + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + formatUploadRef, + parseUploadRef, + uploadBranch, + uploadFilename, + UPLOAD_EXTENSIONS, +} from '../../scripts/lib/upload_refs.mjs'; + +const SET = 'abcdefghijklmnopqrstuvwxyz'.slice(0, 20) + '234567'; +const SHA = 'a'.repeat(64); + +test('a well-formed reference round-trips', () => { + for (const ext of Object.keys(UPLOAD_EXTENSIONS)) { + const text = `upload:${SET}/${SHA}.${ext}`; + const ref = parseUploadRef(` ${text} `); + assert.deepEqual(ref, { set: SET, sha256: SHA, ext }); + assert.equal(formatUploadRef(ref), text); + } +}); + +test('anything else is not a reference', () => { + for (const text of [ + '', + null, + `upload:${SET}/${SHA}.gif`, + `upload:${SET}/${SHA}.PNG`, + `upload:${SET.toUpperCase()}/${SHA}.png`, + `upload:${SET}/${SHA.slice(1)}.png`, + `upload:${SET}1/${SHA}.png`, + `upload:../${SHA}.png`, + `upload:${SET}/${SHA}.png | alt text`, + `https://example.org/upload:${SET}/${SHA}.png`, + `upload:${SET}/${SHA}.png\nupload:${SET}/${SHA}.png`, + ]) { + assert.equal(parseUploadRef(text), null, JSON.stringify(text)); + } +}); + +test('the branch is built from the user id, never from the reference', () => { + const ref = parseUploadRef(`upload:${SET}/${SHA}.pdf`); + assert.equal(uploadBranch(583231, ref), `u/583231/${SET}/${SHA}`); + assert.equal(uploadBranch('583231', ref), `u/583231/${SET}/${SHA}`); + assert.equal(uploadFilename(ref), `${SHA}.pdf`); + for (const bad of ['', '0', '-1', '12a', 'octocat', '1/2', undefined]) { + assert.throws(() => uploadBranch(bad, ref), /not a GitHub user id/, String(bad)); + } + assert.throws(() => uploadBranch(1, { set: '../x', sha256: SHA }), /not an upload reference/); +}); diff --git a/test/scripts/upload_refs_parity.test.mjs b/test/scripts/upload_refs_parity.test.mjs new file mode 100644 index 0000000..3aa9c43 --- /dev/null +++ b/test/scripts/upload_refs_parity.test.mjs @@ -0,0 +1,99 @@ +/** + * The upload service (services/uploads/) deploys from its own directory, so it + * keeps a copy of the `upload:` reference rules in services/uploads/lib/refs.mjs + * instead of importing scripts/lib/upload_refs.mjs. This test fails the moment + * the two disagree on anything the submit page, the service, the scaffolder or + * the cleanup workflow depend on. + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import * as original from '../../scripts/lib/upload_refs.mjs'; +import * as copy from '../../services/uploads/lib/refs.mjs'; + +const SET = 'abcdefghijklmnopqrstuvwx23'; +const SHA = '0123456789abcdef'.repeat(4); + +test('both copies export the same names', () => { + assert.deepEqual(Object.keys(copy).sort(), Object.keys(original).sort()); +}); + +test('the same extension map and the same regexes', () => { + assert.deepEqual(copy.UPLOAD_EXTENSIONS, original.UPLOAD_EXTENSIONS); + assert.equal(Object.isFrozen(copy.UPLOAD_EXTENSIONS), Object.isFrozen(original.UPLOAD_EXTENSIONS)); + for (const name of ['SET_ID_RE', 'SHA256_RE']) { + assert.equal(copy[name].source, original[name].source, name); + assert.equal(copy[name].flags, original[name].flags, name); + } +}); + +test('the same answers for valid and invalid references', () => { + const inputs = [ + ...Object.keys(original.UPLOAD_EXTENSIONS).map((ext) => `upload:${SET}/${SHA}.${ext}`), + ` upload:${SET}/${SHA}.png\n`, + '', + null, + undefined, + 42, + `upload:${SET}/${SHA}.gif`, + `upload:${SET}/${SHA}.PNG`, + `upload:${SET}/${SHA}.jpeg`, + `upload:${SET.toUpperCase()}/${SHA}.png`, + `upload:${SET.slice(1)}/${SHA}.png`, + `upload:${SET}1/${SHA}.png`, + `upload:${SET}/${SHA.slice(1)}.png`, + `upload:${SET}/${SHA.toUpperCase()}.png`, + `upload:../${SHA}.png`, + `upload:4242/${SET}/${SHA}.png`, + `upload:${SET}/${SHA}.png | alt text`, + `upload:${SET}/${SHA}.png/../x`, + `UPLOAD:${SET}/${SHA}.png`, + `https://example.org/upload:${SET}/${SHA}.png`, + ]; + for (const input of inputs) { + assert.deepEqual(copy.parseUploadRef(input), original.parseUploadRef(input), JSON.stringify(input)); + } + // The table really does hold both kinds. + assert.equal(inputs.filter((input) => original.parseUploadRef(input)).length, 5); +}); + +test('the same reference text, file names and branch names', () => { + for (const ext of Object.keys(original.UPLOAD_EXTENSIONS)) { + const ref = { set: SET, sha256: SHA, ext }; + assert.equal(copy.formatUploadRef(ref), original.formatUploadRef(ref)); + assert.equal(copy.uploadFilename(ref), original.uploadFilename(ref)); + for (const id of [1, 4242, '4242', ' 99 ']) { + assert.equal(copy.uploadBranch(id, ref), original.uploadBranch(id, ref)); + } + } + assert.equal(copy.uploadBranch(4242, { set: SET, sha256: SHA }), `u/4242/${SET}/${SHA}`); +}); + +test('both refuse the same bad user ids and references when naming a branch', () => { + const good = { set: SET, sha256: SHA }; + const cases = [ + [0, good], + [-1, good], + ['01', good], + ['42/../7', good], + ['', good], + [null, good], + [1.5, good], + [4242, { set: SET.toUpperCase(), sha256: SHA }], + [4242, { set: '../..', sha256: SHA }], + [4242, { set: SET, sha256: 'nothex' }], + [4242, { set: SET, sha256: `${SHA}/x` }], + ]; + for (const [id, ref] of cases) { + const outcome = (impl) => { + try { + return `ok ${impl.uploadBranch(id, ref)}`; + } catch (error) { + return `threw ${error.message}`; + } + }; + const expected = outcome(original); + assert.match(expected, /^threw /, JSON.stringify([id, ref])); + assert.equal(outcome(copy), expected, JSON.stringify([id, ref])); + } +}); diff --git a/test/scripts/uploads.test.mjs b/test/scripts/uploads.test.mjs new file mode 100644 index 0000000..ce61978 --- /dev/null +++ b/test/scripts/uploads.test.mjs @@ -0,0 +1,230 @@ +/** + * Reading `upload:` references out of the private uploads repository + * (scripts/lib/uploads.mjs). Every request goes to a stub; no network. + */ + +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import test from 'node:test'; + +import { uploadBranch } from '../../scripts/lib/upload_refs.mjs'; +import { notConfiguredWarning, notFoundWarning, uploadResolver } from '../../scripts/lib/uploads.mjs'; + +const TOKEN = 'ghs_uploadsTokenThatMustNeverLeak0123456789'; +const REPO = 'example-org/catalog-uploads'; +const AUTHOR = '123'; +const ENV = { UPLOADS_TOKEN: TOKEN, UPLOADS_REPO: REPO, ISSUE_AUTHOR_ID: AUTHOR }; +const SET = 'abcdefghijklmnopqrstuvwxyz'; + +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]); +const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); +const REF = { set: SET, sha256: sha256(PNG), ext: 'png' }; + +const COMMIT = 'c'.repeat(40); +const TREE = 'd'.repeat(40); +const BLOB = 'e'.repeat(40); + +function jsonResponse(body, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + headers: { get: () => null }, + json: async () => body, + }; +} + +function bytesResponse(bytes, { stream = false } = {}) { + return { + ok: true, + status: 200, + headers: { get: () => null }, + arrayBuffer: async () => bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), + ...(stream + ? { + body: (async function* chunks() { + for (let i = 0; i < bytes.length; i += 4) yield bytes.slice(i, i + 4); + })(), + } + : {}), + }; +} + +/** + * A stand-in for the GitHub REST API's git database holding one file on one + * branch. `overrides` replaces the answer to a step (`ref`, `commit`, `tree`, + * `blob`) with a status or a response. + */ +function fakeGitHub({ + branch = uploadBranch(AUTHOR, REF), + file = PNG, + entry = {}, + overrides = {}, + stream, +} = {}) { + const calls = []; + const base = `https://api.github.com/repos/${REPO}/git/`; + const answer = (step, make) => { + const override = overrides[step]; + if (typeof override === 'number') return jsonResponse({ message: 'x' }, override); + if (override) return override; + return make(); + }; + const fetchImpl = async (url, init) => { + calls.push({ url, init }); + const route = url.startsWith(base) ? url.slice(base.length) : ''; + if (route === `ref/heads/${branch}` || (route.startsWith('ref/heads/') && overrides.ref)) { + return answer('ref', () => jsonResponse({ object: { type: 'commit', sha: COMMIT } })); + } + if (route === `commits/${COMMIT}`) return answer('commit', () => jsonResponse({ tree: { sha: TREE } })); + if (route === `trees/${TREE}`) { + return answer('tree', () => + jsonResponse({ + tree: [{ path: `${REF.sha256}.png`, type: 'blob', sha: BLOB, size: file.length, ...entry }], + }) + ); + } + if (route === `blobs/${BLOB}`) return answer('blob', () => bytesResponse(file, { stream })); + return jsonResponse({ message: 'Not Found' }, 404); + }; + return { calls, fetchImpl }; +} + +const resolverFor = (github, env = ENV) => { + const resolver = uploadResolver(env, { fetchImpl: github.fetchImpl }); + assert.equal(resolver.ok, true, resolver.problem); + return resolver; +}; + +test('a reference resolves ref → commit → tree → blob, re-hashed, with the documented headers', async () => { + const github = fakeGitHub(); + const bytes = await resolverFor(github).resolve(REF, { maxBytes: 1024, field: 'Screenshots' }); + assert.deepEqual([...bytes], [...PNG]); + + const base = `https://api.github.com/repos/${REPO}/git/`; + assert.deepEqual( + github.calls.map((call) => call.url), + [ + `${base}ref/heads/u/${AUTHOR}/${SET}/${REF.sha256}`, + `${base}commits/${COMMIT}`, + `${base}trees/${TREE}`, + `${base}blobs/${BLOB}`, + ] + ); + for (const [index, { init }] of github.calls.entries()) { + assert.equal(init.method, 'GET'); + assert.equal(init.headers.authorization, `Bearer ${TOKEN}`); + assert.equal(init.headers['x-github-api-version'], '2022-11-28'); + assert.equal( + init.headers.accept, + index === 3 ? 'application/vnd.github.raw+json' : 'application/vnd.github+json' + ); + } +}); + +test('the ref is built from ISSUE_AUTHOR_ID alone, so another user’s upload is never found', async () => { + // The file exists, but under user 999. The issue author is 123, and nothing + // in the reference can name a user, so the request goes to 123's ref. + const github = fakeGitHub({ branch: uploadBranch('999', REF) }); + const result = await resolverFor(github).resolve(REF, { maxBytes: 1024, field: 'Screenshots' }); + assert.deepEqual(result, { warning: notFoundWarning('Screenshots', REF) }); + assert.equal(github.calls.length, 1); + assert.ok(github.calls[0].url.includes(`/ref/heads/u/${AUTHOR}/`), github.calls[0].url); + assert.ok(!github.calls[0].url.includes('/u/999/')); +}); + +test('the size cap is judged from the tree entry before the blob is fetched', async () => { + const github = fakeGitHub({ entry: { size: 5000 } }); + await assert.rejects(resolverFor(github).resolve(REF, { maxBytes: 1024 }), /larger than the .* size cap/); + assert.ok(!github.calls.some((call) => call.url.includes('/blobs/')), 'the blob was never requested'); +}); + +test('the size cap is enforced again while the blob streams', async () => { + // The tree claims a small file; the blob is bigger than it said. + const big = new Uint8Array(64).fill(7); + const github = fakeGitHub({ file: big, entry: { size: 8 }, stream: true }); + await assert.rejects(resolverFor(github).resolve(REF, { maxBytes: 16 }), /larger than the .* size cap/); +}); + +test('bytes that do not hash to the reference are rejected', async () => { + const tampered = PNG.slice(); + tampered[11] = 0xff; + const github = fakeGitHub({ file: tampered }); + await assert.rejects( + resolverFor(github).resolve(REF, { maxBytes: 1024 }), + /does not match its upload reference/ + ); +}); + +test('a tree entry that is not a blob is rejected without fetching it', async () => { + const github = fakeGitHub({ entry: { type: 'tree' } }); + await assert.rejects(resolverFor(github).resolve(REF, { maxBytes: 1024 }), /not a single file/); + assert.ok(!github.calls.some((call) => call.url.includes('/blobs/'))); +}); + +test('a 404 at any step, or a tree without the file, is “not found”, never a throw', async () => { + for (const step of ['ref', 'commit', 'tree', 'blob']) { + const github = fakeGitHub({ overrides: { [step]: 404 } }); + const result = await resolverFor(github).resolve(REF, { maxBytes: 1024, field: 'Deck' }); + assert.deepEqual(result, { warning: notFoundWarning('Deck', REF) }, step); + } + const missing = fakeGitHub({ entry: { path: 'other.png' } }); + const result = await resolverFor(missing).resolve(REF, { maxBytes: 1024, field: 'Deck' }); + assert.deepEqual(result, { warning: notFoundWarning('Deck', REF) }); +}); + +test('any other GitHub error is a throw that names the status', async () => { + const github = fakeGitHub({ overrides: { commit: 500 } }); + await assert.rejects(resolverFor(github).resolve(REF, { maxBytes: 1024 }), /GitHub answered HTTP 500/); +}); + +test('the token goes to api.github.com only and never into an error or a warning', async () => { + const hosts = new Set(); + const leaky = async (url) => { + hosts.add(new URL(url).host); + throw new Error(`connect failed with Bearer ${TOKEN}`); + }; + const resolver = uploadResolver(ENV, { fetchImpl: leaky }); + await assert.rejects(resolver.resolve(REF, { maxBytes: 1024 }), (error) => { + assert.ok(!error.message.includes(TOKEN), error.message); + assert.match(error.message, /\*\*\*/); + return true; + }); + assert.deepEqual([...hosts], ['api.github.com']); + + const github = fakeGitHub({ overrides: { ref: 404 } }); + const { warning } = await resolverFor(github).resolve(REF, { maxBytes: 1024, field: 'Screenshots' }); + assert.ok(!warning.includes(TOKEN)); + assert.ok(github.calls.every((call) => new URL(call.url).origin === 'https://api.github.com')); +}); + +test('missing or malformed settings mean “not set up”, and the problem never carries the token', () => { + for (const env of [ + {}, + { ...ENV, UPLOADS_TOKEN: ' ' }, + { ...ENV, UPLOADS_REPO: '' }, + { ...ENV, ISSUE_AUTHOR_ID: '' }, + { ...ENV, UPLOADS_REPO: 'no-slash' }, + { ...ENV, UPLOADS_REPO: 'owner/../x' }, + { ...ENV, UPLOADS_REPO: 'owner/..' }, + { ...ENV, UPLOADS_REPO: 'https://github.com/owner/name' }, + { ...ENV, ISSUE_AUTHOR_ID: 'octocat' }, + { ...ENV, ISSUE_AUTHOR_ID: '0' }, + ]) { + const resolver = uploadResolver(env, { fetchImpl: () => assert.fail('no request without settings') }); + assert.equal(resolver.ok, false, JSON.stringify(env)); + assert.ok(!resolver.problem.includes(TOKEN)); + } +}); + +test('the warnings speak to the submitter, in the singular and the plural', () => { + assert.equal( + notConfiguredWarning('Slide deck', 1), + 'A file attached on the submit page for “Slide deck” could not be collected, because file uploads are not set up on this catalog. To include it, edit this issue and drag the file into the “Slide deck” answer, in place of the line that starts with `upload:`.' + ); + assert.match( + notConfiguredWarning('Screenshots', 2), + /^2 files attached .* drag the files .* lines that start with/ + ); + assert.match(notFoundWarning('Screenshots', REF), /could not be found — .* may have expired/); + assert.ok(notFoundWarning('Screenshots', REF).includes(`upload:${SET}/${REF.sha256}.png`)); +}); diff --git a/test/scripts/uploads_cleanup.test.mjs b/test/scripts/uploads_cleanup.test.mjs new file mode 100644 index 0000000..02cac76 --- /dev/null +++ b/test/scripts/uploads_cleanup.test.mjs @@ -0,0 +1,346 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { cleanup, issueFromDraftBranch } from '../../scripts/uploads_cleanup.mjs'; + +const SET_B = 'b'.repeat(26); +const SET_C = 'c'.repeat(26); +const SET_D = 'd'.repeat(26); +const H1 = '1'.repeat(64); +const H2 = '2'.repeat(64); +const H3 = '3'.repeat(64); +const NOW = Date.parse('2026-10-16T12:00:00Z'); +const DAY = 86400000; +const TOKEN = 'ghs_SECRETUPLOADS'; +const GH_TOKEN = 'ghs_SECRETCATALOG'; + +const refLine = (set, hash, ext = 'png') => `upload:${set}/${hash}.${ext}`; +const entry = (user, set, hash, ageDays = 0) => ({ + ref: `refs/heads/u/${user}/${set}/${hash}`, + object: { sha: `sha-${user}-${set}-${hash}`.slice(0, 40), ageDays }, +}); + +/** A fake GitHub: records every call, serves the listing, issues and commits. */ +function fake({ refs = [], issues = [], deleteStatus = 204 } = {}) { + const calls = []; + const byBranch = new Map(refs.map((r) => [r.ref.replace('refs/heads/', ''), r])); + const bySha = new Map(refs.map((r) => [r.object.sha, r])); + const slice = (list, url) => { + const page = Number(new URL(url).searchParams.get('page') || 1); + return list.slice((page - 1) * 100, page * 100); + }; + const json = (body, status = 200) => new Response(JSON.stringify(body), { status }); + const fetchImpl = async (url, init = {}) => { + const method = init.method || 'GET'; + calls.push({ method, url, auth: init.headers?.authorization }); + const u = new URL(url); + if (method === 'DELETE') { + const branch = u.pathname.split('/git/refs/heads/')[1]; + assert.ok( + byBranch.has(branch) || deleteStatus !== 204, + `DELETE of a branch that is not in the listing: ${branch}` + ); + return new Response(null, { status: deleteStatus }); + } + // Like GitHub: matching-refs ignores per_page/page and always answers in full. + if (u.pathname.endsWith('/git/matching-refs/heads/u/')) return json(refs); + if (u.pathname.includes('/git/commits/')) { + const r = bySha.get(u.pathname.split('/').pop()); + if (!r) return json({}, 404); + return json({ committer: { date: new Date(NOW - r.object.ageDays * DAY).toISOString() } }); + } + if (u.pathname.endsWith('/issues')) return json(slice(issues, url)); + const one = /\/issues\/(\d+)$/.exec(u.pathname); + if (one) { + const found = issues.find((i) => String(i.number) === one[1]); + return found ? json(found) : json({}, 404); + } + return json({}, 404); + }; + return { + fetchImpl, + calls, + deletes: () => calls.filter((c) => c.method === 'DELETE').map((c) => c.url.split('/git/refs/heads/')[1]), + }; +} + +const env = (extra) => ({ + UPLOADS_TOKEN: TOKEN, + GITHUB_TOKEN: GH_TOKEN, + UPLOADS_REPO: 'org/uploads', + CATALOG_REPO: 'org/catalog', + ...extra, +}); + +const run = (gh, e, out = []) => + cleanup({ env: env(e), fetch: gh.fetchImpl, now: () => NOW, log: (l) => out.push(l) }); + +test("closed: deletes only the author's referenced sets, never another author's identical set id", async () => { + const gh = fake({ + refs: [entry(7, SET_B, H1), entry(7, SET_B, H2), entry(7, SET_C, H1), entry(8, SET_B, H1)], + }); + const body = `### Screenshots\n\n${refLine(SET_B, H1)} | alt text\n${refLine(SET_B, H2, 'pdf')}\n`; + const result = await run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7', ISSUE_BODY: body }); + assert.deepEqual(gh.deletes().sort(), [`u/7/${SET_B}/${H1}`, `u/7/${SET_B}/${H2}`].sort()); + assert.equal(result.deleted.length, 2); +}); + +test('closed: issue text naming refs that are not in the listing never produces a DELETE', async () => { + const gh = fake({ refs: [entry(7, SET_B, H1)] }); + const body = [refLine(SET_C, H3), refLine(SET_D, H1), 'u/7/x/y', '../../heads/main'].join('\n'); + await run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7', ISSUE_BODY: body }); + assert.deepEqual(gh.deletes(), []); +}); + +test('closed: a body that does not look like an author id is refused before any call deletes', async () => { + const gh = fake({ refs: [entry(7, SET_B, H1)] }); + await assert.rejects( + run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7/../8', ISSUE_BODY: refLine(SET_B, H1) }), + /user id/ + ); + assert.deepEqual(gh.deletes(), []); +}); + +test('malformed refs in the listing are ignored', async () => { + const gh = fake({ + refs: [ + entry(7, SET_B, H1), + { ref: 'refs/heads/u/7/short/' + H1, object: { sha: 'a' } }, + { ref: `refs/heads/u/abc/${SET_B}/${H1}`, object: { sha: 'b' } }, + { ref: `refs/heads/u/7/${SET_B}/${H1}/extra`, object: { sha: 'c' } }, + { ref: `refs/heads/u/7/${SET_B}/${'a'.repeat(64).toUpperCase()}`, object: { sha: 'd' } }, + { ref: 'refs/heads/main', object: { sha: 'e' } }, + { ref: `refs/tags/u/7/${SET_B}/${H1}`, object: { sha: 'f' } }, + ], + }); + const result = await run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7', ISSUE_BODY: refLine(SET_B, H1) }); + assert.deepEqual(gh.deletes(), [`u/7/${SET_B}/${H1}`]); + assert.equal(result.ignored, 6); +}); + +test('sweep: keeps referenced and young sets, deletes old unreferenced ones', async () => { + const gh = fake({ + refs: [ + entry(7, SET_B, H1, 30), // old, referenced by an open issue of user 7 + entry(7, SET_C, H1, 30), // old, unreferenced -> delete + entry(7, SET_C, H2, 20), // same set, also old + entry(7, SET_D, H1, 3), // young + entry(8, SET_B, H2, 30), // same set id as a referenced one, but another author -> delete + entry(9, SET_D, H3, 30), // old set with one young member is kept + entry(9, SET_D, H1, 1), + ], + issues: [ + { number: 1, user: { id: 7 }, body: `text\n${refLine(SET_B, H1)} | alt\n` }, + { number: 2, user: { id: 9 }, pull_request: {}, body: refLine(SET_C, H1) }, + ], + }); + const result = await run(gh, { MODE: 'sweep' }); + assert.deepEqual( + gh.deletes().sort(), + [`u/7/${SET_C}/${H1}`, `u/7/${SET_C}/${H2}`, `u/8/${SET_B}/${H2}`].sort() + ); + assert.equal(result.kept, 3); +}); + +test('sweep: honours MAX_AGE_DAYS', async () => { + const gh = fake({ refs: [entry(7, SET_C, H1, 5)] }); + await run(gh, { MODE: 'sweep', MAX_AGE_DAYS: '3' }); + assert.equal(gh.deletes().length, 1); +}); + +test('sweep: an unreadable commit keeps the set; a failed issue listing deletes nothing', async () => { + const unreadable = fake({ + refs: [{ ref: `refs/heads/u/7/${SET_C}/${H1}`, object: { sha: 'missing', ageDays: 99 } }], + }); + const orig = unreadable.fetchImpl; + const gh = { + ...unreadable, + fetchImpl: (u, i) => + u.includes('/git/commits/') ? Promise.resolve(new Response('{}', { status: 404 })) : orig(u, i), + }; + await run(gh, { MODE: 'sweep' }); + assert.deepEqual(unreadable.deletes(), []); + + const broken = fake({ refs: [entry(7, SET_C, H1, 99)] }); + const o2 = broken.fetchImpl; + await assert.rejects( + cleanup({ + env: env({ MODE: 'sweep' }), + now: () => NOW, + fetch: (u, i) => + u.includes('/issues') ? Promise.resolve(new Response('{}', { status: 500 })) : o2(u, i), + }), + /failed: 500/ + ); + assert.deepEqual(broken.deletes(), []); +}); + +test('matching-refs is read in one unpaginated call: 250 refs terminate and are all deleted', async () => { + const refs = []; + for (let i = 0; i < 250; i += 1) refs.push(entry(7, SET_B, i.toString(16).padStart(64, '0'), 30)); + refs.push(entry(8, SET_B, H1, 30)); + for (const mode of ['closed', 'sweep']) { + const gh = fake({ refs }); + const result = await run(gh, { MODE: mode, ISSUE_AUTHOR_ID: '7', ISSUE_BODY: refLine(SET_B, H1) }); + const listings = gh.calls.filter((c) => c.url.includes('/git/matching-refs/')); + assert.equal(listings.length, 1, mode); + assert.ok(!/[?&](per_page|page)=/.test(listings[0].url), listings[0].url); + assert.equal(result.listed, 251, mode); + const expected = refs.filter((r) => mode === 'sweep' || r.ref.includes('/u/7/')); + assert.equal(gh.deletes().length, expected.length, mode); + assert.equal(new Set(gh.deletes()).size, expected.length, mode); + } +}); + +test('pagination: lists and issues beyond one page are all considered', async () => { + const refs = []; + for (let i = 0; i < 130; i += 1) refs.push(entry(7, 'e'.repeat(26), i.toString(16).padStart(64, '0'), 30)); + const issues = []; + for (let i = 0; i < 100; i += 1) issues.push({ number: i, user: { id: 1 }, body: '' }); + issues.push({ number: 101, user: { id: 7 }, body: refLine('e'.repeat(26), '0'.repeat(64)) }); + const gh = fake({ refs, issues }); + const result = await run(gh, { MODE: 'sweep' }); + // The only reference sits on page 2 of the issues; the whole set is kept. + assert.equal(result.listed, 130); + assert.deepEqual(gh.deletes(), []); + + const closed = fake({ refs }); + await run(closed, { + MODE: 'closed', + ISSUE_AUTHOR_ID: '7', + ISSUE_BODY: refLine('e'.repeat(26), '0'.repeat(64)), + }); + assert.equal(closed.deletes().length, 130); +}); + +for (const status of [404, 422]) { + test(`a ${status} on delete means already gone, not a failure`, async () => { + const gh = fake({ refs: [entry(7, SET_B, H1)], deleteStatus: status }); + const result = await run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7', ISSUE_BODY: refLine(SET_B, H1) }); + assert.equal(result.gone, 1); + assert.equal(result.failed, 0); + }); +} + +test('another delete error is counted as a failure', async () => { + const gh = fake({ refs: [entry(7, SET_B, H1)], deleteStatus: 403 }); + const result = await run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7', ISSUE_BODY: refLine(SET_B, H1) }); + assert.equal(result.failed, 1); +}); + +test('DRY_RUN makes no DELETE request but reports what would go', async () => { + const gh = fake({ refs: [entry(7, SET_B, H1)] }); + const result = await run(gh, { + MODE: 'closed', + ISSUE_AUTHOR_ID: '7', + ISSUE_BODY: refLine(SET_B, H1), + DRY_RUN: 'true', + }); + assert.equal(gh.calls.filter((c) => c.method === 'DELETE').length, 0); + assert.equal(result.deleted.length, 1); + assert.match(result.lines.join('\n'), /would delete: 1/); +}); + +test('tokens never appear in the output, and each token goes only to its own repository', async () => { + const out = []; + const gh = fake({ refs: [entry(7, SET_C, H1, 30)], issues: [] }); + const result = await run(gh, { MODE: 'sweep' }, out); + const text = [...out, ...result.lines].join('\n'); + assert.ok(!text.includes(TOKEN) && !text.includes(GH_TOKEN)); + for (const c of gh.calls) { + const expected = c.url.includes('/org/catalog/') ? `Bearer ${GH_TOKEN}` : `Bearer ${TOKEN}`; + assert.equal(c.auth, expected, c.url); + } +}); + +test('a missing token or bad mode is refused', async () => { + const gh = fake(); + await assert.rejects( + cleanup({ env: { MODE: 'closed', UPLOADS_REPO: 'o/r' }, fetch: gh.fetchImpl }), + /UPLOADS_TOKEN/ + ); + await assert.rejects(run(gh, { MODE: 'nuke' }), /MODE/); +}); + +test('closed: a set another open issue by the same author still references is kept', async () => { + const gh = fake({ + refs: [entry(7, SET_B, H1), entry(7, SET_C, H1)], + issues: [ + { number: 5, user: { id: 7 }, body: refLine(SET_B, H1) }, + { number: 6, user: { id: 8 }, body: refLine(SET_C, H1) }, // another author: no protection + ], + }); + const body = `${refLine(SET_B, H1)}\n${refLine(SET_C, H1)}`; + const result = await run(gh, { MODE: 'closed', ISSUE_AUTHOR_ID: '7', ISSUE_NUMBER: '9', ISSUE_BODY: body }); + assert.deepEqual(gh.deletes(), [`u/7/${SET_C}/${H1}`]); + assert.equal(result.kept, 1); +}); + +test('issueFromDraftBranch: good, malformed and fork-like branch names', () => { + assert.equal(issueFromDraftBranch('entry/water-routing-41'), 41); + assert.equal(issueFromDraftBranch('entry/a-1'), 1); + for (const bad of [ + 'entry/water', + 'entry/-41', + 'entry/Water-41', + 'entry/a/b-41', + 'feature/water-41', + 'refs/heads/entry/a-41', + 'entry/a-0', + 'entry/a-41 ', + 'entry/a-41\nx', + '', + null, + undefined, + 'entry/../x-41', + 'entry/a-99999999999', + ]) { + assert.equal(issueFromDraftBranch(bad), null, String(bad)); + } +}); + +test('pr-closed: the issue is found from the branch, fetched, and handled as closed', async () => { + const gh = fake({ + refs: [entry(7, SET_B, H1), entry(8, SET_B, H1), entry(7, SET_C, H1)], + issues: [ + { number: 41, user: { id: 7 }, body: refLine(SET_B, H1), state: 'closed' }, + { number: 42, user: { id: 7 }, body: refLine(SET_C, H1) }, + ], + }); + const result = await run(gh, { MODE: 'pr-closed', PR_BRANCH: 'entry/water-41' }); + assert.deepEqual(gh.deletes(), [`u/7/${SET_B}/${H1}`]); + assert.equal(result.mode, 'pr-closed'); + assert.ok(gh.calls.some((c) => c.url.endsWith('/org/catalog/issues/41'))); +}); + +test('pr-closed: a branch that is not a draft branch does nothing, and a PR number is refused', async () => { + const gh = fake({ + refs: [entry(7, SET_B, H1)], + issues: [{ number: 41, user: { id: 7 }, body: refLine(SET_B, H1) }], + }); + const result = await run(gh, { MODE: 'pr-closed', PR_BRANCH: 'main' }); + assert.deepEqual(gh.deletes(), []); + assert.deepEqual(result.deleted, []); + assert.ok(!gh.calls.some((c) => c.url.includes('/issues/'))); + + const pr = fake({ + refs: [entry(7, SET_B, H1)], + issues: [{ number: 41, pull_request: {}, user: { id: 7 }, body: refLine(SET_B, H1) }], + }); + await assert.rejects(run(pr, { MODE: 'pr-closed', PR_BRANCH: 'entry/x-41' }), /pull request/); + assert.deepEqual(pr.deletes(), []); +}); + +test('pr-closed: an issue that is still open keeps its files and says why', async () => { + const gh = fake({ + refs: [entry(7, SET_B, H1)], + issues: [{ number: 41, user: { id: 7 }, body: refLine(SET_B, H1), state: 'open' }], + }); + const out = []; + const result = await run(gh, { MODE: 'pr-closed', PR_BRANCH: 'entry/water-41' }, out); + assert.deepEqual(gh.deletes(), []); + assert.deepEqual(result.deleted, []); + assert.equal(result.failed, 0); + assert.match(out.join('\n'), /#41 is still open/); + assert.deepEqual(result.lines, out); +}); diff --git a/test/scripts/uploads_url.test.mjs b/test/scripts/uploads_url.test.mjs new file mode 100644 index 0000000..95be6a6 --- /dev/null +++ b/test/scripts/uploads_url.test.mjs @@ -0,0 +1,75 @@ +/** + * `submit.uploads_url`, the switch for file uploads on /submit/: the address + * check `npm run validate` applies (scripts/lib/uploads_url.mjs), its + * dependency on `submit.signin_url`, and the key surviving a run of either + * configurator, which has no question for it. + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as jsYaml from 'js-yaml'; + +import { uploadsUrlProblem } from '../../scripts/lib/uploads_url.mjs'; +import { applyAnswers, answersFromConfig } from '../../assets/js/configurator/answers.js'; +import { defaultConfig } from '../../assets/js/configurator/default-config.js'; +import { renderFiles } from '../../assets/js/configurator/render-files.js'; + +const SIGNIN = 'https://catalog-signin.vercel.app'; + +test('blank, missing or off is fine, with or without a sign-in service', () => { + for (const value of [undefined, null, '', ' ', false]) { + assert.equal(uploadsUrlProblem(value, ''), '', String(value)); + assert.equal(uploadsUrlProblem(value, SIGNIN), '', String(value)); + } +}); + +test('the upload service’s https origin is accepted next to a sign-in service', () => { + for (const value of [ + 'https://catalog-uploads.vercel.app', + 'https://catalog-uploads.vercel.app/', + ' https://uploads.example.org:8443 ', + 'http://localhost:3001', + 'http://127.0.0.1:3001', + ]) { + assert.equal(uploadsUrlProblem(value, SIGNIN), '', value); + } +}); + +test('an address without a sign-in service is a problem: only signed-in submitters can upload', () => { + for (const signin of [undefined, null, '', ' ', false]) + assert.match( + uploadsUrlProblem('https://catalog-uploads.vercel.app', signin), + /needs submit\.signin_url: uploads work only for signed-in submitters/, + String(signin) + ); +}); + +test('the same address rules as signin_url apply, worded for the upload service', () => { + const cases = { + 'http://catalog-uploads.vercel.app': /must start with https/, + 'javascript:alert(1)': /must start with https/, + 'catalog-uploads.vercel.app': /not a web address.*upload service's address/, + 'https://catalog-uploads.vercel.app/api/upload': + /no path or query — use "https:\/\/catalog-uploads\.vercel\.app"/, + 'https://catalog-uploads.vercel.app/?x=1': /no path or query/, + 'https://user:pass@catalog-uploads.vercel.app': /no path or query/, + }; + for (const [value, pattern] of Object.entries(cases)) + assert.match(uploadsUrlProblem(value, SIGNIN), pattern, value); + assert.match(uploadsUrlProblem(42, SIGNIN), /in quotes/); + assert.match(uploadsUrlProblem(['https://a.example'], SIGNIN), /in quotes/); + // The address problem is reported before the missing sign-in one. + assert.match(uploadsUrlProblem('http://x.example', ''), /must start with https/); +}); + +test('a hand-set uploads_url survives the configurators, which never ask about it', () => { + const base = defaultConfig(); + base.site.submit = { + ...base.site.submit, + signin_url: SIGNIN, + uploads_url: 'https://catalog-uploads.vercel.app', + }; + const config = applyAnswers(base, { ...answersFromConfig(base), siteName: 'Renamed' }); + const site = jsYaml.load(renderFiles(config)['_data/site.yml']); + assert.equal(site.submit.uploads_url, 'https://catalog-uploads.vercel.app'); + assert.equal(site.submit.signin_url, SIGNIN); +}); diff --git a/test/scripts/uploads_workflows.test.mjs b/test/scripts/uploads_workflows.test.mjs new file mode 100644 index 0000000..29861b6 --- /dev/null +++ b/test/scripts/uploads_workflows.test.mjs @@ -0,0 +1,182 @@ +/** + * The workflow wiring around uploaded files: the scaffold step gets a + * read-only token for the uploads repository (and nothing else does), the + * cleanup workflow mints the only write token, and with UPLOADS_REPO unset + * the feature is off with a clean summary. + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import YAML from 'yaml'; + +import { evaluate, runs } from '../helpers/workflow_expr.mjs'; + +const DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '../..', '.github', 'workflows'); +const load = (name) => YAML.parse(fs.readFileSync(path.join(DIR, name), 'utf8')); +const entry = load('new-entry.yml'); +const cleanup = load('uploads-cleanup.yml'); +const step = (job, id) => job.steps.find((s) => s.id === id); +const TOKEN_ACTION = /^actions\/create-github-app-token@[0-9a-f]{40}$/; +const SHA_COMMENT = /uses: actions\/create-github-app-token@[0-9a-f]{40} # v\d+\.\d+\.\d+/; + +test('new-entry: the token step runs only when UPLOADS_REPO is set and is read-only', () => { + const token = step(entry.jobs.scaffold, 'uploads-token'); + assert.equal(token.if, "vars.UPLOADS_REPO != ''"); + assert.match(token.uses, TOKEN_ACTION); + assert.equal(token.with['permission-contents'], 'read'); + assert.equal(token.with['app-id'], '${{ secrets.UPLOADS_APP_ID }}'); + assert.equal(token.with['private-key'], '${{ secrets.UPLOADS_APP_PRIVATE_KEY }}'); + assert.ok(Object.keys(token.with).filter((k) => k.startsWith('permission-')).length === 1, 'only contents'); + assert.match(fs.readFileSync(path.join(DIR, 'new-entry.yml'), 'utf8'), SHA_COMMENT); + const split = step(entry.jobs.scaffold, 'uploads-repo'); + for (const v of [true, false]) { + assert.equal(runs(token, { vars: { UPLOADS_REPO: v ? 'o/r' : '' } }), v); + assert.equal(runs(split, { vars: { UPLOADS_REPO: v ? 'o/r' : '' } }), v); + } +}); + +test('new-entry: a failed token mint never fails the submission; the scaffolder runs with an empty token', () => { + const steps = entry.jobs.scaffold.steps; + for (const id of ['uploads-repo', 'uploads-token']) { + assert.equal(step(entry.jobs.scaffold, id)['continue-on-error'], true, id); + } + // No condition of its own between the mint and the scaffolder: it runs, and + // a failed mint's output is simply empty (scripts/lib/uploads.mjs warns). + const scaffold = step(entry.jobs.scaffold, 'scaffold'); + assert.equal(scaffold.if, undefined); + assert.equal(runs(scaffold, { vars: { UPLOADS_REPO: 'o/r' } }), true); + const between = steps.slice( + steps.indexOf(step(entry.jobs.scaffold, 'uploads-repo')), + steps.indexOf(scaffold) + ); + for (const s of between) assert.equal(s['continue-on-error'], true, s.id ?? s.name); +}); + +test('new-entry: UPLOADS_TOKEN reaches exactly one step, the scaffolder, just before it', () => { + const steps = Object.values(entry.jobs).flatMap((j) => j.steps); + const holders = steps.filter((s) => s.env && 'UPLOADS_TOKEN' in s.env); + assert.equal(holders.length, 1); + assert.equal(holders[0].id, 'scaffold'); + assert.equal(holders[0].env.UPLOADS_TOKEN, '${{ steps.uploads-token.outputs.token }}'); + assert.equal(holders[0].env.UPLOADS_REPO, '${{ vars.UPLOADS_REPO }}'); + assert.equal(holders[0].env.ISSUE_AUTHOR_ID, '${{ github.event.issue.user.id }}'); + const text = fs.readFileSync(path.join(DIR, 'new-entry.yml'), 'utf8'); + assert.equal(text.match(/steps\.uploads-token\.outputs\.token/g).length, 1); + const names = entry.jobs.scaffold.steps.map((s) => s.id ?? s.name); + assert.equal(names.indexOf('uploads-token') + 1, names.indexOf('scaffold')); +}); + +test('cleanup: triggers, empty top-level permissions, concurrency', () => { + assert.deepEqual(Object.keys(cleanup.on).sort(), [ + 'issues', + 'pull_request', + 'schedule', + 'workflow_dispatch', + ]); + assert.deepEqual(cleanup.on.issues.types, ['closed']); + assert.deepEqual(cleanup.on.pull_request.types, ['closed']); + assert.equal(cleanup.on.schedule.length, 1); + const [minute, hour] = cleanup.on.schedule[0].cron.split(' '); + assert.notEqual(minute, '0'); + assert.match(hour, /^\d+$/); + assert.deepEqual(cleanup.permissions, {}); +}); + +test('cleanup: one concurrency group per closed issue or pull request, and one for sweeps', () => { + // GitHub keeps one pending run per group, so a shared group would cancel + // queued close runs; each close keys its own group. + assert.equal(cleanup.concurrency['cancel-in-progress'], false); + const m = /^uploads-cleanup-\$\{\{ (.+) \}\}$/.exec(cleanup.concurrency.group); + assert.ok(m, cleanup.concurrency.group); + const group = (event) => `uploads-cleanup-${evaluate(m[1], { github: { event } })}`; + assert.equal(group({ issue: { number: 41 } }), 'uploads-cleanup-41'); + assert.equal(group({ pull_request: { number: 42 } }), 'uploads-cleanup-42'); + assert.equal(group({ schedule: '47 4 * * *' }), 'uploads-cleanup-sweep'); + assert.equal(group({ inputs: { dry_run: true } }), 'uploads-cleanup-sweep'); +}); + +test('cleanup: the real job needs UPLOADS_REPO and, for a closed issue, the entry label', () => { + const job = cleanup.jobs.cleanup; + assert.deepEqual(job.permissions, { contents: 'read', issues: 'read' }); + const labelled = { event: { issue: { labels: [{ name: 'content:new-entry' }] } } }; + const plain = { event: { issue: { labels: [{ name: 'bug' }] } } }; + const ctx = (base, event_name, repo) => ({ github: { event_name, ...base }, vars: { UPLOADS_REPO: repo } }); + assert.equal(runs(job, ctx(labelled, 'issues', 'o/r')), true); + assert.equal(runs(job, ctx(plain, 'issues', 'o/r')), false); + assert.equal(runs(job, ctx(plain, 'schedule', 'o/r')), true); + assert.equal(runs(job, ctx(plain, 'workflow_dispatch', 'o/r')), true); + assert.equal(runs(job, ctx(labelled, 'issues', '')), false); + assert.equal(runs(job, ctx(plain, 'schedule', '')), false); +}); + +test('cleanup: the turned-off job runs exactly when the feature is off, and says why', () => { + const off = cleanup.jobs['turned-off']; + const ctx = (event_name, repo, name = 'content:new-entry') => ({ + github: { event_name, event: { issue: { labels: [{ name }] } } }, + vars: { UPLOADS_REPO: repo }, + }); + assert.equal(runs(off, ctx('schedule', '')), true); + assert.equal(runs(off, ctx('issues', '')), true); + assert.equal(runs(off, ctx('issues', '', 'bug')), false); + assert.equal(runs(off, ctx('schedule', 'o/r')), false); + assert.match(off.steps[0].run, /UPLOADS_REPO[\s\S]*GITHUB_STEP_SUMMARY/); + assert.equal(off.permissions, undefined); +}); + +test('cleanup: the write token is minted here, scoped to contents write, and reaches only the script', () => { + const job = cleanup.jobs.cleanup; + const token = step(job, 'uploads-token'); + assert.match(token.uses, TOKEN_ACTION); + assert.equal(token.with['permission-contents'], 'write'); + assert.match(fs.readFileSync(path.join(DIR, 'uploads-cleanup.yml'), 'utf8'), SHA_COMMENT); + const holders = job.steps.filter((s) => s.env && 'UPLOADS_TOKEN' in s.env); + assert.equal(holders.length, 1); + assert.equal(holders[0].run, 'node scripts/uploads_cleanup.mjs'); + assert.equal( + holders[0].env.MODE, + "${{ github.event_name == 'issues' && 'closed' || github.event_name == 'pull_request' && 'pr-closed' || 'sweep' }}" + ); + // Issue text only ever reaches a step through env:. + for (const s of job.steps) assert.ok(!/github\.event\.issue/.test(s.run ?? ''), s.name); +}); + +test('both workflows pin the app-token action to the same commit', () => { + const pins = [entry.jobs.scaffold, cleanup.jobs.cleanup].map((j) => step(j, 'uploads-token').uses); + assert.equal(pins[0], pins[1]); +}); + +test('cleanup: a closed draft pull request runs only for a labelled same-repo entry/ branch', () => { + const job = cleanup.jobs.cleanup; + const off = cleanup.jobs['turned-off']; + const pr = ({ label = 'content:new-entry', head = 'org/catalog', ref = 'entry/water-41' } = {}) => ({ + github: { + event_name: 'pull_request', + repository: 'org/catalog', + event: { pull_request: { labels: [{ name: label }], head: { ref, repo: { full_name: head } } } }, + }, + vars: { UPLOADS_REPO: 'o/r' }, + }); + assert.equal(runs(job, pr()), true); + assert.equal(runs(job, pr({ label: 'bug' })), false); + assert.equal(runs(job, pr({ head: 'stranger/catalog' })), false); + assert.equal(runs(job, pr({ ref: 'feature/x' })), false); + const unset = { ...pr(), vars: { UPLOADS_REPO: '' } }; + assert.equal(runs(job, unset), false); + assert.equal(runs(off, unset), true); + assert.equal( + runs(off, { ...unset, github: { ...unset.github, event: pr({ head: 'x/y' }).github.event } }), + false + ); +}); + +test('cleanup: the pull request branch reaches the script through env only', () => { + const run = cleanup.jobs.cleanup.steps.find((s) => s.run === 'node scripts/uploads_cleanup.mjs'); + assert.equal(run.env.PR_BRANCH, '${{ github.event.pull_request.head.ref }}'); + assert.match(run.env.MODE, /pull_request' && 'pr-closed'/); + for (const s of cleanup.jobs.cleanup.steps) + assert.ok(!/github\.event\.pull_request/.test(s.run ?? ''), s.name); + for (const s of cleanup.jobs.cleanup.steps) + assert.ok(!/head\.(sha|ref)/.test(JSON.stringify(s.with ?? {})), s.name); +});