ci: add visual regression checks for Gitea 1.27 and 28

.github/workflows/release.yml

@@ -13,7 +13,13 @@ permissions:
   contents: write
 
 jobs:
+  visual:
+    uses: ./.github/workflows/visual.yml
+    permissions:
+      contents: read
+
   build:
+    needs: visual
     runs-on: ubuntu-24.04
     env:
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'

.github/workflows/visual.yml

@@ -0,0 +1,41 @@
+name: Visual regression
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  visual:
+    name: Gitea ${{ matrix.gitea }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        gitea: ['1.27.3', '28.2.0']
+    env:
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
+      VISUAL_GITEA_VERSION: ${{ matrix.gitea }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # The approved visual baseline is pinned to a full commit SHA.
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 24
+          cache: npm
+      - run: npm ci
+      - name: Install the pinned Chromium browser and its system dependencies
+        run: npx playwright install --with-deps chromium
+      - name: Compare the candidate with the approved visual baseline
+        run: npm run test:visual
+      - name: Upload screenshots, diffs, traces and Gitea logs
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: visual-gitea-${{ matrix.gitea }}
+          path: shots/visual/${{ matrix.gitea }}/
+          if-no-files-found: warn
+          retention-days: 14

README.md

@@ -161,10 +161,16 @@ Captures and reports are local artifacts: `shots/`, `docs/reference/`, `docs/pag
 ### CI / releases
 
 [`.github/workflows/release.yml`](.github/workflows/release.yml) runs on every push and pull request:
-1. lint;
-2. build;
-3. package;
-4. upload the package as a workflow artifact.
+
+1. visual regression on Gitea 1.27.3 and 28.2.0 (light/dark, desktop/mobile), with screenshot diffs and reports;
+2. lint;
+3. build;
+4. package;
+5. upload the package as a workflow artifact.
+
+The visual jobs must pass before packaging or publishing. They create their own fixed test data and
+compare the candidate with an explicitly approved source revision in the same CI environment.
+See [`tools/visual/README.md`](tools/visual/README.md) for local runs, reports and baseline updates.
 
 Pushing a tag `v*` builds with `--strict`, which enforces the 340 KB budget, and publishes a GitHub Release with the `.zip` and `.tar.gz` attached:
 

package.json

@@ -12,6 +12,7 @@
     "seed": "node tools/seed/seed.mjs",
     "shoot": "node tools/shoot/shoot.mjs",
     "smoke": "node tools/shoot/smoke.mjs",
+    "test:visual": "node tools/visual/run.mjs",
     "package": "node build/package.mjs"
   },
   "devDependencies": {

tools/visual/README.md

@@ -0,0 +1,58 @@
+# Visual regression tests
+
+Every pull request, push to `main` and release tag runs this suite on **Gitea 1.27.3 and 28.2.0**.
+The build/package/release job requires both visual jobs to pass.
+
+The suite builds the approved commit in `baseline.json` and the candidate, installs their complete
+packages into two disposable Gitea containers, and seeds both through the API. It uses no developer
+database, migrations from GitHub, service credentials or pre-existing Gitea installation. Both servers
+bind to random loopback ports and are removed after the test run; their logs remain in the report.
+
+For each version, Playwright compares screenshots of the reference and candidate in the **same browser
+and operating system**, using light/dark themes and 1440px/390px viewports. This avoids maintaining PNGs
+specific to a developer's OS. The reference commit stays fixed until an intentional visual change is
+reviewed; it is not automatically replaced by the PR base or by the candidate.
+
+Coverage includes repository home, issue list, issue conversation, PR conversation, unified/split
+diffs, comment menus, reaction pickers and the inline reply editor. Menu checks also hit-test every
+item (including the last one), so clipping/stacking failures cannot pass merely because an element
+exists in the DOM. Mobile split diffs check horizontal scroll restoration after the dropdown closes.
+
+## Run locally
+
+Requires Node 24 and Docker. The runner only creates and removes its own `theme-visual-*` containers.
+
+```sh
+npm ci
+npx playwright install chromium
+npm run test:visual                         # Gitea 28.2.0
+VISUAL_GITEA_VERSION=1.27.3 npm run test:visual
+npm run test:visual -- --project=light-390 --grep 'PR split menus'
+```
+
+`SHOOT_CHROMIUM` can select an installed Chromium executable locally. CI always installs the browser
+matching the pinned `playwright` dependency. Screenshots normalize relative timestamps and disable
+animations; Git commits have fixed authors/dates, external network requests are blocked in the browser,
+and Gitea's footer timing display is disabled.
+
+## Investigate a failure
+
+Download the `visual-gitea-<version>` Actions artifact. It contains the HTML report, reference/actual
+PNGs, pixel diffs on comparison failures, traces, JSON results and both Gitea logs. A failed or missing
+comparison fails the job; CI never uses `--update-snapshots` to accept new output.
+
+```sh
+npx playwright show-report shots/visual/28.2.0/report
+npx playwright show-trace <path-to-trace.zip>
+```
+
+The pixel threshold permits minor antialiasing differences (80 pixels at a per-pixel threshold of
+0.15); it does not permit broad layout/color changes.
+
+## Approve an intentional visual change
+
+First inspect the failing report and actual/diff images for every affected version, theme and viewport.
+After reviewing the intended result, set `baseline.json`'s `ref` to the full SHA of the commit containing
+that theme change and update its description. Commit this explicit baseline change and rerun CI.
+That commit must be available in the repository history fetched by Actions. Changing the fixture or
+test suite itself applies identically to reference and candidate, so it normally needs no baseline change.

tools/visual/baseline.json

@@ -0,0 +1,4 @@
+{
+  "ref": "bd39dda59ab7cea9af727c9d9e71b59f76797072",
+  "description": "v0.2.2, including the reviewed fix for issue #6"
+}

tools/visual/fixture.mjs

@@ -0,0 +1,153 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import net from 'node:net';
+import {randomBytes} from 'node:crypto';
+import {execFileSync, spawnSync} from 'node:child_process';
+
+export const OWNER = 'visual-owner';
+export const PASSWORD = 'visual-fixture-password-2026'; // Disposable, loopback-only test instances.
+export const REPO = `/${OWNER}/theme-playground`;
+const DATE = '2026-01-01T00:00:00Z';
+const identity = {name: 'Visual Owner', email: 'owner@example.invalid'};
+const commit = {author: identity, committer: identity, dates: {author: DATE, committer: DATE}};
+const docker = (...args) => execFileSync('docker', args, {encoding: 'utf8'}).trim();
+
+async function freePort() {
+  const server = net.createServer();
+  await new Promise((resolve, reject) => server.once('error', reject).listen(0, '127.0.0.1', resolve));
+  const port = server.address().port;
+  await new Promise((resolve) => server.close(resolve));
+  return port;
+}
+
+export async function startGitea({version, label, packageDir, directory, servers}) {
+  const data = path.join(directory, label, 'data');
+  const custom = path.join(data, 'gitea');
+  fs.mkdirSync(path.join(custom, 'conf'), {recursive: true});
+  for (const folder of ['public', 'templates']) fs.cpSync(path.join(packageDir, folder), path.join(custom, folder), {recursive: true});
+  const url = `http://127.0.0.1:${await freePort()}`;
+  fs.writeFileSync(path.join(custom, 'conf/app.ini'), `APP_NAME = Theme visual tests
+RUN_MODE = prod
+[database]
+DB_TYPE = sqlite3
+PATH = /data/gitea/gitea.db
+[repository]
+ROOT = /data/git/repositories
+DEFAULT_BRANCH = main
+[server]
+APP_DATA_PATH = /data/gitea
+ROOT_URL = ${url}/
+DISABLE_SSH = true
+OFFLINE_MODE = true
+[security]
+INSTALL_LOCK = true
+SECRET_KEY = ${randomBytes(32).toString('hex')}
+[service]
+DISABLE_REGISTRATION = true
+[mailer]
+ENABLED = false
+[cron]
+ENABLED = false
+[actions]
+ENABLED = false
+[ui]
+THEMES = gitea-auto,github-auto,github-light,github-dark
+DEFAULT_THEME = github-auto
+[other]
+SHOW_FOOTER_TEMPLATE_LOAD_TIME = false
+[log]
+MODE = console
+LEVEL = Warn
+`);
+  const name = `theme-visual-${label}-${randomBytes(5).toString('hex')}`;
+  console.log(`Starting Gitea ${version} (${label}) at ${url}`);
+  docker('run', '-d', '--name', name, '--label', 'purpose=gitea-theme-visual-ci',
+    '-e', `USER_UID=${process.getuid?.() || 1000}`, '-e', 'USER_GID=1000',
+    '-p', `127.0.0.1:${new URL(url).port}:3000`,
+    '--mount', `type=bind,src=${data},dst=/data`, `docker.gitea.com/gitea:${version}`);
+  const server = {name, url, data, label};
+  servers.push(server); // Cleanup also runs when readiness or seeding fails.
+  let ready = false;
+  for (let attempt = 0; attempt < 120; attempt++) {
+    try {
+      const response = await fetch(`${url}/api/v1/version`, {signal: AbortSignal.timeout(2000)});
+      if (response.ok && (await response.json()).version === version) { ready = true; break; }
+    } catch { /* Gitea is initializing its own disposable database. */ }
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  if (!ready) throw new Error(`Gitea ${version} (${label}) did not become ready`);
+  docker('exec', '--user', 'git', name, 'gitea', 'admin', 'user', 'create',
+    '--config', '/data/gitea/conf/app.ini', '--username', OWNER, '--password', PASSWORD,
+    '--email', identity.email, '--admin', '--must-change-password=false');
+  await seed(server);
+  return server;
+}
+
+async function seed({url, name}) {
+  const flush = () => docker('exec', '--user', 'git', name, 'gitea', 'manager', 'flush-queues', '--config', '/data/gitea/conf/app.ini', '--timeout', '10s');
+  async function api(method, route, body, user = OWNER) {
+    const response = await fetch(`${url}/api/v1${route}`, {
+      method,
+      headers: {authorization: `Basic ${Buffer.from(`${user}:${PASSWORD}`).toString('base64')}`, 'content-type': 'application/json'},
+      body: body === undefined ? undefined : JSON.stringify(body),
+      signal: AbortSignal.timeout(20_000),
+    });
+    if (!response.ok) throw new Error(`${method} ${route}: ${response.status} ${(await response.text()).slice(0, 400)}`);
+    return response.status === 204 ? null : response.json();
+  }
+  await api('POST', '/admin/users', {username: 'visual-reviewer', full_name: 'Visual Reviewer', email: 'reviewer@example.invalid', password: PASSWORD, must_change_password: false});
+  await api('POST', '/user/repos', {name: 'theme-playground', description: 'A small repository for visual regression tests.', private: false, auto_init: false, default_branch: 'main'});
+  const colors = 'package colors\n\nfunc Palette() []string {\n\treturn []string{\n\t\t"default",\n\t\t"muted",\n\t}\n}\n';
+  const layout = 'export const spacing = {\n  small: 4,\n  medium: 8,\n};\n';
+  const readme = '# Theme playground\n\nA stable fixture for GitHub theme visual tests.\n\n## Components\n\n- Buttons, navigation and repository files\n- Issue comments and pull request reviews\n\n| Theme | Schemes |\n| --- | --- |\n| GitHub | Light and dark |\n\n> [!NOTE]\n> Review menus must remain visible outside their thread.\n';
+  const repo = `/repos${REPO}`;
+  await api('POST', `${repo}/contents`, {...commit, branch: 'main', message: 'Add visual fixtures', files: Object.entries({'README.md': readme, 'src/colors.go': colors, 'src/layout.ts': layout}).map(([file, content]) => ({operation: 'create', path: file, content: Buffer.from(content).toString('base64')}))});
+  const label = await api('POST', `${repo}/labels`, {name: 'bug', color: 'd73a4a', description: 'Something is not working'});
+  await api('POST', `${repo}/issues`, {title: 'Keep review menus fully visible', body: 'Comment menus should extend beyond the review thread.\n\n- [x] Desktop\n- [ ] Mobile\n\n```css\n.review-thread { overflow: visible; }\n```', labels: [label.id]});
+  await api('POST', `${repo}/issues/1/comments`, {body: 'Please check light and dark themes, including the last menu item.'}, 'visual-reviewer');
+  for (const [file, content] of [['src/colors.go', colors.replace('"muted",', '"accent",\n\t\t"success",')], ['src/layout.ts', layout.replace('small: 4', 'small: 8')]]) {
+    const previous = await api('GET', `${repo}/contents/${file}?ref=main`);
+    await api('PUT', `${repo}/contents/${file}`, {...commit, branch: file.endsWith('.go') ? 'main' : 'review-menu', ...(file.endsWith('.go') ? {new_branch: 'review-menu'} : {}), sha: previous.sha, message: `Update ${file}`, content: Buffer.from(content).toString('base64')});
+  }
+  // Finish push hooks before creating the PR, so a late hook cannot add an extra timeline event.
+  flush();
+  await api('POST', `${repo}/pulls`, {base: 'main', head: 'review-menu', title: 'Refine palette and spacing', body: 'Keep the thread actions usable in every diff view.'});
+  await api('POST', `${repo}/pulls/2/reviews`, {event: 'COMMENT', body: 'A couple of small review comments.', comments: [
+    {path: 'src/colors.go', new_position: 6, body: 'Could we sort these colors alphabetically?'},
+    {path: 'src/layout.ts', new_position: 2, body: 'Please keep the spacing scale consistent.'},
+  ]}, 'visual-reviewer');
+  flush();
+}
+
+export async function login(browser, server, directory, label) {
+  const context = await browser.newContext({locale: 'en-US'});
+  await context.addCookies([{name: 'lang', value: 'en-US', domain: new URL(server.url).hostname, path: '/'}]);
+  try {
+    const page = await context.newPage();
+    await page.goto(`${server.url}/user/login`, {waitUntil: 'load'});
+    await page.fill('#user_name', OWNER);
+    await page.fill('#password', PASSWORD);
+    await Promise.all([
+      page.waitForURL((url) => !url.pathname.startsWith('/user/login')),
+      page.locator('form.ui.form button.ui.primary.button').first().click(),
+    ]);
+    const storageState = path.join(directory, `${label}-session.json`);
+    await context.storageState({path: storageState});
+    return {url: server.url, storageState};
+  } finally { await context.close(); }
+}
+
+export function cleanup(servers, output) {
+  for (const server of servers.reverse()) {
+    try {
+      const logs = spawnSync('docker', ['logs', server.name], {encoding: 'utf8'});
+      fs.writeFileSync(path.join(output, `${server.label}.log`), (logs.stdout || '') + (logs.stderr || ''));
+    } catch { /* Preserve the original failure. */ }
+    // The image creates root-owned SSH files in /data even when Gitea uses USER_UID.
+    // Restore ownership inside this disposable container before removing its bind-mounted directory.
+    try {
+      docker('exec', '--user', 'root', server.name, 'chown', '-R', `${process.getuid?.() ?? 1000}:${process.getgid?.() ?? 1000}`, '/data');
+    } catch (error) { console.error(`Could not restore fixture ownership for ${server.name}: ${error.message}`); }
+    try { docker('rm', '-f', server.name); } catch (error) { console.error(`Could not remove test container ${server.name}: ${error.message}`); }
+  }
+}

tools/visual/playwright.config.mjs

@@ -0,0 +1,39 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {defineConfig} from 'playwright/test';
+
+if (!process.env.VISUAL_STATE) throw new Error('Run npm run test:visual to prepare the isolated Gitea fixtures.');
+const state = JSON.parse(fs.readFileSync(process.env.VISUAL_STATE, 'utf8'));
+const output = path.resolve('shots/visual', state.version);
+
+export default defineConfig({
+  testDir: import.meta.dirname,
+  testMatch: 'theme.spec.mjs',
+  timeout: 30_000,
+  workers: 1,
+  retries: 0,
+  forbidOnly: !!process.env.CI,
+  updateSnapshots: 'none',
+  outputDir: path.join(output, 'test-results'),
+  snapshotPathTemplate: path.join(output, 'reference', '{projectName}', '{testName}', '{arg}{ext}'),
+  reporter: [
+    ['list'],
+    ['html', {outputFolder: path.join(output, 'report'), open: 'never'}],
+    ['json', {outputFile: path.join(output, 'results.json')}],
+  ],
+  expect: {toHaveScreenshot: {animations: 'disabled', caret: 'hide', threshold: 0.15, maxDiffPixels: 80}},
+  use: {
+    baseURL: state.candidate.url,
+    storageState: state.candidate.storageState,
+    locale: 'en-US',
+    timezoneId: 'UTC',
+    reducedMotion: 'reduce',
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+    launchOptions: process.env.SHOOT_CHROMIUM ? {executablePath: process.env.SHOOT_CHROMIUM} : {},
+  },
+  projects: ['light', 'dark'].flatMap((colorScheme) => [1440, 390].map((width) => ({
+    name: `${colorScheme}-${width}`,
+    use: {colorScheme, viewport: {width, height: width === 390 ? 844 : 1000}},
+  }))),
+});

tools/visual/run.mjs

@@ -0,0 +1,51 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {execFileSync, spawnSync} from 'node:child_process';
+import {launchBrowser} from '../shoot/lib/browser.mjs';
+import {startGitea, login, cleanup, REPO} from './fixture.mjs';
+
+const root = path.resolve(import.meta.dirname, '../..');
+process.chdir(root);
+const version = process.env.VISUAL_GITEA_VERSION || '28.2.0';
+if (!/^\d+\.\d+\.\d+$/.test(version)) throw new Error('VISUAL_GITEA_VERSION must be an exact Gitea version.');
+const baseline = JSON.parse(fs.readFileSync(new URL('./baseline.json', import.meta.url), 'utf8'));
+if (!/^[a-f0-9]{40}$/.test(baseline.ref)) throw new Error('The visual baseline must pin an approved, full commit SHA.');
+const output = path.resolve('shots/visual', version);
+fs.rmSync(output, {recursive: true, force: true});
+fs.mkdirSync(output, {recursive: true});
+fs.mkdirSync('.cache/visual', {recursive: true});
+const directory = fs.mkdtempSync(path.resolve('.cache/visual/run-'));
+const referenceSource = path.join(directory, 'reference-source');
+fs.mkdirSync(referenceSource);
+const run = (command, args, cwd = root) => execFileSync(command, args, {cwd, stdio: 'inherit', env: {...process.env, PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1', COPYFILE_DISABLE: '1'}});
+const servers = [];
+let browser;
+try {
+  // Render the approved source in the same browser/OS as the candidate, rather than comparing cross-OS PNGs.
+  console.log(`Visual baseline: ${baseline.description} (${baseline.ref})`);
+  const archive = execFileSync('git', ['archive', '--format=tar', baseline.ref], {maxBuffer: 64 * 1024 * 1024});
+  execFileSync('tar', ['-xf', '-', '-C', referenceSource], {input: archive});
+  run('npm', ['ci', '--ignore-scripts', '--no-audit'], referenceSource);
+  for (const source of [referenceSource, root]) {
+    run(process.execPath, ['build/build.mjs', '--strict'], source);
+    run(process.execPath, ['build/package.mjs'], source);
+  }
+  const reference = await startGitea({version, label: 'reference', packageDir: path.join(referenceSource, 'dist/package/gitea-theme-github'), directory, servers});
+  const candidate = await startGitea({version, label: 'candidate', packageDir: path.join(root, 'dist/package/gitea-theme-github'), directory, servers});
+  browser = await launchBrowser();
+  const state = {version, repo: REPO, baseline, reference: await login(browser, reference, directory, 'reference'), candidate: await login(browser, candidate, directory, 'candidate')};
+  await browser.close();
+  browser = null;
+  const stateFile = path.join(directory, 'state.json');
+  fs.writeFileSync(stateFile, JSON.stringify(state));
+  fs.writeFileSync(path.join(output, 'environment.json'), JSON.stringify({version, baseline, candidate: execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim()}, null, 2));
+  const result = spawnSync(process.execPath, ['node_modules/playwright/cli.js', 'test', '--config', 'tools/visual/playwright.config.mjs', ...process.argv.slice(2)], {
+    stdio: 'inherit', env: {...process.env, VISUAL_STATE: stateFile},
+  });
+  if (result.error) throw result.error;
+  process.exitCode = result.status ?? 1;
+} finally {
+  if (browser) await browser.close();
+  cleanup(servers, output);
+  fs.rmSync(directory, {recursive: true, force: true});
+}