From 7d1ef4e81ced33caa7132da09ff016dae65d5554 Mon Sep 17 00:00:00 2001 From: Thomas Bouffard <27200110+tbouffard@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:53:07 +0200 Subject: [PATCH 1/5] chore(deps-dev): bump playwright from 1.58.2 to 1.61.1 --- package-lock.json | 62 +++++++++++++---------------------------------- package.json | 2 +- 2 files changed, 18 insertions(+), 46 deletions(-) diff --git a/package-lock.json b/package-lock.json index b733701a08..e213a1dd95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,7 +56,7 @@ "minimist": "~1.2.8", "npm-run-all": "~4.1.5", "pinst": "~3.0.0", - "playwright": "~1.58.2", + "playwright": "~1.61.1", "postcss": "~8.5.14", "postcss-cli": "~11.0.1", "prettier": "~3.8.1", @@ -2902,9 +2902,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2922,9 +2919,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2942,9 +2936,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2962,9 +2953,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2982,9 +2970,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3002,9 +2987,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -13986,12 +13968,13 @@ } }, "node_modules/playwright": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", - "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.58.2" + "playwright-core": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -14004,10 +13987,11 @@ } }, "node_modules/playwright-core": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", - "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", "dev": true, + "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" }, @@ -17326,9 +17310,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17350,9 +17331,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17374,9 +17352,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17398,9 +17373,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -27511,19 +27483,19 @@ } }, "playwright": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", - "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", "dev": true, "requires": { "fsevents": "2.3.2", - "playwright-core": "1.58.2" + "playwright-core": "1.61.1" } }, "playwright-core": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", - "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", "dev": true }, "pluralize": { diff --git a/package.json b/package.json index aad06b4fd6..401cbfb0d8 100644 --- a/package.json +++ b/package.json @@ -151,7 +151,7 @@ "minimist": "~1.2.8", "npm-run-all": "~4.1.5", "pinst": "~3.0.0", - "playwright": "~1.58.2", + "playwright": "~1.61.1", "postcss": "~8.5.14", "postcss-cli": "~11.0.1", "prettier": "~3.8.1", From a80c5ca27cdd3f0e5fc8f3a5c08a953b493da6d0 Mon Sep 17 00:00:00 2001 From: Thomas Bouffard <27200110+tbouffard@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:56:13 +0200 Subject: [PATCH 2/5] e2e tests workflow: restore usage of Node 24 which now works with the new pw version --- .github/workflows/test-e2e.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index e10f13ebd2..79f58292da 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -71,22 +71,8 @@ jobs: - name: Free disk space on Ubuntu runner if: runner.os == 'Linux' uses: ./.github/actions/free-runner-disk-space-ubuntu -# START temporary workaround -# Browser download failure with Node 24. See https://github.com/microsoft/playwright/issues/41185 and https://github.com/microsoft/playwright-cli/issues/419#issuecomment-4646958277 (that mentions that it should work with Node 22) -# - name: Build Setup -# uses: ./.github/actions/build-setup - - name: Setup node - uses: actions/setup-node@v6 - with: -# node-version-file: '.nvmrc' - node-version: '22' - registry-url: ${{ inputs.registry-url }} - - name: Install dependencies - uses: bahmutov/npm-install@v1 - # if: inputs.install-dependencies == 'true' - with: - install-command: npm ci --ignore-scripts --prefer-offline --audit false -# END OF temporary workaround + - name: Build Setup + uses: ./.github/actions/build-setup - name: Install ${{matrix.browser}} uses: ./.github/actions/install-playwright-browser with: From ba8f58d4b8fa3b89b849cae62823dcbc54428887 Mon Sep 17 00:00:00 2001 From: Thomas Bouffard <27200110+tbouffard@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:35:34 +0200 Subject: [PATCH 3/5] test-npm-package.yml: restore usage of Node 24 which now works with the new pw version --- .github/workflows/test-npm-package.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-npm-package.yml b/.github/workflows/test-npm-package.yml index 1c44e2c947..0c61ecb1b5 100644 --- a/.github/workflows/test-npm-package.yml +++ b/.github/workflows/test-npm-package.yml @@ -47,22 +47,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 - # START temporary workaround - # Browser download failure with Node 24. See https://github.com/microsoft/playwright/issues/41185 and https://github.com/microsoft/playwright-cli/issues/419#issuecomment-4646958277 (that mentions that it should work with Node 22) - # - name: Build Setup - # uses: ./.github/actions/build-setup - - name: Setup node - uses: actions/setup-node@v6 - with: - # node-version-file: '.nvmrc' - node-version: '22' - registry-url: ${{ inputs.registry-url }} - - name: Install dependencies - uses: bahmutov/npm-install@v1 - # if: inputs.install-dependencies == 'true' - with: - install-command: npm ci --ignore-scripts --prefer-offline --audit false - # END OF temporary workaround + - name: Build Setup + uses: ./.github/actions/build-setup - name: Build npm package run: npm pack - name: List the size of the npm package bundles From e2a072f08994bb38aed5a41b1446460752c798d3 Mon Sep 17 00:00:00 2001 From: Thomas Bouffard <27200110+tbouffard@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:37:02 +0200 Subject: [PATCH 4/5] run tests involving playwright on macos-15. webkit is not up to date on macos-14 --- .github/workflows/test-e2e.yml | 6 +++--- .github/workflows/test-performance.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 79f58292da..f4347d9639 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -54,16 +54,16 @@ jobs: # we want to run the full build on all os: don't cancel running jobs even if one fails fail-fast: false matrix: - os: [macos-14, ubuntu-24.04, windows-2022] + os: [macos-15, ubuntu-24.04, windows-2022] browser: [chromium, firefox, chrome] include: # only test WebKit on macOS - - os: macos-14 + - os: macos-15 browser: webkit # only test Edge on Windows - os: windows-2022 browser: msedge - - os: macos-14 + - os: macos-15 browser: chrome steps: - name: Checkout diff --git a/.github/workflows/test-performance.yml b/.github/workflows/test-performance.yml index ca52026a7b..eacd464bed 100644 --- a/.github/workflows/test-performance.yml +++ b/.github/workflows/test-performance.yml @@ -31,7 +31,7 @@ jobs: # triggerUncaughtException(err, true /* fromPromise */); # ^ # cdpSession.detach: Browser closed. - os: [macos-14, ubuntu-24.04] + os: [macos-15, ubuntu-24.04] env: # Performance tests rely on chromium API browser: chromium From 92972f21653243e65e534316d6a2ec6cb91eec48 Mon Sep 17 00:00:00 2001 From: Thomas Bouffard <27200110+tbouffard@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:34:13 +0200 Subject: [PATCH 5/5] test(e2e): bump macos webkit image snapshot thresholds Update the macOS webkit failure thresholds from the e2e-webkit-test-results-macos-15 report. The new webkit engine on macos-15 produces larger label and font rendering diffs, so the reference thresholds no longer cover the observed differences and the tests fail on CI. Each threshold is set just above the observed diff (rounded up to 2 decimals) and its comment updated to the exact measured percentage. A new dedicated entry is added for group.02.in.collaboration.with.label, which previously relied on the webkit default. Several label/font related snapshots increased by more than 0.3 points (labels.01.general jumps from 1.79% to 4.65%), which likely reflects a real rendering change in the new engine rather than pixel noise. Also document the visual regression threshold mechanism and the report-driven update procedure in CLAUDE.md so the task can be reproduced. --- CLAUDE.md | 27 +++++++++++++++++++ test/e2e/bpmn.colors.test.ts | 4 +-- .../e2e/bpmn.rendering.ignore.options.test.ts | 8 +++--- test/e2e/bpmn.rendering.test.ts | 20 +++++++++----- test/e2e/diagram.navigation.fit.test.ts | 2 +- test/e2e/style.api.test.ts | 2 +- 6 files changed, 48 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 57b50312c3..5925984e86 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -186,6 +186,33 @@ GraphConfigurator **E2E Pattern:** Heavy use of image snapshots (in `__image_snapshots__/`) to verify visual rendering correctness across features. +### E2E Visual Regression Thresholds + +E2E tests compare a freshly rendered screenshot against a reference image snapshot using `jest-image-snapshot` (SSIM comparison). A test fails when the measured difference exceeds its allowed `failureThreshold` (a percentage of differing pixels). Because rendering differs slightly by browser engine and OS (mostly font rendering, a few pixels, not visible to a human), thresholds are configured per browser family and per platform. + +**How thresholds are resolved** (see `test/e2e/helpers/visu/image-snapshot-config.ts`): +- `MultiBrowserImageSnapshotThresholds` holds a default per browser family (`chromium`, `firefox`, `webkit`), passed to `super({ chromium, firefox, webkit })`. +- Each test file subclasses it and overrides `getChromiumThresholds()` / `getFirefoxThresholds()` / `getWebkitThresholds()` to return a `Map` of per-snapshot overrides. +- Each map entry is keyed by the snapshot identifier (the BPMN diagram name, sometimes with a suffix like `.ignored` / `.not-ignored`) and holds per-platform values: `{ linux?, macos?, windows? }`. +- At runtime, the browser family selects the map, `getSimplePlatformName()` selects the platform key, and the value found there wins. If there is no dedicated entry or no matching platform key, the browser-family default is used. +- On CI, macOS runs the `webkit` family, so webkit macOS thresholds come from the `macos` key inside `getWebkitThresholds()`. + +**Threshold value convention:** +- Values are written as `X / 100` (a percentage expressed as a fraction), for example `macos: 0.53 / 100`. +- Add a trailing comment with the actual observed diff percentage from the report, for example `macos: 0.53 / 100, // 0.5228413635451014%`. The fit test file prefixes it with `max` because one entry covers several test variations sharing the snapshot key. +- Set the value by rounding the observed diff UP to 2 decimals (ceil), so the threshold sits just above the observed diff. +- Keep entries ordered by snapshot key to match the surrounding file. + +**Updating thresholds from a CI test-results report** (recurring task): +1. The report bundle (downloaded artifact) contains `index-single-page.html` (jest-html-reporters) and a `__diff_output__/` folder with `-diff.png` per failure. +2. Parse `index-single-page.html`. Each failed block contains: the suite/title, `was % different from snapshot ... Failure threshold was set to %`, and a `__diff_output__/-diff.png` link. The relative path (minus `-diff.png`) identifies the snapshot; note some names contain hyphens (`.not-ignored`) and the fit tests live in nested subfolders, so capture the full path up to `-diff.png`. +3. Group by snapshot key and keep the MAX actual diff (several test variations can share one threshold key, e.g. `with.outside.labels` in the fit tests). +4. For each failing snapshot, locate the matching entry in the relevant test file's `getWebkitThresholds()` (or the correct browser/platform) and set `macos: ceil2(actual) / 100, // %`. If no entry exists (the failure was against the browser-family default), add one in key order. +5. Files that currently carry webkit macOS thresholds: `bpmn.rendering.test.ts`, `bpmn.rendering.ignore.options.test.ts`, `bpmn.colors.test.ts`, `diagram.navigation.fit.test.ts`, `style.api.test.ts` (a test file may hold several threshold subclasses, one per configurator). +6. Reporting: flag any entry where the increase (`new threshold - old threshold`) exceeds 0.3 percentage points. A large jump usually signals a real rendering change or regression worth a human look, not just pixel noise. + +Note (from the class JSDoc): prefer NOT adding a threshold for a new test until it actually fails on CI. Discrepancies mostly come from labels; if labels are not part of what the test verifies, remove labels from the BPMN diagram instead of raising a threshold. + ## Key Architectural Patterns ### Converter Pattern diff --git a/test/e2e/bpmn.colors.test.ts b/test/e2e/bpmn.colors.test.ts index 58f01124f4..29f41627dc 100644 --- a/test/e2e/bpmn.colors.test.ts +++ b/test/e2e/bpmn.colors.test.ts @@ -77,7 +77,7 @@ class ImageSnapshotThresholdsModelColors extends MultiBrowserImageSnapshotThresh [ 'elements.colors.02.labels', { - macos: 0.41 / 100, // 0.40831372531949794% + macos: 1.04 / 100, // 1.033599854269418% }, ], ]); @@ -140,7 +140,7 @@ class ImageSnapshotThresholdsIgnoreBpmnColors extends MultiBrowserImageSnapshotT [ 'elements.colors.02.labels', { - macos: 0.49 / 100, // 0.483122009334358% + macos: 1.13 / 100, // 1.1249364408232543% }, ], ]); diff --git a/test/e2e/bpmn.rendering.ignore.options.test.ts b/test/e2e/bpmn.rendering.ignore.options.test.ts index d27437d03f..6d28c80740 100644 --- a/test/e2e/bpmn.rendering.ignore.options.test.ts +++ b/test/e2e/bpmn.rendering.ignore.options.test.ts @@ -70,13 +70,13 @@ class ImageSnapshotThresholdsActivityLabelBounds extends MultiBrowserImageSnapsh [ 'activities.with.wrongly.positioned.labels.not-ignored', { - macos: 0.44 / 100, // 0.4382175377357411% + macos: 1.17 / 100, // 1.1642294732319813% }, ], [ 'activities.with.wrongly.positioned.labels.ignored', { - macos: 1.5 / 100, // 1.4951298719464878% + macos: 2.01 / 100, // 2.0057344642194885% }, ], ]); @@ -135,13 +135,13 @@ class ImageSnapshotThresholdsLabelStyles extends MultiBrowserImageSnapshotThresh [ 'labels.with.font.styles.not-ignored', { - macos: 0.31 / 100, // 0.30621380597637415% + macos: 0.59 / 100, // 0.584031268762264% }, ], [ 'labels.with.font.styles.ignored', { - macos: 0.38 / 100, // 0.37988509633168904% + macos: 0.84 / 100, // 0.8385178209979638% }, ], ]); diff --git a/test/e2e/bpmn.rendering.test.ts b/test/e2e/bpmn.rendering.test.ts index c195068557..dfcb16b948 100644 --- a/test/e2e/bpmn.rendering.test.ts +++ b/test/e2e/bpmn.rendering.test.ts @@ -207,36 +207,42 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { macos: 0.41 / 100, // 0.40487233108913445% }, ], + [ + 'group.02.in.collaboration.with.label', + { + macos: 0.16 / 100, // 0.15458864441817433% + }, + ], [ 'labels.01.general', { // high value due to font rendering discrepancies with chromium rendering - macos: 1.79 / 100, // 1.7833883910028492% + macos: 4.65 / 100, // 4.644599369259983% }, ], [ 'labels.02.position.and.line.breaks', { // TODO possible rendering issue so high threshold value - macos: 6.11 / 100, // 6.105183205727094% + macos: 6.65 / 100, // 6.6443181311426125% }, ], [ 'labels.03.default.position', { - macos: 0.64 / 100, // 0.6346061558805904% + macos: 1.05 / 100, // 1.0442324603631814% }, ], [ 'labels.04.fonts', { - macos: 0.71 / 100, // 0.703880504764276% + macos: 1.11 / 100, // 1.1087684739990888% }, ], [ 'labels.05.default.position.activities', { - macos: 1.2 / 100, // 1.192492604936246% + macos: 2.31 / 100, // 2.303108822951705% }, ], [ @@ -248,13 +254,13 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { [ 'pools.02.vertical.with.lanes', { - macos: 0.24 / 100, // 0.23336351480325318% + macos: 0.47 / 100, // 0.4618019396829043% }, ], [ 'pools.03.black.box', { - macos: 0.36 / 100, // 0.3576987596416892% + macos: 0.53 / 100, // 0.5228413635451014% }, ], ]); diff --git a/test/e2e/diagram.navigation.fit.test.ts b/test/e2e/diagram.navigation.fit.test.ts index 038613e93a..5d9a3420d9 100644 --- a/test/e2e/diagram.navigation.fit.test.ts +++ b/test/e2e/diagram.navigation.fit.test.ts @@ -121,7 +121,7 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { [ 'with.outside.labels', { - macos: 0.39 / 100, // max 0.38104004012843307% + macos: 0.51 / 100, // max 0.5058234845605303% }, ], ]); diff --git a/test/e2e/style.api.test.ts b/test/e2e/style.api.test.ts index 9da4241081..0e3ec6e462 100644 --- a/test/e2e/style.api.test.ts +++ b/test/e2e/style.api.test.ts @@ -117,7 +117,7 @@ class StyleImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { [ 'font.color.opacity', { - macos: 0.2 / 100, // 0.18895676780704695% + macos: 0.33 / 100, // 0.324776594957199% }, ], [