diff --git a/playwright-tests/DIAGNOSTICS.md b/playwright-tests/DIAGNOSTICS.md new file mode 100644 index 0000000..e61eead --- /dev/null +++ b/playwright-tests/DIAGNOSTICS.md @@ -0,0 +1,17 @@ +# Failure diagnostics + +The normal Playwright configuration retains traces from every failed attempt, including attempt zero. The shared fixture observes both standard pages and contexts created by `newContext()`; popup pages inherit their account role. Reused suite contexts are observed again for each test attempt. + +Failed lookup/setup reports attach public usernames and addresses, timing, gateway URLs, HTTP outcomes, transport failures, pending requests, and sanitized lookup/injection/receipt responses. They do not read local storage or copy signed request bodies. Screenshots and UI state are captured before explicit recipient-page cleanup. The transfer fixture waits for both bounded setup results before closing either account and preserves the original error if cleanup fails. + +`createAndSignInUser()` has a 90-second total setup budget for navigation, username availability and registration. It requires the active Chats screen and expected visible username; an error toast fails setup promptly. This budget leaves time inside the five-minute scenario timeout for evidence and cleanup. It does not repair application registration recovery ([web-client-v2 #1742](https://github.com/Liberdus/web-client-v2/issues/1742)). + +Recipient lookup observes the request before entering the username, then gives the existing UI result up to ten seconds to become `found`. It does not retype, retry the lookup, or accept a negative result. + +Run the offline harness from `playwright-tests`: + +```sh +npm run test:diagnostics +``` + +Two cases deliberately fail attempt zero and pass their retry: a negative lookup and a missing registration confirmation in `beforeAll`. The report verifier checks the original trace, correct recipient screenshot, popup error, registration/cleanup evidence, and absence of the injected secret marker. Additional cases cover delayed success, explicit registration rejection, null/stalled receipts and transport failure. These routes do not create real accounts or contact a gateway. diff --git a/playwright-tests/diagnostics.config.js b/playwright-tests/diagnostics.config.js new file mode 100644 index 0000000..e5236df --- /dev/null +++ b/playwright-tests/diagnostics.config.js @@ -0,0 +1,12 @@ +const { defineConfig } = require('@playwright/test'); + +// Offline tests of the test harness itself; no network setup or real accounts. +module.exports = defineConfig({ + testDir: './harness-tests', + retries: 1, + workers: 1, + timeout: 20_000, + reporter: [['list'], ['json', { outputFile: './test-results/diagnostics-report.json' }]], + outputDir: './test-results/diagnostics', + use: { baseURL: 'http://diagnostics.test/', headless: true, trace: 'retain-on-failure' }, +}); diff --git a/playwright-tests/fixtures/base.js b/playwright-tests/fixtures/base.js index b322068..a988bc9 100644 --- a/playwright-tests/fixtures/base.js +++ b/playwright-tests/fixtures/base.js @@ -5,14 +5,22 @@ */ const { test: baseTest, expect } = require('@playwright/test'); const { toastCloserScript } = require('../helpers/toastHelpers'); +const { TestDiagnostics } = require('../helpers/testDiagnostics'); // Extend base test to override context fixture const test = baseTest.extend({ - context: async ({ browser }, use) => { + diagnostics: [async ({ browser }, use) => { + const diagnostics = TestDiagnostics.current(); + for (const context of browser.contexts()) diagnostics.observeContext(context); + await use(diagnostics); + await diagnostics.finish(); + }, { auto: true }], + context: async ({ browser, diagnostics }, use) => { const context = await browser.newContext(); await context.addInitScript(toastCloserScript); + diagnostics.observeContext(context); await use(context); - await context.close(); + await diagnostics.close([context], null); }, }); diff --git a/playwright-tests/fixtures/newUserFixture.js b/playwright-tests/fixtures/newUserFixture.js index b5fb773..6469e1c 100644 --- a/playwright-tests/fixtures/newUserFixture.js +++ b/playwright-tests/fixtures/newUserFixture.js @@ -6,7 +6,7 @@ exports.test = base.extend({ const username = generateUsername(browserName); await use(username); }, - page: async ({ context, username }, use, testInfo) => { + page: async ({ context, username, diagnostics }, use, testInfo) => { // Attach the username to the test report for easy access await testInfo.attach('username.txt', { body: username, @@ -21,12 +21,12 @@ exports.test = base.extend({ await dialog.dismiss(); } }); + diagnostics.identify(page, { role: 'sender' }); await createAndSignInUser(page, username); await use(page); - await page.close(); - await context.close(); + await diagnostics.capturePage(page, 'before-page-close'); } }); diff --git a/playwright-tests/harness-tests/diagnostics.spec.js b/playwright-tests/harness-tests/diagnostics.spec.js new file mode 100644 index 0000000..ec80427 --- /dev/null +++ b/playwright-tests/harness-tests/diagnostics.spec.js @@ -0,0 +1,185 @@ +const fs = require('fs'); +const http = require('node:http'); +const { test, expect } = require('../fixtures/base'); +const { newContext } = require('../helpers/toastHelpers'); +const { AccountSetup } = require('../helpers/accountSetup'); +const { RecipientLookup } = require('../helpers/recipientLookup'); +const { TestDiagnostics } = require('../helpers/testDiagnostics'); + +class HarnessPage { + static lookup = `sender +
`; + + static registration = ` + `; + + static async streamingServer() { + const server = http.createServer((request, response) => { + response.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); + response.flushHeaders(); + setTimeout(() => response.end(JSON.stringify({ address: 'streamed-recipient-address' })), 200); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return server; + } + + static async open(page, html) { + await page.route('http://diagnostics.test/', route => route.fulfill({ contentType: 'text/html', body: html })); + await page.goto(''); + } + + static report(info, name) { + const attachment = info.attachments.find(item => item.name === name); + expect(attachment).toBeTruthy(); + return JSON.parse(attachment.body ? attachment.body.toString() : fs.readFileSync(attachment.path, 'utf8')); + } + + static async register(page, outcome) { + await page.route('**/inject', route => route.fulfill({ json: { success: outcome !== 'rejected', txId: 'public-registration-id' } })); + await page.route('**/transaction/*', async route => { + if (outcome === 'stalled') return; // Deliberately leave the receipt request pending. + if (outcome === 'success') await new Promise(resolve => setTimeout(resolve, 150)); + await route.fulfill({ json: { transaction: outcome === 'success' ? { success: true } : null } }); + }); + await page.route('http://diagnostics.test/', route => route.fulfill({ contentType: 'text/html', body: this.registration })); + } +} + +test('slow lookup succeeds without retyping or substituting a backend result', async ({ page, diagnostics }) => { + let requests = 0; + await page.route('**/address/*', async route => { + requests++; + await new Promise(resolve => setTimeout(resolve, 200)); + await route.fulfill({ json: { address: 'public-recipient-address' } }); + }); + await HarnessPage.open(page, HarnessPage.lookup); + await RecipientLookup.fill(page, { username: 'recipient', address: 'public-recipient-address' }); + expect(requests).toBe(1); + expect([...diagnostics.requests.values()][0].status).toBe(200); +}); + +test('attempt zero retains lookup, popup and pre-close evidence when retry passes', async ({ browser, page, diagnostics }, info) => { + const context = await newContext(browser); + const recipientPage = await context.newPage(); + diagnostics.identify(recipientPage, { role: 'recipient', username: 'recipient' }); + await context.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '

Recipient

' })); + await recipientPage.goto('http://diagnostics.test/recipient'); + const popupPromise = recipientPage.waitForEvent('popup'); + await recipientPage.evaluate(() => window.open('/popup')); + const popup = await popupPromise; + await popup.waitForLoadState(); + await popup.evaluate(() => { console.error('popup diagnostic marker'); }); + await page.route('**/address/*', route => route.fulfill({ status: info.retry === 0 ? 503 : 200, json: info.retry === 0 ? { error: 'temporary lookup failure', privateKey: 'DO_NOT_CAPTURE' } : { address: 'public-recipient-address' } })); + await HarnessPage.open(page, HarnessPage.lookup); + try { + await RecipientLookup.fill(page, { username: 'recipient', signedInAt: Date.now() }); + } finally { + await diagnostics.capturePage(recipientPage, 'recipient-before-close'); + await recipientPage.close(); + await context.close(); + } +}); + +for (const outcome of ['success', 'rejected', 'missing', 'stalled']) { + test(`account setup ${outcome} is bounded and preserves registration evidence`, async ({ page, diagnostics }, info) => { + const deadline = AccountSetup.timeout; + AccountSetup.timeout = 1_000; + try { + await HarnessPage.register(page, outcome); + const setup = new AccountSetup(page, 'testaccount').run(); + if (outcome === 'success') { + expect((await setup).username).toBe('testaccount'); + return; + } + await expect(setup).rejects.toThrow(/testaccount at registration-confirmation/); + const report = HarnessPage.report(info, 'account-setup-failure'); + expect(report.details.some(item => item.kind === 'account-setup-failure' && item.elapsedMs < 3_000)).toBe(true); + const inject = report.requests.find(item => item.url.endsWith('/inject')); + expect(inject.transaction.username).toBe('testaccount'); + expect(JSON.stringify(report)).not.toContain('DO_NOT_CAPTURE'); + expect(report.captures[0].state.createModalVisible).toBe(true); + if (outcome === 'missing') expect(report.requests.find(item => item.url.includes('/transaction/')).body.transaction).toBeNull(); + if (outcome === 'stalled') expect(report.requests.find(item => item.url.includes('/transaction/')).outcome).toBe('pending'); + if (outcome === 'rejected') expect(report.captures[0].state.toasts).toContain('Registration rejected'); + const originalError = new Error('original setup error'); + await diagnostics.close([{ close: async () => { throw new Error('simulated cleanup failure'); } }], originalError); + expect(diagnostics.errors.some(item => item.kind === 'cleanup-error')).toBe(true); + } finally { AccountSetup.timeout = deadline; } + }); +} + +test('report waits for a response body after headers arrive', async ({ page, diagnostics }, info) => { + const server = await HarnessPage.streamingServer(); + const url = `http://127.0.0.1:${server.address().port}/address/recipient`; + try { + await HarnessPage.open(page, HarnessPage.lookup); + const response = page.waitForResponse(url); + await page.evaluate(url => { void fetch(url); }, url); + await response; + await diagnostics.attach('streaming-response'); + const report = HarnessPage.report(info, 'streaming-response'); + expect(report.requests.find(item => item.url === url).body.address).toBe('streamed-recipient-address'); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + } +}); + +test('transport errors are distinct from HTTP responses', async ({ page, diagnostics }) => { + await page.route('**/address/*', route => route.abort('connectionrefused')); + await HarnessPage.open(page, HarnessPage.lookup); + await page.fill('#chatRecipient', 'recipient'); + await expect(page.locator('#chatRecipientError')).toHaveText('not found'); + const request = [...diagnostics.requests.values()][0]; + expect(request.outcome).toBe('transport-error'); + expect(request.error).toContain('ERR_CONNECTION_REFUSED'); + expect(request.status).toBeUndefined(); +}); + + +test.describe('recipient setup in beforeAll', () => { + test.beforeAll(async ({ browser }, info) => { + const context = await newContext(browser); + const page = await context.newPage(); + const diagnostics = TestDiagnostics.current(); + diagnostics.identify(page, { role: 'recipient' }); + const deadline = AccountSetup.timeout; + AccountSetup.timeout = 1_000; + let setupError; + try { + await HarnessPage.register(page, info.retry === 0 ? 'missing' : 'success'); + await new AccountSetup(page, 'hookrecipient').run(); + } catch (error) { + setupError = error; + throw error; + } finally { + AccountSetup.timeout = deadline; + const resources = [context]; + if (setupError) resources.push({ close: async () => { throw new Error('simulated hook cleanup error'); } }); + await diagnostics.close(resources, setupError); + } + }); + + test('recovers on a fresh attempt without masking hook setup failure', async () => {}); +}); diff --git a/playwright-tests/harness-tests/verify-report.js b/playwright-tests/harness-tests/verify-report.js new file mode 100644 index 0000000..b2502e7 --- /dev/null +++ b/playwright-tests/harness-tests/verify-report.js @@ -0,0 +1,34 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +class DiagnosticsReport { + static verify() { + const report = JSON.parse(fs.readFileSync(path.join(__dirname, '../test-results/diagnostics-report.json'), 'utf8')); + const specs = report.suites.flatMap(suite => [...suite.specs, ...suite.suites.flatMap(child => child.specs)]); + const cases = specs.flatMap(spec => spec.tests); + const flaky = cases.find(test => test.results.length === 2 && test.results[0].status === 'failed'); + assert(flaky, 'Expected controlled attempt-zero failure'); + assert.equal(flaky.results[1].status, 'passed'); + const failed = flaky.results[0]; + const trace = failed.attachments.find(item => item.name === 'trace'); + assert(trace && fs.statSync(trace.path).size > 0, 'Failed initial attempt must retain a trace'); + const attachment = failed.attachments.find(item => item.name === 'failure-diagnostics'); + const evidence = JSON.parse(attachment.body ? Buffer.from(attachment.body, 'base64').toString() : fs.readFileSync(attachment.path, 'utf8')); + assert.equal(evidence.retry, 0); + assert(evidence.requests.some(item => item.status === 503 && item.outcome === 'http-error')); + assert(evidence.errors.some(item => item.text === 'popup diagnostic marker')); + assert(evidence.captures.some(item => item.stage === 'recipient-before-close' && item.identity.username === 'recipient')); + assert(failed.attachments.some(item => item.name.includes('recipient-before-close') && item.contentType === 'image/png')); + assert(!JSON.stringify(evidence).includes('DO_NOT_CAPTURE')); + const hook = cases.find(test => test.results[0].error?.message.includes('hookrecipient')); + assert(hook, 'Expected failed beforeAll registration evidence'); + assert.equal(hook.results[1].status, 'passed'); + assert(hook.results[0].error.message.includes('registration-confirmation')); + assert(hook.results[0].attachments.some(item => item.name === 'account-setup-failure')); + assert(hook.results[0].attachments.some(item => item.name === 'cleanup-diagnostics')); + assert.equal(report.stats.unexpected, 0); + console.log('Verified failed-attempt trace, HTTP evidence, popup errors, redaction and pre-close recipient screenshot.'); + } +} +DiagnosticsReport.verify(); diff --git a/playwright-tests/helpers/accountSetup.js b/playwright-tests/helpers/accountSetup.js new file mode 100644 index 0000000..e8bb5f8 --- /dev/null +++ b/playwright-tests/helpers/accountSetup.js @@ -0,0 +1,64 @@ +const { expect } = require('@playwright/test'); +const { TestDiagnostics } = require('./testDiagnostics'); + +class AccountSetup { + // Covers navigation, username lookup and several confirmation cycles, while leaving + // most of the five-minute scenario budget available for assertions and diagnostics. + static timeout = 90_000; + + constructor(page, username) { + this.page = page; + this.username = username; + this.started = Date.now(); + this.stage = 'open-create-account'; + this.diagnostics = TestDiagnostics.current(); + this.diagnostics.identify(page, { username, registrationStartedAt: this.started }); + } + + remaining() { + const timeout = AccountSetup.timeout - (Date.now() - this.started); + if (timeout <= 0) throw new Error('Account setup deadline exceeded'); + return timeout; + } + + async submit() { + const { page, username } = this; + await page.goto('', { waitUntil: 'domcontentloaded', timeout: this.remaining() }); + await page.locator('#createAccountButton').click({ timeout: this.remaining() }); + await expect(page.locator('#createAccountModal')).toBeVisible({ timeout: this.remaining() }); + this.stage = 'username-availability'; + await page.locator('#newUsername').pressSequentially(username, { timeout: this.remaining() }); + await expect(page.locator('#newUsernameAvailable')).toHaveText('available', { timeout: Math.min(10_000, this.remaining()) }); + this.stage = 'submit-registration'; + await page.locator('#createAccountForm button[type="submit"]').click({ timeout: this.remaining() }); + } + + async run() { + try { + await this.submit(); + this.stage = 'registration-confirmation'; + // Success is stable signed-in UI and the intended account, not a transient toast. + await this.page.waitForFunction(username => { + const errors = [...document.querySelectorAll('.toast.error.show')]; + if (errors.length) throw new Error(`Registration error: ${errors.map(el => el.textContent).join('; ')}`); + const chats = document.querySelector('#chatsScreen.active'); + const name = document.querySelector('.app-name'); + return chats?.getClientRects().length > 0 && getComputedStyle(chats).visibility === 'visible' && + name?.getClientRects().length > 0 && getComputedStyle(name).visibility === 'visible' && name.textContent.trim() === username; + }, this.username, { timeout: this.remaining() }); + const request = [...this.diagnostics.requests.values()].find(entry => entry.transaction?.username === this.username); + const identity = { username: this.username, address: request?.transaction.address, registrationStartedAt: this.started, signedInAt: Date.now() }; + this.diagnostics.identify(this.page, identity); + this.diagnostics.details.push({ kind: 'registration-complete', ...identity }); + return identity; + } catch (cause) { + const detail = { kind: 'account-setup-failure', username: this.username, stage: this.stage, elapsedMs: Date.now() - this.started }; + this.diagnostics.details.push(detail); + await this.diagnostics.capturePage(this.page, this.stage); + await this.diagnostics.attach('account-setup-failure'); + throw new Error(`Account setup failed for ${this.username} at ${this.stage} after ${detail.elapsedMs}ms (deadline ${AccountSetup.timeout}ms): ${cause.message}`, { cause }); + } + } +} + +module.exports = { AccountSetup }; diff --git a/playwright-tests/helpers/recipientLookup.js b/playwright-tests/helpers/recipientLookup.js new file mode 100644 index 0000000..689d419 --- /dev/null +++ b/playwright-tests/helpers/recipientLookup.js @@ -0,0 +1,21 @@ +const { expect } = require('@playwright/test'); +const { TestDiagnostics } = require('./testDiagnostics'); + +class RecipientLookup { + static async fill(page, recipient) { + const diagnostics = TestDiagnostics.current(); + diagnostics.observeContext(page.context()); + diagnostics.details.push({ kind: 'recipient-lookup', sender: await page.locator('.app-name').textContent(), recipient, startedAt: Date.now() }); + try { + await page.fill('#chatRecipient', recipient.username); + await expect(page.locator('#chatRecipientError')).toHaveText('found', { timeout: 10_000 }); + await expect(page.locator('#newChatForm button[type="submit"]')).toBeEnabled(); + } catch (error) { + await diagnostics.capturePage(page, 'recipient-lookup'); + await diagnostics.attach('recipient-lookup-failure'); + throw error; + } + } +} + +module.exports = { RecipientLookup }; diff --git a/playwright-tests/helpers/testDiagnostics.js b/playwright-tests/helpers/testDiagnostics.js new file mode 100644 index 0000000..592929b --- /dev/null +++ b/playwright-tests/helpers/testDiagnostics.js @@ -0,0 +1,213 @@ +const { test } = require('@playwright/test'); + +/** Owns diagnostics for one test attempt, including test-created contexts and popups. */ +class TestDiagnostics { + static attempts = new WeakMap(); + static contextOwners = new WeakMap(); + static identities = new WeakMap(); + + static current() { + const info = test.info(); + if (!this.attempts.has(info)) this.attempts.set(info, new TestDiagnostics(info)); + return this.attempts.get(info); + } + + constructor(info) { + this.info = info; + this.started = Date.now(); + this.pages = new Map(); + this.contextListeners = new Map(); + this.requests = new Map(); + this.bodyReads = new Set(); + this.errors = []; + this.captures = new Map(); + this.details = []; + } + + observeContext(context) { + const previous = TestDiagnostics.contextOwners.get(context); + if (previous === this) return; + if (previous) previous.detachContext(context); + TestDiagnostics.contextOwners.set(context, this); + const onPage = page => this.observePage(page); + context.on('page', onPage); + this.contextListeners.set(context, onPage); + for (const page of context.pages()) this.observePage(page); + } + + observePage(page) { + if (this.pages.has(page)) return this.pages.get(page).identity; + const identity = { page: this.pages.size + 1, ...TestDiagnostics.identities.get(page.context()) }; + const listeners = { + console: message => { + if (message.type() === 'error') this.errors.push({ ...identity, kind: 'console', text: TestDiagnostics.cleanText(message.text()), at: Date.now() }); + }, + pageerror: error => this.errors.push({ ...identity, kind: 'pageerror', text: TestDiagnostics.cleanText(error.message), at: Date.now() }), + request: request => this.recordRequest(request, identity), + response: response => { + const read = this.recordResponse(response).catch(error => { + this.errors.push({ ...identity, kind: 'diagnostic-error', text: TestDiagnostics.cleanText(error.message) }); + }); + this.bodyReads.add(read); + void read.then(() => this.bodyReads.delete(read)); + }, + requestfailed: request => { + const entry = this.requests.get(request); + if (entry) Object.assign(entry, { outcome: 'transport-error', error: request.failure()?.errorText, finishedAt: Date.now() }); + else this.errors.push({ ...identity, kind: 'requestfailed', url: new URL(request.url()).origin + new URL(request.url()).pathname, error: request.failure()?.errorText }); + }, + }; + for (const [event, listener] of Object.entries(listeners)) page.on(event, listener); + this.pages.set(page, { identity, listeners }); + return identity; + } + + identify(page, identity) { + this.observeContext(page.context()); + Object.assign(this.observePage(page), identity); + TestDiagnostics.identities.set(page.context(), { ...TestDiagnostics.identities.get(page.context()), ...identity }); + } + + recordRequest(request, identity) { + const url = new URL(request.url()); + if (!/\/(address|inject|transaction)(\/|$)/.test(url.pathname)) return; + const entry = { + ...identity, url: url.origin + url.pathname + (url.searchParams.has('appReceiptId') ? `?appReceiptId=${url.searchParams.get('appReceiptId')}` : ''), + method: request.method(), startedAt: Date.now(), outcome: 'pending', + }; + if (url.pathname.endsWith('/inject')) { + try { + const payload = request.postDataJSON(); + const tx = typeof payload.tx === 'string' ? JSON.parse(payload.tx) : payload.tx; + // Keep only public registration identity, never signed payloads or message contents. + entry.transaction = { type: tx.type }; + if (tx.type === 'register') Object.assign(entry.transaction, { username: tx.alias, address: tx.from, timestamp: tx.timestamp }); + } catch (error) { + entry.requestBodyError = TestDiagnostics.cleanText(error.message); + } + } + this.requests.set(request, entry); + } + + async recordResponse(response) { + const entry = this.requests.get(response.request()); + if (!entry) return; + Object.assign(entry, { + status: response.status(), outcome: response.ok() ? 'response' : 'http-error', respondedAt: Date.now(), bodyState: 'pending', + headers: { date: response.headers().date, server: response.headers().server, etag: response.headers().etag }, + }); + try { + const body = await TestDiagnostics.bounded(response.text(), 2_000); + try { entry.body = TestDiagnostics.cleanValue(JSON.parse(body)); } + catch { entry.body = TestDiagnostics.cleanText(body); } + entry.finishedAt = Date.now(); + entry.bodyState = 'complete'; + } catch (error) { + entry.bodyError = TestDiagnostics.cleanText(error.message); + entry.bodyState = 'unavailable'; + } + } + + static cleanText(text) { + return String(text).replace(/\b(?:0x)?[a-f\d]{64,}\b/gi, '[redacted hex]').slice(0, 4_000); + } + + static cleanValue(value) { + if (Array.isArray(value)) return value.slice(0, 30).map(item => this.cleanValue(item)); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, + /secret|private.?key|seed|password|signature|pqpublic|encrypted|message|localstorage/i.test(key) + ? '[redacted]' : this.cleanValue(item)])); + } + return typeof value === 'string' ? value.slice(0, 4_000) : value; + } + + static async bounded(promise, timeout) { + let timer; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Diagnostic operation exceeded ${timeout}ms`)), timeout); + })]); + } finally { clearTimeout(timer); } + } + + async capturePage(page, stage) { + if (page.isClosed()) return; + const identity = this.observePage(page); + try { + const state = await TestDiagnostics.bounded(page.evaluate(() => { + const visible = element => !!element && element.getClientRects().length > 0 && getComputedStyle(element).visibility === 'visible'; + const create = document.querySelector('#createAccountForm button[type="submit"]'); + const next = document.querySelector('#newChatForm button[type="submit"]'); + return { + title: document.title, + username: document.querySelector('.app-name')?.textContent, + createModalVisible: visible(document.querySelector('#createAccountModal.active')), + createDisabled: create?.disabled, + toasts: [...document.querySelectorAll('.toast.show')].map(el => el.textContent), + recipient: document.querySelector('#chatRecipient')?.value, + recipientStatus: document.querySelector('#chatRecipientError')?.textContent, + continueDisabled: next?.disabled, + }; + }), 2_000); + const screenshot = await page.screenshot({ timeout: 2_000 }); + this.captures.set(page, { identity, stage, url: page.url(), state: TestDiagnostics.cleanValue(state), screenshot }); + } catch (error) { + this.errors.push({ ...identity, kind: 'diagnostic-error', stage, text: TestDiagnostics.cleanText(error.message) }); + } + } + + async attach(name) { + try { + const { info } = this; + // Each body read has a two-second deadline. Preserve headers-only responses + // as explicitly unavailable instead of racing report serialization. + await Promise.allSettled([...this.bodyReads]); + await info.attach(name, { + body: JSON.stringify({ + test: info.title, retry: info.retry, worker: info.workerIndex, browser: info.project.name, + target: info.project.use.baseURL, startedAt: this.started, + pages: [...this.pages.values()].map(value => value.identity), details: this.details, + requests: [...this.requests.values()], errors: this.errors, + captures: [...this.captures.values()].map(({ screenshot, ...capture }) => capture), + }, null, 2), contentType: 'application/json', + }); + for (const { identity, stage, screenshot } of this.captures.values()) { + await info.attach(`${name}-page-${identity.page}-${stage}`, { body: screenshot, contentType: 'image/png' }); + } + } catch (error) { + // Evidence must never replace the application's/assertion's original error. + console.error(`Could not attach diagnostics: ${TestDiagnostics.cleanText(error.message)}`); + } + } + + detachContext(context) { + const listener = this.contextListeners.get(context); + if (listener) context.off('page', listener); + this.contextListeners.delete(context); + for (const [page, { listeners }] of this.pages) { + if (page.context() !== context) continue; + for (const [event, listener] of Object.entries(listeners)) page.off(event, listener); + } + } + + async finish() { + if (this.info.status !== this.info.expectedStatus) { + await Promise.all([...this.pages.keys()].map(page => this.capturePage(page, 'failure'))); + await this.attach('failure-diagnostics'); + } + for (const context of this.contextListeners.keys()) this.detachContext(context); + } + + // Teardown errors are reported separately; an assertion/setup error stays primary. + async close(resources, primaryError) { + const results = await Promise.allSettled(resources.map(resource => TestDiagnostics.bounded(resource.close(), 5_000))); + const errors = results.filter(result => result.status === 'rejected').map(result => result.reason); + if (!errors.length) return; + this.errors.push(...errors.map(error => ({ kind: 'cleanup-error', text: TestDiagnostics.cleanText(error.message) }))); + await this.attach('cleanup-diagnostics'); + if (!primaryError && this.info.status === this.info.expectedStatus) throw errors[0]; + } +} + +module.exports = { TestDiagnostics }; diff --git a/playwright-tests/helpers/toastHelpers.js b/playwright-tests/helpers/toastHelpers.js index a8b558c..371adc7 100644 --- a/playwright-tests/helpers/toastHelpers.js +++ b/playwright-tests/helpers/toastHelpers.js @@ -1,3 +1,5 @@ +const { TestDiagnostics } = require('./testDiagnostics'); + /** * The toast-closing script. Exported so it can be used with both * page.addInitScript() and context.addInitScript(). @@ -117,6 +119,7 @@ async function injectToastCloserToContext(context) { */ async function newContext(browser, options) { const context = await browser.newContext(options); + TestDiagnostics.current().observeContext(context); await context.addInitScript(toastCloserScript); return context; } diff --git a/playwright-tests/helpers/userHelpers.js b/playwright-tests/helpers/userHelpers.js index bf68d21..849defe 100644 --- a/playwright-tests/helpers/userHelpers.js +++ b/playwright-tests/helpers/userHelpers.js @@ -1,4 +1,5 @@ const { expect } = require("@playwright/test"); +const { AccountSetup } = require("./accountSetup"); async function createUser(page, username) { await page.goto('', { waitUntil: 'networkidle' }); @@ -15,14 +16,7 @@ async function createUser(page, username) { // Helper to create and sign in a new user async function createAndSignInUser(page, username) { - await createUser(page, username); - // expect loading toast to appear - await expect(page.locator('.toast.loading.show')).toBeVisible({ timeout: 20_000 }); - // wait for the loading toast to disappear - await page.waitForSelector('.toast.loading.show', { state: 'detached' }); - await expect(page.locator('#chatsScreen')).toBeVisible({ timeout: 20_000 }); - const appName = await page.locator('.app-name').textContent(); - await expect(appName.trim()).toBe(username); + return new AccountSetup(page, username).run(); } function signInAccountCard(page, username) { diff --git a/playwright-tests/package.json b/playwright-tests/package.json index e139725..355f7dc 100644 --- a/playwright-tests/package.json +++ b/playwright-tests/package.json @@ -4,7 +4,8 @@ "description": "", "main": "index.js", "scripts": { - "test": "playwright test --project=chromium" + "test": "playwright test --project=chromium", + "test:diagnostics": "playwright test -c diagnostics.config.js && node harness-tests/verify-report.js" }, "keywords": [], "author": "", diff --git a/playwright-tests/playwright.config.ts b/playwright-tests/playwright.config.ts index 3264602..a33218c 100644 --- a/playwright-tests/playwright.config.ts +++ b/playwright-tests/playwright.config.ts @@ -29,10 +29,10 @@ export default defineConfig({ /* Base URL to use in actions like `await page.goto('/')`. */ baseURL: 'https://liberdus.com/dev/', - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + /* Keep the failing attempt, including attempt zero and custom browser contexts. */ + trace: 'retain-on-failure', headless: process.env.CI ? true : false, - screenshot: 'on-first-failure', + screenshot: 'only-on-failure', launchOptions: { slowMo: 1000 } }, // Fetch network parameters once before the test run and cache them for sync consumption in tests diff --git a/playwright-tests/tests/attachments.e2e.test.js b/playwright-tests/tests/attachments.e2e.test.js index cb8c02d..6ef87dd 100644 --- a/playwright-tests/tests/attachments.e2e.test.js +++ b/playwright-tests/tests/attachments.e2e.test.js @@ -2,6 +2,7 @@ const { test, expect } = require('../fixtures/newUserFixture'); const { createAndSignInUser, generateUsername } = require('../helpers/userHelpers'); const { newContext } = require('../helpers/toastHelpers'); const { AttachmentMenu } = require('../helpers/attachmentMenu'); +const { TestDiagnostics } = require('../helpers/testDiagnostics'); const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); @@ -397,6 +398,7 @@ test.describe('File Attachment Tests', () => { // Reuse the recipient context that was created in beforeAll const recipientPage = await testRecipient.context.newPage(); + let recipientError; try { // Start waiting for the named download before clicking @@ -453,8 +455,12 @@ test.describe('File Attachment Tests', () => { } finally { // Clean up is handled in afterAll } + } catch (error) { + recipientError = error; + throw error; } finally { - await recipientPage.close(); + await TestDiagnostics.current().capturePage(recipientPage, 'recipient-before-close'); + await TestDiagnostics.current().close([recipientPage], recipientError); } }); @@ -530,6 +536,7 @@ test.describe('File Attachment Tests', () => { // STEP 2: RECIPIENT RECEIVES AND DOWNLOADS ALL ATTACHMENTS const recipientPage = await testRecipient.context.newPage(); + let recipientError; try { // Sign in as recipient @@ -580,9 +587,12 @@ test.describe('File Attachment Tests', () => { // Byte-by-byte comparison expect(downloadedFileBuffer.equals(originalFileBuffer)).toBe(true); } + } catch (error) { + recipientError = error; + throw error; } finally { - // Close recipient page - await recipientPage.close(); + await TestDiagnostics.current().capturePage(recipientPage, 'recipient-before-close'); + await TestDiagnostics.current().close([recipientPage], recipientError); // Clean up is handled in afterAll } diff --git a/playwright-tests/tests/smoke.e2e.test.js b/playwright-tests/tests/smoke.e2e.test.js index d846a15..76970ac 100644 --- a/playwright-tests/tests/smoke.e2e.test.js +++ b/playwright-tests/tests/smoke.e2e.test.js @@ -3,32 +3,39 @@ const { test, expect } = require('../fixtures/newUserFixture'); const { createAndSignInUser, generateUsername } = require('../helpers/userHelpers'); +const { RecipientLookup } = require('../helpers/recipientLookup'); +const { newContext } = require('../helpers/toastHelpers'); +const { TestDiagnostics } = require('../helpers/testDiagnostics'); + let RECIPIENT; test.describe('Tests requiring recipient user', () => { // Create the recipient user once before all tests test.beforeAll(async ({ browser, browserName }) => { - const page = await browser.newPage(); + const context = await newContext(browser); + const page = await context.newPage(); const recipientName = generateUsername(browserName); - await createAndSignInUser(page, recipientName); - RECIPIENT = recipientName; - await page.close(); + const diagnostics = TestDiagnostics.current(); + diagnostics.identify(page, { role: 'recipient' }); + let setupError; + try { + RECIPIENT = await createAndSignInUser(page, recipientName); + } catch (error) { + setupError = error; + throw error; + } finally { + await diagnostics.close([context], setupError); + } }); test('should open New Chat modal, start a chat, and send a message', async ({ page }) => { - const recipient = RECIPIENT; - await page.click('#newChatButton'); await expect(page.locator('#newChatModal')).toBeVisible(); - await page.fill('#chatRecipient', recipient); - await page.waitForTimeout(3_000); - const recipientStatus = await page.locator('#chatRecipientError').textContent().catch(() => ''); - expect(recipientStatus).toBe('found'); + await RecipientLookup.fill(page, RECIPIENT); const continueBtn = page.locator('#newChatForm button[type="submit"]'); - await expect(continueBtn).toBeEnabled(); await continueBtn.click(); await expect(page.locator('#chatModal')).toBeVisible(); @@ -50,18 +57,14 @@ test.describe('Tests requiring recipient user', () => { }); test('should send LIB to contact with no memo, and check history', async ({ page }) => { - const recipient = RECIPIENT; + const recipient = RECIPIENT.username; const amount = 20; // Open New Chat await page.click('#newChatButton'); await expect(page.locator('#newChatModal')).toBeVisible(); - await page.fill('#chatRecipient', recipient); - await page.waitForTimeout(3_000); - const recipientStatus = await page.locator('#chatRecipientError').textContent().catch(() => ''); - expect(recipientStatus).toBe('found'); + await RecipientLookup.fill(page, RECIPIENT); const continueBtn = page.locator('#newChatForm button[type="submit"]'); - await expect(continueBtn).toBeEnabled(); await continueBtn.click(); await expect(page.locator('#chatModal')).toBeVisible(); diff --git a/playwright-tests/tests/transfers.e2e.test.js b/playwright-tests/tests/transfers.e2e.test.js index 08502df..90be782 100644 --- a/playwright-tests/tests/transfers.e2e.test.js +++ b/playwright-tests/tests/transfers.e2e.test.js @@ -6,13 +6,15 @@ const { newContext } = require('../helpers/toastHelpers'); // Setup test fixture with two users const test = base.extend({ - users: async ({ browser, browserName }, use, testInfo) => { + users: async ({ browser, browserName, diagnostics }, use, testInfo) => { const ctxA = await newContext(browser); const ctxB = await newContext(browser); const pageA = await ctxA.newPage(); const pageB = await ctxB.newPage(); const userA = generateUsername(browserName); const userB = generateUsername(browserName); + diagnostics.identify(pageA, { role: 'account-a' }); + diagnostics.identify(pageB, { role: 'account-b' }); // Attach both usernames to the report await testInfo.attach('test-users.json', { @@ -20,12 +22,15 @@ const test = base.extend({ contentType: 'application/json' }); + let setupError; try { - // Create both users in parallel - await Promise.all([ + // Let both bounded setups finish so either account retains its own evidence. + const setup = await Promise.allSettled([ createAndSignInUser(pageA, userA), createAndSignInUser(pageB, userB) ]); + const failure = setup.find(result => result.status === 'rejected'); + if (failure) throw failure.reason; // Get initial balances const [balanceA, balanceB] = await Promise.all([ @@ -47,10 +52,15 @@ const test = base.extend({ balance: parseFloat(balanceB) } }); + } catch (error) { + setupError = error; + throw error; } finally { - // Ensure we close the pages and contexts even if test fails - await ctxA.close(); - await ctxB.close(); + await Promise.all([ + diagnostics.capturePage(pageA, 'transfer-cleanup'), + diagnostics.capturePage(pageB, 'transfer-cleanup'), + ]); + await diagnostics.close([ctxA, ctxB], setupError); } } });