diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..fe12857 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,27 @@ +name: Build and Test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + name: Build and Test (Node ${{ matrix.node }}) + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + node: [20, 22, 24] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: ${{ matrix.node }} + cache: npm + - run: npm ci + - run: npm test diff --git a/README.md b/README.md index 5f445ca..f25e991 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,7 @@ The security level is specified in the URL path (`minnow` or `shark`), which int ### Scripts - `npm run build` - Compile TypeScript to JavaScript +- `npm test` - Build and run chat regression tests with mocked LiteLLM responses - `npm start` - Run compiled application - `npm run dev` - Run with ts-node for development - `npm run watch` - Watch mode for TypeScript compilation @@ -294,6 +295,15 @@ npm run build Output is compiled to the `dist/` directory. +### Testing + +```bash +npm ci +npm test +``` + +Tests use local fixtures and mock LiteLLM requests, so Docker and API keys are not needed. CI runs the build and tests on Node.js 20, 22, and 24. + ### Type Safety The application uses TypeScript with strict mode enabled. All request/response types are validated at runtime using Zod schemas, ensuring type safety throughout the application lifecycle. diff --git a/package.json b/package.json index a2f1207..d217b10 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "build": "tsc", "start": "node dist/server.js", "dev": "ts-node src/server.ts", - "watch": "tsc --watch" + "watch": "tsc --watch", + "test": "npm run build && node --test test/*.test.cjs" }, "keywords": [], "author": "", diff --git a/src/routes/chat.ts b/src/routes/chat.ts index 5b8cb26..03382cc 100644 --- a/src/routes/chat.ts +++ b/src/routes/chat.ts @@ -86,7 +86,7 @@ export async function chatHandler(req: Request, res: Response): Promise { if (!pathParseResult.success) { res.status(400).json({ error: 'Invalid path parameter', - message: `Security level must be 'minnow' or 'shark'. ${pathParseResult.error.errors.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ')}` + message: `Security level must be 'minnow' or 'shark'. ${pathParseResult.error.issues.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ')}` }); return; } @@ -101,7 +101,7 @@ export async function chatHandler(req: Request, res: Response): Promise { if (!queryParseResult.success) { res.status(400).json({ error: 'Invalid query parameters', - message: queryParseResult.error.errors.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ') + message: queryParseResult.error.issues.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ') }); return; } @@ -114,7 +114,7 @@ export async function chatHandler(req: Request, res: Response): Promise { if (!bodyParseResult.success) { res.status(400).json({ error: 'Invalid request body', - message: bodyParseResult.error.errors.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ') || 'Request body must contain "messages" field (string, array, or JSON string representing an array)' + message: bodyParseResult.error.issues.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ') || 'Request body must contain "messages" field (string, array, or JSON string representing an array)' }); return; } diff --git a/test/chat.test.cjs b/test/chat.test.cjs new file mode 100644 index 0000000..a0a5d97 --- /dev/null +++ b/test/chat.test.cjs @@ -0,0 +1,114 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +// Every provider request is mocked; no API credentials or running LiteLLM required. +process.env.LITELLM_SERVER_URL = 'http://litellm.invalid'; +const { chatHandler } = require('../dist/routes/chat'); + +async function chat({ level = 'minnow', query = {}, body = { messages: 'Hello' } } = {}) { + const res = { + statusCode: 200, + status(code) { + this.statusCode = code; + return this; + }, + json(data) { + this.body = data; + return this; + }, + }; + await chatHandler({ params: { level }, query, body }, res); + return res; +} + +const invalidRequests = [ + ['invalid fish level', { level: 'secure' }, 'Invalid path parameter', /level:/], + ['unknown model', { query: { model: 'not-configured' } }, 'Invalid query parameters', /model: Model must be one of/], + ['invalid domain', { query: { domain: 'unknown' } }, 'Invalid query parameters', /domain:/], + ['unknown query key', { query: { extra: 'value' } }, 'Invalid query parameters', /extra/], + ['repeated model query', { query: { model: ['gpt-5-mini', 'gpt-5-mini'] } }, 'Invalid query parameters', /model:/], + ['empty message list', { body: { messages: [] } }, 'Invalid request body', /messages:/], + ['non-string content', { body: { messages: [{ role: 'user', content: 42 }] } }, 'Invalid request body', /messages\.0\.content:/], + ['missing message role', { body: { messages: [{ content: 'Hello' }] } }, 'Invalid request body', /messages\.0\.role:/], +]; + +for (const [name, request, error, message] of invalidRequests) { + test(`returns a structured 400 for ${name}`, async (t) => { + const provider = t.mock.method(globalThis, 'fetch', async () => { + throw new Error('Invalid requests must not reach LiteLLM'); + }); + const res = await chat(request); + assert.equal(res.statusCode, 400); + assert.equal(res.body.error, error); + assert.match(res.body.message, message); + assert.equal(provider.mock.callCount(), 0); + }); +} + +for (const domain of ['general', 'finance', 'medicine', 'vacation-rental', 'taxes']) { + for (const [fish, promptLevel] of [['minnow', 'insecure'], ['shark', 'secure']]) { + test(`preserves ${domain}/${fish} prompt and model routing`, async (t) => { + const completion = { choices: [{ message: { role: 'assistant', content: 'Hello back' } }] }; + const provider = t.mock.method(globalThis, 'fetch', async () => ({ + ok: true, + json: async () => completion, + })); + const messages = [{ role: 'user', content: 'Hello' }]; + const model = 'claude-3-5-haiku-latest'; + const res = await chat({ level: fish, query: { domain, model }, body: { messages } }); + assert.equal(res.statusCode, 200); + assert.deepEqual(res.body, completion); + assert.equal(provider.mock.callCount(), 1); + const [url, options] = provider.mock.calls[0].arguments; + assert.equal(url, 'http://litellm.invalid/v1/chat/completions'); + assert.equal(options.method, 'POST'); + assert.deepEqual(options.headers, { 'Content-Type': 'application/json' }); + const prompt = readFileSync(path.join(__dirname, '../src/domains', domain, `${promptLevel}.txt`), 'utf8').trim(); + assert.deepEqual(JSON.parse(options.body), { + model, + messages: [{ role: 'system', content: prompt }, ...messages], + }); + }); + } +} + +const message = { role: 'user', content: 'Hello' }; +for (const [name, messages] of [ + ['plain text', 'Hello'], + ['message array', [message]], + ['JSON array', JSON.stringify([message])], + ['JSON object', JSON.stringify(message)], +]) { + test(`normalizes ${name} with the default model and domain`, async (t) => { + const provider = t.mock.method(globalThis, 'fetch', async () => ({ + ok: true, + json: async () => ({ choices: [] }), + })); + const res = await chat({ body: { messages } }); + assert.equal(res.statusCode, 200); + assert.equal(provider.mock.callCount(), 1); + const sent = JSON.parse(provider.mock.calls[0].arguments[1].body); + assert.equal(sent.model, 'gpt-5-mini'); + assert.deepEqual(sent.messages, [ + { role: 'system', content: readFileSync(path.join(__dirname, '../src/domains/general/insecure.txt'), 'utf8').trim() }, + message, + ]); + }); +} + +test('preserves LiteLLM error status and response text', async (t) => { + const provider = t.mock.method(globalThis, 'fetch', async () => ({ + ok: false, + status: 503, + text: async () => 'Provider temporarily unavailable', + })); + const res = await chat(); + assert.equal(res.statusCode, 503); + assert.deepEqual(res.body, { + error: 'LiteLLM server error', + message: 'Provider temporarily unavailable', + }); + assert.equal(provider.mock.callCount(), 1); +}); diff --git a/tsconfig.json b/tsconfig.json index b432dc9..bf3117c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "target": "ES2020", - "module": "commonjs", + "module": "Node16", "lib": ["ES2020"], "outDir": "./dist", "rootDir": "./src", @@ -10,7 +10,7 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "resolveJsonModule": true, - "moduleResolution": "node" + "moduleResolution": "Node16" }, "include": ["src/**/*"], "exclude": ["node_modules", "dist"]