From b69b1a1bf5140bb056132ae7256ebbb7d448b11b Mon Sep 17 00:00:00 2001 From: solomon2773 Date: Thu, 27 Aug 2026 15:36:09 -0500 Subject: [PATCH] feat: add guarded AI ticket auto-resolution (#8) --- docs/guides/ai-support.md | 27 +++- .../migrations/z_12_ai_support/migration.sql | 2 + prisma/schema.prisma | 17 +- src/app/admin/settings/page.tsx | 12 ++ src/components/ticket-thread.tsx | 17 +- src/lib/actions/client.ts | 19 +++ src/lib/ai-policy.test.ts | 70 ++++++++ src/lib/ai-policy.ts | 47 ++++++ src/lib/i18n.ts | 2 + src/lib/services/ai.ts | 150 +++++++++++++++++- src/lib/settings.ts | 3 + tests/integration/ai-support.test.ts | 126 +++++++++++++++ 12 files changed, 468 insertions(+), 24 deletions(-) create mode 100644 prisma/migrations/z_12_ai_support/migration.sql create mode 100644 src/lib/ai-policy.test.ts create mode 100644 src/lib/ai-policy.ts create mode 100644 tests/integration/ai-support.test.ts diff --git a/docs/guides/ai-support.md b/docs/guides/ai-support.md index 7e00642..6d3a065 100644 --- a/docs/guides/ai-support.md +++ b/docs/guides/ai-support.md @@ -2,8 +2,9 @@ OpenHosting can draft ticket replies for your staff and classify incoming tickets, using **your own** API key. Nothing is sent to a model provider unless -you enable a provider and switch a feature on, and nothing an AI writes reaches -a customer without a person pressing send. +you enable a provider and switch a feature on. Reply drafts remain staff-reviewed; +the separate auto-resolution feature is an explicit opt-in with stricter source, +priority, and confidence gates. ## Setting it up @@ -25,6 +26,8 @@ a customer without a person pressing send. | Draft sign-off | — | Appended verbatim, e.g. `— The support team` | | Classify new tickets | off | Sets department and priority when a ticket is created | | Minimum triage confidence | 0.7 | Below this the customer's own choices are kept | +| Auto-resolve grounded tier-1 tickets | off | Posts and closes only a non-urgent answer grounded in published articles | +| Minimum auto-resolve confidence | 0.92 | Below this the ticket remains open for staff | ## Reply drafts @@ -66,11 +69,27 @@ Two deliberate limits: support form that fails because an AI is unavailable would be worse than no classification at all. +## Automatic tier-1 resolution + +Auto-resolution is off by default and is independent from reply drafts and +triage. It only runs for a newly opened ticket that is not high priority. The +model must return a structured answer above the configured confidence threshold, +mark the request as not requiring a person, and cite one or more article IDs +that still exist in the published knowledgebase. Invented, unpublished, or +missing citations reject the answer. + +Account changes, refunds, disputes, outages, security/abuse reports, uncertain +diagnoses, and any request needing an external action must stay with staff. A +qualifying answer is posted with an **Automated** badge, the ticket is closed in +the same database transaction, and `ticket.auto_resolved` records its confidence +and source article IDs in the audit log. If a staff/customer update races the +model response, the optimistic write fails and no automated message is posted. + ## Costs and privacy - You are billed by your provider for the tokens each feature uses. A reply - draft sends your published knowledgebase plus the ticket thread; a - classification sends the subject and first message only. + draft and auto-resolution send published knowledgebase content plus relevant + ticket text; classification sends the subject and first message. - Ticket content and knowledgebase articles are sent to the provider you configured, when a feature runs. Customer passwords, payment details and API keys are never part of a prompt. diff --git a/prisma/migrations/z_12_ai_support/migration.sql b/prisma/migrations/z_12_ai_support/migration.sql new file mode 100644 index 0000000..07e4ffa --- /dev/null +++ b/prisma/migrations/z_12_ai_support/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "TicketMessage" +ADD COLUMN "isAiGenerated" BOOLEAN NOT NULL DEFAULT false; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1b8a5f2..3299597 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -583,14 +583,15 @@ model Ticket { } model TicketMessage { - id String @id @default(cuid()) - ticketId String - ticket Ticket @relation(fields: [ticketId], references: [id], onDelete: Cascade) - userId String - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - message String - attachments TicketAttachment[] - createdAt DateTime @default(now()) + id String @id @default(cuid()) + ticketId String + ticket Ticket @relation(fields: [ticketId], references: [id], onDelete: Cascade) + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + message String + isAiGenerated Boolean @default(false) + attachments TicketAttachment[] + createdAt DateTime @default(now()) } // Database storage remains the default; S3-compatible storage keeps only a key. diff --git a/src/app/admin/settings/page.tsx b/src/app/admin/settings/page.tsx index 08d7477..3de1c3c 100644 --- a/src/app/admin/settings/page.tsx +++ b/src/app/admin/settings/page.tsx @@ -105,6 +105,18 @@ const SECTIONS: Array<{ type: "text", help: "Below this the customer's own choices are kept.", }, + { + key: "ai_auto_resolve", + label: "Auto-resolve grounded tier-1 tickets", + type: "checkbox", + help: "Opt-in. Only closes non-urgent new tickets when the answer cites published knowledgebase articles above the confidence threshold.", + }, + { + key: "ai_auto_resolve_min_confidence", + label: "Minimum auto-resolve confidence (0-1)", + type: "text", + help: "Defaults to 0.92. Lower thresholds increase the risk of an incorrect automatic answer.", + }, ], }, { diff --git a/src/components/ticket-thread.tsx b/src/components/ticket-thread.tsx index 7d456dd..9a2f181 100644 --- a/src/components/ticket-thread.tsx +++ b/src/components/ticket-thread.tsx @@ -29,8 +29,9 @@ export async function TicketThread({
{ticket.messages.map((message) => { - const isStaff = Boolean(message.user.roleId); - const isViewer = message.userId === viewerId; + const isAi = message.isAiGenerated; + const isStaff = Boolean(message.user.roleId) || isAi; + const isViewer = message.userId === viewerId && !isAi; return (

- {message.user.firstName} {message.user.lastName} - {isStaff && ( + {isAi + ? t("ticket.aiSupport") + : `${message.user.firstName} ${message.user.lastName}`} + {isAi ? ( + + {t("ticket.automated")} + + ) : isStaff ? ( {t("ticket.staff")} - )} + ) : null} {isViewer && !isStaff && ( {t("ticket.you")} )} diff --git a/src/lib/actions/client.ts b/src/lib/actions/client.ts index de1fdfa..cc8717a 100644 --- a/src/lib/actions/client.ts +++ b/src/lib/actions/client.ts @@ -251,6 +251,25 @@ export async function createTicket( } catch { // classification is best-effort } + // Optional tier-1 resolution is separately gated by published sources and a + // high confidence threshold. Provider failure never blocks ticket creation. + try { + const { autoResolveTicket } = await import("@/lib/services/ai"); + const resolution = await autoResolveTicket(ticket.id); + if (resolution) { + await audit("ticket.auto_resolved", { + userId: user.id, + targetType: "ticket", + targetId: ticket.id, + metadata: { + confidence: resolution.confidence, + sourceArticleIds: resolution.sourceArticleIds, + }, + }); + } + } catch { + // automatic resolution is best-effort; the open ticket remains available + } redirect(`/dashboard/tickets/${ticket.id}`); } diff --git a/src/lib/ai-policy.test.ts b/src/lib/ai-policy.test.ts new file mode 100644 index 0000000..008fc38 --- /dev/null +++ b/src/lib/ai-policy.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from "vitest"; +import { parseGroundedAutoResolution } from "@/lib/ai-policy"; + +const published = new Set(["kb-1", "kb-2"]); + +describe("AI auto-resolution policy", () => { + test("accepts a high-confidence answer grounded in published articles", () => { + expect( + parseGroundedAutoResolution( + { + answer: "Follow the documented reset steps, then sign in again.", + confidence: 0.96, + requiresHuman: false, + sourceArticleIds: ["kb-1"], + }, + 0.92, + published, + ), + ).toEqual({ + answer: "Follow the documented reset steps, then sign in again.", + confidence: 0.96, + sourceArticleIds: ["kb-1"], + }); + }); + + test("rejects escalation, low confidence, and uncited answers", () => { + const base = { + answer: "This answer is long enough to pass the content length check.", + confidence: 0.96, + requiresHuman: false, + sourceArticleIds: ["kb-1"], + }; + expect( + parseGroundedAutoResolution( + { ...base, requiresHuman: true }, + 0.92, + published, + ), + ).toBeNull(); + expect( + parseGroundedAutoResolution( + { ...base, confidence: 0.91 }, + 0.92, + published, + ), + ).toBeNull(); + expect( + parseGroundedAutoResolution( + { ...base, sourceArticleIds: [] }, + 0.92, + published, + ), + ).toBeNull(); + }); + + test("rejects references to unpublished or invented articles", () => { + expect( + parseGroundedAutoResolution( + { + answer: "This answer is long enough but cites an unknown source.", + confidence: 0.99, + requiresHuman: false, + sourceArticleIds: ["kb-missing"], + }, + 0.92, + published, + ), + ).toBeNull(); + }); +}); diff --git a/src/lib/ai-policy.ts b/src/lib/ai-policy.ts new file mode 100644 index 0000000..eccc6bc --- /dev/null +++ b/src/lib/ai-policy.ts @@ -0,0 +1,47 @@ +export type GroundedAutoResolution = { + answer: string; + confidence: number; + sourceArticleIds: string[]; +}; + +export function parseGroundedAutoResolution( + value: unknown, + minimumConfidence: number, + publishedArticleIds: ReadonlySet, +): GroundedAutoResolution | null { + if (!value || typeof value !== "object") return null; + const candidate = value as Record; + if (candidate.requiresHuman !== false) return null; + if ( + typeof candidate.confidence !== "number" || + !Number.isFinite(candidate.confidence) || + candidate.confidence < minimumConfidence || + candidate.confidence > 1 + ) { + return null; + } + if (typeof candidate.answer !== "string") return null; + const answer = candidate.answer.trim(); + if (answer.length < 20 || answer.length > 4_000) return null; + if (!Array.isArray(candidate.sourceArticleIds)) return null; + const sourceArticleIds = [ + ...new Set( + candidate.sourceArticleIds.filter( + (id): id is string => + typeof id === "string" && publishedArticleIds.has(id), + ), + ), + ]; + if ( + sourceArticleIds.length === 0 || + sourceArticleIds.length !== candidate.sourceArticleIds.length + ) { + return null; + } + + return { + answer, + confidence: candidate.confidence, + sourceArticleIds, + }; +} diff --git a/src/lib/i18n.ts b/src/lib/i18n.ts index 4924f26..a56dc83 100644 --- a/src/lib/i18n.ts +++ b/src/lib/i18n.ts @@ -118,6 +118,8 @@ const en = { "ticket.closed": "This ticket is closed.", "ticket.attachments": "Attachments (up to 3 files, 5 MB each)", "ticket.staff": "Staff", + "ticket.aiSupport": "AI support", + "ticket.automated": "Automated", "ticket.you": "(you)", "account.title": "Account settings", diff --git a/src/lib/services/ai.ts b/src/lib/services/ai.ts index c0dad2a..b349667 100644 --- a/src/lib/services/ai.ts +++ b/src/lib/services/ai.ts @@ -4,6 +4,10 @@ import { getAiDriver } from "@/lib/extensions/registry"; import { extensionConfig } from "@/lib/extensions/types"; import type { AiMessage } from "@/lib/extensions/types"; import { getSetting, getSettings } from "@/lib/settings"; +import { + parseGroundedAutoResolution, + type GroundedAutoResolution, +} from "@/lib/ai-policy"; /** * AI support features. @@ -47,28 +51,45 @@ export async function aiReplyDraftsEnabled(): Promise { return aiConfigured(); } +type KnowledgebaseArticle = { id: string; title: string; body: string }; + /** Published articles only — unpublished drafts are not company policy yet. */ -async function knowledgebaseContext(limit = 40): Promise { - const articles = await db.kbArticle.findMany({ +async function publishedKnowledgebaseArticles( + limit = 40, +): Promise { + return db.kbArticle.findMany({ where: { published: true }, - select: { title: true, body: true }, + select: { id: true, title: true, body: true }, orderBy: { updatedAt: "desc" }, take: limit, }); +} + +function formatKnowledgebase(articles: KnowledgebaseArticle[]): string { if (articles.length === 0) return "(The knowledgebase is empty.)"; return articles - .map((a) => `## ${a.title}\n${a.body.slice(0, 4000)}`) + .map( + (article) => + `## ${article.title} [article_id=${article.id}]\n${article.body.slice(0, 4000)}`, + ) .join("\n\n"); } +async function knowledgebaseContext(limit = 40): Promise { + return formatKnowledgebase(await publishedKnowledgebaseArticles(limit)); +} + function threadMessages( - messages: { message: string; userId: string }[], + messages: { message: string; userId: string; isAiGenerated?: boolean }[], customerId: string, ): AiMessage[] { // The customer speaks as "user"; staff replies are the assistant's own past // turns, which is exactly the shape a chat model expects. return messages.map((m) => ({ - role: m.userId === customerId ? ("user" as const) : ("assistant" as const), + role: + m.userId === customerId && !m.isAiGenerated + ? ("user" as const) + : ("assistant" as const), content: m.message, })); } @@ -86,7 +107,10 @@ export async function draftTicketReply(ticketId: string): Promise where: { id: ticketId }, include: { user: { select: { id: true, firstName: true } }, - messages: { orderBy: { createdAt: "asc" }, select: { message: true, userId: true } }, + messages: { + orderBy: { createdAt: "asc" }, + select: { message: true, userId: true, isAiGenerated: true }, + }, }, }); if (!ticket) return null; @@ -188,3 +212,115 @@ export async function triageTicket(ticketId: string): Promise { if (result.confidence < threshold) return null; return result as Triage; } + +export async function autoResolveTicket( + ticketId: string, +): Promise { + if ((await getSetting("ai_auto_resolve")) !== "true") return null; + const provider = await activeProvider(); + if (!provider?.driver.completeJson) return null; + + const [ticket, articles, staff] = await Promise.all([ + db.ticket.findUnique({ + where: { id: ticketId }, + include: { + messages: { + orderBy: { createdAt: "asc" }, + take: 1, + select: { message: true }, + }, + }, + }), + publishedKnowledgebaseArticles(), + db.user.findFirst({ + where: { roleId: { not: null } }, + orderBy: { createdAt: "asc" }, + select: { id: true }, + }), + ]); + if (!ticket || ticket.status !== "OPEN" || ticket.priority === "HIGH") { + return null; + } + if (!staff || articles.length === 0) return null; + + const configuredThreshold = Number( + await getSetting("ai_auto_resolve_min_confidence"), + ); + const threshold = + Number.isFinite(configuredThreshold) && + configuredThreshold >= 0 && + configuredThreshold <= 1 + ? configuredThreshold + : 0.92; + const schema = { + type: "object", + properties: { + answer: { type: "string" }, + confidence: { type: "number" }, + requiresHuman: { type: "boolean" }, + sourceArticleIds: { type: "array", items: { type: "string" } }, + }, + required: [ + "answer", + "confidence", + "requiresHuman", + "sourceArticleIds", + ], + additionalProperties: false, + }; + const system = [ + "Decide whether a newly opened hosting support ticket can be fully answered from the published knowledgebase.", + "Set requiresHuman=true for account changes, refunds, billing disputes, outages, security issues, abuse, uncertain diagnoses, missing facts, or any request that needs an external action.", + "Only set requiresHuman=false when the answer is static tier-1 guidance stated directly in the articles.", + "When answering, do not claim that any action was performed. Match the customer's language.", + "sourceArticleIds must contain every article used and may contain only the article_id values shown below.", + "confidence is 0 to 1. Be conservative because a qualifying answer will be posted automatically and close the ticket.", + "Answer as JSON only.", + "", + "# Published knowledgebase", + formatKnowledgebase(articles), + ].join("\n"); + const raw = await provider.driver.completeJson(provider.config, { + system, + maxTokens: 4_000, + schema, + messages: [ + { + role: "user", + content: `Subject: ${ticket.subject}\nDepartment: ${ticket.department}\nPriority: ${ticket.priority}\n\n${ticket.messages[0]?.message ?? ""}`.slice( + 0, + 8_000, + ), + }, + ], + }); + const resolution = parseGroundedAutoResolution( + raw, + threshold, + new Set(articles.map((article) => article.id)), + ); + if (!resolution) return null; + + const applied = await db.$transaction(async (tx) => { + const closed = await tx.ticket.updateMany({ + where: { + id: ticket.id, + status: "OPEN", + updatedAt: ticket.updatedAt, + }, + data: { status: "CLOSED" }, + }); + if (closed.count !== 1) return false; + await tx.ticketMessage.create({ + data: { + ticketId: ticket.id, + userId: staff.id, + message: resolution.answer, + isAiGenerated: true, + }, + }); + return true; + }); + + return applied ? resolution : null; +} diff --git a/src/lib/settings.ts b/src/lib/settings.ts index 38e8553..c9a7821 100644 --- a/src/lib/settings.ts +++ b/src/lib/settings.ts @@ -37,6 +37,9 @@ export const SETTING_DEFAULTS: Record = { ai_auto_triage: "false", // below this confidence the customer's own choices are left alone (0-1) ai_triage_min_confidence: "0.7", + // close a new tier-1 ticket only when the model cites published KB articles + ai_auto_resolve: "false", + ai_auto_resolve_min_confidence: "0.92", // ── Fraud prevention ── fraud_review_all: "false", // external risk score (0-99) at or above which orders go to manual review diff --git a/tests/integration/ai-support.test.ts b/tests/integration/ai-support.test.ts new file mode 100644 index 0000000..a182c71 --- /dev/null +++ b/tests/integration/ai-support.test.ts @@ -0,0 +1,126 @@ +import { afterAll, beforeEach, expect, test, vi } from "vitest"; +import { db } from "@/lib/db"; +import { autoResolveTicket } from "@/lib/services/ai"; + +const aiMocks = vi.hoisted(() => ({ + completeJson: vi.fn(), +})); + +vi.mock("@/lib/extensions/registry", () => ({ + getAiDriver: () => ({ + slug: "test-ai", + name: "Test AI", + configFields: [], + complete: vi.fn(), + completeJson: aiMocks.completeJson, + }), +})); + +beforeEach(async () => { + await db.user.deleteMany(); + await db.role.deleteMany(); + await db.extension.deleteMany(); + await db.kbArticle.deleteMany(); + await db.setting.deleteMany(); + vi.clearAllMocks(); +}); + +afterAll(async () => { + await db.$disconnect(); +}); + +async function setupAutoResolution(priority: "LOW" | "MEDIUM" | "HIGH") { + const role = await db.role.create({ + data: { name: "AI test staff", permissions: ["tickets"] }, + }); + await db.user.create({ + data: { + email: "ai-staff@example.test", + password: "not-used", + firstName: "Support", + lastName: "Team", + roleId: role.id, + }, + }); + const customer = await db.user.create({ + data: { + email: "ai-customer@example.test", + password: "not-used", + firstName: "Customer", + lastName: "Test", + }, + }); + const article = await db.kbArticle.create({ + data: { + title: "Reset a password", + slug: "reset-a-password", + body: "Use the reset form and follow the emailed link.", + published: true, + }, + }); + await db.extension.create({ + data: { + type: "AI", + slug: "test-ai", + name: "Test AI", + enabled: true, + }, + }); + await db.setting.createMany({ + data: [ + { key: "ai_auto_resolve", value: "true" }, + { key: "ai_auto_resolve_min_confidence", value: "0.92" }, + ], + }); + const ticket = await db.ticket.create({ + data: { + userId: customer.id, + subject: "How do I reset my password?", + priority, + messages: { + create: { + userId: customer.id, + message: "Please tell me the documented password reset steps.", + }, + }, + }, + }); + return { article, ticket }; +} + +test("posts and closes a grounded high-confidence tier-1 answer", async () => { + const { article, ticket } = await setupAutoResolution("LOW"); + aiMocks.completeJson.mockResolvedValue({ + answer: "Use the reset form and follow the link sent to your email address.", + confidence: 0.98, + requiresHuman: false, + sourceArticleIds: [article.id], + }); + + expect(await autoResolveTicket(ticket.id)).toEqual( + expect.objectContaining({ confidence: 0.98 }), + ); + + const updated = await db.ticket.findUniqueOrThrow({ + where: { id: ticket.id }, + include: { messages: { orderBy: { createdAt: "asc" } } }, + }); + expect(updated.status).toBe("CLOSED"); + expect(updated.messages).toHaveLength(2); + expect(updated.messages[1]).toEqual( + expect.objectContaining({ + isAiGenerated: true, + message: "Use the reset form and follow the link sent to your email address.", + }), + ); +}); + +test("never sends a high-priority ticket to the auto-resolution model", async () => { + const { ticket } = await setupAutoResolution("HIGH"); + + expect(await autoResolveTicket(ticket.id)).toBeNull(); + expect(aiMocks.completeJson).not.toHaveBeenCalled(); + expect( + await db.ticket.findUniqueOrThrow({ where: { id: ticket.id } }), + ).toEqual(expect.objectContaining({ status: "OPEN" })); +});