From 12ea6853c9152dcad7efa95b8537388850bba6b8 Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Tue, 15 Sep 2026 13:21:21 +0000 Subject: [PATCH 1/5] FE-1666: Expose a browser host for creating experiments (#9676) --- .changeset/browser-experiment-host.md | 5 + .changeset/typed-experiment-host.md | 5 + libs/@hashintel/petrinaut-core/src/ai.test.ts | 24 + libs/@hashintel/petrinaut-core/src/ai.ts | 11 +- .../petrinaut-core/src/experiments.ts | 8 + .../src/experiments/host.test.ts | 106 ++++ .../petrinaut-core/src/experiments/host.ts | 165 ++++++ libs/@hashintel/petrinaut-core/src/index.ts | 9 + libs/@hashintel/petrinaut/docs/experiments.md | 14 + .../src/react/experiment-host/context.ts | 15 + .../prepare-experiment.test.ts | 98 ++++ .../experiment-host/prepare-experiment.ts | 129 +++++ .../src/react/experiment-host/provider.tsx | 124 ++++ .../experiment-host/run-experiment.test.ts | 547 ++++++++++++++++++ .../react/experiment-host/run-experiment.ts | 372 ++++++++++++ .../src/react/experiments/context.ts | 20 +- .../src/react/experiments/provider.test.tsx | 126 ++++ .../src/react/experiments/provider.tsx | 112 ++-- .../react/experiments/sweep-optimization.ts | 123 ++++ .../react/experiments/sweep-session.test.ts | 43 +- .../src/react/experiments/sweep-session.ts | 12 +- libs/@hashintel/petrinaut/src/react/index.ts | 1 + .../src/react/optimizations/context.ts | 2 + .../src/react/optimizations/provider.tsx | 1 + .../create-sweep-trial-evaluator.test.ts | 22 + .../provider/create-sweep-trial-evaluator.ts | 4 +- .../src/react/petrinaut-provider.tsx | 7 +- .../experiments/experiment-results.tsx | 12 +- .../experiments/sweep-optimizer.test.ts | 8 +- .../experiments/sweep-optimizer.ts | 124 +--- .../diagrams/ai-experiments-architecture.d2 | 13 + .../content/diagrams/ai-experiments-flow.d2 | 14 + .../experiments/ai-created-experiments.mdx | 108 ++++ 33 files changed, 2215 insertions(+), 169 deletions(-) create mode 100644 .changeset/browser-experiment-host.md create mode 100644 .changeset/typed-experiment-host.md create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/host.test.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/host.ts create mode 100644 libs/@hashintel/petrinaut/src/react/experiment-host/context.ts create mode 100644 libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.test.ts create mode 100644 libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.ts create mode 100644 libs/@hashintel/petrinaut/src/react/experiment-host/provider.tsx create mode 100644 libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts create mode 100644 libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts create mode 100644 libs/@hashintel/petrinaut/src/react/experiments/sweep-optimization.ts create mode 100644 libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-architecture.d2 create mode 100644 libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-flow.d2 create mode 100644 libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx diff --git a/.changeset/browser-experiment-host.md b/.changeset/browser-experiment-host.md new file mode 100644 index 000000000000..a773a06c3089 --- /dev/null +++ b/.changeset/browser-experiment-host.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Expose `ExperimentHostContext.runExperiment` with validation, progress callbacks, cancellation, and captured results for up to 100,000 runs. diff --git a/.changeset/typed-experiment-host.md b/.changeset/typed-experiment-host.md new file mode 100644 index 000000000000..d5f656422bd6 --- /dev/null +++ b/.changeset/typed-experiment-host.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut-core": patch +--- + +Add validated request and result schemas and a `runExperiment` host interface for simulations and optimizations with up to 100,000 final runs. diff --git a/libs/@hashintel/petrinaut-core/src/ai.test.ts b/libs/@hashintel/petrinaut-core/src/ai.test.ts index 06f9ac185f07..9c4ef2010aee 100644 --- a/libs/@hashintel/petrinaut-core/src/ai.test.ts +++ b/libs/@hashintel/petrinaut-core/src/ai.test.ts @@ -1,3 +1,4 @@ +import { rolldown } from "rolldown"; import { describe, expect, test } from "vitest"; import { z } from "zod"; @@ -27,6 +28,29 @@ const createInstance = () => }); describe("Petrinaut AI core exports", () => { + test("keeps execution runtimes out of the AI entry before tree shaking", async () => { + const build = await rolldown({ + input: "src/ai.ts", + external: /^[^./]/u, + treeshake: false, + logLevel: "silent", + }); + try { + const { output } = await build.generate({ format: "esm" }); + const modules = output.flatMap((chunk) => + chunk.type === "chunk" ? Object.keys(chunk.modules) : [], + ); + expect(modules.length).toBeGreaterThan(0); + expect( + modules.filter((moduleId) => + /\/(?:hir|simulation\/monte-carlo\/runtime)\//u.test(moduleId), + ), + ).toEqual([]); + } finally { + await build.close(); + } + }); + test("tool metadata stays aligned with input schemas and has no execute", () => { expect(Object.keys(petrinautAiTools).sort()).toEqual( Object.keys(petrinautAiToolInputSchemas).sort(), diff --git a/libs/@hashintel/petrinaut-core/src/ai.ts b/libs/@hashintel/petrinaut-core/src/ai.ts index 867ba5871c96..6fbc5884c9e5 100644 --- a/libs/@hashintel/petrinaut-core/src/ai.ts +++ b/libs/@hashintel/petrinaut-core/src/ai.ts @@ -13,6 +13,15 @@ import { typedKeys } from "./lib/typed-entries"; import type { Petrinaut } from "./instance"; +export { + petrinautExperimentRequestSchema, + petrinautExperimentResultSchema, + type PetrinautExperimentRequest, + type PetrinautExperimentProgress, + type PetrinautExperimentResult, + type PetrinautExperimentHost, +} from "./experiments/host"; + export { arcEndpointSchema, colorSchema, @@ -141,7 +150,7 @@ const getLatestNetDefinitionToolInputSchema = z const getNetCompilationErrorsToolInputSchema = z .strictObject({}) .describe( - "Validate the current Petrinaut net snapshot and return its TypeScript and HIR diagnostics.", + "Validate the current Petrinaut net snapshot and return its TypeScript and HIR diagnostics. Saved scenario and metric compilation is checked separately when creating an experiment.", ); export const setNetTitleToolInputSchema = z diff --git a/libs/@hashintel/petrinaut-core/src/experiments.ts b/libs/@hashintel/petrinaut-core/src/experiments.ts index 8463b1f4f2fb..52062a18387e 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments.ts @@ -8,6 +8,14 @@ * deferred `load`: a heavy backend is imported the first time selection * reaches it and stays out of bundles that never use it. */ +export { + petrinautExperimentRequestSchema, + petrinautExperimentResultSchema, + type PetrinautExperimentRequest, + type PetrinautExperimentProgress, + type PetrinautExperimentResult, + type PetrinautExperimentHost, +} from "./experiments/host"; export type { ExperimentAssessment, ExperimentBlocker, diff --git a/libs/@hashintel/petrinaut-core/src/experiments/host.test.ts b/libs/@hashintel/petrinaut-core/src/experiments/host.test.ts new file mode 100644 index 000000000000..e8ba1fc1d47e --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/host.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { + petrinautExperimentRequestSchema, + petrinautExperimentResultSchema, +} from "../experiments"; + +const simulation = { + name: "Baseline", + scenarioId: "baseline", + scenarioParameterValues: { enabled: { mode: "fixed", value: true } }, + runCount: 10, + seed: 42, + dt: 0.1, + maxTime: 10, + metricIds: ["cost"], + execution: { mode: "simulate" }, +}; + +const optimization = { + ...simulation, + scenarioParameterValues: { capacity: { mode: "range", min: 1, max: 10 } }, + execution: { + mode: "optimize", + objectiveMetricId: "cost", + direction: "minimize", + steps: 5, + runsPerStep: 5, + }, +}; + +describe("experiment host schemas", () => { + it("accepts typed fixed values and bounded optimization", () => { + expect(petrinautExperimentRequestSchema.parse(simulation)).toEqual( + simulation, + ); + expect(petrinautExperimentRequestSchema.parse(optimization)).toEqual( + optimization, + ); + }); + + it.each([ + { + ...simulation, + scenarioParameterValues: optimization.scenarioParameterValues, + }, + { ...optimization, scenarioParameterValues: {} }, + { + ...optimization, + scenarioParameterValues: { capacity: { mode: "range", min: 10, max: 1 } }, + }, + { + ...optimization, + execution: { ...optimization.execution, runsPerStep: 11 }, + }, + { + ...optimization, + execution: { ...optimization.execution, objectiveMetricId: "missing" }, + }, + { ...simulation, metricIds: ["cost", "cost"] }, + { ...simulation, runCount: 0 }, + { ...simulation, runCount: 100_001 }, + { ...simulation, dt: 1e-10 }, + { ...simulation, dt: 11 }, + { + ...simulation, + scenarioParameterValues: { enabled: { mode: "fixed", value: "false" } }, + }, + ])("rejects invalid bounds or ambiguous requests", (request) => { + expect(petrinautExperimentRequestSchema.safeParse(request).success).toBe( + false, + ); + }); + + it.each([simulation, optimization])( + "accepts a 100,000-run final budget", + (request) => { + expect( + petrinautExperimentRequestSchema.parse({ + ...request, + runCount: 100_000, + }).runCount, + ).toBe(100_000); + }, + ); + + it("exposes an object JSON schema for tool providers", () => { + const schema = z.toJSONSchema(petrinautExperimentRequestSchema); + expect(schema.type).toBe("object"); + expect(schema.properties).toHaveProperty("execution"); + expect(schema.additionalProperties).toBe(false); + }); + + it("rejects non-finite metric results", () => { + expect( + petrinautExperimentResultSchema.safeParse({ + status: "complete", + experimentId: "experiment", + name: "Baseline", + runsCompleted: 10, + metrics: [{ id: "cost", label: "Cost", value: NaN }], + }).success, + ).toBe(false); + }); +}); diff --git a/libs/@hashintel/petrinaut-core/src/experiments/host.ts b/libs/@hashintel/petrinaut-core/src/experiments/host.ts new file mode 100644 index 000000000000..21aedfe5aef2 --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/host.ts @@ -0,0 +1,165 @@ +import { z } from "zod"; + +import type { AbortSignalLike } from "../environment"; + +const parameterValueSchema = z.union([z.number().finite(), z.boolean()]); +const parameterInputSchema = z.discriminatedUnion("mode", [ + z.strictObject({ mode: z.literal("fixed"), value: parameterValueSchema }), + z.strictObject({ + mode: z.literal("range"), + min: z.number().finite(), + max: z.number().finite(), + }), +]); + +export const petrinautExperimentRequestSchema = z + .strictObject({ + name: z.string().min(1).max(120), + scenarioId: z.string().min(1), + scenarioParameterValues: z + .record(z.string().min(1), parameterInputSchema) + .describe( + "Scenario parameter identifiers mapped to typed fixed values or optimization ranges. Omitted parameters use their saved defaults.", + ), + runCount: z.number().int().min(1).max(100_000), + seed: z.number().int().min(0).max(4294967295), + dt: z.number().positive().max(1_000_000), + maxTime: z.number().positive().max(1_000_000), + metricIds: z.array(z.string().min(1)).min(1).max(20), + execution: z.discriminatedUnion("mode", [ + z.strictObject({ mode: z.literal("simulate") }), + z.strictObject({ + mode: z.literal("optimize"), + objectiveMetricId: z.string().min(1), + direction: z.enum(["minimize", "maximize"]), + steps: z.number().int().min(1).max(100), + runsPerStep: z.number().int().min(1).max(1000), + }), + ]), + }) + .superRefine((request, context) => { + if ( + request.dt > request.maxTime || + request.maxTime / request.dt > 1_000_000 + ) { + context.addIssue({ + code: "custom", + path: ["dt"], + message: + "Time step must fit the duration and keep each run within 1,000,000 steps.", + }); + } + if (new Set(request.metricIds).size !== request.metricIds.length) { + context.addIssue({ + code: "custom", + path: ["metricIds"], + message: "Metric IDs must be unique.", + }); + } + let rangeCount = 0; + for (const [parameterId, parameter] of Object.entries( + request.scenarioParameterValues, + )) { + if (parameter.mode === "range") { + rangeCount += 1; + if (parameter.min >= parameter.max) { + context.addIssue({ + code: "custom", + path: ["scenarioParameterValues", parameterId], + message: "Range minimum must be less than its maximum.", + }); + } + } + } + if (request.execution.mode === "simulate" && rangeCount > 0) { + context.addIssue({ + code: "custom", + path: ["scenarioParameterValues"], + message: "Simulation requires fixed parameter values.", + }); + } + if (request.execution.mode === "optimize") { + if (rangeCount === 0) { + context.addIssue({ + code: "custom", + path: ["scenarioParameterValues"], + message: "Optimization requires at least one parameter range.", + }); + } + if (!request.metricIds.includes(request.execution.objectiveMetricId)) { + context.addIssue({ + code: "custom", + path: ["execution", "objectiveMetricId"], + message: "The objective must be included in metricIds.", + }); + } + if (request.execution.runsPerStep > request.runCount) { + context.addIssue({ + code: "custom", + path: ["execution", "runsPerStep"], + message: "Runs per step must not exceed runCount.", + }); + } + if (request.execution.steps * request.execution.runsPerStep > 10_000) { + context.addIssue({ + code: "custom", + path: ["execution"], + message: "Optimization must request at most 10,000 search runs.", + }); + } + } + }) + .describe( + "Run an experiment in the connected Petrinaut host using a saved scenario and saved metrics. Returns the final result after bounded simulation or optimization finishes. Read the net first to obtain scenario and metric IDs and scenario parameter identifiers. The host validates and compiles a frozen model before running and reports progress through onProgress.", + ); + +export type PetrinautExperimentRequest = z.infer< + typeof petrinautExperimentRequestSchema +>; + +export type PetrinautExperimentProgress = { + experimentId: string; + name: string; + phase: "validating" | "running" | "optimizing" | "refining"; + runsCompleted: number; + runsTarget: number; + step?: number; + steps?: number; +}; + +export const petrinautExperimentResultSchema = z.strictObject({ + status: z.enum(["complete", "cancelled", "error"]), + experimentId: z.string().nullable(), + name: z.string(), + message: z.string().optional(), + runsCompleted: z.number().int().nonnegative(), + metrics: z.array( + z.strictObject({ + id: z.string(), + label: z.string(), + value: z.number().finite().nullable(), + }), + ), + optimization: z + .strictObject({ + parameters: z.record(z.string(), parameterValueSchema), + objectiveValue: z.number().finite().nullable(), + stepsCompleted: z.number().int().nonnegative(), + }) + .optional(), +}); + +export type PetrinautExperimentResult = z.infer< + typeof petrinautExperimentResultSchema +>; + +export type PetrinautExperimentHost = { + /** Reports updates through onProgress and resolves with a captured terminal result. */ + runExperiment: ( + request: PetrinautExperimentRequest, + options?: { + signal?: AbortSignalLike; + onProgress?: (progress: PetrinautExperimentProgress) => void; + }, + ) => Promise; +}; diff --git a/libs/@hashintel/petrinaut-core/src/index.ts b/libs/@hashintel/petrinaut-core/src/index.ts index 801431f154c0..03f5ae7b12fe 100644 --- a/libs/@hashintel/petrinaut-core/src/index.ts +++ b/libs/@hashintel/petrinaut-core/src/index.ts @@ -235,6 +235,15 @@ export type { PetrinautDocName, } from "./ai"; +export { + petrinautExperimentRequestSchema, + petrinautExperimentResultSchema, + type PetrinautExperimentHost, + type PetrinautExperimentProgress, + type PetrinautExperimentRequest, + type PetrinautExperimentResult, +} from "./experiments"; + // --- Simulation --- export { PETRINAUT_DEFAULT_SEED, diff --git a/libs/@hashintel/petrinaut/docs/experiments.md b/libs/@hashintel/petrinaut/docs/experiments.md index b474d834695e..c2e53c562f1a 100644 --- a/libs/@hashintel/petrinaut/docs/experiments.md +++ b/libs/@hashintel/petrinaut/docs/experiments.md @@ -11,6 +11,20 @@ Experiments live under the **Simulate** [global mode](drawing-a-net.md#global-mo 3. Fill in the configuration (see below). 4. Click **Run** -- **Create sweep** when a value is swept, **Optimize** when the in-browser optimizer will search it. The button reads **Starting** (or **Creating**) while the experiment starts. +### Experiments started by an integration + +An application embedding Petrinaut can start an experiment with a saved +scenario and metrics, including a search over numeric parameter ranges. +These experiments appear in the Experiments list. +Each request supports up to 100,000 simulation runs, or 100,000 final runs +at the best parameter values after optimization. + +While the request runs, parameter changes and removal are locked. You can +inspect the charts. Controls become available when the request finishes. +Later exploration leaves the result returned to the application unchanged. +An optimization finishes after its search and the final runs at its best +parameter values. Keep the browser page open until the request finishes. + ### Configuration | Setting | Default | Notes | diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/context.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/context.ts new file mode 100644 index 000000000000..8bc0b26f2aa9 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/context.ts @@ -0,0 +1,15 @@ +import { createContext } from "react"; + +import type { PetrinautExperimentHost } from "@hashintel/petrinaut-core/experiments"; + +export const ExperimentHostContext = createContext({ + runExperiment: (request) => + Promise.resolve({ + status: "error", + experimentId: null, + name: request.name, + message: "Experiments are unavailable in this host", + runsCompleted: 0, + metrics: [], + }), +}); diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.test.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.test.ts new file mode 100644 index 000000000000..0c1174ff719f --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.test.ts @@ -0,0 +1,98 @@ +import { expect, it } from "vitest"; + +import { sirModel } from "@hashintel/petrinaut-core/examples"; + +import { prepareExperiment } from "./prepare-experiment"; + +it.each(["simulate", "optimize"] as const)( + "retains selected metric distributions for %s requests", + (mode) => { + const { input } = prepareExperiment( + { + name: "SIR experiment", + scenarioId: "scenario__seasonal_flu", + scenarioParameterValues: + mode === "optimize" + ? { infected_ratio: { mode: "range", min: 0.01, max: 0.1 } } + : {}, + runCount: 100_000, + seed: 42, + dt: 1, + maxTime: 10, + metricIds: ["metric__infected_fraction"], + execution: + mode === "optimize" + ? { + mode, + objectiveMetricId: "metric__infected_fraction", + direction: "minimize", + steps: 3, + runsPerStep: 8, + } + : { mode }, + }, + sirModel.petriNetDefinition, + sirModel.title, + ); + expect(input.runCount).toBe(100_000); + expect(input.metricSpecs).toEqual([ + { + id: "metric__infected_fraction", + label: sirModel.petriNetDefinition.metrics![0]!.name, + kind: "expression", + code: sirModel.petriNetDefinition.metrics![0]!.code, + sampleRuns: "all", + runOutput: { type: "distribution" }, + }, + ]); + }, +); + +it("selects one scenario and objective from a model with several scenarios and metrics", () => { + const definition = structuredClone(sirModel.petriNetDefinition); + definition.metrics = [ + ...(definition.metrics ?? []), + { + id: "population", + name: "Population", + code: "return state.places.Susceptible.count;", + }, + ]; + const { input, optimization } = prepareExperiment( + { + name: "SIR optimization", + scenarioId: "scenario__seasonal_flu", + scenarioParameterValues: { + population: { mode: "fixed", value: 100 }, + infected_ratio: { mode: "range", min: 0.01, max: 0.1 }, + }, + runCount: 25, + seed: 42, + dt: 1, + maxTime: 10, + metricIds: ["metric__infected_fraction", "population"], + execution: { + mode: "optimize", + objectiveMetricId: "metric__infected_fraction", + direction: "minimize", + steps: 3, + runsPerStep: 8, + }, + }, + definition, + sirModel.title, + ); + + expect(input.metricSpecs.map((metric) => metric.id)).toEqual([ + "metric__infected_fraction", + "population", + ]); + expect( + optimization?.model.definition.scenarios?.map((scenario) => scenario.id), + ).toEqual(["scenario__seasonal_flu"]); + expect( + optimization?.model.definition.metrics?.map((metric) => metric.id), + ).toEqual(["metric__infected_fraction"]); + expect(definition.scenarios).toHaveLength(4); + expect(definition.metrics).toHaveLength(2); +}); diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.ts new file mode 100644 index 000000000000..cc657a227b8b --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/prepare-experiment.ts @@ -0,0 +1,129 @@ +import { petrinautExperimentRequestSchema } from "@hashintel/petrinaut-core/experiments"; + +import { + buildParameterAxis, + type ExperimentParameterAxis, +} from "../experiments/parameter-grid"; +import { buildSweepOptimizationInput } from "../experiments/sweep-optimization"; + +import type { CreateExperimentInput } from "../experiments/context"; +import type { SDCPN } from "@hashintel/petrinaut-core"; +import type { PetrinautExperimentRequest } from "@hashintel/petrinaut-core/experiments"; +import type { PetrinautOptimizationInput } from "@hashintel/petrinaut-core/optimization"; + +export const prepareExperiment = ( + rawRequest: PetrinautExperimentRequest, + definition: SDCPN, + title: string, +): { + request: PetrinautExperimentRequest; + input: CreateExperimentInput; + fixedValues: Record; + optimization: PetrinautOptimizationInput | null; +} => { + const request = petrinautExperimentRequestSchema.parse(rawRequest); + const scenario = definition.scenarios?.find( + (candidate) => candidate.id === request.scenarioId, + ); + if (!scenario) { + throw new Error(`Scenario "${request.scenarioId}" does not exist`); + } + for (const identifier of Object.keys(request.scenarioParameterValues)) { + if ( + !scenario.scenarioParameters.some( + (parameter) => parameter.identifier === identifier, + ) + ) { + throw new Error(`Scenario parameter "${identifier}" does not exist`); + } + } + const metricFor = (id: string) => { + const metric = definition.metrics?.find((candidate) => candidate.id === id); + if (!metric) { + throw new Error(`Metric "${id}" does not exist`); + } + return metric; + }; + const metricSpecs: CreateExperimentInput["metricSpecs"] = + request.metricIds.map((id) => { + const metric = metricFor(id); + return { + id: metric.id, + label: metric.name, + kind: "expression", + code: metric.code, + sampleRuns: "all", + runOutput: { type: "distribution" }, + }; + }); + const parameterAxes: ExperimentParameterAxis[] = []; + const fixedScenarioValues: Record = {}; + const scenarioParameterValues: CreateExperimentInput["scenarioParameterValues"] = + {}; + const fixedValues: Record = {}; + for (const parameter of scenario.scenarioParameters) { + const selected = request.scenarioParameterValues[parameter.identifier]; + if (selected?.mode === "range") { + const outcome = buildParameterAxis(parameter, selected); + if (!outcome.ok) { + throw new Error(outcome.error); + } + scenarioParameterValues[parameter.identifier] = selected; + parameterAxes.push(outcome.axis); + } else { + const value = + selected?.value ?? + (parameter.type === "boolean" + ? parameter.default !== 0 + : parameter.default); + if ( + (parameter.type === "boolean" && typeof value !== "boolean") || + (parameter.type !== "boolean" && typeof value !== "number") || + (parameter.type === "integer" && !Number.isInteger(value)) || + (parameter.type === "ratio" && + (typeof value !== "number" || value < 0 || value > 1)) + ) { + throw new Error( + `Parameter "${parameter.identifier}" requires a ${parameter.type} value`, + ); + } + fixedValues[parameter.identifier] = value; + fixedScenarioValues[parameter.identifier] = Number(value); + scenarioParameterValues[parameter.identifier] = { + mode: "fixed", + value: String(value), + }; + } + } + const input: CreateExperimentInput = { + name: request.name, + scenarioId: scenario.id, + scenarioParameterValues, + runCount: request.runCount, + seed: request.seed, + dt: request.dt, + maxTime: request.maxTime, + metricSpecs, + computeBackend: "cpu", + }; + const execution = request.execution; + const optimization = + execution.mode === "optimize" + ? buildSweepOptimizationInput({ + title, + definition, + experiment: { + ...input, + scenario, + parameterAxes, + scenarioParameterValues: fixedScenarioValues, + constraints: [], + constraintPolicy: null, + }, + metric: metricFor(execution.objectiveMetricId), + objective: execution, + runsPerStep: execution.runsPerStep, + }) + : null; + return { request, input, fixedValues, optimization }; +}; diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/provider.tsx b/libs/@hashintel/petrinaut/src/react/experiment-host/provider.tsx new file mode 100644 index 000000000000..c2009b00ce04 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/provider.tsx @@ -0,0 +1,124 @@ +/** + * @layerRoot react.experiment-host + * @role Runs bounded host requests over experiments and their optimizer + */ +import { use, useEffect, useRef, useState } from "react"; + +import { createReadableStore } from "@hashintel/petrinaut-core"; + +import { ExperimentsContext } from "../experiments/context"; +import { useStableCallback } from "../hooks/use-stable-callback"; +import { PetrinautInstanceContext } from "../instance-context"; +import { LanguageClientContext } from "../lsp/context"; +import { NotificationsContext } from "../notifications/context"; +import { OptimizationsContext } from "../optimizations/context"; +import { SDCPNContext } from "../state/sdcpn-context"; +import { ExperimentHostContext } from "./context"; +import { runExperiment as runHostExperiment } from "./run-experiment"; + +import type { ExperimentRecord } from "../experiments/context"; +import type { OptimizationRecord } from "../optimizations/context"; +import type { PetrinautExperimentHost } from "@hashintel/petrinaut-core/experiments"; +import type { PropsWithChildren } from "react"; + +export const ExperimentHostProvider = ({ children }: PropsWithChildren) => { + const { petriNetDefinition, extensions, title } = use(SDCPNContext); + const instance = use(PetrinautInstanceContext); + const experimentsContext = use(ExperimentsContext); + const optimizationsContext = use(OptimizationsContext); + const languageClient = use(LanguageClientContext); + const { addNotification } = use(NotificationsContext); + const [experiments] = useState(() => + createReadableStore([]), + ); + const [optimizations] = useState(() => + createReadableStore([]), + ); + const controllers = useRef(new Set()); + useEffect( + () => experiments.set(experimentsContext.experiments), + [experiments, experimentsContext.experiments], + ); + useEffect( + () => optimizations.set(optimizationsContext.optimizations), + [optimizations, optimizationsContext.optimizations], + ); + useEffect(() => { + const active = controllers.current; + return () => { + for (const controller of active) { + controller.abort(); + } + active.clear(); + }; + }, []); + const runExperiment: PetrinautExperimentHost["runExperiment"] = + useStableCallback(async (request, options) => { + const controller = new AbortController(); + const cancel = () => controller.abort(); + options?.signal?.addEventListener("abort", cancel, { once: true }); + if (options?.signal?.aborted) { + cancel(); + } + controllers.current.add(controller); + try { + const result = await runHostExperiment( + { + definition: instance?.definition.get() ?? petriNetDefinition, + extensions: instance?.extensions ?? extensions, + title, + experiments, + optimizations, + actions: { + createExperiment: experimentsContext.createExperiment, + navigateSweep: experimentsContext.navigateSweep, + cancelExperiment: experimentsContext.cancelExperiment, + createOptimization: optimizationsContext.createOptimization, + cancelOptimization: optimizationsContext.cancelOptimization, + }, + validate: async (definition, settings) => { + const diagnostics = await languageClient.requestDiagnostics( + definition, + settings, + ); + if (diagnostics.errorCount > 0) { + throw new Error( + [...diagnostics.byUri.values()] + .flat() + .filter((diagnostic) => diagnostic.severity === 1) + .map((diagnostic) => diagnostic.message) + .join("\n"), + ); + } + }, + }, + request, + { ...options, signal: controller.signal }, + ); + addNotification({ + message: + result.status === "complete" + ? `${result.name} complete` + : (result.message ?? `${result.name} stopped`), + tone: + result.status === "complete" + ? "success" + : result.status === "error" + ? "error" + : "neutral", + }); + return result; + } finally { + options?.signal?.removeEventListener("abort", cancel); + controllers.current.delete(controller); + } + }); + const [value] = useState(() => ({ + runExperiment, + })); + return ( + + {children} + + ); +}; diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts new file mode 100644 index 000000000000..ff048ba11919 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts @@ -0,0 +1,547 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createReadableStore, + DEFAULT_PETRINAUT_EXTENSIONS, +} from "@hashintel/petrinaut-core"; + +import { sweepCellObjective } from "../experiments/sweep-cell-objective"; +import { prepareExperiment } from "./prepare-experiment"; +import { runExperiment } from "./run-experiment"; + +import type { + CreateExperimentOptions, + ExperimentRecord, + SweepVisitedCell, +} from "../experiments/context"; +import type { OptimizationRecord } from "../optimizations/context"; +import type { ExperimentHostDependencies } from "./run-experiment"; +import type { + MonteCarloUserDefinedMetricFrame, + SDCPN, +} from "@hashintel/petrinaut-core"; +import type { PetrinautExperimentRequest } from "@hashintel/petrinaut-core/experiments"; + +const makeDefinition = (): SDCPN => ({ + places: [], + transitions: [], + types: [], + parameters: [], + differentialEquations: [], + scenarios: [ + { + id: "scenario", + name: "Scenario", + scenarioParameters: [ + { identifier: "rate", type: "real", default: 0.5 }, + { identifier: "count", type: "integer", default: 3 }, + { identifier: "enabled", type: "boolean", default: 0 }, + ], + parameterOverrides: {}, + initialState: { type: "per_place", content: {} }, + }, + ], + metrics: [{ id: "metric", name: "Metric", code: "return 1;" }], +}); + +const makeRequest = (optimize = false): PetrinautExperimentRequest => ({ + name: "Requested experiment", + scenarioId: "scenario", + scenarioParameterValues: { + count: { mode: "fixed", value: 7 }, + ...(optimize ? { rate: { mode: "range" as const, min: 0, max: 1 } } : {}), + }, + runCount: 25, + seed: 1, + dt: 1, + maxTime: 10, + metricIds: ["metric"], + execution: optimize + ? { + mode: "optimize", + objectiveMetricId: "metric", + direction: "maximize", + steps: 2, + runsPerStep: 4, + } + : { mode: "simulate" }, +}); + +const makeRecord = (optimize = false): ExperimentRecord => ({ + id: "experiment", + name: "Requested experiment", + createdAt: 0, + scenarioId: "scenario", + scenarioName: "Scenario", + scenario: makeDefinition().scenarios![0]!, + scenarioParameterValues: {}, + constraints: [], + constraintPolicy: null, + runCount: 25, + seed: 1, + dt: 1, + maxTime: 10, + status: optimize ? "idle" : "running", + error: null, + metricSpecs: [], + computeBackend: "cpu", + computeBackendFallbackReason: null, + startedAt: 0, + finishedAt: null, + progress: { + activeRuns: 25, + advancedRuns: 0, + allFinished: false, + completedRuns: 0, + erroredRuns: 0, + frameNumber: 0, + runCount: 25, + time: 0, + }, + metricFrames: [], + latestMetricFramesById: {}, + sweepBatches: [], + parameterAxes: optimize + ? [{ identifier: "rate", min: 0, max: 1, integer: false, stepCount: 50 }] + : [], + sweep: optimize + ? { + selection: { rate: { from: 0, to: 50 } }, + selectionKey: "initial", + runsCompleted: 0, + runsSampled: 0, + runTarget: null, + computing: false, + visited: [], + } + : null, +}); + +const makeStudy = (): OptimizationRecord => ({ + id: "optimization", + input: prepareExperiment(makeRequest(true), makeDefinition(), "Net") + .optimization!, + createdAt: 0, + origin: { kind: "sweep", experimentId: "experiment" }, + status: "running", + error: null, + runId: "run", + lastSeq: 0, + requestedTrials: 2, + completedTrials: 0, + prunedTrials: 0, + failedTrials: 0, + trials: [], + best: null, + importance: null, +}); + +const createHarness = (optimize = false) => { + const experiments = createReadableStore([]); + const optimizations = createReadableStore([]); + const record = makeRecord(optimize); + const study = makeStudy(); + const refinement = Promise.withResolvers(); + let released = false; + let createOptions: CreateExperimentOptions | undefined; + const actions: ExperimentHostDependencies["actions"] = { + createExperiment: vi.fn< + ExperimentHostDependencies["actions"]["createExperiment"] + >(async (_input, options) => { + createOptions = options; + void options?.ownership?.finished.then(() => { + released = true; + }); + experiments.set([record]); + return record; + }), + createOptimization: vi.fn(async () => { + optimizations.set([study]); + return study.id; + }), + cancelExperiment: vi.fn(() => { + refinement.resolve(null); + }), + cancelOptimization: vi.fn(), + navigateSweep: vi.fn(() => refinement.promise), + }; + const dependencies: ExperimentHostDependencies = { + definition: makeDefinition(), + extensions: DEFAULT_PETRINAUT_EXTENSIONS, + title: "Net", + validate: vi.fn(async () => {}), + experiments, + optimizations, + actions, + }; + return { + dependencies, + experiments, + optimizations, + actions, + record, + study, + refinement, + get released() { + return released; + }, + get createOptions() { + return createOptions; + }, + }; +}; + +const completeSearch = (harness: ReturnType) => { + harness.optimizations.set([ + { + ...harness.study, + status: "complete", + completedTrials: 2, + best: { trial: 1, parameters: { rate: 0.251 }, objective: 7 }, + }, + ]); +}; + +describe("runExperiment", () => { + it("uses the sweep's rounded integer bounds for optimization", () => { + const request = makeRequest(true); + request.scenarioParameterValues = { + count: { mode: "range", min: 1.2, max: 7.8 }, + enabled: { mode: "fixed", value: true }, + }; + const { optimization } = prepareExperiment( + request, + makeDefinition(), + "Net", + ); + expect(optimization?.scenario.parameterBindings).toMatchObject({ + count: { + kind: "optimize", + domain: { kind: "integer", minimum: 1, maximum: 8, step: 1 }, + }, + enabled: { kind: "fixed", value: true }, + rate: { kind: "fixed", value: 0.5 }, + }); + }); + + it("coalesces progress outside store publication and ignores unchanged values", async () => { + const harness = createHarness(); + const onProgress = vi.fn(); + const pending = runExperiment(harness.dependencies, makeRequest(), { + onProgress, + }); + await vi.waitFor(() => expect(onProgress).toHaveBeenCalledOnce()); + onProgress.mockClear(); + + for (const completedRuns of [1, 2, 3]) { + harness.experiments.set([ + { + ...harness.record, + progress: { ...harness.record.progress!, completedRuns }, + }, + ]); + } + expect(onProgress).not.toHaveBeenCalled(); + await Promise.resolve(); + expect(onProgress).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ phase: "running", runsCompleted: 3 }), + ); + + harness.experiments.set([...harness.experiments.get()]); + await Promise.resolve(); + expect(onProgress).toHaveBeenCalledOnce(); + + harness.experiments.set([{ ...harness.record, status: "complete" }]); + await pending; + const callsAtCompletion = onProgress.mock.calls.length; + harness.experiments.set([harness.record]); + await Promise.resolve(); + expect(onProgress).toHaveBeenCalledTimes(callsAtCompletion); + }); + + it("publishes optimization phase changes even when no runs have completed", async () => { + const harness = createHarness(true); + const onProgress = vi.fn(); + const pending = runExperiment(harness.dependencies, makeRequest(true), { + onProgress, + }); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + expect(onProgress).toHaveBeenCalledWith( + expect.objectContaining({ phase: "optimizing", runsCompleted: 0 }), + ); + completeSearch(harness); + await vi.waitFor(() => + expect(harness.actions.navigateSweep).toHaveBeenCalledOnce(), + ); + expect(onProgress).toHaveBeenCalledWith( + expect.objectContaining({ phase: "refining", runsCompleted: 0 }), + ); + harness.refinement.resolve(null); + await pending; + }); + + it("captures the final distribution mean and releases its controls after completion", async () => { + const harness = createHarness(); + const onProgress = vi.fn(); + const pending = runExperiment(harness.dependencies, makeRequest(), { + onProgress, + }); + await vi.waitFor(() => + expect(harness.actions.createExperiment).toHaveBeenCalledOnce(), + ); + expect(harness.released).toBe(false); + const frame: MonteCarloUserDefinedMetricFrame = { + metricId: "metric", + label: "Metric", + outputType: "distribution", + frameNumber: 10, + time: 10, + bins: [ + [1, 5], + [6, 20], + ], + value: null, + frameValue: null, + timeValue: null, + runSampleCount: 25, + timeSampleCount: 25, + }; + harness.experiments.set([ + { + ...harness.record, + status: "complete", + progress: { + ...harness.record.progress!, + activeRuns: 0, + completedRuns: 25, + allFinished: true, + }, + metricFrames: [frame], + }, + ]); + const result = await pending; + frame.bins = [[100, 25]]; + expect(result).toMatchObject({ + status: "complete", + experimentId: "experiment", + runsCompleted: 25, + metrics: [{ id: "metric", value: 5 }], + }); + expect(harness.released).toBe(true); + expect(onProgress).toHaveBeenCalledWith( + expect.objectContaining({ experimentId: "experiment", phase: "running" }), + ); + }); + + it("keeps the validated snapshot and fixed parameters while the editor changes", async () => { + const harness = createHarness(true); + const validation = Promise.withResolvers(); + harness.dependencies.validate = vi.fn(() => validation.promise); + const pending = runExperiment(harness.dependencies, makeRequest(true)); + harness.dependencies.definition.scenarios![0]!.scenarioParameters[1]!.default = 99; + harness.dependencies.definition.metrics![0]!.code = "return 99;"; + validation.resolve(); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + expect(harness.createOptions?.definition?.metrics?.[0]?.code).toBe( + "return 1;", + ); + const [optimizationInput, options] = + vi.mocked(harness.actions.createOptimization).mock.calls[0] ?? []; + expect(optimizationInput).toMatchObject({ + scenario: { + parameterBindings: { + count: { kind: "fixed", value: 7 }, + enabled: { kind: "fixed", value: false }, + }, + }, + execution: { seedsPerTrial: 4 }, + }); + expect(options).toMatchObject({ + sweep: { refineOnSettle: false }, + }); + harness.createOptions?.ownership?.cancel(); + expect((await pending).status).toBe("cancelled"); + }); + + it("waits for the refined distribution and returns its mean at the quantized point", async () => { + const harness = createHarness(true); + const pending = runExperiment(harness.dependencies, makeRequest(true)); + let settled = false; + void pending.then(() => { + settled = true; + }); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + completeSearch(harness); + await vi.waitFor(() => + expect(harness.actions.navigateSweep).toHaveBeenCalledOnce(), + ); + expect(settled).toBe(false); + expect(harness.released).toBe(false); + const refinedFrame: MonteCarloUserDefinedMetricFrame = { + metricId: "metric", + label: "Metric", + outputType: "distribution", + frameNumber: 10, + time: 10, + bins: [ + [4, 5], + [9, 20], + ], + value: null, + frameValue: null, + timeValue: null, + runSampleCount: 25, + timeSampleCount: 25, + }; + harness.experiments.set([ + { ...harness.record, metricFrames: [refinedFrame] }, + ]); + harness.refinement.resolve({ + position: { rate: 13 }, + runsCompleted: 25, + means: { metric: sweepCellObjective([refinedFrame], "metric")! }, + sampleCounts: { metric: 25 }, + }); + expect(harness.experiments.get()[0]?.metricFrames).toEqual([refinedFrame]); + expect(await pending).toMatchObject({ + status: "complete", + runsCompleted: 25, + metrics: [{ id: "metric", value: 8 }], + optimization: { + parameters: { rate: 0.26, count: 7, enabled: false }, + objectiveValue: 8, + stepsCompleted: 2, + }, + }); + expect(harness.released).toBe(true); + }); + + it("cancels both the optimizer and final refinement without returning success", async () => { + const harness = createHarness(true); + const controller = new AbortController(); + const pending = runExperiment(harness.dependencies, makeRequest(true), { + signal: controller.signal, + }); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + completeSearch(harness); + await vi.waitFor(() => + expect(harness.actions.navigateSweep).toHaveBeenCalledOnce(), + ); + controller.abort(); + expect(await pending).toMatchObject({ status: "cancelled", metrics: [] }); + expect(harness.actions.cancelOptimization).toHaveBeenCalledWith( + "optimization", + ); + expect(harness.actions.cancelExperiment).toHaveBeenCalledWith("experiment"); + expect(harness.released).toBe(true); + }); + + it("does not create an experiment after snapshot validation fails", async () => { + const harness = createHarness(); + harness.dependencies.validate = vi.fn(async () => { + throw new Error("Transition lambda must return a number"); + }); + expect( + await runExperiment(harness.dependencies, makeRequest()), + ).toMatchObject({ + status: "error", + experimentId: null, + message: "Transition lambda must return a number", + }); + expect(harness.actions.createExperiment).not.toHaveBeenCalled(); + }); + + it("returns the metric compile error when no optimization trial produces a value", async () => { + const harness = createHarness(true); + const pending = runExperiment(harness.dependencies, makeRequest(true)); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + harness.createOptions?.ownership?.onError?.( + 'Metric "Metric" did not compile: Cannot find name "missingValue"', + ); + harness.optimizations.set([ + { ...harness.study, status: "complete", prunedTrials: 2 }, + ]); + expect(await pending).toMatchObject({ + status: "error", + message: + 'Metric "Metric" did not compile: Cannot find name "missingValue"', + }); + }); + + it("returns a final refinement error before React has published the failed record", async () => { + const harness = createHarness(true); + const pending = runExperiment(harness.dependencies, makeRequest(true)); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + completeSearch(harness); + await vi.waitFor(() => + expect(harness.actions.navigateSweep).toHaveBeenCalledOnce(), + ); + harness.createOptions?.ownership?.onError?.("1 of 17 runs failed"); + harness.refinement.resolve(null); + expect(await pending).toMatchObject({ + status: "error", + message: "1 of 17 runs failed", + metrics: [], + }); + }); + + it("settles cancellation while validation is still pending", async () => { + const harness = createHarness(); + const controller = new AbortController(); + const validation = Promise.withResolvers(); + harness.dependencies.validate = () => validation.promise; + const pending = runExperiment(harness.dependencies, makeRequest(), { + signal: controller.signal, + }); + controller.abort(); + expect(await pending).toMatchObject({ + status: "cancelled", + experimentId: null, + }); + validation.resolve(); + expect(harness.actions.createExperiment).not.toHaveBeenCalled(); + }); + + it.each>([ + { scenarioId: "missing" }, + { metricIds: ["missing"] }, + { + scenarioParameterValues: { + missing: { mode: "fixed" as const, value: 1 }, + }, + }, + { + scenarioParameterValues: { + count: { mode: "fixed" as const, value: 1.5 }, + }, + }, + { + scenarioParameterValues: { + enabled: { mode: "fixed" as const, value: 1 }, + }, + }, + ])("rejects invalid references and typed parameters: %j", async (patch) => { + const harness = createHarness(); + expect( + ( + await runExperiment(harness.dependencies, { + ...makeRequest(), + ...patch, + }) + ).status, + ).toBe("error"); + expect(harness.actions.createExperiment).not.toHaveBeenCalled(); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts new file mode 100644 index 000000000000..696c8e584e0c --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts @@ -0,0 +1,372 @@ +import { axisValueAt } from "../experiments/parameter-grid"; +import { sweepCellObjective } from "../experiments/sweep-cell-objective"; +import { sweepPointFor } from "../optimizations/provider/create-sweep-trial-evaluator"; +import { prepareExperiment } from "./prepare-experiment"; + +import type { + ExperimentRecord, + ExperimentsActionsValue, +} from "../experiments/context"; +import type { + OptimizationRecord, + OptimizationsContextValue, +} from "../optimizations/context"; +import type { + PetrinautExtensionSettings, + ReadableStore, + SDCPN, +} from "@hashintel/petrinaut-core"; +import type { + PetrinautExperimentHost, + PetrinautExperimentProgress, + PetrinautExperimentResult, +} from "@hashintel/petrinaut-core/experiments"; + +export type ExperimentHostDependencies = { + definition: SDCPN; + extensions: PetrinautExtensionSettings; + title: string; + validate: ( + definition: SDCPN, + extensions: PetrinautExtensionSettings, + ) => Promise; + experiments: ReadableStore; + optimizations: ReadableStore; + actions: Pick< + ExperimentsActionsValue, + "createExperiment" | "navigateSweep" | "cancelExperiment" + > & + Pick< + OptimizationsContextValue, + "createOptimization" | "cancelOptimization" + >; +}; + +const abortable = ( + operation: Promise, + signal: AbortSignal, +): Promise => + new Promise((resolve, reject) => { + const cancel = () => reject(signal.reason); + signal.addEventListener("abort", cancel, { once: true }); + if (signal.aborted) { + cancel(); + } + void operation + .then(resolve, reject) + .finally(() => signal.removeEventListener("abort", cancel)); + }); + +const waitForRecord = ( + store: ReadableStore, + id: string, + predicate: (record: Record) => boolean, + signal: AbortSignal, +): Promise => + new Promise((resolve, reject) => { + let off = () => {}; + const cancel = () => { + off(); + reject(signal.reason); + }; + const check = (records: readonly Record[]) => { + const record = records.find((candidate) => candidate.id === id); + if (record && predicate(record)) { + off(); + signal.removeEventListener("abort", cancel); + resolve(record); + } + }; + off = store.subscribe(check); + signal.addEventListener("abort", cancel, { once: true }); + if (signal.aborted) { + cancel(); + } else { + check(store.get()); + } + }); + +export const runExperiment = async ( + dependencies: ExperimentHostDependencies, + rawRequest: Parameters[0], + options?: Parameters[1], +): Promise => { + const controller = new AbortController(); + const { signal } = controller; + const cancel = () => controller.abort(); + options?.signal?.addEventListener("abort", cancel, { once: true }); + if (options?.signal?.aborted) { + cancel(); + } + const ownership = Promise.withResolvers(); + const requestName = rawRequest.name; + let searchRuns = 0; + let experimentId: string | null = null; + let optimizationId: string | null = null; + let executionError: string | null = null; + const getExecutionError = () => executionError; + let phase: PetrinautExperimentProgress["phase"] = "validating"; + let progressQueued = false; + let settled = false; + let lastProgress: PetrinautExperimentProgress | undefined; + const publishProgress = () => { + if (progressQueued || settled || !options?.onProgress) { + return; + } + progressQueued = true; + // Store publication can happen in a host's React effect. Deliver outside + // that effect so streaming updates do not form a cascading state update. + queueMicrotask(() => { + progressQueued = false; + if (settled) { + return; + } + const experiment = dependencies.experiments + .get() + .find((record) => record.id === experimentId); + if (!experiment) { + return; + } + const study = dependencies.optimizations + .get() + .find((record) => record.id === optimizationId); + const progress: PetrinautExperimentProgress = { + experimentId: experiment.id, + name: experiment.name, + phase, + runsCompleted: + experiment.sweep?.runsCompleted ?? + experiment.progress?.completedRuns ?? + 0, + runsTarget: phase === "optimizing" ? searchRuns : experiment.runCount, + ...(study + ? { + step: Math.min( + study.requestedTrials, + study.completedTrials + + study.prunedTrials + + study.failedTrials + + 1, + ), + steps: study.requestedTrials, + } + : {}), + }; + if ( + lastProgress?.experimentId === progress.experimentId && + lastProgress.name === progress.name && + lastProgress.phase === progress.phase && + lastProgress.runsCompleted === progress.runsCompleted && + lastProgress.runsTarget === progress.runsTarget && + lastProgress.step === progress.step && + lastProgress.steps === progress.steps + ) { + return; + } + lastProgress = progress; + options.onProgress?.(progress); + }); + }; + const offExperiments = dependencies.experiments.subscribe(publishProgress); + const offOptimizations = + dependencies.optimizations.subscribe(publishProgress); + const stopCompute = () => { + if (optimizationId) { + dependencies.actions.cancelOptimization(optimizationId); + } + if (experimentId) { + dependencies.actions.cancelExperiment(experimentId); + } + }; + signal.addEventListener("abort", stopCompute); + + try { + signal.throwIfAborted(); + const definition = structuredClone(dependencies.definition); + const extensions = structuredClone(dependencies.extensions); + const { request, input, fixedValues, optimization } = prepareExperiment( + rawRequest, + definition, + dependencies.title, + ); + searchRuns = + request.execution.mode === "optimize" + ? request.execution.runsPerStep + : request.runCount; + await abortable(dependencies.validate(definition, extensions), signal); + signal.throwIfAborted(); + experimentId = ( + await abortable( + dependencies.actions.createExperiment(input, { + definition, + extensions, + ownership: { + signal, + finished: ownership.promise, + cancel, + onError: (message) => { + executionError = message; + }, + }, + }), + signal, + ) + ).id; + signal.throwIfAborted(); + phase = optimization ? "optimizing" : "running"; + publishProgress(); + + if (optimization && request.execution.mode === "optimize") { + const experiment = await waitForRecord( + dependencies.experiments, + experimentId, + (record) => record.status !== "initializing", + signal, + ); + if (experiment.status === "error" || experiment.status === "cancelled") { + throw new Error( + experiment.error ?? "Experiment stopped before optimization started", + ); + } + optimizationId = await dependencies.actions.createOptimization( + optimization, + { + sweep: { + experimentId, + axes: experiment.parameterAxes, + metricId: request.execution.objectiveMetricId, + refineOnSettle: false, + }, + }, + ); + signal.throwIfAborted(); + const study = await waitForRecord( + dependencies.optimizations, + optimizationId, + (record) => + record.status !== "initializing" && record.status !== "running", + signal, + ); + if (study.status !== "complete") { + if (study.status === "cancelled") { + cancel(); + signal.throwIfAborted(); + } + throw new Error(study.error ?? "Optimization failed"); + } + if (!study.best) { + throw new Error( + getExecutionError() ?? + "Optimization finished without a finite objective", + ); + } + const point = sweepPointFor( + experiment.parameterAxes, + study.best.parameters, + ); + if (!point) { + throw new Error( + "The best point does not match the experiment parameters", + ); + } + phase = "refining"; + publishProgress(); + const cell = await dependencies.actions.navigateSweep( + experimentId, + point, + ); + signal.throwIfAborted(); + if (!cell || cell.runsCompleted < request.runCount) { + throw new Error( + getExecutionError() ?? + "The best point did not finish its requested runs", + ); + } + const parameters = { ...fixedValues }; + for (const axis of experiment.parameterAxes) { + const position = cell.position[axis.identifier]; + if (position === undefined) { + throw new Error( + `The best point is missing parameter "${axis.identifier}"`, + ); + } + parameters[axis.identifier] = axisValueAt(axis, position); + } + return { + status: "complete", + experimentId, + name: request.name, + runsCompleted: cell.runsCompleted, + metrics: input.metricSpecs.map((metric) => ({ + id: metric.id, + label: metric.label, + value: cell.means[metric.id] ?? null, + })), + optimization: { + parameters, + objectiveValue: + cell.means[request.execution.objectiveMetricId] ?? null, + stepsCompleted: + study.completedTrials + study.prunedTrials + study.failedTrials, + }, + }; + } + + const completed = await waitForRecord( + dependencies.experiments, + experimentId, + (record) => + record.status === "complete" || + record.status === "error" || + record.status === "cancelled", + signal, + ); + if (completed.status === "cancelled") { + cancel(); + signal.throwIfAborted(); + } + if ( + completed.status === "error" || + (completed.progress?.erroredRuns ?? 0) > 0 + ) { + throw new Error(completed.error ?? "Some experiment runs failed"); + } + return { + status: "complete", + experimentId, + name: request.name, + runsCompleted: completed.progress?.completedRuns ?? 0, + metrics: input.metricSpecs.map((metric) => ({ + id: metric.id, + label: metric.label, + value: sweepCellObjective(completed.metricFrames, metric.id), + })), + }; + } catch (error) { + const cancelled = signal.aborted; + stopCompute(); + const record = dependencies.experiments + .get() + .find((candidate) => candidate.id === experimentId); + return { + status: cancelled ? "cancelled" : "error", + experimentId, + name: requestName, + message: cancelled + ? "Experiment cancelled" + : error instanceof Error + ? error.message + : String(error), + runsCompleted: + record?.sweep?.runsCompleted ?? record?.progress?.completedRuns ?? 0, + metrics: [], + }; + } finally { + settled = true; + offExperiments(); + offOptimizations(); + signal.removeEventListener("abort", stopCompute); + options?.signal?.removeEventListener("abort", cancel); + ownership.resolve(); + } +}; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/context.ts b/libs/@hashintel/petrinaut/src/react/experiments/context.ts index c28e60467c17..76873f5a1c04 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/context.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/context.ts @@ -24,6 +24,8 @@ import type { MonteCarloUserDefinedMetricFrame, MonteCarloWorkerProgress, Scenario, + SDCPN, + PetrinautExtensionSettings, } from "@hashintel/petrinaut-core"; import type { PetrinautOptimizationConstraintPolicy } from "@hashintel/petrinaut-core/optimization"; @@ -105,8 +107,21 @@ export type CreateExperimentInput = { constraintPolicy?: PetrinautOptimizationConstraintPolicy; }; +export type CreateExperimentOptions = { + definition?: SDCPN; + extensions?: PetrinautExtensionSettings; + ownership?: { + signal: AbortSignal; + finished: Promise; + cancel: () => void; + onError?: (message: string) => void; + }; +}; + export type ExperimentRecord = { id: string; + /** Compute controls are held until a host request captures its result. */ + requestActive?: boolean; name: string; createdAt: number; scenarioId: string | null; @@ -245,7 +260,10 @@ export type ExperimentsContextValue = { * once a sweep's session is registered (or a plain run's backend selection * has begun). Selects nothing: the caller decides what opens, and when. */ - createExperiment: (input: CreateExperimentInput) => Promise; + createExperiment: ( + input: CreateExperimentInput, + options?: CreateExperimentOptions, + ) => Promise; cancelExperiment: (experimentId: string) => void; removeExperiment: (experimentId: string) => void; /** Moves a sweep's navigator; compute follows the selection up to the run count. */ diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx index 6113943de8d9..6d148ff4692c 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx @@ -421,6 +421,132 @@ describe("buildSweepAxes", () => { }); describe("ExperimentsProvider", () => { + it("keeps user navigation from changing a host-controlled sweep", async () => { + const worker = new FakeMonteCarloWorker(); + const { getValue, renderResult } = renderExperimentsProvider(worker); + const controller = new AbortController(); + const finished = Promise.withResolvers(); + let experimentId = ""; + try { + await act(async () => { + experimentId = ( + await getValue().createExperiment( + { + name: "Owned sweep", + scenarioId: "scenario", + scenarioParameterValues: { + rate: { mode: "range", min: 0, max: 1 }, + }, + runCount: 1, + seed: 42, + dt: 1, + maxTime: 10, + metricSpecs: CONSTANT_METRIC_SPEC, + }, + { + definition: { + ...EMPTY_SDCPN, + scenarios: [ + { + id: "scenario", + name: "Scenario", + scenarioParameters: [ + { identifier: "rate", type: "real", default: 0.5 }, + ], + parameterOverrides: {}, + initialState: { type: "per_place", content: {} }, + }, + ], + }, + ownership: { + signal: controller.signal, + finished: finished.promise, + cancel: () => controller.abort(), + }, + }, + ) + ).id; + }); + const selection = { rate: { from: 10, to: 10 } }; + await act(async () => { + getValue().setSweepSelection(experimentId, selection); + }); + expect(getValue().experiments[0]?.sweep?.selection).toEqual({ + rate: { from: 0, to: 50 }, + }); + expect(worker.sent).toHaveLength(0); + await act(async () => { + finished.resolve(); + }); + await act(async () => { + getValue().setSweepSelection(experimentId, selection); + await flushWorkerSetup(); + }); + expect(getValue().experiments[0]?.sweep?.selection).toEqual(selection); + } finally { + finished.resolve(); + renderResult.unmount(); + } + }); + + it("protects a host request until capture, while keeping its cancel action available", async () => { + const worker = new FakeMonteCarloWorker(); + const { getValue, getNavigationState, renderResult } = + renderExperimentsProvider(worker); + const controller = new AbortController(); + const finished = Promise.withResolvers(); + const cancel = vi.fn(() => controller.abort()); + let experimentId = ""; + try { + await act(async () => { + experimentId = ( + await getValue().createExperiment( + { + name: "Owned experiment", + scenarioId: null, + scenarioParameterValues: {}, + runCount: 1, + seed: 42, + dt: 1, + maxTime: 10, + metricSpecs: CONSTANT_METRIC_SPEC, + }, + { + ownership: { + signal: controller.signal, + finished: finished.promise, + cancel, + }, + }, + ) + ).id; + await flushWorkerSetup(); + }); + expect(getNavigationState().simulateResource).toBeNull(); + expect(getValue().experiments[0]?.requestActive).toBe(true); + await act(async () => { + getValue().removeExperiment(experimentId); + }); + expect(getValue().experiments).toHaveLength(1); + await act(async () => { + getValue().cancelExperiment(experimentId); + }); + expect(cancel).toHaveBeenCalledOnce(); + expect(getValue().experiments[0]?.status).toBe("cancelled"); + await act(async () => { + finished.resolve(); + }); + expect(getValue().experiments[0]?.requestActive).toBe(false); + await act(async () => { + getValue().removeExperiment(experimentId); + }); + expect(getValue().experiments).toHaveLength(0); + } finally { + finished.resolve(); + renderResult.unmount(); + } + }); + it("leaves the creation overlay open until the caller selects the created experiment", async () => { const worker = new FakeMonteCarloWorker(); const { getNavigationState, getValue, renderResult } = diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx index 05c92f84c398..9614a615de1c 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx @@ -32,6 +32,7 @@ import { NotificationsContext } from "../notifications/context"; import { SDCPNContext } from "../state/sdcpn-context"; import { type ExperimentComputeBackend, + type CreateExperimentOptions, type ExperimentRecord, ExperimentsActionsContext, type ExperimentsActionsValue, @@ -143,6 +144,9 @@ export const ExperimentsProvider: React.FC = ({ new Map(), ); const sweepSessionsRef = useRef(new Map()); + const ownershipRef = useRef( + new Map>(), + ); /** Backends an experiment chose, disposed with the experiment. */ const backendsRef = useRef(new Map()); const [experiments, setExperiments] = useState([]); @@ -208,6 +212,9 @@ export const ExperimentsProvider: React.FC = ({ experimentId: string, patch: Partial, ) => { + if (patch.error) { + ownershipRef.current.get(experimentId)?.onError?.(patch.error); + } // Stamped here rather than at each call site, so no path — completion, a // worker error, cancellation — can finish an experiment without // recording when it stopped. @@ -285,15 +292,20 @@ export const ExperimentsProvider: React.FC = ({ error: event.message, status: "error", }); - addNotification({ - message: `${experimentName} failed: ${event.message}`, - tone: "error", - }); + if (!ownershipRef.current.has(experimentId)) { + addNotification({ + message: `${experimentName} failed: ${event.message}`, + tone: "error", + }); + } } else { sync(); } - if (event.type === "complete") { + if ( + event.type === "complete" && + !ownershipRef.current.has(experimentId) + ) { addNotification({ message: `${experimentName} complete`, tone: "success", @@ -346,6 +358,7 @@ export const ExperimentsProvider: React.FC = ({ seed: experiment.seed, // Nothing computes until a control moves or an optimizer navigates. startComputing: false, + requireSuccessfulRuns: experiment.requestActive === true, // Leading-edge, so the first frames publish instantly; while a // batch streams, ~10 re-renders a second read as live on a chart // and leave the rest of the UI most of each frame's budget. @@ -403,22 +416,51 @@ export const ExperimentsProvider: React.FC = ({ error: message, status: "error", }); - addNotification({ - message: `${experiment.name} failed: ${message}`, - tone: "error", - }); + if (!ownershipRef.current.has(experimentId)) { + addNotification({ + message: `${experiment.name} failed: ${message}`, + tone: "error", + }); + } }, }); sweepSessionsRef.current.set(experimentId, session); }; + const cancelExperiment: ExperimentsContextValue["cancelExperiment"] = ( + experimentId, + ) => { + const ownership = ownershipRef.current.get(experimentId); + if (ownership && !ownership.signal.aborted) { + ownership.cancel(); + return; + } + const pendingRegistration = + pendingRegistrationsRef.current.get(experimentId); + if (pendingRegistration) { + pendingRegistrationsRef.current.delete(experimentId); + pendingRegistration.abortController.abort(); + patchExperiment(experimentId, { status: "cancelled" }); + return; + } + + if (sweepSessionsRef.current.has(experimentId)) { + disposeSweepSession(experimentId); + patchExperiment(experimentId, { status: "cancelled" }); + return; + } + + registrationsRef.current.get(experimentId)?.handle.cancel(); + }; + const createExperiment: ExperimentsContextValue["createExperiment"] = async ( input, + options, ) => { assertExperimentInput(input); - const sdcpn = petriNetDefinitionRef.current; - const experimentExtensions = extensionsRef.current; + const sdcpn = options?.definition ?? petriNetDefinitionRef.current; + const experimentExtensions = options?.extensions ?? extensionsRef.current; const scenario = input.scenarioId ? ((sdcpn.scenarios ?? []).find(({ id }) => id === input.scenarioId) ?? null) @@ -442,6 +484,7 @@ export const ExperimentsProvider: React.FC = ({ sdcpn: experimentSdcpn, requestScenarioHir: languageClientRef.current.requestScenarioHir, }); + options?.ownership?.signal.throwIfAborted(); const experimentId = generateUuid(); const experiment = newExperimentRecord({ @@ -453,6 +496,18 @@ export const ExperimentsProvider: React.FC = ({ fixedScenarioValues: compiled.fixedScenarioValues, scenario: compiled.scenario, }); + const ownership = options?.ownership; + if (ownership) { + experiment.requestActive = true; + ownershipRef.current.set(experimentId, ownership); + const cancel = () => cancelExperiment(experimentId); + ownership.signal.addEventListener("abort", cancel, { once: true }); + void ownership.finished.finally(() => { + ownership.signal.removeEventListener("abort", cancel); + ownershipRef.current.delete(experimentId); + patchExperiment(experimentId, { requestActive: false }); + }); + } setExperiments((prev) => [experiment, ...prev]); const abortController = new AbortController(); @@ -566,10 +621,12 @@ export const ExperimentsProvider: React.FC = ({ const message = errorMessage(error); patchExperiment(experimentId, { error: message, status: "error" }); - addNotification({ - message: `${experiment.name} failed: ${message}`, - tone: "error", - }); + if (!ownershipRef.current.has(experimentId)) { + addNotification({ + message: `${experiment.name} failed: ${message}`, + tone: "error", + }); + } } }; @@ -578,30 +635,12 @@ export const ExperimentsProvider: React.FC = ({ return experiment; }; - const cancelExperiment: ExperimentsContextValue["cancelExperiment"] = ( + const removeExperiment: ExperimentsContextValue["removeExperiment"] = ( experimentId, ) => { - const pendingRegistration = - pendingRegistrationsRef.current.get(experimentId); - if (pendingRegistration) { - pendingRegistrationsRef.current.delete(experimentId); - pendingRegistration.abortController.abort(); - patchExperiment(experimentId, { status: "cancelled" }); + if (ownershipRef.current.has(experimentId)) { return; } - - if (sweepSessionsRef.current.has(experimentId)) { - disposeSweepSession(experimentId); - patchExperiment(experimentId, { status: "cancelled" }); - return; - } - - registrationsRef.current.get(experimentId)?.handle.cancel(); - }; - - const removeExperiment: ExperimentsContextValue["removeExperiment"] = ( - experimentId, - ) => { disposeSweepSession(experimentId); disposeExperimentHandle(experimentId); setExperiments((prev) => @@ -619,6 +658,9 @@ export const ExperimentsProvider: React.FC = ({ experimentId, selection, ) => { + if (ownershipRef.current.has(experimentId)) { + return; + } sweepSessionsRef.current.get(experimentId)?.setSelection(selection); }; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-optimization.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-optimization.ts new file mode 100644 index 000000000000..54dc217c3dfa --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-optimization.ts @@ -0,0 +1,123 @@ +import { petrinautOptimizationInputSchema } from "@hashintel/petrinaut-core/optimization"; + +import type { ExperimentRecord } from "./context"; +import type { + Metric, + Scenario, + ScenarioParameter, + SDCPN, +} from "@hashintel/petrinaut-core"; +import type { + PetrinautOptimizationDirection, + PetrinautOptimizationInput, + PetrinautOptimizationParameterBinding, +} from "@hashintel/petrinaut-core/optimization"; + +/** What the manifest reads of the experiment: its execution, axes, constraints and the scenario it compiled. */ +export type SweepOptimizationExperiment = Pick< + ExperimentRecord, + | "name" + | "seed" + | "dt" + | "maxTime" + | "parameterAxes" + | "scenarioParameterValues" + | "constraints" + | "constraintPolicy" +> & { scenario: Scenario }; + +/** + * The manifest of a study searching the experiment's swept axes for the best + * value of one of its metrics. Bindings come from `experiment.scenario`: an + * axis is an optimize binding over its interval, every other parameter is + * fixed at the value the experiment was created with, so the trials' values + * — which the evaluator judges the experiment's parameter constraints + * against — match what the sweep simulates. An ad-hoc record's generated + * scenario has only axes, so every binding is optimize. The experiment's + * constraints and pass threshold ride the manifest as they are. Throws with + * the schema's message when the experiment cannot be a study (a step budget + * over the cap, say). + */ +export const buildSweepOptimizationInput = ({ + title, + definition, + experiment, + name = experiment.name, + metric, + objective, + runsPerStep, +}: { + title: string; + definition: SDCPN; + experiment: SweepOptimizationExperiment; + name?: string; + metric: Metric; + objective: { direction: PetrinautOptimizationDirection; steps: number }; + /** Runs each point computes before its value is read. */ + runsPerStep: number; +}): PetrinautOptimizationInput => { + const { scenario } = experiment; + const fixedValueFor = (parameter: ScenarioParameter): number | boolean => { + const value = + experiment.scenarioParameterValues[parameter.identifier] ?? + parameter.default; + return parameter.type === "boolean" ? value !== 0 : value; + }; + const parameterBindings: Record< + string, + PetrinautOptimizationParameterBinding + > = {}; + for (const parameter of scenario.scenarioParameters) { + const axis = experiment.parameterAxes.find( + (candidate) => candidate.identifier === parameter.identifier, + ); + if (axis === undefined) { + parameterBindings[parameter.identifier] = { + kind: "fixed", + value: fixedValueFor(parameter), + }; + } else if (axis.integer) { + parameterBindings[parameter.identifier] = { + kind: "optimize", + domain: { + kind: "integer", + minimum: axis.min, + maximum: axis.max, + step: 1, + scale: "linear", + }, + }; + } else { + parameterBindings[parameter.identifier] = { + kind: "optimize", + domain: { + kind: "continuous", + minimum: axis.min, + maximum: axis.max, + scale: "linear", + }, + }; + } + } + const { constraints, constraintPolicy } = experiment; + return petrinautOptimizationInputSchema.parse({ + kind: "petrinaut-optimization", + version: 1, + name, + model: { + title, + definition: { ...definition, scenarios: [scenario], metrics: [metric] }, + }, + scenario: { id: scenario.id, parameterBindings }, + objective: { metricId: metric.id, direction: objective.direction }, + ...(constraints.length > 0 ? { constraints } : {}), + ...(constraints.length > 0 && constraintPolicy ? { constraintPolicy } : {}), + execution: { + seed: experiment.seed, + dt: experiment.dt, + maxTime: experiment.maxTime, + seedsPerTrial: runsPerStep, + }, + study: { trials: objective.steps, sampler: "tpe" }, + }); +}; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts index 98d826a1eb00..ffe2e604bd78 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts @@ -190,7 +190,7 @@ function makeFakeBatch(request: { listener(progress); } }, - complete() { + complete(erroredRuns = 0) { if (!started) { throw new Error("batch completed before start()"); } @@ -201,8 +201,8 @@ function makeFakeBatch(request: { activeRuns: 0, advancedRuns: request.runCount, allFinished: true, - completedRuns: request.runCount, - erroredRuns: 0, + completedRuns: request.runCount - erroredRuns, + erroredRuns, frameNumber: 0, runCount: request.runCount, time: 0, @@ -222,7 +222,10 @@ function makeHarness( runCount: number, initialSelection?: SweepSelection, options: Partial< - Pick + Pick< + CreateSweepSessionOptions, + "startComputing" | "axes" | "requireSuccessfulRuns" + > > = {}, ) { const batches: ReturnType[] = []; @@ -784,6 +787,38 @@ describe("navigateTo", () => { session.dispose(); }); + it("refuses a host's final refinement when a run in the last rung errors", async () => { + const { session, batches, updates, onError, settle } = makeHarness( + 25, + point(0, 0), + { + startComputing: false, + requireSuccessfulRuns: true, + }, + ); + const trial = session.navigateTo(point(1, 1), { runCap: 8 }); + await settle(); + batches[0]!.stream([frame(8, [[2, 8]])]); + batches[0]!.complete(); + await settle(); + expect((await trial)?.runsCompleted).toBe(8); + + const refinement = session.navigateTo(point(1, 1)); + await settle(); + expect(batches[1]!.request.runCount).toBe(17); + batches[1]!.stream([frame(16, [[2, 16]])]); + batches[1]!.complete(1); + await expect(refinement).rejects.toThrow("1 of 17 runs failed"); + await settle(); + expect(onError).toHaveBeenCalledExactlyOnceWith("1 of 17 runs failed"); + expect(updates.at(-1)).toMatchObject({ + failed: true, + computing: false, + runsCompleted: 8, + }); + session.dispose(); + }); + it("resolves null when another navigation supersedes it, and on dispose", async () => { const { session, settle } = makeHarness(100, point(0, 0), { startComputing: false, diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts index 62eeb2cd1756..c3f046585e84 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts @@ -147,6 +147,8 @@ export type CreateSweepSessionOptions = { * publishes once as idle and waits for a selection. Defaults to on. */ startComputing?: boolean; + /** Refuse partial results when a batch finishes with errored runs. */ + requireSuccessfulRuns?: boolean; instantiateBatch: InstantiateSweepBatch; onUpdate: (update: SweepSessionUpdate) => void; /** @@ -565,7 +567,15 @@ export function createSweepSession( const unsubscribeProgress = handle.progress.subscribe(() => onLiveTick()); const unsubscribeEvents = handle.events.subscribe((event) => { if (event.type === "complete") { - resolveDone("complete"); + if (options.requireSuccessfulRuns && event.progress.erroredRuns > 0) { + fail( + loopGeneration, + `${event.progress.erroredRuns} of ${event.progress.runCount} runs failed`, + ); + resolveDone("stopped"); + } else { + resolveDone("complete"); + } } else if (event.type === "cancelled") { resolveDone("stopped"); } else { diff --git a/libs/@hashintel/petrinaut/src/react/index.ts b/libs/@hashintel/petrinaut/src/react/index.ts index f13cdcfbcaeb..8127388cbbed 100644 --- a/libs/@hashintel/petrinaut/src/react/index.ts +++ b/libs/@hashintel/petrinaut/src/react/index.ts @@ -92,6 +92,7 @@ export { ExperimentsContext, isExperimentActive, } from "./experiments/context"; +export { ExperimentHostContext } from "./experiment-host/context"; export type { CreateExperimentInput, ExperimentRecord, diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/context.ts b/libs/@hashintel/petrinaut/src/react/optimizations/context.ts index d1adaecf0f27..65191280e57a 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/context.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/context.ts @@ -134,6 +134,8 @@ type CreateOptimizationOptions = { axes: readonly ExperimentParameterAxis[]; /** The experiment metric the objective reads at each point. */ metricId: string; + /** A host can await its own final refinement after the search finishes. */ + refineOnSettle?: boolean; }; }; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx b/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx index 7233f295c1d7..d6d8b29b1055 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx @@ -389,6 +389,7 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { experimentId: sweep.experimentId, axes: sweep.axes, metricId: sweep.metricId, + refineOnSettle: sweep.refineOnSettle, navigateSweep: (experimentId, selection, navigateOptions) => experimentsActionsRef.current.navigateSweep( experimentId, diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts index 5ecb6c771f81..e50291ae4f8f 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts @@ -68,6 +68,28 @@ describe("sweepPointFor", () => { }); describe("createSweepTrialEvaluator", () => { + it("leaves final refinement to the host", async () => { + const navigateSweep = vi.fn().mockResolvedValue({ + position: { rate: 25 }, + runsCompleted: 8, + means: { infected: 2 }, + }); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE], + metricId: "infected", + refineOnSettle: false, + navigateSweep, + }); + await evaluator.evaluateTrial(request({ rate: 0.5 })); + evaluator.settle({ trial: 1, parameters: { rate: 0.5 }, objective: 2 }); + expect(navigateSweep).toHaveBeenCalledExactlyOnceWith( + "exp", + { rate: { from: 25, to: 25 } }, + { runCap: 8 }, + ); + }); + it("navigates the sweep to the trial's point with the manifest's runs per trial and reads the metric there", async () => { const navigateSweep = vi.fn().mockResolvedValue({ position: { rate: 25, days: 5 }, diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts index 9a63cd564dab..2e5f64cd0621 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts @@ -134,6 +134,7 @@ export const createSweepTrialEvaluator = ({ experimentId, axes, metricId, + refineOnSettle = true, navigateSweep, }: { experimentId: string; @@ -141,6 +142,7 @@ export const createSweepTrialEvaluator = ({ axes: readonly ExperimentParameterAxis[]; /** The experiment metric the study optimizes. */ metricId: string; + refineOnSettle?: boolean; navigateSweep: ExperimentsActionsValue["navigateSweep"]; }): SweepTrialEvaluator => { /** Where the sweep is parked once the study is over; "none" while it runs. */ @@ -234,7 +236,7 @@ export const createSweepTrialEvaluator = ({ } parked = next; const point = bestPoint ?? lastPoint; - if (point !== null) { + if (point !== null && refineOnSettle) { // The sweep may be gone already (its experiment removed): nothing // left to park. void navigateSweep(experimentId, point).catch(() => undefined); diff --git a/libs/@hashintel/petrinaut/src/react/petrinaut-provider.tsx b/libs/@hashintel/petrinaut/src/react/petrinaut-provider.tsx index 9aeb6eee6be0..318e89062c50 100644 --- a/libs/@hashintel/petrinaut/src/react/petrinaut-provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/petrinaut-provider.tsx @@ -1,3 +1,4 @@ +import { ExperimentHostProvider } from "./experiment-host/provider"; import { ExperimentsProvider } from "./experiments/provider"; import { LanguageClientProvider } from "./lsp/provider"; import { @@ -85,7 +86,11 @@ export const PetrinautProvider: React.FC = ({ > - {children} + + + {children} + + diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx index b5b24395fbf4..c362e5f69b1c 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx @@ -324,12 +324,18 @@ export const experimentResultsModel = ( const displayStatus: ExperimentDisplayStatus = following ? "optimizing" : experiment.status; - const canCancel = isExperimentActive(experiment) || following !== null; + const canCancel = + experiment.requestActive === true || + isExperimentActive(experiment) || + following !== null; // The navigator and the surface only display while the selection is not // the user's to move: a study drives it, or the sweep was cancelled and its // session is gone. A failed selection locks nothing — the next selection // computes afresh — and a sweep never completes. - const locked = following !== null || experiment.status === "cancelled"; + const locked = + experiment.requestActive === true || + following !== null || + experiment.status === "cancelled"; const tone: ChartCardTone = following ? "optimizing" : "default"; // The study's displays are there from the drawer's first frame for an // experiment created with Optimize, whatever the study's status. @@ -378,6 +384,7 @@ export const experimentResultsModel = ( iconName="stop" tooltip="Stop optimizing; the sweep keeps its point" data-sweep-optimizing + disabled={experiment.requestActive} onClick={optimizer.stop} > Stop @@ -514,6 +521,7 @@ export const experimentResultsModel = ( tone="neutral" size="sm" prefix={} + disabled={experiment.requestActive} onClick={() => { optimizer.discard(); actions.removeExperiment(experiment.id); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts index db6de071c43b..ba15ed3452b4 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts @@ -4,6 +4,10 @@ import { synthesizeAdHocOptimization } from "@hashintel/petrinaut-core"; import { sirModel } from "@hashintel/petrinaut-core/examples"; import { buildAdHocSweepAxes } from "../../../../../../react/experiments/parameter-grid"; +import { + buildSweepOptimizationInput, + type SweepOptimizationExperiment, +} from "../../../../../../react/experiments/sweep-optimization"; import { sirOptimizationConstraints, sirOptimizationMetric, @@ -11,10 +15,8 @@ import { } from "../../../../../../react/optimizations/sir-optimization-input.fixtures"; import { makeExperiment } from "./experiments-story-fixtures"; import { - buildSweepOptimizationInput, startSweepStudy, sweepOptimizationMetric, - type SweepOptimizationExperiment, type SweepStudyStarter, } from "./sweep-optimizer"; @@ -236,7 +238,7 @@ describe("buildSweepOptimizationInput", () => { metricId: "infected", direction: "maximize", }); - expect(input.name).toBe("Mixed sweep · Maximize Infected"); + expect(input.name).toBe("Mixed sweep"); }); it("throws the schema's rejection when the steps exceed the trial cap", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts index f18408fc7f75..67f6fe6b069b 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts @@ -8,9 +8,8 @@ */ import { use } from "react"; -import { petrinautOptimizationInputSchema } from "@hashintel/petrinaut-core/optimization"; - import { EXPERIMENT_RUN_LADDER } from "../../../../../../react/experiments/parameter-grid"; +import { buildSweepOptimizationInput } from "../../../../../../react/experiments/sweep-optimization"; import { currentTrialNumber, isOptimizationActive, @@ -27,17 +26,8 @@ import type { OptimizationRecord, OptimizationsContextValue, } from "../../../../../../react/optimizations/context"; -import type { - Metric, - Scenario, - ScenarioParameter, - SDCPN, -} from "@hashintel/petrinaut-core"; -import type { - PetrinautOptimizationDirection, - PetrinautOptimizationInput, - PetrinautOptimizationParameterBinding, -} from "@hashintel/petrinaut-core/optimization"; +import type { Metric, SDCPN } from "@hashintel/petrinaut-core"; +import type { PetrinautOptimizationDirection } from "@hashintel/petrinaut-core/optimization"; /** What the Create Experiment drawer's Objective section decides. */ export type SweepObjective = { @@ -90,113 +80,6 @@ export const sweepOptimizationMetric = ( }; }; -/** What the manifest reads of the experiment: its execution, axes, constraints and the scenario it compiled. */ -export type SweepOptimizationExperiment = Pick< - ExperimentRecord, - | "name" - | "seed" - | "dt" - | "maxTime" - | "parameterAxes" - | "scenarioParameterValues" - | "constraints" - | "constraintPolicy" -> & { scenario: Scenario }; - -/** - * The manifest of a study searching the experiment's swept axes for the best - * value of one of its metrics. Bindings come from `experiment.scenario`: an - * axis is an optimize binding over its interval, every other parameter is - * fixed at the value the experiment was created with, so the trials' values - * — which the evaluator judges the experiment's parameter constraints - * against — match what the sweep simulates. An ad-hoc record's generated - * scenario has only axes, so every binding is optimize. The experiment's - * constraints and pass threshold ride the manifest as they are. Throws with - * the schema's message when the experiment cannot be a study (a step budget - * over the cap, say). - */ -export const buildSweepOptimizationInput = ({ - title, - definition, - experiment, - metric, - objective, - runsPerStep, -}: { - title: string; - definition: SDCPN; - experiment: SweepOptimizationExperiment; - metric: Metric; - objective: Pick; - /** Runs each point computes before its value is read. */ - runsPerStep: number; -}): PetrinautOptimizationInput => { - const { scenario } = experiment; - const fixedValueFor = (parameter: ScenarioParameter): number | boolean => { - const value = - experiment.scenarioParameterValues[parameter.identifier] ?? - parameter.default; - return parameter.type === "boolean" ? value !== 0 : value; - }; - const parameterBindings: Record< - string, - PetrinautOptimizationParameterBinding - > = {}; - for (const parameter of scenario.scenarioParameters) { - const axis = experiment.parameterAxes.find( - (candidate) => candidate.identifier === parameter.identifier, - ); - if (axis === undefined) { - parameterBindings[parameter.identifier] = { - kind: "fixed", - value: fixedValueFor(parameter), - }; - } else if (axis.integer) { - parameterBindings[parameter.identifier] = { - kind: "optimize", - domain: { - kind: "integer", - minimum: axis.min, - maximum: axis.max, - step: 1, - scale: "linear", - }, - }; - } else { - parameterBindings[parameter.identifier] = { - kind: "optimize", - domain: { - kind: "continuous", - minimum: axis.min, - maximum: axis.max, - scale: "linear", - }, - }; - } - } - const { constraints, constraintPolicy } = experiment; - return petrinautOptimizationInputSchema.parse({ - kind: "petrinaut-optimization", - version: 1, - name: `${experiment.name} · ${directionWord(objective.direction)} ${metric.name}`, - model: { - title, - definition: { ...definition, scenarios: [scenario], metrics: [metric] }, - }, - scenario: { id: scenario.id, parameterBindings }, - objective: { metricId: metric.id, direction: objective.direction }, - ...(constraints.length > 0 ? { constraints } : {}), - ...(constraints.length > 0 && constraintPolicy ? { constraintPolicy } : {}), - execution: { - seed: experiment.seed, - dt: experiment.dt, - maxTime: experiment.maxTime, - seedsPerTrial: runsPerStep, - }, - study: { trials: objective.steps, sampler: "tpe" }, - }); -}; - /** What starting a study reads beside the experiment: the net, its title and the optimizations action. */ export type SweepStudyStarter = { title: string; @@ -229,6 +112,7 @@ export const startSweepStudy = async ( title, definition, experiment: { ...experiment, scenario: experiment.scenario }, + name: `${experiment.name} · ${directionWord(objective.direction)} ${spec.label}`, metric: sweepOptimizationMetric(spec, definition), objective, runsPerStep: SWEEP_OPTIMIZATION_RUNS_PER_STEP, diff --git a/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-architecture.d2 b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-architecture.d2 new file mode 100644 index 000000000000..df167007a646 --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-architecture.d2 @@ -0,0 +1,13 @@ +direction: down +vars: {d2-config: {theme-id: 0}} +caller: "Host caller" +contract: "Zod contract" +host: "Browser host" +validate: "Snapshot checks" +runtime: "Experiments + optimizer" +workers: "CPU workers" +caller <-> host: "request / result" +contract -> host: "schema" {style.stroke-dash: 4} +host -> validate: "prepare" +host <-> runtime: "run / cancel / progress" +runtime -> workers: "batches" diff --git a/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-flow.d2 b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-flow.d2 new file mode 100644 index 000000000000..193276099077 --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiments-flow.d2 @@ -0,0 +1,14 @@ +direction: down +vars: {d2-config: {theme-id: 0}} +validate: "Capture + validate" +create: "Create + lock inputs" +simulate: "Simulate" +optimize: "Search + refine" +result: "Capture result" +finish: "Unlock + notify" +validate -> create: "valid" +create -> simulate +create -> optimize +simulate -> result: "done / cancelled" +optimize -> result: "done / cancelled" +result -> finish diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx new file mode 100644 index 000000000000..3dbc47f4bb2f --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx @@ -0,0 +1,108 @@ +--- +title: Browser experiment host +description: Validate and run bounded experiment requests in the browser. +sidebar_order: 40 +attachTo: react.experiment-host +--- + +`PetrinautExperimentHost` runs ordinary experiments from bounded requests. +The browser validates the captured model, reports progress, and returns a +captured result. Optimization uses the same request and experiment record. + +![Browser experiment architecture](@diagrams/ai-experiments-architecture.svg) + +## Request and result + +The [core contract](layer:core.experiments) exports Zod schemas and the host interface +from `@hashintel/petrinaut-core/experiments`. The [browser provider](layer:react.experiment-host) +implements `runExperiment(request, { signal, onProgress })` and exposes it +through `ExperimentHostContext` from `@hashintel/petrinaut/react`. + +Use saved scenario and metric IDs, and scenario parameter identifiers, from +the document definition: + +```json +{ + "name": "Capacity check", + "scenarioId": "saved-scenario-id", + "scenarioParameterValues": {}, + "runCount": 100, + "seed": 42, + "dt": 0.1, + "maxTime": 60, + "metricIds": ["saved-metric-id"], + "execution": { "mode": "simulate" } +} +``` + +Omitted parameters use saved defaults. Simulation accepts fixed values such +as `{ "mode": "fixed", "value": 4 }`. Optimization needs a numeric range such +as `{ "mode": "range", "min": 1, "max": 8 }` and this execution object: + +```json +{ + "mode": "optimize", + "objectiveMetricId": "saved-metric-id", + "direction": "minimize", + "steps": 10, + "runsPerStep": 8 +} +``` + +`runCount` is the simulation budget or the final budget at the best point, +up to 100,000 runs. +`runsPerStep` controls each search trial. The schema bounds runs, steps, +duration, and metric count. The host resolves references, checks parameter +types, and runs [snapshot diagnostics](doc:ai/snapshot-diagnostics) before +compiling the selected scenario and metrics. Execution uses CPU workers. + +The promise resolves with `status` (`complete`, `cancelled`, or `error`), +experiment ID, name, completed runs, and metric values. Optimization adds +selected parameter values, objective value, and completed steps. Failures +include a message. `onProgress` receives updates while the request runs. +The promise resolves with the terminal result; it contains no progress history. + +## Execution and ownership + +The host uses the same experiment runtime, sweep optimization input builder, +and optimizer as the creation drawer. The drawer returns a record when work +starts. The host awaits completion, including final refinement, and captures +the result for its caller. Its cancellation and input ownership cover that +request lifetime. AI tools and other integrations use this same host API. + +![Browser experiment flow](@diagrams/ai-experiments-flow.svg) + +The host captures the model before asynchronous validation. Simulation runs +to its budget. Optimization searches the ranges, then refines the best point +to the requested final budget before returning. + +While the request runs, parameter changes, competing execution, and removal +are blocked. The drawer still allows chart inspection. Aborting the request +cancels its owned work. Completion captures the result, unlocks the controls, +and shows a notification. Later exploration leaves the captured result unchanged. + +Experiments retain metric distributions, including at the optimizer's best +point. The drawer shows heatmaps and histograms. Result values summarize the +last sampled distribution as a mean. See [experiment execution](doc:simulation/experiments), +[sweep orchestration](doc:experiments/sweep-orchestration), and the +[browser optimizer](doc:optimizer/browser-runtime). + +:::danger[Important: host lifetime] +Execution requires the mounted browser host. Durable jobs, recovery after +reload, and background notifications need a persistence contract. +Tracked in [FE-1666](https://linear.app/hash/issue/FE-1666). +::: + +:::danger[Important: objective semantics] +Objectives and returned metrics use the last sampled frame. Runs that ended +earlier can be absent. Terminal or time-integrated objectives need an explicit +reduction contract and matching backend behavior. +Tracked in [FE-1666](https://linear.app/hash/issue/FE-1666). +::: + +:::danger[Important: orchestration refactor] +The coordinator reads experiment and optimization state from React providers. +Extract a reusable implementation behind `PetrinautExperimentHost` when a +second host needs this execution flow. +Tracked in [FE-1666](https://linear.app/hash/issue/FE-1666). +::: From e5638422e3843f5e397134ec474f7a760a80fb67 Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Tue, 15 Sep 2026 13:21:22 +0000 Subject: [PATCH 2/5] FE-1667: Run experiments from Petrinaut AI chat (#9678) --- .changeset/experiment-ai-tool.md | 5 + .changeset/small-experiments-chat.md | 5 + .../test/compiler-feedback.integration.ts | 3 +- .../integration/compiler-feedback.test.ts | 2 +- .../test/root-creation.integration.ts | 5 +- .../test/typed-state.integration.ts | 2 +- libs/@hashintel/petrinaut-core/src/ai.ts | 7 + libs/@hashintel/petrinaut-core/src/index.ts | 1 + .../@hashintel/petrinaut/docs/ai-assistant.md | 43 +- libs/@hashintel/petrinaut/docs/experiments.md | 20 +- .../src/ui/views/Editor/editor-view.test.tsx | 130 +++++ .../src/ui/views/Editor/editor-view.tsx | 35 +- .../Editor/panels/ai-assistant-panel.test.tsx | 347 +++++++++++++- .../Editor/panels/ai-assistant-panel.tsx | 109 ++++- .../panels/ai-assistant-panel/README.md | 14 + .../ai-assistant-contents.tsx | 23 + .../experiment-card.test.tsx | 152 ++++++ .../ai-assistant-contents/experiment-card.tsx | 443 ++++++++++++++++++ .../get-message-render-items.ts | 8 + .../format-diagnostics-for-ai.test.ts | 4 +- .../format-diagnostics-for-ai.ts | 2 +- .../Editor/panels/ai-assistant-panel/types.ts | 7 + .../content/ai/experiment-chat.mdx | 27 ++ .../content/diagrams/ai-experiment-chat.d2 | 12 + .../experiments/ai-created-experiments.mdx | 7 +- 25 files changed, 1352 insertions(+), 61 deletions(-) create mode 100644 .changeset/experiment-ai-tool.md create mode 100644 .changeset/small-experiments-chat.md create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.test.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/README.md create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.test.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.tsx create mode 100644 libs/@local/petrinaut-arch-docs/content/ai/experiment-chat.mdx create mode 100644 libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiment-chat.d2 diff --git a/.changeset/experiment-ai-tool.md b/.changeset/experiment-ai-tool.md new file mode 100644 index 000000000000..0778715ee050 --- /dev/null +++ b/.changeset/experiment-ai-tool.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut-core": patch +--- + +Register the createExperiment tool with its validated request schema for AI integrations. diff --git a/.changeset/small-experiments-chat.md b/.changeset/small-experiments-chat.md new file mode 100644 index 000000000000..a01f78913ff3 --- /dev/null +++ b/.changeset/small-experiments-chat.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Run experiments and optimizations from AI chat with progress, cancellation, and a link to metric distributions. Keep the conversation open while inspecting experiments. diff --git a/apps/brunch-agent/test/compiler-feedback.integration.ts b/apps/brunch-agent/test/compiler-feedback.integration.ts index 2db5b938b373..5187f2251841 100644 --- a/apps/brunch-agent/test/compiler-feedback.integration.ts +++ b/apps/brunch-agent/test/compiler-feedback.integration.ts @@ -34,7 +34,8 @@ import { openBrowserFixture } from "./browser-fixture.ts"; import { browserResultFrom } from "./browser-result.ts"; import { nativeSchemaProvider } from "./native-schema-provider.ts"; -const cleanCompilation = "No errors or warnings found in net function code."; +const cleanCompilation = + "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment."; const output = mkdtempSync(join(tmpdir(), "m7c-compiler-feedback-")); const save = (name: string, value: unknown) => writeFileSync(join(output, `${name}.json`), JSON.stringify(value, null, 2)); diff --git a/apps/brunch-agent/test/integration/compiler-feedback.test.ts b/apps/brunch-agent/test/integration/compiler-feedback.test.ts index 2ea0e6a5e1bf..5c126074ccc8 100644 --- a/apps/brunch-agent/test/integration/compiler-feedback.test.ts +++ b/apps/brunch-agent/test/integration/compiler-feedback.test.ts @@ -37,7 +37,7 @@ test.skipIf(!enabled)( expect(summary.mode).toBe("batched-construction"); expect(summary.dirtyCompilation).toContain("definitelyNotDefined"); expect(summary.cleanCompilation).toBe( - "No errors or warnings found in net function code.", + "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment.", ); expect(summary.repairHash).toMatch(/^[a-f0-9]{64}$/u); expect(summary.layoutHash).toMatch(/^[a-f0-9]{64}$/u); diff --git a/apps/brunch-agent/test/root-creation.integration.ts b/apps/brunch-agent/test/root-creation.integration.ts index f12e4d74afa4..d83c5a0b8bcb 100644 --- a/apps/brunch-agent/test/root-creation.integration.ts +++ b/apps/brunch-agent/test/root-creation.integration.ts @@ -301,7 +301,7 @@ try { save("compilation", result); assert.equal( result.output, - "No errors or warnings found in net function code.", + "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment.", ); return text("Native creation and canonical check completed."); }), @@ -504,7 +504,8 @@ try { requests: contexts.length, applied: records.length, schemaClasses: observedNodeMutationNames, - compilation: "No errors or warnings found in net function code.", + compilation: + "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment.", scope: "Same-session synthetic creation/correction only; reopen assertion follows.", }); diff --git a/apps/brunch-agent/test/typed-state.integration.ts b/apps/brunch-agent/test/typed-state.integration.ts index 74a659b17012..1f54ac36cdd3 100644 --- a/apps/brunch-agent/test/typed-state.integration.ts +++ b/apps/brunch-agent/test/typed-state.integration.ts @@ -631,7 +631,7 @@ try { compilations.push(result); assert.equal( result.output, - "No errors or warnings found in net function code.", + "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment.", "Final corrected net must report clean canonical diagnostics; no scenario execution follows", ); return text( diff --git a/libs/@hashintel/petrinaut-core/src/ai.ts b/libs/@hashintel/petrinaut-core/src/ai.ts index 6fbc5884c9e5..894cb7126d70 100644 --- a/libs/@hashintel/petrinaut-core/src/ai.ts +++ b/libs/@hashintel/petrinaut-core/src/ai.ts @@ -9,6 +9,7 @@ import { type AiCommandActionName, } from "./command-schemas"; import { probabilisticSatellitesSDCPN } from "./examples"; +import { petrinautExperimentRequestSchema } from "./experiments/host"; import { typedKeys } from "./lib/typed-entries"; import type { Petrinaut } from "./instance"; @@ -93,6 +94,7 @@ export const getLatestNetDefinitionToolName = "getLatestNetDefinition"; export const getNetCompilationErrorsToolName = "getNetCompilationErrors"; export const setNetTitleToolName = "setNetTitle"; export const readPetrinautDocToolName = "readPetrinautDoc"; +export const createExperimentToolName = "createExperiment"; export const petrinautDocNames = [ "drawing-a-net", @@ -182,6 +184,7 @@ export const petrinautAiToolInputSchemas = { [getNetCompilationErrorsToolName]: getNetCompilationErrorsToolInputSchema, [setNetTitleToolName]: setNetTitleToolInputSchema, [readPetrinautDocToolName]: readPetrinautDocToolInputSchema, + [createExperimentToolName]: petrinautExperimentRequestSchema, }; export const petrinautAiMutationTools = createToolBundle( @@ -211,6 +214,10 @@ export const petrinautAiTools = { description: getSchemaDescription(readPetrinautDocToolInputSchema), inputSchema: readPetrinautDocToolInputSchema, }, + [createExperimentToolName]: { + description: getSchemaDescription(petrinautExperimentRequestSchema), + inputSchema: petrinautExperimentRequestSchema, + }, } satisfies PetrinautAiTools; export type PetrinautAiToolName = keyof typeof petrinautAiTools; diff --git a/libs/@hashintel/petrinaut-core/src/index.ts b/libs/@hashintel/petrinaut-core/src/index.ts index 03f5ae7b12fe..5794c6d02b97 100644 --- a/libs/@hashintel/petrinaut-core/src/index.ts +++ b/libs/@hashintel/petrinaut-core/src/index.ts @@ -202,6 +202,7 @@ export { colorSchema, componentInstanceSchema, createPetrinautAiWritableCallbacks, + createExperimentToolName, differentialEquationSchema, getLatestNetDefinitionToolName, getNetCompilationErrorsToolName, diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index b828d8223ff6..5ce4f524111a 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -1,6 +1,6 @@ # AI Assistant -Petrinaut has an in-app AI assistant that can build a net from a natural-language description, review or revise an existing one, read TypeScript compilation diagnostics, and consult its own user-guide pages to answer "how do I ..." questions. The host application controls whether the assistant is available -- it is enabled on [demo.petrinaut.org](https://demo.petrinaut.org) and in [HASH](https://hash.ai) and may or may not be enabled in other Petrinaut embeds. +Petrinaut has an in-app AI assistant that can build a net from a natural-language description, review or revise an existing one, read TypeScript compilation diagnostics, run experiments, and consult its own user-guide pages to answer "how do I ..." questions. The host application controls whether the assistant and its tools are available -- it is enabled on [demo.petrinaut.org](https://demo.petrinaut.org) and in [HASH](https://hash.ai) and may or may not be enabled in other Petrinaut embeds. ## Opening the panel @@ -10,7 +10,13 @@ There are two entry points: 2. **File → New → Build with Brunch**. When the host enables its experimental Brunch demo mode, shows net-management controls, and provides an assistant, this creates a fresh empty net, opens the assistant, and offers two chips before the first message: **Interview first** (elicit before inventing missing detail or drawing the net) or **Quick preview** (fill reasonable gaps, mark them as provisional, and wait for assent). **Start blank** creates the same empty net without opening the assistant or showing the first-run prompt. Without that host flag, **New** retains its ordinary direct blank-net behavior. 3. **First-run prompt**. When you load Petrinaut against an empty net, a centred prompt appears. Type a description and its trailing action becomes **Send**; select it to open the panel with your message already in flight. When the host provides Voice mode, the empty prompt instead shows a waveform action titled **Start voice mode**. It opens the same assistant without creating an empty text message. Dismiss the prompt with the **X**, by clicking outside it, or by pressing **Escape**; it is hidden for the rest of the session once dismissed. -The assistant panel only renders in **Edit** mode. Switching to **Simulate** mode hides it; switch back to **Edit** to continue the conversation. The panel resizes by dragging its left edge. Text and voice share the **AI** transcript. Some hosts add a second tab, such as **Workpiece**, for a saved document. Select a tab to switch views, or use the left/right arrow keys while a tab is focused. Switching does not end a response, clear your draft or interrupt Voice; the composer and active controls remain available. +The assistant panel stays open when you switch to **Simulate** or **Notebook** +mode, so you can inspect an experiment while its chat result arrives. The panel +resizes by dragging its left edge. Text and voice share the **AI** transcript. +Some hosts add a second tab, such as **Workpiece**, for a saved document. +Select a tab to switch views, or use the left/right arrow keys while a tab is +focused. Switching does not end a response, clear your draft or interrupt Voice; +the composer and active controls remain available. ## The conversation @@ -168,6 +174,34 @@ For example, the Petrinaut website's worked-model route displays its template ti After applying changes, the assistant may automatically check TypeScript compile diagnostics (you'll see a **Checked net compilation errors** card) and fix problems on its own before continuing. +## Experiments from chat + +When your host enables experiment tools, ask the assistant to run a saved +[scenario](scenarios.md) and measure one or more saved metrics. +For example: "Run 100 simulations of this scenario and show the completed +orders metric." The assistant can also search numeric scenario parameter +ranges to minimize or maximize a metric. + +The experiment appears in a compact card with its status, run count, and +results. Simulation cards use blue; optimization cards use purple and glow +while running. Select **View +experiment** to inspect metric distributions in the Experiments drawer. The +heatmap shows how values spread across runs; click a time step to see its +histogram. Select **Cancel** +to stop its work. The assistant receives the +result when the requested work finishes; an optimization includes the final +runs at its best parameter values. + +While an AI experiment runs, its compute-changing controls are locked. You +can inspect its charts or cancel it. After completion you can explore its +parameters again; the result already recorded in chat stays unchanged. +Experiments run in your current browser session, so keep the page open until +they finish. See [Experiments](experiments.md#experiments-created-by-the-assistant). + +A request with no saved result and no active run shows **Not running**. +Ask the assistant to run a new experiment. **Cancel** is available only for +experiments running in this panel. + ## Read-only behaviour Whether the assistant can change the net depends on the editor state: @@ -182,8 +216,9 @@ The composer stays open in all of these cases, so you can still ask questions, r The assistant can request a fresh TypeScript check of the current net and use the returned errors to revise its code. An unchanged set of errors still counts as a completed check. If checking fails, the assistant receives an -error. The bottom **Diagnostics** tab continues to show diagnostics for the -code you are editing. +error. Experiment creation also checks its selected +scenario and metrics before running. The bottom **Diagnostics** tab continues +to show diagnostics for the code you are editing. ## Host configuration diff --git a/libs/@hashintel/petrinaut/docs/experiments.md b/libs/@hashintel/petrinaut/docs/experiments.md index c2e53c562f1a..d0aac65fa4f1 100644 --- a/libs/@hashintel/petrinaut/docs/experiments.md +++ b/libs/@hashintel/petrinaut/docs/experiments.md @@ -11,19 +11,23 @@ Experiments live under the **Simulate** [global mode](drawing-a-net.md#global-mo 3. Fill in the configuration (see below). 4. Click **Run** -- **Create sweep** when a value is swept, **Optimize** when the in-browser optimizer will search it. The button reads **Starting** (or **Creating**) while the experiment starts. -### Experiments started by an integration +### Experiments created by the assistant -An application embedding Petrinaut can start an experiment with a saved -scenario and metrics, including a search over numeric parameter ranges. -These experiments appear in the Experiments list. +If your host enables experiment tools, the [AI assistant](ai-assistant.md#experiments-from-chat) +can run a saved scenario with saved metrics and fixed parameter values. It +can also optimize numeric parameter ranges. These requests create ordinary +experiments in this list and show a progress card in chat. Each request supports up to 100,000 simulation runs, or 100,000 final runs at the best parameter values after optimization. -While the request runs, parameter changes and removal are locked. You can -inspect the charts. Controls become available when the request finishes. -Later exploration leaves the result returned to the application unchanged. +While the assistant's request runs, parameter changes and removal are locked. +You can inspect the charts and cancel the experiment. When it finishes, the +assistant receives its results and the controls become available again. +Later exploration does not change the result already recorded in chat. + An optimization finishes after its search and the final runs at its best -parameter values. Keep the browser page open until the request finishes. +parameter values. Keep the browser page open until the request finishes; +experiments are not restored after a reload. ### Configuration diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.test.tsx new file mode 100644 index 000000000000..8dfe8fd3ce56 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.test.tsx @@ -0,0 +1,130 @@ +/** @vitest-environment jsdom */ +import { cleanup, render, screen } from "@testing-library/react"; +import { use, useEffect } from "react"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { + EditorContext, + type EditorGlobalMode, +} from "../../../react/state/editor-context"; +import { UserSettingsContext } from "../../../react/state/user-settings-context"; +import { EditorView } from "./editor-view"; + +import type { PetrinautAiAssistant } from "../../petrinaut"; +import type { UIMessageChunk } from "ai"; + +const lifecycle = vi.hoisted(() => ({ + mount: vi.fn(), + cancelPendingRequest: vi.fn(), +})); + +vi.mock("../../../react", () => ({ + usePetrinautCommands: () => ({ applyAutoLayout: vi.fn() }), +})); +vi.mock("../../../react/state/use-selection-cleanup", () => ({ + useSelectionCleanup: () => {}, +})); +vi.mock("../../../react/state/use-is-read-only", () => ({ + useIsReadOnly: () => false, +})); +vi.mock("./panels/ai-assistant-panel", () => ({ + AiAssistantPanel: () => { + useEffect(() => { + lifecycle.mount(); + return lifecycle.cancelPendingRequest; + }, []); + return
Pending experiment
; + }, +})); +vi.mock("./panels/SimulateView/simulate-view", () => ({ + SimulateView: () => ( +
Experiment results
+ ), +})); +vi.mock("../Notebook/notebook-view", () => ({ + NotebookView: () =>
, +})); +vi.mock("../SDCPN/sdcpn-view", () => ({ SDCPNView: () => null })); +vi.mock("./panels/LeftSideBar/panel", () => ({ LeftSideBar: () => null })); +vi.mock("./panels/PropertiesPanel/panel", () => ({ + PropertiesPanel: () => null, +})); +vi.mock("./panels/BottomPanel/panel", () => ({ BottomPanel: () => null })); +vi.mock("./components/BottomBar/bottom-bar", () => ({ BottomBar: () => null })); +vi.mock("./components/TopBar/top-bar", () => ({ TopBar: () => null })); +vi.mock("./components/ai-cta-modal", () => ({ AiCtaModal: () => null })); +vi.mock("./components/import-error-dialog", () => ({ + ImportErrorDialog: () => null, +})); +vi.mock("../../components/walkthrough/walkthrough-dialog", () => ({ + WalkthroughDialog: () => null, +})); +vi.mock("./simulation-creation-drawer", () => ({ + SimulationCreationDrawer: () => null, +})); +vi.mock("./use-editor-commands", () => ({ EditorCommands: () => null })); + +const aiAssistant: PetrinautAiAssistant = { + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: () => + Promise.resolve( + new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + ), + }, +}; + +const EditorAtMode = ({ mode }: { mode: EditorGlobalMode }) => { + const editor = use(EditorContext); + const settings = use(UserSettingsContext); + return ( + + + + + + ); +}; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("EditorView assistant lifecycle", () => { + test("keeps the pending assistant mounted when opening experiment results and changing modes", () => { + const { rerender, unmount } = render(); + const assistant = screen.getByRole("region", { name: "AI assistant" }); + expect(lifecycle.mount).toHaveBeenCalledTimes(1); + + rerender(); + expect(screen.getByRole("region", { name: "Experiments" })).toBeTruthy(); + expect(screen.getByRole("region", { name: "AI assistant" })).toBe( + assistant, + ); + + rerender(); + expect(screen.getByRole("region", { name: "Notebook" })).toBeTruthy(); + expect(screen.getByRole("region", { name: "AI assistant" })).toBe( + assistant, + ); + + rerender(); + expect(screen.getByRole("region", { name: "AI assistant" })).toBe( + assistant, + ); + expect(lifecycle.mount).toHaveBeenCalledTimes(1); + expect(lifecycle.cancelPendingRequest).not.toHaveBeenCalled(); + + unmount(); + expect(lifecycle.cancelPendingRequest).toHaveBeenCalledTimes(1); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx index 4f1318d45974..5552eb3a8132 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx @@ -100,6 +100,7 @@ const formatRelativeTime = (isoTimestamp: string): string => { // overflow, scrollIntoView can still scroll it programmatically — pushing the // TopBar out of view. const rowContainerStyle = css({ + position: "relative", flex: "[1]", minHeight: "[0]", userSelect: "none", @@ -573,25 +574,25 @@ export const EditorView = ({ onCursorModeChange={setCursorMode} hasAiAssistant={aiAssistant !== undefined} /> - - {aiAssistant && ( - - setPendingAiAssistantMessage(null) - } - onInitialInteractionModeConsumed={() => - setPendingAiInteractionMode(null) - } - /> - )} )} + + {aiAssistant && ( + + setPendingAiAssistantMessage(null) + } + onInitialInteractionModeConsumed={() => + setPendingAiInteractionMode(null) + } + /> + )} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 81cd533b4bfb..6b59ddcea956 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -16,6 +16,9 @@ import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS, createJsonDocHandle, + type PetrinautExperimentHost, + type PetrinautExperimentRequest, + type PetrinautExperimentResult, createPetrinaut, getLatestNetDefinitionToolName, setNetTitleToolName, @@ -26,6 +29,7 @@ import { type ErrorTracker, ErrorTrackerContext, } from "../../../../react/error-tracker-context"; +import { ExperimentHostContext } from "../../../../react/experiment-host/context"; import { PetrinautInstanceContext } from "../../../../react/instance-context"; import { DEFAULT_LANGUAGE_CLIENT_CONTEXT, @@ -236,6 +240,7 @@ const renderTestPanel = ({ total: 0, errorCount: 0, }), + experimentHost, }: { aiAssistant: PetrinautAiAssistant; editorContext?: EditorContextValue; @@ -247,6 +252,7 @@ const renderTestPanel = ({ strictMode?: boolean; titleEditable?: boolean; requestDiagnostics?: LanguageClientContextValue["requestDiagnostics"]; + experimentHost?: PetrinautExperimentHost; }) => { const handle = createJsonDocHandle({ id: "ai-assistant-panel-test", @@ -276,20 +282,29 @@ const renderTestPanel = ({ ) => ( - - - - - - - + + Promise.reject(new Error("Experiment host unavailable")), + } + } + > + + + + + + + + ); @@ -5374,3 +5389,307 @@ describe("AiAssistantPanel host interactive tools", () => { } }); }); + +describe("AI experiment requests", () => { + const request: PetrinautExperimentRequest = { + name: "Chat experiment", + scenarioId: "scenario-1", + scenarioParameterValues: {}, + runCount: 8, + seed: 42, + dt: 0.1, + maxTime: 10, + metricIds: ["metric-1"], + execution: { mode: "simulate" }, + }; + const result: PetrinautExperimentResult = { + status: "complete", + experimentId: "experiment-1", + name: request.name, + runsCompleted: 8, + metrics: [{ id: "metric-1", label: "Count", value: 12 }], + }; + const createTransport = () => { + const sendMessages = vi.fn(); + sendMessages.mockImplementationOnce(async () => + streamChunks([ + { type: "start-step" }, + { + type: "tool-input-available", + toolCallId: "experiment-call", + toolName: "createExperiment", + input: request, + }, + { type: "finish-step" }, + { type: "finish", finishReason: "tool-calls" }, + ]), + ); + sendMessages.mockImplementation(async () => + streamChunks(textChunks("done", "Result received")), + ); + return { + transport: { reconnectToStream: async () => null, sendMessages }, + sendMessages, + }; + }; + + test("shows progress and sends exactly one captured result after completion", async () => { + const completion = Promise.withResolvers(); + const createExperiment = vi.fn( + (input, options) => { + options?.onProgress?.({ + experimentId: "experiment-1", + name: input.name, + phase: "running", + runsCompleted: 3, + runsTarget: 8, + }); + return completion.promise; + }, + ); + const { transport, sendMessages } = createTransport(); + renderTestPanel({ + aiAssistant: { transport }, + initialMessage: "Run an experiment", + experimentHost: { runExperiment: createExperiment }, + }); + + const card = await screen.findByRole("region", { + name: "Experiment: Chat experiment", + }); + await waitFor(() => + expect(within(card).getByText("3 of 8 runs")).not.toBeNull(), + ); + expect(sendMessages).toHaveBeenCalledTimes(1); + await act(async () => completion.resolve(result)); + await screen.findByText("Result received"); + expect(within(card).getByText("Finished")).not.toBeNull(); + expect(within(card).getByText("Count")).not.toBeNull(); + expect(within(card).getByText("12")).not.toBeNull(); + expect(sendMessages).toHaveBeenCalledTimes(2); + expect( + sendMessages.mock.calls[1]?.[0].messages.flatMap( + (message) => message.parts, + ), + ).toContainEqual( + expect.objectContaining({ + type: "tool-createExperiment", + output: result, + }), + ); + }); + + test("cancels browser computation through the experiment card", async () => { + const createExperiment = vi.fn( + (input, options) => + new Promise((resolve) => { + options?.onProgress?.({ + experimentId: "experiment-1", + name: input.name, + phase: "running", + runsCompleted: 0, + runsTarget: 8, + }); + options?.signal?.addEventListener("abort", () => + resolve({ + ...result, + status: "cancelled", + runsCompleted: 0, + metrics: [], + }), + ); + }), + ); + const { transport } = createTransport(); + renderTestPanel({ + aiAssistant: { transport }, + initialMessage: "Run an experiment", + experimentHost: { runExperiment: createExperiment }, + }); + const card = await screen.findByRole("region", { + name: "Experiment: Chat experiment", + }); + await waitFor(() => expect(createExperiment).toHaveBeenCalledOnce()); + fireEvent.click(within(card).getByRole("button", { name: "Cancel" })); + await waitFor(() => + expect(within(card).getByText("Cancelled")).not.toBeNull(), + ); + expect(createExperiment.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); + }); + + test("shows observed experiment requests without claiming local execution", async () => { + const runExperiment = vi.fn(); + const { transport, sendMessages } = createTransport(); + renderTestPanel({ + aiAssistant: { + transport, + followMessages: { canReplace: () => true }, + messages: [ + { + id: "observed-experiment", + role: "assistant", + parts: [ + { + type: "tool-createExperiment", + toolCallId: "observed-call", + state: "input-available", + input: request, + }, + ], + }, + ], + }, + experimentHost: { runExperiment }, + }); + const card = await screen.findByRole("region", { + name: "Experiment: Chat experiment", + }); + await act(async () => {}); + expect(within(card).getByText("Not running")).not.toBeNull(); + expect(within(card).queryByRole("button", { name: "Cancel" })).toBeNull(); + expect(card.getAttribute("aria-busy")).toBe("false"); + expect(runExperiment).not.toHaveBeenCalled(); + expect(sendMessages).not.toHaveBeenCalled(); + }); + + test("allows cancellation during validation before the host reports progress", async () => { + const completion = Promise.withResolvers(); + const runExperiment = vi.fn( + (_input, options) => { + options?.signal?.addEventListener("abort", () => + completion.resolve({ + ...result, + status: "cancelled", + experimentId: null, + runsCompleted: 0, + metrics: [], + }), + ); + return completion.promise; + }, + ); + const { transport } = createTransport(); + renderTestPanel({ + aiAssistant: { transport }, + initialMessage: "Run", + experimentHost: { runExperiment }, + }); + const card = await screen.findByRole("region", { + name: "Experiment: Chat experiment", + }); + await waitFor(() => + expect(within(card).getByText("Validating")).not.toBeNull(), + ); + fireEvent.click(within(card).getByRole("button", { name: "Cancel" })); + await waitFor(() => + expect(within(card).getByText("Cancelled")).not.toBeNull(), + ); + expect(runExperiment.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); + }); + + test("clears active experiment indicators if the host rejects", async () => { + const completion = Promise.withResolvers(); + const runExperiment = vi.fn( + () => completion.promise, + ); + const { transport } = createTransport(); + renderTestPanel({ + aiAssistant: { transport }, + initialMessage: "Run", + experimentHost: { runExperiment }, + }); + const card = await screen.findByRole("region", { + name: "Experiment: Chat experiment", + }); + await waitFor(() => + expect(within(card).getByText("Validating")).not.toBeNull(), + ); + await act(async () => completion.reject(new Error("Host failed"))); + await waitFor(() => expect(card.getAttribute("aria-busy")).toBe("false")); + expect(within(card).queryByRole("button", { name: "Cancel" })).toBeNull(); + }); + + test("ignores a late experiment from a replaced conversation with the same tool ID", async () => { + const previous = Promise.withResolvers(); + const current = Promise.withResolvers(); + const createExperiment = vi.fn(); + createExperiment.mockImplementationOnce(() => previous.promise); + createExperiment.mockImplementationOnce((input, options) => { + options?.onProgress?.({ + experimentId: "experiment-2", + name: input.name, + phase: "running", + runsCompleted: 3, + runsTarget: 8, + }); + options?.signal?.addEventListener("abort", () => + current.resolve({ + ...result, + experimentId: "experiment-2", + name: input.name, + status: "cancelled", + runsCompleted: 3, + metrics: [], + }), + ); + return current.promise; + }); + const sendMessages = vi.fn(async () => + streamChunks(textChunks("result", "Result received")), + ); + const config = (conversationId: string): PetrinautAiAssistant => ({ + conversationId, + transport: { reconnectToStream: async () => null, sendMessages }, + messages: [ + { + id: `${conversationId}-request`, + role: "assistant", + parts: [ + { + type: "tool-createExperiment", + toolCallId: "reused-experiment-call", + state: "input-available", + input: { ...request, name: conversationId }, + }, + ], + }, + ], + }); + const { rerenderPanel } = renderTestPanel({ + aiAssistant: config("Previous experiment"), + experimentHost: { runExperiment: createExperiment }, + }); + await waitFor(() => expect(createExperiment).toHaveBeenCalledOnce()); + rerenderPanel(config("Current experiment")); + await waitFor(() => expect(createExperiment).toHaveBeenCalledTimes(2)); + expect(createExperiment.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); + const card = await screen.findByRole("region", { + name: "Experiment: Current experiment", + }); + + await act(async () => { + createExperiment.mock.calls[0]?.[1]?.onProgress?.({ + experimentId: "experiment-1", + name: "Previous experiment", + phase: "running", + runsCompleted: 7, + runsTarget: 8, + }); + previous.resolve({ ...result, name: "Previous experiment" }); + await previous.promise; + }); + expect(within(card).getByText("3 of 8 runs")).not.toBeNull(); + expect(within(card).queryByText("Finished")).toBeNull(); + expect( + screen.queryByRole("region", { name: "Experiment: Previous experiment" }), + ).toBeNull(); + expect(sendMessages).not.toHaveBeenCalled(); + + fireEvent.click(within(card).getByRole("button", { name: "Cancel" })); + expect(createExperiment.mock.calls[1]?.[1]?.signal?.aborted).toBe(true); + await waitFor(() => + expect(within(card).getByText("Cancelled")).not.toBeNull(), + ); + await waitFor(() => expect(sendMessages).toHaveBeenCalledOnce()); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 3f86c40d2f19..d8114fac1bed 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -18,6 +18,10 @@ import { import { aiCommandActionInputSchemas, + createExperimentToolName, + petrinautExperimentRequestSchema, + type PetrinautExperimentProgress, + type PetrinautExperimentResult, type AiCommandActionName, getLatestNetDefinitionToolName, getNetCompilationErrorsToolName, @@ -31,6 +35,7 @@ import { } from "@hashintel/petrinaut-core"; import { ErrorTrackerContext } from "../../../../react/error-tracker-context"; +import { ExperimentHostContext } from "../../../../react/experiment-host/context"; import { useLatest } from "../../../../react/hooks/use-latest"; import { PetrinautInstanceContext } from "../../../../react/instance-context"; import { LanguageClientContext } from "../../../../react/lsp/context"; @@ -507,9 +512,32 @@ const ConversationAiAssistantPanel = ({ const { requestDiagnostics } = use(LanguageClientContext); const requestDiagnosticsRef = useLatest(requestDiagnostics); + const experimentHost = use(ExperimentHostContext); + const experimentControllersRef = useRef(new Map()); + const [experimentStates, setExperimentStates] = useState< + Record< + string, + { + active: boolean; + progress?: PetrinautExperimentProgress; + result?: PetrinautExperimentResult; + } + > + >({}); + const cancelExperiment = useCallback((toolCallId: string) => { + experimentControllersRef.current.get(toolCallId)?.abort(); + }, []); + useEffect(() => { + const controllers = experimentControllersRef.current; + return () => { + for (const controller of controllers.values()) controller.abort(); + controllers.clear(); + }; + }, []); const { hasSelection, + globalMode, isAiAssistantOpen, navigateTo, propertiesPanelWidth, @@ -590,9 +618,6 @@ const ConversationAiAssistantPanel = ({ return readCurrentDiagnostics(instance, requestDiagnosticsRef.current); }, [instance, requestDiagnosticsRef]); - /* eslint-disable react-hooks-js/refs -- See the `"use no memo"` directive - above: the refs are only read when the wrapped transport runs, never during - render. The lint rule can't see that. */ // The wrapper is render-derived from the host transport. Delaying this to an // effect leaves useChat on the previous host for one committed render. // Timing stays outside diagnostics so it tags receipt of the response chunks. @@ -606,7 +631,6 @@ const ConversationAiAssistantPanel = ({ ), [aiAssistant.transport, readDiagnosticsContext], ); - /* eslint-enable react-hooks-js/refs */ // Stream errors (server returned an error chunk, function timed out, etc.) // are otherwise opaque to the user — `useChat` resets `status` to `"ready"` @@ -881,6 +905,61 @@ const ConversationAiAssistantPanel = ({ return; } + if (toolCall.toolName === createExperimentToolName) { + const request = petrinautExperimentRequestSchema.parse(toolCall.input); + const controller = new AbortController(); + experimentControllersRef.current.set(toolCall.toolCallId, controller); + setExperimentStates((states) => ({ + ...states, + [toolCall.toolCallId]: { active: true }, + })); + const isCurrentRequest = () => + generation === submissionGenerationRef.current && + executionConversationId === toolHostIdentityRef.current && + experimentControllersRef.current.get(toolCall.toolCallId) === + controller; + try { + const result = await experimentHost.runExperiment(request, { + signal: controller.signal, + onProgress: (progress) => { + if (!isCurrentRequest()) return; + setExperimentStates((states) => ({ + ...states, + [toolCall.toolCallId]: { active: true, progress }, + })); + }, + }); + if (isCurrentRequest()) { + setExperimentStates((states) => ({ + ...states, + [toolCall.toolCallId]: { active: false, result }, + })); + } + await addAutomaticToolOutput({ + tool: toolCall.toolName, + toolCallId: toolCall.toolCallId, + output: result, + }); + } finally { + if (isCurrentRequest()) { + setExperimentStates((states) => ({ + ...states, + [toolCall.toolCallId]: { + ...states[toolCall.toolCallId], + active: false, + }, + })); + } + if ( + experimentControllersRef.current.get(toolCall.toolCallId) === + controller + ) { + experimentControllersRef.current.delete(toolCall.toolCallId); + } + } + return; + } + if (toolCall.toolName === getLatestNetDefinitionToolName) { await addAutomaticToolOutput({ tool: toolCall.toolName, @@ -1253,6 +1332,10 @@ const ConversationAiAssistantPanel = ({ const submissionConversationIdRef = useRef(conversationId); useLayoutEffect(() => { if (submissionConversationIdRef.current === conversationId) return; + for (const controller of experimentControllersRef.current.values()) + controller.abort(); + experimentControllersRef.current.clear(); + setExperimentStates({}); submissionConversationIdRef.current = conversationId; followedMessagesRef.current = undefined; locallyStreamedToolCallsRef.current.clear(); @@ -1751,6 +1834,8 @@ const ConversationAiAssistantPanel = ({ const generation = submissionGenerationRef.current; automaticToolTerminationRef.current = { generation, kind: "stopped" }; abortAutomaticTools(); + for (const controller of experimentControllersRef.current.values()) + controller.abort(); stopRequestedRef.current = true; if (requestStop !== undefined) { try { @@ -1962,9 +2047,6 @@ const ConversationAiAssistantPanel = ({ stop: stopComposer, submitText, }; - /* eslint-disable react-hooks-js/refs -- The public render prop receives - stable event callbacks that read their refs only when the host invokes - them from an event handler or effect. */ const composerControl = aiAssistant.renderComposerControl?.( composerControlContext, ); @@ -1979,7 +2061,6 @@ const ConversationAiAssistantPanel = ({ setVoiceActive, submitVoiceInput, }); - /* eslint-enable react-hooks-js/refs */ return ( { abortAutomaticTools(); + for (const controller of experimentControllersRef.current.values()) + controller.abort(); + experimentControllersRef.current.clear(); + setExperimentStates({}); submissionGenerationRef.current += 1; // Clearing aborts any in-flight response too, which fires `onFinish` // with `isAbort`. Drop the stop flag first so that handler treats this @@ -2108,7 +2195,11 @@ const ConversationAiAssistantPanel = ({ onSubmit={submitComposerInput} onVoiceDockCollapsedChange={setVoiceDockCollapsed} promptChips={promptChips} - rightOffset={hasSelection ? propertiesPanelWidth + PANEL_MARGIN : 0} + rightOffset={ + globalMode === "edit" && hasSelection + ? propertiesPanelWidth + PANEL_MARGIN + : 0 + } status={status} stopped={stopped} voiceHandoffPending={voiceHandoffPending} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/README.md b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/README.md new file mode 100644 index 000000000000..8f3807b8414f --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/README.md @@ -0,0 +1,14 @@ +--- +layer: ui.views.editor.ai +role: Renders AI conversations and executes client tools against the editor host +--- + +The assistant renders AI SDK messages and dispatches client tools from the +parent `ai-assistant-panel.tsx`. Document tools call the core instance. +Experiment tools call the browser host and show progress until its final result. +The embedding application supplies transport and conversation persistence. + +See the browser host +for execution and cancellation, and +the chat integration +for tool dispatch and presentation. diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 167e835b509d..948acef7b9ef 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -28,6 +28,10 @@ import { HorizontalTabsHeader } from "../../../../components/sub-view/horizontal import { ResizeHandle } from "../../../../resize/resize-handle"; import { AiVoiceModeIcon } from "../../components/ai-voice-mode-button"; import { voiceSetupLabels } from "../../components/voice-session-labels"; +import { + ExperimentCard, + type AiExperimentState, +} from "./ai-assistant-contents/experiment-card"; import { aiFooterMinHeight } from "./ai-assistant-contents/footer-height"; import { getMessageRenderItems } from "./ai-assistant-contents/get-message-render-items"; import { @@ -64,6 +68,8 @@ export type AiAssistantContentsProps = { composerControl?: ReactNode; composerFocusRequest?: number; error?: Error; + experimentStates?: Record; + onCancelExperiment?: (toolCallId: string) => void; input: string; inputMode?: PetrinautAiInputMode; interactiveTools?: readonly PetrinautAiInteractiveTool[]; @@ -431,10 +437,14 @@ const AiAssistantMessage = memo( handlersRef, interactiveTools, message, + experimentStates, + onCancelExperiment, }: { handlersRef: MessageHandlersRef; interactiveTools: readonly PetrinautAiInteractiveTool[]; message: PetrinautAiMessage; + experimentStates?: Record; + onCancelExperiment?: (toolCallId: string) => void; }) => { const role = message.role === "user" ? "user" : "assistant"; const renderItems = getMessageRenderItems(message, interactiveTools); @@ -474,6 +484,15 @@ const AiAssistantMessage = memo( part={item.part} /> ); + case "experiment": + return ( + + ); case "tools": return ( ))} {stopped && !error && !messages.at(-1)?.metadata?.stopped && ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.test.tsx new file mode 100644 index 000000000000..61acb19004d4 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.test.tsx @@ -0,0 +1,152 @@ +/** @vitest-environment jsdom */ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; + +import { ExperimentCard } from "./experiment-card"; + +import type { ExperimentToolPart } from "./experiment-card"; +import type { PetrinautExperimentResult } from "@hashintel/petrinaut-core"; + +afterEach(cleanup); + +const part: ExperimentToolPart = { + type: "tool-createExperiment", + toolCallId: "tool", + state: "input-available", + input: { + name: "Population", + scenarioId: "scenario", + scenarioParameterValues: {}, + runCount: 8, + seed: 1, + dt: 1, + maxTime: 10, + metricIds: ["count"], + execution: { mode: "simulate" }, + }, +}; +const result: PetrinautExperimentResult = { + status: "complete", + experimentId: "experiment", + name: "Population", + runsCompleted: 8, + metrics: [{ id: "count", label: "Count", value: 12 }], +}; + +it("reports actual run progress and removes active indicators when finished", () => { + const progress = { + experimentId: "experiment", + name: "Population", + phase: "running" as const, + runsCompleted: 3, + runsTarget: 8, + }; + const view = render( + , + ); + const card = screen.getByRole("region", { name: "Experiment: Population" }); + expect(card.getAttribute("data-tone")).toBe("simulation"); + expect(card.getAttribute("aria-busy")).toBe("true"); + expect(screen.getByRole("progressbar").getAttribute("aria-valuenow")).toBe( + "3", + ); + expect(screen.getByRole("progressbar").getAttribute("aria-valuemax")).toBe( + "8", + ); + view.rerender( + , + ); + expect(screen.getByRole("progressbar").getAttribute("aria-valuenow")).toBe( + "5", + ); + view.rerender( + , + ); + expect(card.getAttribute("aria-busy")).toBe("false"); + expect(card.getAttribute("data-pending")).toBe("false"); + expect(screen.queryByRole("progressbar")).toBeNull(); + expect(screen.getByRole("status").textContent).toBe("Finished"); +}); + +it("keeps optimization identity while validating and after cancellation", () => { + const optimizationPart: ExperimentToolPart = { + ...part, + input: { + ...part.input, + execution: { + mode: "optimize", + objectiveMetricId: "count", + direction: "maximize", + steps: 3, + runsPerStep: 4, + }, + }, + }; + const view = render( + , + ); + const card = screen.getByRole("region", { name: "Experiment: Population" }); + expect(card.getAttribute("data-tone")).toBe("optimization"); + expect(screen.getByRole("status").textContent).toBe("Validating"); + view.rerender( + , + ); + expect(card.getAttribute("data-tone")).toBe("optimization"); + expect(card.getAttribute("data-pending")).toBe("false"); + expect(screen.getByRole("status").textContent).toBe("Cancelled"); +}); + +it("shows a failed tool without a pending validation label or progress", () => { + render( + , + ); + expect(screen.getByRole("status").textContent).toBe("Failed"); + expect(screen.queryByText("Checking the model")).toBeNull(); + expect(screen.queryByRole("progressbar")).toBeNull(); +}); + +it("only offers cancellation while this panel owns a running request", () => { + const onCancel = vi.fn(); + const view = render(); + const card = screen.getByRole("region", { name: "Experiment: Population" }); + expect(card.getAttribute("aria-busy")).toBe("false"); + expect(screen.getByRole("status").textContent).toBe("Not running"); + expect(screen.queryByRole("button", { name: "Cancel" })).toBeNull(); + expect( + screen.getByText(/Ask the assistant to run a new experiment/), + ).not.toBeNull(); + + view.rerender( + , + ); + expect(screen.getByRole("status").textContent).toBe("Validating"); + fireEvent.click(screen.getByRole("button", { name: "Cancel" })); + expect(onCancel).toHaveBeenCalledWith(part.toolCallId); + + view.rerender( + , + ); + expect(card.getAttribute("aria-busy")).toBe("false"); + expect(screen.queryByRole("button", { name: "Cancel" })).toBeNull(); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.tsx new file mode 100644 index 000000000000..697c9662e2d4 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-card.tsx @@ -0,0 +1,443 @@ +import { use } from "react"; + +import { Button, Icon } from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; + +import { ExperimentsContext } from "../../../../../../react/experiments/context"; +import { EditorContext } from "../../../../../../react/state/editor-context"; + +import type { PetrinautAiMessage } from "../types"; +import type { + PetrinautExperimentProgress, + PetrinautExperimentResult, +} from "@hashintel/petrinaut-core"; + +export type ExperimentToolPart = Extract< + PetrinautAiMessage["parts"][number], + { type: "tool-createExperiment" } +>; + +export type AiExperimentState = { + active: boolean; + progress?: PetrinautExperimentProgress; + result?: PetrinautExperimentResult; +}; + +const cardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2.5", + padding: "2.5", + borderRadius: "xl", + border: "[1px solid]", + borderColor: "blue.a30", + backgroundColor: "blue.s10", + backgroundImage: + "[linear-gradient(135deg, var(--colors-blue-a10), transparent 70%)]", + fontSize: "sm", + color: "neutral.s90", + transition: "[border-color 200ms ease, box-shadow 200ms ease]", + "&[data-pending=true]": { + boxShadow: + "[0 0 0 1px var(--colors-blue-a15), 0 0 14px var(--colors-blue-a20)]", + }, + "&[data-tone=optimization]": { + borderColor: "purple.a30", + backgroundColor: "purple.s10", + backgroundImage: + "[linear-gradient(135deg, var(--colors-purple-a10), transparent 70%)]", + boxShadow: "[0 2px 12px var(--colors-purple-a10)]", + }, + "&[data-tone=optimization][data-pending=true]": { + animationName: "[petrinautOptimizingGlow]", + animationDuration: "[2.8s]", + animationTimingFunction: "ease-in-out", + animationIterationCount: "[infinite]", + }, + "@media (prefers-reduced-motion: reduce)": { + transition: "[none]", + "&[data-tone=optimization][data-pending=true]": { + animationName: "[none]", + boxShadow: + "[0 0 0 1px var(--colors-purple-a15), 0 0 14px var(--colors-purple-a20)]", + }, + }, +}); + +const headerStyle = css({ + display: "flex", + alignItems: "flex-start", + gap: "2", +}); +const iconStyle = css({ + display: "grid", + placeItems: "center", + width: "[28px]", + height: "[28px]", + marginTop: "0.5", + flexShrink: "0", + borderRadius: "lg", + color: "blue.s100", + backgroundColor: "blue.a15", + "[data-tone=optimization] &": { + color: "purple.s100", + backgroundColor: "purple.a15", + }, +}); +const kindStyle = css({ + fontSize: "xs", + fontWeight: "medium", + color: "blue.s100", + lineHeight: "[16px]", + "[data-tone=optimization] &": { color: "purple.s100" }, +}); +const titleStyle = css({ + display: "block", + fontSize: "sm", + fontWeight: "semibold", + lineHeight: "[20px]", + color: "neutral.s120", + overflowWrap: "anywhere", +}); +const metadataStyle = css({ + display: "flex", + alignItems: "center", + columnGap: "1.5", + flexWrap: "wrap", + "& > :not(:first-child)::before": { + content: '"·"', + marginRight: "1.5", + color: "neutral.s60", + }, +}); +const statusStyle = css({ + display: "inline-flex", + alignItems: "center", + gap: "1.5", + width: "[fit-content]", + padding: "[2px 6px]", + flexShrink: "0", + marginTop: "0.5", + borderRadius: "full", + fontSize: "xs", + fontWeight: "medium", + lineHeight: "[14px]", + color: "blue.s110", + backgroundColor: "blue.a10", + "&[data-status=pending]": { + "[data-tone=optimization] &": { + color: "purple.s110", + backgroundColor: "purple.a10", + }, + }, + "&[data-status=complete]": { + color: "green.s110", + backgroundColor: "green.a10", + }, + "&[data-status=error]": { color: "red.s110", backgroundColor: "red.a10" }, + "&[data-status=cancelled]": { + color: "neutral.s90", + backgroundColor: "neutral.a10", + }, +}); +const statusDotStyle = css({ + width: "[5px]", + height: "[5px]", + borderRadius: "full", + backgroundColor: "[currentColor]", + animationName: "pulse", + animationDuration: "[1.6s]", + animationTimingFunction: "ease-in-out", + animationIterationCount: "[infinite]", + "@media (prefers-reduced-motion: reduce)": { animationName: "[none]" }, +}); +const detailStyle = css({ + fontSize: "xs", + color: "neutral.s90", + fontVariantNumeric: "tabular-nums", + overflowWrap: "anywhere", +}); +const progressTrackStyle = css({ + height: "[5px]", + borderRadius: "full", + overflow: "hidden", + backgroundColor: "blue.a15", + "[data-tone=optimization] &": { backgroundColor: "purple.a15" }, +}); +const progressFillStyle = css({ + height: "[100%]", + borderRadius: "full", + backgroundColor: "blue.s90", + transition: "[width 280ms ease-out]", + "[data-tone=optimization] &": { backgroundColor: "purple.s90" }, + "@media (prefers-reduced-motion: reduce)": { transition: "[none]" }, +}); +const metricsStyle = css({ + display: "flex", + flexWrap: "wrap", + columnGap: "4", + rowGap: "2", + flex: "[1 1 100px]", + minWidth: "[0]", + margin: "0", +}); +const metricStyle = css({ + display: "flex", + flexDirection: "column", + minWidth: "[0]", + overflowWrap: "anywhere", + "& > dt": { fontSize: "xs", color: "neutral.s90", lineHeight: "[16px]" }, + "& > dd": { + margin: "0", + fontSize: "lg", + lineHeight: "[24px]", + fontWeight: "semibold", + fontVariantNumeric: "tabular-nums", + letterSpacing: "[-0.02em]", + color: "neutral.s120", + }, +}); +const actionsStyle = css({ + display: "flex", + alignItems: "center", + flexWrap: "wrap", + gap: "2", + marginLeft: "[auto]", +}); + +export const ExperimentCard = ({ + part, + state, + onCancel, +}: { + part: ExperimentToolPart; + state?: AiExperimentState; + onCancel?: (toolCallId: string) => void; +}) => { + const { experiments } = use(ExperimentsContext); + const { navigateTo } = use(EditorContext); + const result = + part.state === "output-available" ? part.output : state?.result; + const progress = state?.progress; + const experimentId = result?.experimentId ?? progress?.experimentId; + const name = + result?.name ?? progress?.name ?? part.input?.name ?? "Experiment"; + const errorText = part.state === "output-error" ? part.errorText : undefined; + const pending = state?.active === true && !result && !errorText; + const optimization = + part.input?.execution?.mode === "optimize" || + result?.optimization !== undefined || + progress?.phase === "optimizing" || + progress?.phase === "refining"; + const statusKind = errorText + ? "error" + : (result?.status ?? (pending ? "pending" : "unavailable")); + const status = errorText + ? "Failed" + : result + ? { complete: "Finished", cancelled: "Cancelled", error: "Failed" }[ + result.status + ] + : !pending + ? "Not running" + : progress?.phase === "optimizing" + ? "Optimizing" + : progress?.phase === "refining" + ? "Refining" + : progress?.phase === "running" + ? "Running" + : "Validating"; + const progressValue = progress + ? Math.min(progress.runsTarget, Math.max(0, progress.runsCompleted)) + : 0; + const progressPercent = + progress && progress.runsTarget > 0 + ? (progressValue / progress.runsTarget) * 100 + : 0; + const available = + experimentId && + experiments.some((experiment) => experiment.id === experimentId); + + return ( +
+
+ +
+ {name} +
+ + {optimization ? "Optimization" : "Simulation"} + + {(result || pending) && ( + + {result + ? `${result.runsCompleted} runs` + : progress + ? `${progress.runsCompleted} of ${progress.runsTarget} runs` + : "Checking the model"} + + )} +
+
+
+ {pending ? ( +
+
+ {progress && pending && ( +
+
+
+
+ {progress.phase === "optimizing" && + progress.step !== undefined && + progress.steps !== undefined && ( + + Step {progress.step} of {progress.steps} + + )} + {progress.phase === "refining" && ( + Refining the best result + )} +
+ )} + {!pending && !result && !errorText && ( +

+ No result is available for this request. Ask the assistant to run a + new experiment. +

+ )} + {(result?.message ?? errorText) && ( +

+ {result?.message ?? errorText} +

+ )} + {result?.metrics.length || + available || + (pending && onCancel) || + result?.experimentId ? ( +
+ {result && result.metrics.length > 0 && ( +
+ {result.metrics.map((metric) => ( +
+
{metric.label}
+
+ {metric.value === null + ? "No value" + : metric.value.toLocaleString(undefined, { + maximumSignificantDigits: 6, + })} +
+
+ ))} +
+ )} + {(available || (pending && onCancel) || result?.experimentId) && ( +
+ {available && ( + + )} + {pending && onCancel && ( + + )} + {result?.experimentId && !available && ( + + Results saved in chat; experiment is no longer open. + + )} +
+ )} +
+ ) : null} +
+ ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/get-message-render-items.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/get-message-render-items.ts index d7170d4e9101..efd0690c3387 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/get-message-render-items.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/get-message-render-items.ts @@ -8,6 +8,7 @@ import { isToolPart, toToolRenderItem, type ToolRenderItem } from "./tool-list"; import type { PetrinautAiInteractiveTool } from "../../../../../types/ai-interactive-tool"; import type { PetrinautAiMessage } from "../types"; +import type { ExperimentToolPart } from "./experiment-card"; export type MessagePart = PetrinautAiMessage["parts"][number]; export type TextPart = Extract; @@ -16,6 +17,7 @@ export type ReasoningMessagePart = Extract; export type MessageRenderItem = | { type: "reasoning"; key: string; part: ReasoningMessagePart } | { type: "text"; key: string; part: TextPart } + | { type: "experiment"; key: string; part: ExperimentToolPart } | { type: "tools"; key: string; tools: ToolRenderItem[] }; export const isPartActive = ( @@ -67,6 +69,12 @@ export const getMessageRenderItems = ( return; } + if (part.type === "tool-createExperiment") { + flushTools(); + items.push({ type: "experiment", key: part.toolCallId, part }); + return; + } + if (isToolPart(part)) { const tool = toToolRenderItem(message, part, interactiveTools); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.test.ts index c8e2a24b72d9..50a4c015f036 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.test.ts @@ -56,7 +56,9 @@ describe("formatDiagnosticsForAi", () => { definition, diagnosticsByUri: new Map(), }), - ).toBe("No errors or warnings found in net function code."); + ).toBe( + "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment.", + ); }); test("formats transition and differential-equation diagnostics", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.ts index 0ddaa881f4fa..0ed8bdaf469e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/format-diagnostics-for-ai.ts @@ -62,7 +62,7 @@ export const formatDiagnosticsForAi = ({ ); if (diagnostics.length === 0) { - return "No errors or warnings found in net function code."; + return "No errors or warnings found in net function code. Scenario and metric compilation is checked when creating an experiment."; } const shownDiagnostics = diagnostics.slice(0, maxDiagnostics); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/types.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/types.ts index f7905126cedc..03d7d77d3d4d 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/types.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/types.ts @@ -1,5 +1,8 @@ import type { AiToolCall, AiToolOutput } from "./tool-summaries"; import type { + createExperimentToolName, + PetrinautExperimentRequest, + PetrinautExperimentResult, getLatestNetDefinitionToolName, getNetCompilationErrorsToolName, PetrinautAiCommandToolInput, @@ -34,6 +37,10 @@ type PetrinautAiUiTools = { output: AiToolOutput; }; } & { + [createExperimentToolName]: { + input: PetrinautExperimentRequest; + output: PetrinautExperimentResult; + }; [getLatestNetDefinitionToolName]: { input: PetrinautAiToolInput; output: { diff --git a/libs/@local/petrinaut-arch-docs/content/ai/experiment-chat.mdx b/libs/@local/petrinaut-arch-docs/content/ai/experiment-chat.mdx new file mode 100644 index 000000000000..04a8a8d01fce --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/ai/experiment-chat.mdx @@ -0,0 +1,27 @@ +--- +title: Experiments in AI chat +description: Dispatch experiment tools and show live progress in the conversation. +attachTo: ui.views.editor.ai +sidebar_order: 20 +--- + +Petrinaut AI dispatches `createExperiment` to the [browser experiment host](doc:experiments/ai-created-experiments). +Its chat card shows live progress while the client tool waits for its final result. + +![Experiment chat flow](@diagrams/ai-experiment-chat.svg) + +The tool definition in `petrinautAiTools` supplies the canonical request schema. +The embedding application admits the tool and preserves its call ID through +transport and continuation. Browser progress updates the existing card; +completion supplies one captured tool result to the same conversation. + +Simulation cards use blue. Optimization cards use purple and glow while +running. Progress transitions respect reduced-motion preferences. **Cancel** +aborts the request; **View experiment** opens the drawer for chart inspection. +The chat stays mounted across Edit, Simulate, and Notebook modes so opening +an experiment preserves the active conversation. + +The [host guide](doc:experiments/ai-created-experiments#execution-and-ownership) +describes input locks, final refinement, result semantics, and browser lifetime. +The [user guide](https://github.com/hashintel/hash/blob/main/libs/%40hashintel/petrinaut/docs/ai-assistant.md#experiments-from-chat) +describes the controls. diff --git a/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiment-chat.d2 b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiment-chat.d2 new file mode 100644 index 000000000000..e21e6bd7871d --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-experiment-chat.d2 @@ -0,0 +1,12 @@ +vars: {d2-config: {theme-id: 0}} +grid-rows: 2 +grid-columns: 2 +horizontal-gap: 160 +vertical-gap: 120 +transport: "AI transport" +chat: "AI chat\nDispatch + card" +drawer: "Experiment drawer" +host: "Browser host" +transport <-> chat: "call / result + ID" +chat <-> host: "run / cancel\nprogress / result" +chat -> drawer: "open" diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx index 3dbc47f4bb2f..0bdb83a249c3 100644 --- a/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx +++ b/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx @@ -1,6 +1,6 @@ --- -title: Browser experiment host -description: Validate and run bounded experiment requests in the browser. +title: AI-created experiments +description: Run experiments from AI chat through the browser host. sidebar_order: 40 attachTo: react.experiment-host --- @@ -11,6 +11,9 @@ captured result. Optimization uses the same request and experiment record. ![Browser experiment architecture](@diagrams/ai-experiments-architecture.svg) +The [AI chat integration](doc:ai/experiment-chat) exposes this host through +`petrinautAiTools.createExperiment` and displays its progress in the conversation. + ## Request and result The [core contract](layer:core.experiments) exports Zod schemas and the host interface From 01f59f9f45a34b1c8dc2bcf25943df0851d6d9bd Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Tue, 15 Sep 2026 13:41:05 +0000 Subject: [PATCH 3/5] FE-1668: Demonstrate AI experiment and optimization integration (#9654) --- .../main/app/ai-experiments-demo/README.md | 27 ++ .../ai-experiments-demo.tsx | 62 +++ .../create-experiment-demo-transport.test.ts | 357 ++++++++++++++++++ .../create-experiment-demo-transport.ts | 177 +++++++++ .../src/routes/ai-experiments.tsx | 7 + .../content/ai/client-integration.mdx | 57 +++ .../content/diagrams/ai-client-integration.d2 | 11 + .../experiments/ai-created-experiments.mdx | 30 ++ 8 files changed, 728 insertions(+) create mode 100644 apps/petrinaut-website/src/main/app/ai-experiments-demo/README.md create mode 100644 apps/petrinaut-website/src/main/app/ai-experiments-demo/ai-experiments-demo.tsx create mode 100644 apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.test.ts create mode 100644 apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.ts create mode 100644 apps/petrinaut-website/src/routes/ai-experiments.tsx create mode 100644 libs/@local/petrinaut-arch-docs/content/ai/client-integration.mdx create mode 100644 libs/@local/petrinaut-arch-docs/content/diagrams/ai-client-integration.d2 diff --git a/apps/petrinaut-website/src/main/app/ai-experiments-demo/README.md b/apps/petrinaut-website/src/main/app/ai-experiments-demo/README.md new file mode 100644 index 000000000000..fa6659ba1b97 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/ai-experiments-demo/README.md @@ -0,0 +1,27 @@ +--- +layer: website.ai-experiments +role: Demonstrates experiment client tools with scripted chat and real browser execution +--- + +The `/ai-experiments` route supplies a saved model and a scripted AI SDK +transport to Petrinaut. The transport issues `createExperiment`; the editor's +host validates and runs it, and its chat renders progress and the result. + +The conversation first describes a fixed experiment recipe, then starts it +when asked to run or optimize. Replies stream with short pauses; run progress +comes from the experiment host. Completion messages use the returned metric +and selected parameter values, including cancellation and missing results. + +The SIR example is an illustrative model. Its results are computed simulations, +not measured flu data. The metric is the mean infected share in the last +populated distribution frame; runs that ended earlier may be absent. Searching +the initial infected share demonstrates parameter search, not a health intervention. + +This demo exercises Petrinaut's tool contract and presentation. Brunch owns +its production tool selection, transport integration, and agent behavior. +See AI client integration for the responsibility +boundary and AI-created experiments +for the demo steps. + +The user guide covers [AI chat](https://github.com/hashintel/hash/blob/main/libs/%40hashintel/petrinaut/docs/ai-assistant.md#experiments-from-chat) +and [experiment controls](https://github.com/hashintel/hash/blob/main/libs/%40hashintel/petrinaut/docs/experiments.md#experiments-created-by-the-assistant). diff --git a/apps/petrinaut-website/src/main/app/ai-experiments-demo/ai-experiments-demo.tsx b/apps/petrinaut-website/src/main/app/ai-experiments-demo/ai-experiments-demo.tsx new file mode 100644 index 000000000000..b88ffa011d11 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/ai-experiments-demo/ai-experiments-demo.tsx @@ -0,0 +1,62 @@ +import { useState } from "react"; + +import { createJsonDocHandle } from "@hashintel/petrinaut-core"; +import { sirModel } from "@hashintel/petrinaut-core/examples"; +import { Petrinaut, type PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +import { BrowserOptimizationProvider } from "../optimization-demo/browser-optimization-provider"; +import { createExperimentDemoTransport } from "./create-experiment-demo-transport"; + +const introduction: PetrinautAiMessage[] = [ + { + id: "experiment-demo-introduction", + role: "assistant", + parts: [ + { + type: "text", + text: "Let's explore an illustrative outbreak model. Ask how to compare its runs or search different starting infection levels.\n\nFor optimization, enable **Parameter sweeps** and **In-browser optimization** in Settings → Simulation.", + }, + ], + }, +]; + +export const AiExperimentsDemo = () => { + const [handle] = useState(() => + createJsonDocHandle({ + id: "ai-experiments-demo", + initial: structuredClone(sirModel.petriNetDefinition), + }), + ); + const [transport] = useState(createExperimentDemoTransport); + + return ( +
+
+ SIR model demo. Scripted conversation · live simulation + results. +
+
+ + + +
+
+ ); +}; diff --git a/apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.test.ts b/apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.test.ts new file mode 100644 index 000000000000..69f0261f265a --- /dev/null +++ b/apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.test.ts @@ -0,0 +1,357 @@ +import { readUIMessageStream, type UIMessageChunk } from "ai"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +import { + petrinautExperimentRequestSchema, + type PetrinautExperimentResult, +} from "@hashintel/petrinaut-core"; + +import { createExperimentDemoTransport } from "./create-experiment-demo-transport"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +const startTurn = (messages: PetrinautAiMessage[], abortSignal?: AbortSignal) => + createExperimentDemoTransport().sendMessages({ + chatId: "demo", + messages, + trigger: "submit-message", + messageId: undefined, + abortSignal, + }); + +const collect = async ( + reader: ReadableStreamDefaultReader, + chunks: UIMessageChunk[] = [], +) => { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) return chunks; + chunks.push(chunk.value); + } +}; + +const readTurn = async (messages: PetrinautAiMessage[]) => { + const stream = await startTurn(messages); + const reading = collect(stream.getReader()); + await vi.runAllTimersAsync(); + return reading; +}; + +const responseText = (chunks: UIMessageChunk[]) => + chunks + .flatMap((chunk) => (chunk.type === "text-delta" ? [chunk.delta] : [])) + .join(""); + +const userMessage = (text: string): PetrinautAiMessage => ({ + id: `user-${text}`, + role: "user", + parts: [{ type: "text", text }], +}); + +const completedMessage = ( + output: Partial = {}, +): PetrinautAiMessage => ({ + id: "completed", + role: "assistant", + parts: [ + { type: "text", text: "Starting the experiment." }, + { + type: "tool-createExperiment", + toolCallId: "experiment-call", + state: "output-available", + input: petrinautExperimentRequestSchema.parse({ + name: "Outbreak baseline", + scenarioId: "scenario__seasonal_flu", + scenarioParameterValues: {}, + runCount: 256, + seed: 42, + dt: 0.01, + maxTime: 20, + metricIds: ["metric__infected_fraction"], + execution: { mode: "simulate" }, + }), + output: { + status: "complete", + experimentId: "experiment-1", + name: "Outbreak baseline", + runsCompleted: 173, + metrics: [ + { id: "metric__unrelated", label: "Other", value: 0.99 }, + { + id: "metric__infected_fraction", + label: "Infected Fraction", + value: 0.123456, + }, + ], + ...output, + }, + }, + ], +}); + +describe("experiment integration demo", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + test.each([ + ["run", "simulate", "Outbreak baseline"], + [ + "Run 256 simulations for 20 model-time units.", + "simulate", + "Outbreak baseline", + ], + ["optimize", "optimize", "Starting infection search"], + [ + "Optimize for the lowest infected share.", + "optimize", + "Starting infection search", + ], + ] as const)( + "emits a valid bounded request for %s", + async (prompt, mode, name) => { + const chunks = await readTurn([userMessage(prompt)]); + const call = chunks.find( + (chunk) => chunk.type === "tool-input-available", + ); + expect(call?.toolName).toBe("createExperiment"); + const request = petrinautExperimentRequestSchema.parse(call?.input); + expect(request).toMatchObject({ + name, + runCount: 256, + maxTime: 20, + execution: { mode }, + }); + expect(responseText(chunks)).toContain("256"); + expect(responseText(chunks)).toContain("20 model-time units"); + }, + ); + + test.each([ + ["How can I see the variation in this outbreak model?", "distributions"], + ["Can we compare different starting infection levels?", "1% to 20%"], + ["How do I run an experiment?", "distributions"], + ["What does optimize do for an experiment?", "1% to 20%"], + ] as const)("plans without launching for %s", async (prompt, phrase) => { + const chunks = await readTurn([userMessage(prompt)]); + expect(chunks.some((chunk) => chunk.type === "tool-input-available")).toBe( + false, + ); + expect(responseText(chunks)).toContain(phrase); + }); + + test.each([ + "Write a poem", + "Optimize my website", + "Run a shell command", + "Do not run an experiment", + ])("guides unsupported requests without launching: %s", async (prompt) => { + const chunks = await readTurn([userMessage(prompt)]); + expect(chunks.some((chunk) => chunk.type === "tool-input-available")).toBe( + false, + ); + expect(responseText(chunks)).toContain("fixed recipe"); + }); + + test("states the fixed recipe when a requested count differs", async () => { + const chunks = await readTurn([userMessage("Run 10 simulations")]); + expect(responseText(chunks)).toContain("This demo runs 256 simulations"); + }); + + test("pauses, streams words, then launches after its explanation", async () => { + const stream = await startTurn([userMessage("run")]); + const chunks: UIMessageChunk[] = []; + const reading = collect(stream.getReader(), chunks); + await vi.advanceTimersByTimeAsync(349); + expect(chunks).toEqual([{ type: "start-step" }]); + await vi.advanceTimersByTimeAsync(1); + expect(responseText(chunks)).toBe("This "); + await vi.advanceTimersByTimeAsync(300); + expect(responseText(chunks)).toMatch(/^This demo runs/); + expect(chunks.some((chunk) => chunk.type === "tool-input-available")).toBe( + false, + ); + await vi.runAllTimersAsync(); + await reading; + expect( + chunks.findIndex((chunk) => chunk.type === "tool-input-available"), + ).toBeGreaterThan(chunks.findIndex((chunk) => chunk.type === "text-end")); + }); + + test("reports the actual completed count and matching metric", async () => { + const chunks = await readTurn([userMessage("run"), completedMessage()]); + expect(responseText(chunks)).toContain( + "173 runs finished. Last sampled mean infected share: 12.35%.", + ); + expect(responseText(chunks)).not.toContain("99%"); + expect(chunks.some((chunk) => chunk.type === "tool-input-available")).toBe( + false, + ); + }); + + test("reports the selected parameter from the completed optimization", async () => { + const chunks = await readTurn([ + userMessage("optimize"), + completedMessage({ + optimization: { + parameters: { infected_ratio: 0.077 }, + objectiveValue: 0.123456, + stepsCompleted: 4, + }, + }), + ]); + expect(responseText(chunks)).toContain( + "The search selected 7.7% initially infected.", + ); + expect(responseText(chunks)).toContain("173 runs finished."); + expect(responseText(chunks)).toContain("12.35%"); + }); + + test.each([ + { metrics: [] }, + { + metrics: [ + { + id: "metric__infected_fraction", + label: "Infected Fraction", + value: null, + }, + ], + }, + ])( + "reports missing metrics without inventing a value", + async ({ metrics }) => { + const chunks = await readTurn([ + userMessage("run"), + completedMessage({ metrics }), + ]); + expect(responseText(chunks)).toContain( + "No infected-share metric was returned.", + ); + expect(responseText(chunks)).not.toContain("mean infected share:"); + }, + ); + + test("preserves zero values and does not format a boolean as a selected share", async () => { + const chunks = await readTurn([ + userMessage("optimize"), + completedMessage({ + metrics: [ + { + id: "metric__infected_fraction", + label: "Infected Fraction", + value: 0, + }, + ], + optimization: { + parameters: { infected_ratio: false }, + objectiveValue: 0, + stepsCompleted: 4, + }, + }), + ]); + expect(responseText(chunks)).toContain("infected share: 0%."); + expect(responseText(chunks)).not.toContain("search selected"); + }); + + test.each(["cancelled", "error"] as const)( + "reports a host %s result", + async (status) => { + const chunks = await readTurn([ + userMessage("run"), + completedMessage({ + status, + message: "The host stopped the experiment.", + }), + ]); + expect(responseText(chunks)).toContain( + status === "cancelled" ? "was cancelled" : "could not finish", + ); + expect(responseText(chunks)).toContain("173 completed runs"); + expect(responseText(chunks)).toContain( + "The host stopped the experiment.", + ); + expect(responseText(chunks)).not.toContain("Last sampled mean"); + }, + ); + + test("continues the same message and starts fresh for a later user request", async () => { + const completed = completedMessage(); + const history = [userMessage("run"), completed]; + const reply = await readTurn(history); + let continuedMessage = completed; + for await (const update of readUIMessageStream({ + message: structuredClone(completed), + stream: new ReadableStream({ + start(controller) { + for (const chunk of reply) controller.enqueue(chunk); + controller.close(); + }, + }), + terminateOnError: true, + })) { + continuedMessage = update; + } + expect(continuedMessage.id).toBe(completed.id); + expect( + continuedMessage.parts.filter( + (part) => part.type === "tool-createExperiment", + ), + ).toHaveLength(1); + expect( + continuedMessage.parts.filter((part) => part.type === "text"), + ).toHaveLength(2); + const next = await readTurn([...history, userMessage("optimize")]); + expect( + next.find((chunk) => chunk.type === "tool-input-available")?.toolCallId, + ).not.toBe("experiment-call"); + expect(next.some((chunk) => chunk.type === "tool-input-available")).toBe( + true, + ); + }); + + test.each([0, 800])( + "aborts at %s ms without launching a later tool", + async (duration) => { + const cancellation = new AbortController(); + const stream = await startTurn([userMessage("run")], cancellation.signal); + const chunks: UIMessageChunk[] = []; + const reading = collect(stream.getReader(), chunks).catch( + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(duration); + cancellation.abort(); + await vi.runAllTimersAsync(); + expect(await reading).toMatchObject({ name: "AbortError" }); + expect( + chunks.some((chunk) => chunk.type === "tool-input-available"), + ).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }, + ); + + test("honors an already aborted signal", async () => { + const cancellation = new AbortController(); + cancellation.abort(); + const stream = await startTurn([userMessage("run")], cancellation.signal); + await expect(stream.getReader().read()).rejects.toMatchObject({ + name: "AbortError", + }); + expect(vi.getTimerCount()).toBe(0); + }); + + test("cancels pending typing when its reader closes", async () => { + const stream = await startTurn([userMessage("run")]); + const reader = stream.getReader(); + const chunks: UIMessageChunk[] = []; + const reading = collect(reader, chunks); + await vi.advanceTimersByTimeAsync(800); + await reader.cancel(); + await vi.runAllTimersAsync(); + await reading; + expect(chunks.some((chunk) => chunk.type === "text-delta")).toBe(true); + expect(chunks.some((chunk) => chunk.type === "tool-input-available")).toBe( + false, + ); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.ts b/apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.ts new file mode 100644 index 000000000000..0831618fddc6 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/ai-experiments-demo/create-experiment-demo-transport.ts @@ -0,0 +1,177 @@ +import { generateId, type UIMessageChunk } from "ai"; + +import { + createExperimentToolName, + type PetrinautExperimentRequest, + type PetrinautExperimentResult, +} from "@hashintel/petrinaut-core"; + +import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui"; + +const percentage = (value: number) => + `${(value * 100).toLocaleString("en", { maximumFractionDigits: 2 })}%`; + +const describeResult = (result: PetrinautExperimentResult): string => { + if (result.status !== "complete") { + const status = + result.status === "cancelled" ? "was cancelled" : "could not finish"; + return `The experiment ${status} after ${result.runsCompleted} completed runs. ${result.message ?? "Send another request to try again."}`; + } + const infectedShare = result.metrics.find( + (metric) => metric.id === "metric__infected_fraction", + )?.value; + const selectedShare = result.optimization?.parameters.infected_ratio; + const selection = + typeof selectedShare === "number" + ? `The search selected ${percentage(selectedShare)} initially infected. ` + : ""; + const metric = + infectedShare == null + ? "No infected-share metric was returned." + : `Last sampled mean infected share: ${percentage(infectedShare)}.`; + return `${selection}${result.runsCompleted} runs finished. ${metric} Open the experiment to explore the distribution.`; +}; + +const delay = (duration: number, signal: AbortSignal): Promise => + new Promise((resolve, reject) => { + let timer: ReturnType; + const abort = () => { + clearTimeout(timer); + reject(signal.reason); + }; + timer = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(); + }, duration); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }); + +export const createExperimentDemoTransport = (): PetrinautAiChatTransport => ({ + reconnectToStream: () => Promise.resolve(null), + sendMessages: ({ messages, abortSignal }) => { + const lastUserIndex = messages.findLastIndex( + (message) => + message.role === "user" && + message.id !== "petrinaut-diagnostics-context", + ); + const prompt = + messages[lastUserIndex]?.parts + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(" ") ?? ""; + const output = messages + .slice(lastUserIndex + 1) + .flatMap((message) => message.parts) + .find( + (part) => + part.type === "tool-createExperiment" && + part.state === "output-available", + ); + const concernsModel = + /\b(outbreak|infected|infection|scenario|simulations?|experiments?)\b/i.test( + prompt, + ); + const command = prompt.trim().replace(/^please\s+/i, ""); + const optimize = + /^optimi[sz]e\b/i.test(command) && + (concernsModel || /^optimi[sz]e[.!]?$/i.test(command)); + const simulate = + /^run\b/i.test(command) && (concernsModel || /^run[.!]?$/i.test(command)); + const planning = + /\b(how|what|can we|compare)\b/i.test(prompt) && concernsModel; + const declined = /\b(don['’]t|do not|stop|cancel)\b/i.test(prompt); + let text = + "Try “run” for the outbreak baseline, or “optimize” to search starting infection levels. This demo uses a fixed recipe."; + let request: PetrinautExperimentRequest | undefined; + if (output) { + text = describeResult(output.output); + } else if ((optimize || simulate) && !declined) { + request = { + name: optimize ? "Starting infection search" : "Outbreak baseline", + scenarioId: "scenario__seasonal_flu", + scenarioParameterValues: { + population: { mode: "fixed", value: 100 }, + infected_ratio: optimize + ? { mode: "range", min: 0.01, max: 0.2 } + : { mode: "fixed", value: 0.05 }, + }, + runCount: 256, + seed: 42, + dt: 0.01, + maxTime: 20, + metricIds: ["metric__infected_fraction"], + execution: optimize + ? { + mode: "optimize", + objectiveMetricId: "metric__infected_fraction", + direction: "minimize", + steps: 4, + runsPerStep: 8, + } + : { mode: "simulate" }, + }; + text = optimize + ? "This demo searches starting infection shares from 1% to 20% in a population of 100. I'll score four candidates with 8 runs each, then check the selected candidate with 256 runs over 20 model-time units." + : "This demo runs 256 simulations over 20 model-time units, starting with 5 infected people out of 100. I'll keep the infected-share distribution."; + } else if (planning && !declined) { + text = /\b(starting|initial|compare|optimi\w*)\b/i.test(prompt) + ? "We can compare starting infection shares from 1% to 20%, then check the selected candidate with 256 runs. Send “optimize” to search for the lowest last sampled mean infected share." + : "We can repeat the Seasonal Flu scenario with 5 infected people out of 100, then compare the infected-share distributions. Send “run” to start 256 simulations."; + } + const cancellation = new AbortController(); + const abort = () => cancellation.abort(abortSignal?.reason); + abortSignal?.addEventListener("abort", abort, { once: true }); + if (abortSignal?.aborted) abort(); + let readerCancelled = false; + return Promise.resolve( + new ReadableStream({ + async start(controller) { + const { signal } = cancellation; + const textId = generateId(); + try { + signal.throwIfAborted(); + controller.enqueue({ type: "start-step" }); + await delay(350, signal); + controller.enqueue({ type: "text-start", id: textId }); + const words = text.match(/\S+\s*/g) ?? []; + for (const word of words) { + signal.throwIfAborted(); + controller.enqueue({ + type: "text-delta", + id: textId, + delta: word, + }); + await delay( + /[.!?][”"]?\s*$/.test(word) + ? 170 + : Math.min(55, 30 + word.length * 1.5), + signal, + ); + } + controller.enqueue({ type: "text-end", id: textId }); + if (request) { + await delay(250, signal); + controller.enqueue({ + type: "tool-input-available", + toolCallId: generateId(), + toolName: createExperimentToolName, + input: request, + }); + } + controller.enqueue({ type: "finish-step" }); + controller.enqueue({ type: "finish" }); + controller.close(); + } catch (error) { + if (!readerCancelled) controller.error(error); + } finally { + abortSignal?.removeEventListener("abort", abort); + } + }, + cancel() { + readerCancelled = true; + cancellation.abort(); + }, + }), + ); + }, +}); diff --git a/apps/petrinaut-website/src/routes/ai-experiments.tsx b/apps/petrinaut-website/src/routes/ai-experiments.tsx new file mode 100644 index 000000000000..812e3ca4d205 --- /dev/null +++ b/apps/petrinaut-website/src/routes/ai-experiments.tsx @@ -0,0 +1,7 @@ +import { createFileRoute } from "@tanstack/react-router"; + +import { AiExperimentsDemo } from "../main/app/ai-experiments-demo/ai-experiments-demo"; + +export const Route = createFileRoute("/ai-experiments")({ + component: AiExperimentsDemo, +}); diff --git a/libs/@local/petrinaut-arch-docs/content/ai/client-integration.mdx b/libs/@local/petrinaut-arch-docs/content/ai/client-integration.mdx new file mode 100644 index 000000000000..7d6dce189ec8 --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/ai/client-integration.mdx @@ -0,0 +1,57 @@ +--- +title: AI client integration +description: What Petrinaut provides and what each integrating application owns. +attachTo: core.experiments +sidebar_order: 10 +--- + +Petrinaut provides experiment schemas, browser execution, progress, and captured +results. An integrating application selects the tools its agent may call and +decides how to continue after each result. + +![Client and host responsibilities](@diagrams/ai-client-integration.svg) + +## Brunch integration + +Brunch sends client tool calls through its AI SDK transport and resumes the +conversation with a `client-tool-result` signal correlated by `toolCallId`. +This supports browser experiments: Petrinaut owns the document and compute; +Brunch owns the conversation. See the [route contract](https://github.com/hashintel/hash/blob/main/apps/brunch-agent/README.md#panel-and-voice-conversation-route) +and [package topology](https://github.com/hashintel/hash/blob/main/libs/%40hashintel/brunch-agent/docs/reference/architecture/topology.md). + +Brunch selects tools by integration mode. Its configured panel includes +`readPetrinautDoc`, `getLatestNetDefinition`, and `mutate_petrinet`; other modes +admit individual construction tools. These explicit catalogs let Brunch adopt +experiments independently. + +| Owner | Responsibility | +| ------------------------------------------------------ | --------------------------------------------------------------------------------------------- | +| [Core](layer:core.experiments) | Define request/result schemas and host/progress types. | +| [Browser host](doc:experiments/ai-created-experiments) | Validate, run, cancel, protect inputs, and capture results. | +| [AI chat](doc:ai/experiment-chat) | Dispatch calls, show progress, and return terminal results. | +| Integrating application | Select tools, carry schemas, preserve call identity, persist history, and continue the agent. | + +To adopt experiments, register `createExperiment` in Brunch's tool catalog and +client-result admission boundary. Supply the canonical input schema to the +model and return the browser result with the original call ID. Test continuation +through the real Flue transport. Progress remains local while the call waits. + +:::danger[Important: tool schema conversion] +Several legacy headless construction tools use an open Valibot object and +embed JSON Schema in the description, with Zod validation at runtime. +The document and compilation readers already pass canonical schemas directly. +An experiment adapter should also preserve its nested input schema. +The [legacy conversion](https://github.com/hashintel/hash/blob/main/libs/%40hashintel/brunch-agent/packages/plugin-sdcpn/src/tools/petrinaut-construction.ts) +needs a separate integration change, tracked in +[FE-1484](https://linear.app/hash/issue/FE-1484/ai-assisted-parameterization-and-creation-of-experiments). +::: + +## Integration demo + +The [website demo](layer:website.ai-experiments) uses scripted conversation and +real browser execution. It exercises Petrinaut's contract and chat presentation. +Brunch retains ownership of production prompts, tool selection, persistence, +and result interpretation. Its production catalog is unchanged. + +Follow the [demo steps](doc:experiments/ai-created-experiments#try-the-demo) to +verify Petrinaut's part. Brunch adoption needs its own transport integration test. diff --git a/libs/@local/petrinaut-arch-docs/content/diagrams/ai-client-integration.d2 b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-client-integration.d2 new file mode 100644 index 000000000000..b287c2fe5cc1 --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/diagrams/ai-client-integration.d2 @@ -0,0 +1,11 @@ +direction: down +vars: {d2-config: {theme-id: 0}} +app: "Application agent" +transport: "Client transport" +demo: "Scripted demo" +chat: "Petrinaut AI chat" +host: "Browser host" +app <-> transport: "calls / results" +transport <-> chat +demo -> chat: "demo only" {style.stroke-dash: 4} +chat <-> host: "request / progress / result" diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx index 0bdb83a249c3..583ce9df74b2 100644 --- a/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx +++ b/libs/@local/petrinaut-arch-docs/content/experiments/ai-created-experiments.mdx @@ -90,6 +90,36 @@ last sampled distribution as a mean. See [experiment execution](doc:simulation/e [sweep orchestration](doc:experiments/sweep-orchestration), and the [browser optimizer](doc:optimizer/browser-runtime). +## Try the demo + +From the repository root: + +```sh +turbo run dev --filter @apps/petrinaut-website +``` + +Open [the experiment demo](http://localhost:5173/ai-experiments), using the +port printed by the server if it differs. + +1. Open the AI assistant and ask **How can I see the variation in this outbreak model?** + Read the baseline proposal, then send **run**. Watch the card, select + **View experiment**, and click a heatmap time step to see its histogram. +2. Enable **Parameter sweeps** and **In-browser optimization** in Settings. + Ask to compare starting infection levels, then send **optimize**. Inspect + progress, the final result, and the best point's distribution. +3. Select **Cancel** during an optimization and confirm its card stops updating. +4. After completion, change a parameter in the drawer and confirm the earlier + chat result stays unchanged. + +The [demo layer](layer:website.ai-experiments) streams scripted replies with +short pauses. Completion figures come from the host's returned result. +The SIR example produces simulated model data and is not calibrated to measured +flu data. Searching initial infection levels demonstrates parameter search. + +The demo requires no LLM or Brunch server. The website development command +builds its Petrinaut dependencies before starting. [AI client integration](doc:ai/client-integration) +describes Brunch adoption and application responsibilities. + :::danger[Important: host lifetime] Execution requires the mounted browser host. Durable jobs, recovery after reload, and background notifications need a persistence contract. From e3c62d6f69c6c05d299d8b4b39a1ce7cdda97fb5 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 16:51:30 +0200 Subject: [PATCH 4/5] Remove the model-facing Brunch question marker Co-authored-by: Cursor --- .../src/agents/chat-agent/tool-catalogue.ts | 6 - .../runbook/schema-carrier-probe.ts | 7 +- .../admission-controls.integration.ts | 23 +--- .../integration/admission-controls.test.ts | 16 +-- .../history-retention.integration.ts | 29 +--- .../test/integration/petrinaut-chat-result.ts | 1 + .../integration/petrinaut-chat.integration.ts | 35 ++--- .../test/integration/petrinaut-chat.test.ts | 28 ++-- .../workpiece-revisions.integration.ts | 4 +- .../brunch-panel-transport.ts | 4 +- .../use-flue-chat-history.ts | 4 +- .../voice-interview/canonical-speech.test.ts | 14 ++ .../docs/reference/architecture/topology.md | 1 - .../brunch-agent/packages/core/src/flue.ts | 29 +--- .../brunch-agent/packages/core/src/index.ts | 3 + .../packages/core/src/prompts/SYSTEM.md | 2 - .../packages/core/src/question-marker.ts | 9 +- .../core/test/question-marker.test.ts | 41 ++---- .../core/test/update-workpiece.test.ts | 9 +- .../transport-aisdk/test/ui-stream.test.ts | 125 +++++++++--------- 20 files changed, 166 insertions(+), 224 deletions(-) diff --git a/apps/brunch-agent/src/agents/chat-agent/tool-catalogue.ts b/apps/brunch-agent/src/agents/chat-agent/tool-catalogue.ts index c50545a6eacf..4bae474f5836 100644 --- a/apps/brunch-agent/src/agents/chat-agent/tool-catalogue.ts +++ b/apps/brunch-agent/src/agents/chat-agent/tool-catalogue.ts @@ -47,12 +47,6 @@ export const ordinaryBrunchToolCatalogue: readonly OrdinaryBrunchToolCatalogueEn executionOwner: "flue", role: "substrate", }, - { - name: "brunch_mark_question", - definitionOwner: "brunch-core", - executionOwner: "brunch-app", - role: "workpiece", - }, { name: "mutate_workpiece", definitionOwner: "brunch-core", diff --git a/apps/brunch-agent/src/evaluations/runbook/schema-carrier-probe.ts b/apps/brunch-agent/src/evaluations/runbook/schema-carrier-probe.ts index 484681301fab..e4fb6960ba78 100644 --- a/apps/brunch-agent/src/evaluations/runbook/schema-carrier-probe.ts +++ b/apps/brunch-agent/src/evaluations/runbook/schema-carrier-probe.ts @@ -17,6 +17,7 @@ import { clientToolHistoryFrom, clientToolResultSignal, } from "@hashintel/brunch-agent-transport-aisdk"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "@hashintel/brunch-agent/question-marker"; import { petrinautAiTools, type PetrinautAiToolInput, @@ -159,8 +160,10 @@ try { }); assert.deepEqual(generatedAddType.parameters, canonicalSchema); assert( - generatedTools.some((tool) => tool.name === "brunch_mark_question"), - "Question marker missing", + !generatedTools.some((tool) => + BRUNCH_QUESTION_TOOL_NAMES.some((name) => name === tool.name), + ), + "Legacy question marker must not be mounted", ); assert.deepEqual(headless.definition().types, [ petrinautAiTools.addType.inputSchema.parse(nestedType), diff --git a/apps/brunch-agent/test/integration/admission-controls.integration.ts b/apps/brunch-agent/test/integration/admission-controls.integration.ts index bcfb5e887344..e20c81ad2ccb 100644 --- a/apps/brunch-agent/test/integration/admission-controls.integration.ts +++ b/apps/brunch-agent/test/integration/admission-controls.integration.ts @@ -25,7 +25,7 @@ import { VALIDATED_CONSTRUCTION_MODE, } from "@hashintel/brunch-agent-plugin-sdcpn/flue"; import { snapshotToUiMessages } from "@hashintel/brunch-agent-transport-aisdk"; -import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "@hashintel/brunch-agent/question-marker"; import { CLIENT_TOOL_RESULT_SIGNAL, @@ -70,7 +70,7 @@ const browserNames: ReadonlySet = new Set([ const project = (history: FlueConversationSnapshot) => snapshotToUiMessages(history, { clientToolNames: browserNames, - hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), + hiddenToolNames: new Set(BRUNCH_QUESTION_TOOL_NAMES), }); const faux = fauxProvider({ provider: "anthropic", @@ -152,20 +152,10 @@ const run = async () => { const observations = []; try { for (const names of [ - [BRUNCH_QUESTION_TOOL_NAME, "addType"], - ["addType", BRUNCH_QUESTION_TOOL_NAME], ["mutate_workpiece", "addType"], ["addType", "mutate_workpiece"], - [BRUNCH_QUESTION_TOOL_NAME, "mutate_workpiece", "addType"], - [BRUNCH_QUESTION_TOOL_NAME, "addType", "mutate_workpiece"], - ["mutate_workpiece", BRUNCH_QUESTION_TOOL_NAME, "addType"], - ["mutate_workpiece", "addType", BRUNCH_QUESTION_TOOL_NAME], - ["addType", BRUNCH_QUESTION_TOOL_NAME, "mutate_workpiece"], - ["addType", "mutate_workpiece", BRUNCH_QUESTION_TOOL_NAME], ["addType", "unmounted_admission_probe"], ["addType"], - [BRUNCH_QUESTION_TOOL_NAME], - ["mutate_workpiece", BRUNCH_QUESTION_TOOL_NAME], ]) { caseId = names.join("-"); const client = clientFor(); @@ -310,12 +300,7 @@ const run = async () => { const message = fauxAssistantMessage( [ fauxText(text), - ...(abort - ? [makeCall("addType")] - : [ - makeCall("mutate_workpiece"), - makeCall(BRUNCH_QUESTION_TOOL_NAME), - ]), + ...(abort ? [makeCall("addType")] : [makeCall("mutate_workpiece")]), ], { stopReason: "toolUse" }, ); @@ -391,7 +376,7 @@ const run = async () => { }); } const rejected = observations.find( - (observation) => observation.caseId === "brunch_mark_question-addType", + (observation) => observation.caseId === "mutate_workpiece-addType", )!; const priorIds = new Set( rejected.seeded.messages.map((message) => message.id), diff --git a/apps/brunch-agent/test/integration/admission-controls.test.ts b/apps/brunch-agent/test/integration/admission-controls.test.ts index 1cd7be59968c..7171af4337fa 100644 --- a/apps/brunch-agent/test/integration/admission-controls.test.ts +++ b/apps/brunch-agent/test/integration/admission-controls.test.ts @@ -31,12 +31,12 @@ beforeAll(async () => { }); test("production rejects every mixed proposal before publishing or partially executing it", () => { - expect(result.observations).toHaveLength(14); + expect(result.observations).toHaveLength(4); const mixed = result.observations.filter( ({ generated }) => generated.length > 1 && generated.some((call) => call.name === "addType"), ); - expect(mixed).toHaveLength(11); + expect(mixed).toHaveLength(3); for (const observation of mixed) { expect(observation.pendingMutationIds).toEqual([]); expect(observation.after).toEqual(observation.before); @@ -138,7 +138,7 @@ test("every failed submission is attributable from the server output by stage, s } }); -test("production still settles revisions and noninteractive markers without browser results", () => { +test("production settles revisions without browser results", () => { for (const observation of result.observations) { expect(observation.seed.error).toBeNull(); const revision = observation.seeded.messages @@ -151,16 +151,6 @@ test("production still settles revisions and noninteractive markers without brow output: { revisionId: `${observation.caseId}-old-revision`, ordinal: 1 }, }); } - for (const caseId of [ - "brunch_mark_question", - "mutate_workpiece-brunch_mark_question", - ]) { - const observation = result.observations.find( - (entry) => entry.caseId === caseId, - )!; - expect(observation.attempt.error).toBeNull(); - expect(observation.providerCallsBeforeClientResult).toBe(2); - } }); test("an independently admitted browser mutation waits for its correlated result and does not reapply", () => { diff --git a/apps/brunch-agent/test/integration/history-retention.integration.ts b/apps/brunch-agent/test/integration/history-retention.integration.ts index cef953e8fcd5..4514c1656879 100644 --- a/apps/brunch-agent/test/integration/history-retention.integration.ts +++ b/apps/brunch-agent/test/integration/history-retention.integration.ts @@ -20,7 +20,7 @@ import { snapshotToUiMessages, CLIENT_TOOL_RESULT_SIGNAL, } from "@hashintel/brunch-agent-transport-aisdk"; -import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "@hashintel/brunch-agent/question-marker"; import { agentOwnershipHeaders, @@ -350,7 +350,7 @@ const tools = (name: string, input: Record, id: string) => const project = (snapshot: FlueConversationSnapshot) => snapshotToUiMessages(snapshot, { clientToolNames: new Set([READ_PETRINAUT_DOCS_TOOL_NAME]), - hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), + hiddenToolNames: new Set(BRUNCH_QUESTION_TOOL_NAMES), }); const status = async (operation: () => Promise) => { try { @@ -431,11 +431,6 @@ try { ); responses.push( tools("ping", { note: "a4-early-ping" }, "a4-ping-early"), - tools( - BRUNCH_QUESTION_TOOL_NAME, - { question: "Which synthetic record follows?" }, - "a4-question", - ), tools( READ_PETRINAUT_DOCS_TOOL_NAME, { doc: "ai-assistant" }, @@ -515,13 +510,7 @@ try { .filter((part) => part.type === "dynamic-tool"); assert.deepEqual( publicTools.map((part) => part.toolCallId), - [ - "a4-ping-early", - "a4-question", - "a4-doc-early", - "a4-ping-middle", - "a4-doc-late", - ], + ["a4-ping-early", "a4-doc-early", "a4-ping-middle", "a4-doc-late"], ); for (const suffix of ["early", "middle"]) { const ping = publicTools.find( @@ -531,15 +520,11 @@ try { assert.deepEqual(ping.input, { note: `a4-${suffix}-ping` }); assert.deepEqual(ping.output, { ok: true, note: `a4-${suffix}-ping` }); } - const marker = publicTools.find( - (part) => part.toolCallId === "a4-question", - ); - assert(marker?.state === "output-available"); - assert.deepEqual(marker.output, { marked: true }); assert( - before.messages + !before.messages .flatMap((message) => message.parts) .some((part) => part.type === "data-brunch-question"), + "New responses must not create question markers", ); const clientResults = clientToolHistoryFrom(before.messages).results; assert.deepEqual( @@ -702,7 +687,7 @@ try { ? compactions.some( (event) => !event.isError && - event.messagesBefore === 20 && + event.messagesBefore === 18 && event.messagesAfter === 3, ) : compactions.some( @@ -710,7 +695,7 @@ try { !event.isError && event.messagesAfter < event.messagesBefore, ), silentOverflow - ? "Silent overflow must fold the known 20-message window to 3" + ? "Silent overflow must fold the known 18-message window to 3" : "Actual successful folding must reduce runtime context messages", ); assert( diff --git a/apps/brunch-agent/test/integration/petrinaut-chat-result.ts b/apps/brunch-agent/test/integration/petrinaut-chat-result.ts index b1a70cb413ca..3dd04e44c418 100644 --- a/apps/brunch-agent/test/integration/petrinaut-chat-result.ts +++ b/apps/brunch-agent/test/integration/petrinaut-chat-result.ts @@ -21,6 +21,7 @@ export interface PetrinautChatResult { readonly resumedStatus: number; readonly resumedText: string; readonly resumedFinish: UIMessageChunk | undefined; + readonly questionResponseProviderCalls: number; readonly questionMarkerLive: unknown; readonly questionMarkerHistory: unknown; readonly questionToolVisibleLive: boolean; diff --git a/apps/brunch-agent/test/integration/petrinaut-chat.integration.ts b/apps/brunch-agent/test/integration/petrinaut-chat.integration.ts index f0db8e9b64b7..c2f706756c4f 100644 --- a/apps/brunch-agent/test/integration/petrinaut-chat.integration.ts +++ b/apps/brunch-agent/test/integration/petrinaut-chat.integration.ts @@ -10,6 +10,7 @@ import { fauxText, fauxThinking, fauxToolCall, + type Provider, } from "@earendil-works/pi-ai"; import { createFlueClient, FlueApiError } from "@flue/sdk"; @@ -21,7 +22,7 @@ import { import { ELICITATION_SKILL_NAME } from "@hashintel/brunch-agent/flue"; import { BRUNCH_QUESTION_DATA_NAME, - BRUNCH_QUESTION_TOOL_NAME, + BRUNCH_QUESTION_TOOL_NAMES, } from "@hashintel/brunch-agent/question-marker"; import { PING_TOOL_NAME } from "../../src/agents/chat-agent/tools/ping.ts"; @@ -113,13 +114,22 @@ const questionToolVisibleInHistory = ( ): boolean => messages .flatMap((message) => message.parts) - .some((part) => part.type === `tool-${BRUNCH_QUESTION_TOOL_NAME}`); + .some((part) => + BRUNCH_QUESTION_TOOL_NAMES.some((name) => part.type === `tool-${name}`), + ); const faux = fauxProvider({ provider: "anthropic", models: [{ id: CHAT_MODEL_ID, reasoning: true }], }); -installFauxProvider(faux.provider); +let providerCallCount = 0; +installFauxProvider({ + ...faux.provider, + streamSimple(model, context, options) { + providerCallCount += 1; + return faux.provider.streamSimple(model, context, options); + }, +} satisfies Provider); const application = await loadBuiltBrunchApplication(); try { @@ -134,14 +144,14 @@ try { const panelTransport = createFlueChatTransport({ client: historyClient, clientToolNames, - hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), + hiddenToolNames: new Set(BRUNCH_QUESTION_TOOL_NAMES), }); const projectHistory = ( snapshot: Awaited>, ) => snapshotToUiMessages(snapshot, { clientToolNames, - hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), + hiddenToolNames: new Set(BRUNCH_QUESTION_TOOL_NAMES), }); if (process.env.BRUNCH_RESUME_PHASE === "1") { @@ -237,16 +247,6 @@ try { ], { stopReason: "toolUse" }, ), - fauxAssistantMessage( - [ - fauxToolCall( - BRUNCH_QUESTION_TOOL_NAME, - { question }, - { id: "tool-question-1" }, - ), - ], - { stopReason: "toolUse" }, - ), fauxAssistantMessage([ fauxText( `The guide says the assistant can read its own documentation pages. ${question}`, @@ -343,6 +343,7 @@ try { ], }, ] as UIMessage[]; + const questionResponseCallStart = providerCallCount; const resumedChunks = await chunksFrom( await panelTransport.sendMessages({ trigger: "submit-message", @@ -451,12 +452,14 @@ try { .map((chunk) => chunk.delta) .join(""), resumedFinish: resumedChunks.at(-1), + questionResponseProviderCalls: + providerCallCount - questionResponseCallStart, questionMarkerLive: questionMarkerFromChunks(resumedChunks), questionMarkerHistory: questionMarkerFromHistory(historyMessages), questionToolVisibleLive: resumedChunks.some( (chunk) => chunk.type === "tool-input-available" && - chunk.toolName === BRUNCH_QUESTION_TOOL_NAME, + BRUNCH_QUESTION_TOOL_NAMES.some((name) => name === chunk.toolName), ), questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), historyUserEntryCount: userEntryIds.length, diff --git a/apps/brunch-agent/test/integration/petrinaut-chat.test.ts b/apps/brunch-agent/test/integration/petrinaut-chat.test.ts index c507aeef030f..4ae71d981f7b 100644 --- a/apps/brunch-agent/test/integration/petrinaut-chat.test.ts +++ b/apps/brunch-agent/test/integration/petrinaut-chat.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { expect, test } from "vitest"; import { READ_PETRINAUT_DOCS_TOOL_NAME } from "@hashintel/brunch-agent-plugin-sdcpn/flue"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "@hashintel/brunch-agent/question-marker"; import { runNodeScript } from "./run-node-script"; @@ -74,15 +75,13 @@ test("the browser transport streams the mounted Flue agent through server and cl type: "finish", finishReason: "stop", }); - expect(result.questionMarkerLive).toEqual({ - question: "Which documentation page should we inspect next?", - toolCallId: "tool-question-1", - }); + expect(result.resumedText).toContain( + "Which documentation page should we inspect next?", + ); + expect(result.questionResponseProviderCalls).toBe(1); + expect(result.questionMarkerLive).toBeUndefined(); expect(result.questionToolVisibleLive).toBe(false); - expect(result.questionMarkerHistory).toEqual({ - question: "Which documentation page should we inspect next?", - toolCallId: "tool-question-1", - }); + expect(result.questionMarkerHistory).toBeUndefined(); expect(result.questionToolVisibleHistory).toBe(false); expect(result.historyUserEntryCount).toBe(1); expect(result.historyClientToolResultCount).toBe(1); @@ -123,7 +122,9 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(result.interviewerToolNames).toContain( READ_PETRINAUT_DOCS_TOOL_NAME, ); - expect(result.interviewerToolNames).toContain("brunch_mark_question"); + expect(result.interviewerToolNames).not.toEqual( + expect.arrayContaining([...BRUNCH_QUESTION_TOOL_NAMES]), + ); expect(result.interviewerToolNames).not.toContain("brunch_ask"); expect(result.interviewerToolNames).not.toContain("sweep"); expect(result.interviewerToolNames).not.toContain("brunch_sweep"); @@ -168,10 +169,7 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(resumeResult.historyUserText).toContain( "Run the FE-1435 transport probe.", ); - expect(resumeResult.questionMarkerHistory).toEqual({ - question: "Which documentation page should we inspect next?", - toolCallId: "tool-question-1", - }); + expect(resumeResult.questionMarkerHistory).toBeUndefined(); expect(resumeResult.questionToolVisibleHistory).toBe(false); expect(resumeResult.transcript).toContain("tool ping"); expect(resumeResult.transcript).toContain( @@ -179,7 +177,9 @@ test("the browser transport streams the mounted Flue agent through server and cl ); expect(resumeResult.transcript).toContain("tool activate_skill"); expect(resumeResult.transcript).toContain("tool read_skill_resource"); - expect(resumeResult.transcript).toContain("tool brunch_mark_question"); + for (const markerName of BRUNCH_QUESTION_TOOL_NAMES) { + expect(resumeResult.transcript).not.toContain(`tool ${markerName}`); + } } finally { await rm(dbDirectory, { recursive: true, force: true }); } diff --git a/apps/brunch-agent/test/integration/workpiece-revisions.integration.ts b/apps/brunch-agent/test/integration/workpiece-revisions.integration.ts index d0c454318f07..35d51f2f4d7f 100644 --- a/apps/brunch-agent/test/integration/workpiece-revisions.integration.ts +++ b/apps/brunch-agent/test/integration/workpiece-revisions.integration.ts @@ -132,10 +132,8 @@ const probe = async () => { const mixed = []; for (const names of [ - ["brunch_mark_question", "addType"], ["mutate_workpiece", "addType"], - ["brunch_mark_question", "mutate_workpiece", "addType"], - ["addType", "mutate_workpiece", "brunch_mark_question"], + ["addType", "mutate_workpiece"], ]) { const caseId = names.join("-"); const typeInput = { diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index be6ffd06d48c..d88974d81c4f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -3,7 +3,7 @@ import { FlueChatAdmissionError, } from "@hashintel/brunch-agent-transport-aisdk"; import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; -import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "@hashintel/brunch-agent/question-marker"; import { sweepOutputSchema } from "../brunch-sweep-output"; import { brunchClientToolNames } from "./brunch-client-tools"; @@ -373,7 +373,7 @@ export const createBrunchPanelTransport = ( ...(options?.mapClientToolInput === undefined ? {} : { mapClientToolInput: options.mapClientToolInput }), - hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), + hiddenToolNames: new Set(BRUNCH_QUESTION_TOOL_NAMES), onAdmission: (event) => { tracker.recordAdmission(event); options?.onAdmission?.(event.admission); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts index cff00f5cc8df..abd58441b598 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts @@ -1,7 +1,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { snapshotToUiMessages } from "@hashintel/brunch-agent-transport-aisdk"; -import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "@hashintel/brunch-agent/question-marker"; import { brunchClientToolNames } from "./brunch-client-tools"; @@ -46,7 +46,7 @@ const projectPetrinautMessages = ( dynamicClientToolNames, validatedClientToolNames, ...(mapClientToolInput === undefined ? {} : { mapClientToolInput }), - hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), + hiddenToolNames: new Set(BRUNCH_QUESTION_TOOL_NAMES), }) as PetrinautAiMessage[]; export const useFlueChatHistory = ( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index 50807c4ba97e..4be82493ca68 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -164,6 +164,20 @@ describe("canonical speech selection", () => { }); }); + test("keeps an unmarked question in full-response speech without enabling repeat", () => { + const text = "The batch is ready. Which operator confirms the batch next?"; + const selection = selectCanonicalSpeech([ + { + id: "assistant-unmarked-question", + role: "assistant", + parts: [{ type: "text", text, state: "done" }], + }, + ]); + + expect(selection.segments.map((segment) => segment.text)).toEqual([text]); + expect(selection.questionSegment).toBeUndefined(); + }); + test.each([ { name: "missing exact finalized prose", diff --git a/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md b/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md index 6eae2ff12e08..6d1ed3de6750 100644 --- a/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md +++ b/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md @@ -150,7 +150,6 @@ those remain with their definition owners. | `task` | Flue | Flue | substrate delegation | | `activate_skill` | Flue | Flue | skill activation | | `read_skill_resource` | Flue | Flue | skill resource read | -| `brunch_mark_question` | Brunch core | Brunch app | question relay metadata | | `mutate_workpiece` | Brunch core | Brunch app | durable full-revision write with recorded delta | | `read_petrinaut_net` | Petrinaut Core | Petrinaut website | current document read | | `read_petrinaut_docs` | Petrinaut Core | Petrinaut website | user-guide read | diff --git a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts index 2e944556eeb6..5457acabb61d 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts @@ -2,7 +2,6 @@ import { type AgentDispatchRequest, type CompactionConfig, defineTool, - useDataWriter, useModel, usePersistentState, useSkill, @@ -12,13 +11,6 @@ import { import * as v from "valibot"; import systemPrompt from "./prompts/SYSTEM.md?raw"; -import { - BRUNCH_QUESTION_DATA_NAME, - BRUNCH_QUESTION_TOOL_NAME, - BrunchQuestionDataSchema, - BrunchQuestionInputSchema, - type BrunchQuestionData, -} from "./question-marker"; import { ELICITATION_SKILL_NAME, elicitationSkill, @@ -61,7 +53,7 @@ const _preparedWorkpieceDeliveryIsDispatchable = ( * Mount the contributions owned by Brunch core and return its system prompt. * * Core contributes the always-on universal prompt, one `elicitation` - * capability skill, the question marker, and durable workpiece revisions. + * capability skill and durable workpiece revisions. */ export function useBrunchAgent( model: string, @@ -73,10 +65,6 @@ export function useBrunchAgent( ): string { useModel(model, compaction === undefined ? undefined : { compaction }); useSkill(elicitationSkill); - const writeQuestion = useDataWriter(BRUNCH_QUESTION_DATA_NAME, { - schema: BrunchQuestionDataSchema, - }); - useTool(createBrunchQuestionMarkerTool(writeQuestion)); const [revision, setRevision] = usePersistentState( workpieceRevisionStateKey, null, @@ -92,21 +80,6 @@ export function useBrunchAgent( return systemPrompt.replace(/^\s+|\s+$/gu, ""); } -export const createBrunchQuestionMarkerTool = ( - writeQuestion: (question: BrunchQuestionData) => void, -) => - defineTool({ - name: BRUNCH_QUESTION_TOOL_NAME, - description: - "Mark the exact text of a direct question for accessible replay. Call this immediately before including that exact question in ordinary assistant prose. This marker does not ask or answer the question itself.", - input: BrunchQuestionInputSchema, - output: v.object({ marked: v.literal(true) }), - run({ data, toolCallId }) { - writeQuestion({ question: data.question, toolCallId }); - return { output: { marked: true as const } }; - }, - }); - /** Successful settlement carriage; optional Markdown admits retained pointer-only results. */ export const updateWorkpieceOutputSchema = v.object({ ...workpieceRevisionPointerSchema.entries, diff --git a/libs/@hashintel/brunch-agent/packages/core/src/index.ts b/libs/@hashintel/brunch-agent/packages/core/src/index.ts index 4cfd52cd8d92..cfbd5f04cc04 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/index.ts @@ -30,8 +30,11 @@ export { export { BRUNCH_QUESTION_DATA_NAME, BRUNCH_QUESTION_TOOL_NAME, + BRUNCH_QUESTION_TOOL_NAMES, BrunchQuestionDataSchema, BrunchQuestionInputSchema, + LEGACY_BRUNCH_QUESTION_TOOL_NAME, + LEGACY_QUESTION_REPLAY_TOOL_NAME, parseBrunchQuestionData, type BrunchQuestionData, } from "./question-marker"; diff --git a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md index 1e973409b543..c0cd5b3e8621 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md +++ b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md @@ -10,8 +10,6 @@ Establish what the result must help the person decide, answer, compare, explain, Use the person's vocabulary and follow their active account rather than traversing a schema, template, or target representation. For practice-based accounts, prefer concrete remembered cases. Do not open with a battery of independent questions; deepen one answerable thread at a time and group questions only when they share one frame. -Before asking the person a direct question, call `brunch_mark_question` with the exact question text. Then include the exact same question text in ordinary assistant prose. The marker only makes that text available for accessible replay; it does not wait for or accept the answer, so continue the same response normally after calling it. Do not mark headings, rhetorical questions, or prose that you will not present verbatim. - Activate `elicitation` when progress requires source-side knowledge that cannot be responsibly inferred from the available account, including substantive interviewing, consequential corrections, or consulting a source. In a non-interactive conversation, use the supplied account as the complete input: report a blocking gap and the smallest question a later interactive conversation must answer, without asking it or inventing an answer. ## Authorship and uncertainty diff --git a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts index ba6194c63f50..0f07838d2a78 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts @@ -1,6 +1,13 @@ import * as v from "valibot"; -export const BRUNCH_QUESTION_TOOL_NAME = "brunch_mark_question"; +export const LEGACY_BRUNCH_QUESTION_TOOL_NAME = "brunch_mark_question"; +export const LEGACY_QUESTION_REPLAY_TOOL_NAME = "mark_question_for_replay"; +/** @deprecated Retained only for source compatibility with historical projections. */ +export const BRUNCH_QUESTION_TOOL_NAME = LEGACY_BRUNCH_QUESTION_TOOL_NAME; +export const BRUNCH_QUESTION_TOOL_NAMES = [ + LEGACY_BRUNCH_QUESTION_TOOL_NAME, + LEGACY_QUESTION_REPLAY_TOOL_NAME, +] as const; export const BRUNCH_QUESTION_DATA_NAME = "brunch-question"; const NonBlankStringSchema = v.pipe( diff --git a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts index 42e0f59d22ba..1102a668f431 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts @@ -1,21 +1,26 @@ import * as v from "valibot"; -import { describe, expect, test, vi } from "vitest"; +import { describe, expect, test } from "vitest"; -import { createBrunchQuestionMarkerTool } from "../src/flue"; import { BRUNCH_QUESTION_DATA_NAME, BRUNCH_QUESTION_TOOL_NAME, + BRUNCH_QUESTION_TOOL_NAMES, BrunchQuestionDataSchema, BrunchQuestionInputSchema, + LEGACY_BRUNCH_QUESTION_TOOL_NAME, + LEGACY_QUESTION_REPLAY_TOOL_NAME, parseBrunchQuestionData, - type BrunchQuestionData, } from "../src/question-marker"; -import type { FlueLogger } from "@flue/runtime"; - -describe("the Brunch question marker", () => { - test("defines one non-interactive tool and data-part identity", () => { +describe("legacy Brunch question markers", () => { + test("preserves historical tool and data identities for projection compatibility", () => { expect(BRUNCH_QUESTION_TOOL_NAME).toBe("brunch_mark_question"); + expect(LEGACY_BRUNCH_QUESTION_TOOL_NAME).toBe("brunch_mark_question"); + expect(LEGACY_QUESTION_REPLAY_TOOL_NAME).toBe("mark_question_for_replay"); + expect(BRUNCH_QUESTION_TOOL_NAMES).toEqual([ + "brunch_mark_question", + "mark_question_for_replay", + ]); expect(BRUNCH_QUESTION_DATA_NAME).toBe("brunch-question"); }); @@ -35,28 +40,6 @@ describe("the Brunch question marker", () => { ).toEqual({ question, toolCallId: "tool-question-1" }); }); - test("writes the exact marker without terminating or waiting for an answer", async () => { - const writeQuestion = vi.fn<(question: BrunchQuestionData) => void>(); - const tool = createBrunchQuestionMarkerTool(writeQuestion); - - const result = await tool.run({ - data: { question: "Which line should run this order?" }, - log: { - error: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - }, - toolCallId: "tool-question-1", - }); - - expect(writeQuestion).toHaveBeenCalledOnce(); - expect(writeQuestion).toHaveBeenCalledWith({ - question: "Which line should run this order?", - toolCallId: "tool-question-1", - }); - expect(result).toEqual({ output: { marked: true } }); - }); - test.each([ { question: "" }, { question: " " }, diff --git a/libs/@hashintel/brunch-agent/packages/core/test/update-workpiece.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/update-workpiece.test.ts index b4ef9364eb29..a1226cc03885 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/update-workpiece.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/update-workpiece.test.ts @@ -13,6 +13,7 @@ import { elicitationSkill, workpieceMarkdownByteCeiling, } from "../src/flue"; +import { BRUNCH_QUESTION_TOOL_NAMES } from "../src/question-marker"; import { deriveWorkpieceMutation } from "../src/update-workpiece"; import { workpieceRevisionStateKey, @@ -176,9 +177,11 @@ test("captures the persistent-state setter at render and writes from run", async const mounted = vi .mocked(useTool) .mock.calls.map(([definition]) => definition); - expect(mounted.map((definition) => definition.name)).toContain( - "brunch_mark_question", - ); + const mountedNames = mounted.map((definition) => definition.name); + for (const markerName of BRUNCH_QUESTION_TOOL_NAMES) { + expect(mountedNames).not.toContain(markerName); + expect(prompt).not.toContain(markerName); + } const revisionTool = mounted.find( (definition) => definition.name === MUTATE_WORKPIECE_TOOL_NAME, ); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts index df2f8166f212..87d12871f81a 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts @@ -127,71 +127,74 @@ test("projects data and metadata onto the AI SDK stream", () => { }); }); -test("hides an implementation tool while preserving its data marker", () => { - const written = project( - [ - { - type: "message-started", - conversationId: "conversation-1", - messageId: "message-1", - submissionId: "submission-1", - turnId: "turn-1", - position: position(0), - }, - { - type: "tool-input", - conversationId: "conversation-1", - messageId: "message-1", - toolCallId: "tool-question-1", - toolName: "brunch_mark_question", - input: { question: "Which line should run this order?" }, - position: position(1), - }, - { - type: "data-part", - conversationId: "conversation-1", - messageId: "message-1", - name: "brunch-question", - data: { - question: "Which line should run this order?", +test.each(["brunch_mark_question", "mark_question_for_replay"])( + "hides historical implementation tool $markerToolName while preserving its data marker", + (markerToolName) => { + const written = project( + [ + { + type: "message-started", + conversationId: "conversation-1", + messageId: "message-1", + submissionId: "submission-1", + turnId: "turn-1", + position: position(0), + }, + { + type: "tool-input", + conversationId: "conversation-1", + messageId: "message-1", toolCallId: "tool-question-1", + toolName: markerToolName, + input: { question: "Which line should run this order?" }, + position: position(1), }, - position: position(2), - }, - { - type: "tool-output", - conversationId: "conversation-1", + { + type: "data-part", + conversationId: "conversation-1", + messageId: "message-1", + name: "brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + position: position(2), + }, + { + type: "tool-output", + conversationId: "conversation-1", + toolCallId: "tool-question-1", + output: { marked: true }, + position: position(3), + }, + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "completed", + position: position(4), + }, + ], + new Set([markerToolName]), + ); + + expect(written).toContainEqual({ + type: "data-brunch-question", + data: { + question: "Which line should run this order?", toolCallId: "tool-question-1", - output: { marked: true }, - position: position(3), }, - { - type: "submission-settled", - conversationId: "conversation-1", - submissionId: "submission-1", - outcome: "completed", - position: position(4), - }, - ], - new Set(["brunch_mark_question"]), - ); - - expect(written).toContainEqual({ - type: "data-brunch-question", - data: { - question: "Which line should run this order?", - toolCallId: "tool-question-1", - }, - }); - expect( - written.some( - (chunk) => - chunk.type === "tool-input-available" || - chunk.type === "tool-output-available" || - chunk.type === "tool-output-error", - ), - ).toBe(false); -}); + }); + expect( + written.some( + (chunk) => + chunk.type === "tool-input-available" || + chunk.type === "tool-output-available" || + chunk.type === "tool-output-error", + ), + ).toBe(false); + }, +); test("ignores observation catch-up chunks in a submission stream", () => { const written = project([ From 8173c53b2807c685dfd769a255b3bfd712254179 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 16:59:55 +0200 Subject: [PATCH 5/5] Update buffered Voice marker expectation Co-authored-by: Cursor --- .../voice-interview/buffered-admission.integration.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/buffered-admission.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/buffered-admission.integration.test.ts index adc06eb7ec96..ed7630cb60f6 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/buffered-admission.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/buffered-admission.integration.test.ts @@ -52,7 +52,7 @@ const voice = () => { return { bridge, speakCanonical }; }; -test("buffered production output remains silent until approved; marker and ordinary prose survive without speaking tool payloads", () => { +test("buffered production output remains silent until approved; ordinary prose survives without a repeatable question or spoken tool payloads", () => { const sample = result.buffering.find( ({ caseId }) => caseId === "buffered-valid", )!; @@ -68,7 +68,7 @@ test("buffered production output remains silent until approved; marker and ordin ]); expect(speakCanonical).not.toHaveBeenCalled(); const completed = speechFrom(sample.projectedAfter); - expect(completed.questionSegment?.text).toBe(result.question); + expect(completed.questionSegment).toBeUndefined(); expect(completed.segments.map((segment) => segment.text)).toEqual([ sample.text, "Timing remains unknown.",