feat(eval): imperative online-insight commands + BYO execution role (draft)

src/core/eval.tsx

@@ -101,6 +101,7 @@ import type {
   CreateConfigurationBundleInput,
   CreateDatasetInput,
   CreateOnlineEvalInput,
+  CreateOnlineInsightInput,
   EvaluateInput,
   EvaluateResult,
   GetBatchEvaluationResult,
@@ -556,6 +557,62 @@ export class EvalClient implements CoreEvalClient {
       : retryWhileRolePropagates(() => control.send(command));
   }
 
+  async createOnlineInsight(
+    input: CreateOnlineInsightInput,
+    options: CoreOptions,
+  ): Promise<CreateOnlineEvaluationConfigResponse> {
+    const dataSourceConfig =
+      input.agent !== undefined
+        ? await agentDataSource(input.agent, input.endpoint, this.clients, options)
+        : input.dataSourceConfig;
+    const control = this.clients.control(toClientConfig(options));
+
+    const command = new CreateOnlineEvaluationConfigCommand({
+      onlineEvaluationConfigName: input.name,
+      description: input.description,
+      rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters),
+      dataSourceConfig,
+      insights: input.insightIds.map((insightId) => ({ insightId })),
+      clusteringConfig: input.clusteringConfig,
+      evaluationExecutionRoleArn: input.evaluationExecutionRoleArn,
+      enableOnCreate: input.enableOnCreate ?? true,
+    });
+
+    // The role is caller-supplied, so one that cannot be assumed is a real
+    // misconfiguration — fail fast rather than retry as we do for a role we just
+    // provisioned ourselves.
+    return control.send(command);
+  }
+
+  // Insight configs are the same resource as eval configs, so reads and lifecycle
+  // reuse the eval methods; only create/update differ (insights vs evaluators).
+  getOnlineInsight(id: string, options: CoreOptions): Promise<GetOnlineEvaluationConfigResponse> {
+    return this.getOnlineEvaluationConfig(id, options);
+  }
+
+  listOnlineInsights(
+    nextToken: string | undefined,
+    maxResults: number | undefined,
+    options: CoreOptions,
+  ): Promise<ListOnlineEvaluationConfigsResponse> {
+    return this.listOnlineEvaluationConfigs(nextToken, maxResults, options);
+  }
+
+  setOnlineInsightExecutionStatus(
+    id: string,
+    executionStatus: "ENABLED" | "DISABLED",
+    options: CoreOptions,
+  ): Promise<UpdateOnlineEvaluationConfigResponse> {
+    return this.setOnlineEvaluationExecutionStatus(id, executionStatus, options);
+  }
+
+  deleteOnlineInsight(
+    id: string,
+    options: CoreOptions,
+  ): Promise<DeleteOnlineEvaluationConfigResponse> {
+    return this.deleteOnlineEvaluationConfig(id, options);
+  }
+
   // updateOnlineEvaluationConfig fetches the current config and merges the
   // provided fields over it, because UpdateOnlineEvaluationConfig replaces the
   // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than

src/handlers/eval/index.tsx

@@ -5,6 +5,7 @@ import type { AppIO } from "../../io";
 import type { Core } from "../types";
 import { createEvaluatorHandler } from "./evaluator";
 import { createOnlineEvalHandler } from "./online-eval";
+import { createOnlineInsightHandler } from "./online-insight";
 import { createDatasetHandler } from "./dataset";
 import { createBatchEvaluationHandler } from "./batch-evaluation";
 import { createOnDemandHandler } from "./ondemand";
@@ -16,6 +17,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router {
     .default(renderTui(core, io))
     .handler(createEvaluatorHandler(core, io))
     .handler(createOnlineEvalHandler(core, io))
+    .handler(createOnlineInsightHandler(core, io))
     .handler(createDatasetHandler(core, io))
     .handler(createBatchEvaluationHandler(core, io))
     .handler(createOnDemandHandler(core, io))

src/handlers/eval/online-insight/create/index.tsx

@@ -0,0 +1,135 @@
+import z from "zod";
+import type { DataSourceConfig, Filter } from "@aws-sdk/client-bedrock-agentcore-control";
+import { createHandler, flag } from "../../../../router";
+import { InputValidationError } from "../../../../errors";
+import { JsonRendererKey } from "../../../../tui";
+import { SourceResolver, type AppIO } from "../../../../io";
+import type { Core } from "../../../types";
+import { coreOptsFromCtx, parseJsonFlag } from "../../../utils";
+
+const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight.";
+const ARN_PREFIX = "arn:";
+
+export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) =>
+  createHandler({
+    name: "create",
+    description: "create an online insight config",
+    flags: [
+      flag("name", "the name of the online insight config", z.string().optional()),
+      flag(
+        "execution-role-arn",
+        "IAM role the online insight assumes (required; not auto-provisioned)",
+        z.string().optional(),
+      ),
+      flag("agent", "harness ID or runtime ID whose traffic to sample", z.string().optional()),
+      flag(
+        "endpoint",
+        "the agent endpoint qualifier to scope monitoring to (default DEFAULT)",
+        z.string().optional(),
+      ),
+      flag(
+        "data-source-config",
+        "the traces to evaluate (JSON DataSourceConfig; inline, file://<path>, or - for stdin), as an alternative to --agent",
+        z.string().optional(),
+      ),
+      flag(
+        "insight",
+        "insight ID(s) to apply: Builtin.Insight.* identifiers or full ARNs",
+        z.array(z.string()).optional(),
+      ),
+      flag(
+        "clustering-frequency",
+        "insight clustering cadence(s): DAILY, WEEKLY, MONTHLY",
+        z.array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])).optional(),
+      ),
+      flag(
+        "sampling-rate",
+        "percentage of sessions to sample (0.01-100)",
+        z.number().min(0.01).max(100).optional(),
+      ),
+      flag(
+        "session-timeout-minutes",
+        "minutes of inactivity before a session is considered complete (1-1440, default 15)",
+        z.number().int().min(1).max(1440).optional(),
+      ),
+      flag(
+        "filters",
+        "trace filters (JSON Filter[]; inline, file://<path>, or - for stdin)",
+        z.string().optional(),
+      ),
+      flag(
+        "enable-on-create",
+        "whether to enable evaluation immediately (default true; pass false to create it paused)",
+        z.enum(["true", "false"]).optional(),
+      ),
+      flag(
+        "description",
+        "a description of the config's monitoring purpose",
+        z.string().optional(),
+      ),
+    ],
+    handle: async (ctx, flags) => {
+      if (!flags["name"])
+        throw new InputValidationError("required option '--name <name>' not specified");
+      if (!flags["execution-role-arn"])
+        throw new InputValidationError(
+          "required option '--execution-role-arn <execution-role-arn>' not specified",
+        );
+      if (!flags["sampling-rate"])
+        throw new InputValidationError(
+          "required option '--sampling-rate <sampling-rate>' not specified",
+        );
+      if (!flags["insight"] || flags["insight"].length === 0)
+        throw new InputValidationError("required option '--insight <insight...>' not specified");
+
+      for (const id of flags["insight"]) {
+        if (!id.startsWith(BUILTIN_INSIGHT_PREFIX) && !id.startsWith(ARN_PREFIX))
+          throw new InputValidationError(
+            `invalid insight "${id}": must be a ${BUILTIN_INSIGHT_PREFIX}* identifier or a full ARN`,
+          );
+      }
+
+      const hasAgent = flags["agent"] !== undefined;
+      const hasDataSource = flags["data-source-config"] !== undefined;
+      if (hasAgent === hasDataSource)
+        throw new InputValidationError(
+          "specify exactly one of '--agent' or '--data-source-config'",
+        );
+      if (hasDataSource && flags["endpoint"])
+        throw new InputValidationError("'--endpoint' can only be used with '--agent'");
+
+      const source = new SourceResolver({ stdin: io.stdin });
+      const frequencies = flags["clustering-frequency"];
+      const common = {
+        name: flags["name"],
+        description: flags["description"],
+        samplingRate: flags["sampling-rate"],
+        sessionTimeoutMinutes: flags["session-timeout-minutes"],
+        filters: parseJsonFlag<Filter[]>(
+          "filters",
+          await source.resolveText("filters", flags["filters"]),
+        ),
+        insightIds: flags["insight"],
+        clusteringConfig: frequencies ? { frequencies } : undefined,
+        evaluationExecutionRoleArn: flags["execution-role-arn"],
+        enableOnCreate:
+          flags["enable-on-create"] === undefined
+            ? undefined
+            : flags["enable-on-create"] === "true",
+      };
+
+      const response = await core.eval.createOnlineInsight(
+        hasAgent
+          ? { ...common, agent: flags["agent"]!, endpoint: flags["endpoint"] }
+          : {
+              ...common,
+              dataSourceConfig: parseJsonFlag<DataSourceConfig>(
+                "data-source-config",
+                await source.resolveText("data-source-config", flags["data-source-config"]),
+              )!,
+            },
+        coreOptsFromCtx(ctx),
+      );
+      ctx.require(JsonRendererKey).renderJson(response);
+    },
+  });

src/handlers/eval/online-insight/delete/index.tsx

@@ -0,0 +1,20 @@
+import z from "zod";
+import { createHandler, flag } from "../../../../router";
+import { InputValidationError } from "../../../../errors";
+import { JsonRendererKey } from "../../../../tui";
+import type { Core } from "../../../types";
+import { coreOptsFromCtx } from "../../../utils";
+
+export const createDeleteOnlineInsightHandler = (core: Core) =>
+  createHandler({
+    name: "delete",
+    description: "delete an online insight config by id",
+    flags: [flag("id", "the ID of the online insight config to delete", z.string().optional())],
+    handle: async (ctx, flags) => {
+      if (!flags["id"]) throw new InputValidationError("required option '--id <id>' not specified");
+
+      ctx
+        .require(JsonRendererKey)
+        .renderJson(await core.eval.deleteOnlineInsight(flags["id"], coreOptsFromCtx(ctx)));
+    },
+  });

src/handlers/eval/online-insight/get/index.tsx

@@ -0,0 +1,20 @@
+import z from "zod";
+import { createHandler, flag } from "../../../../router";
+import { InputValidationError } from "../../../../errors";
+import { JsonRendererKey } from "../../../../tui";
+import type { Core } from "../../../types";
+import { coreOptsFromCtx } from "../../../utils";
+
+export const createGetOnlineInsightHandler = (core: Core) =>
+  createHandler({
+    name: "get",
+    description: "get an online insight config by id",
+    flags: [flag("id", "the ID of the online insight config", z.string().optional())],
+    handle: async (ctx, flags) => {
+      if (!flags["id"]) throw new InputValidationError("required option '--id <id>' not specified");
+
+      ctx
+        .require(JsonRendererKey)
+        .renderJson(await core.eval.getOnlineInsight(flags["id"], coreOptsFromCtx(ctx)));
+    },
+  });

src/handlers/eval/online-insight/index.tsx

@@ -0,0 +1,19 @@
+import { Router } from "../../../router";
+import type { AppIO } from "../../../io";
+import type { Core } from "../../types";
+import { createCreateOnlineInsightHandler } from "./create";
+import { createGetOnlineInsightHandler } from "./get";
+import { createListOnlineInsightHandler } from "./list";
+import { createPauseOnlineInsightHandler } from "./pause";
+import { createResumeOnlineInsightHandler } from "./resume";
+import { createDeleteOnlineInsightHandler } from "./delete";
+
+export function createOnlineInsightHandler(core: Core, io: AppIO): Router {
+  return new Router("online-insight", "manage AgentCore online insight configs")
+    .handler(createCreateOnlineInsightHandler(core, io))
+    .handler(createGetOnlineInsightHandler(core))
+    .handler(createListOnlineInsightHandler(core))
+    .handler(createPauseOnlineInsightHandler(core))
+    .handler(createResumeOnlineInsightHandler(core))
+    .handler(createDeleteOnlineInsightHandler(core));
+}

src/handlers/eval/online-insight/list/index.tsx

@@ -0,0 +1,23 @@
+import z from "zod";
+import { createHandler, flag } from "../../../../router";
+import { JsonRendererKey } from "../../../../tui";
+import type { Core } from "../../../types";
+import { coreOptsFromCtx } from "../../../utils";
+
+export const createListOnlineInsightHandler = (core: Core) =>
+  createHandler({
+    name: "list",
+    description: "list online insight configs",
+    flags: [
+      flag("next-token", "pagination token returned by a previous request", z.string().optional()),
+      flag("max-results", "maximum number of items to return", z.number().optional()),
+    ],
+    handle: async (ctx, flags) => {
+      const response = await core.eval.listOnlineInsights(
+        flags["next-token"],
+        flags["max-results"],
+        coreOptsFromCtx(ctx),
+      );
+      ctx.require(JsonRendererKey).renderJson(response);
+    },
+  });