From a0b03a423d028d757dccfe4ee4473691dd2568cd Mon Sep 17 00:00:00 2001 From: Jonas Jesus Date: Sun, 25 Oct 2026 21:54:57 -0300 Subject: [PATCH] =?UTF-8?q?fix(grain):=20migrate=20to=20Grain=20API=20v2?= =?UTF-8?q?=20=E2=80=94=20v1=20was=20retired=20(410),=20webhooks=20stopped?= =?UTF-8?q?=20on=20Sep=2030?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Client on /_/public-api/v2 with Public-Api-Version; recordings via POST, hooks are one per type (recording_added / recording_updated) with include. - Webhook payloads only say which recording changed (v2 hooks are unsigned): the indexed row is always re-fetched from Grain by id, never trusted from the POST body. GRAIN_API_KEY keeps the webhook working after a restart (onChange only runs when the connection is saved). - Tools keep their contract (owners = recorders, notes = ai_summary). - Backfill script on v2, with --since and full rows. - grain, slack-mcp, discord-read, discord, registry prefer SUPABASE_SERVICE_ROLE_KEY: the shared Supabase is moving to RLS on and no anon access (tokens and private data were readable/writable with the public anon key). Co-Authored-By: Claude Opus 5.5 (1M context) --- discord-read/server/lib/supabase-client.ts | 8 +- discord/server/lib/supabase.ts | 8 +- grain/scripts/backfill.ts | 162 ++++++++-------- grain/server/constants.ts | 22 ++- grain/server/db/queries.ts | 36 ++-- grain/server/lib/grain-client.ts | 206 ++++++++------------- grain/server/lib/supabase-client.ts | 7 +- grain/server/lib/types.ts | 182 ++++++++++-------- grain/server/main.ts | 112 +++++------ grain/server/tools/get-recording.ts | 19 +- grain/server/tools/get-summary.ts | 18 +- grain/server/tools/get-transcript.ts | 5 +- grain/server/tools/list-recordings.ts | 3 +- grain/server/triggers/publisher.ts | 19 +- registry/server/tools/registry-binding.ts | 16 +- slack-mcp/server/lib/supabase-client.ts | 8 +- 16 files changed, 416 insertions(+), 415 deletions(-) diff --git a/discord-read/server/lib/supabase-client.ts b/discord-read/server/lib/supabase-client.ts index 13ff4db5..e2f83d74 100644 --- a/discord-read/server/lib/supabase-client.ts +++ b/discord-read/server/lib/supabase-client.ts @@ -47,7 +47,8 @@ export function getSupabaseClient(): SupabaseClient | null { } const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { console.warn( @@ -76,7 +77,10 @@ export function getSupabaseClient(): SupabaseClient | null { * Check if Supabase is configured and available */ export function isSupabaseConfigured(): boolean { - return !!process.env.SUPABASE_URL && !!process.env.SUPABASE_ANON_KEY; + return ( + !!process.env.SUPABASE_URL && + !!(process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY) + ); } /** diff --git a/discord/server/lib/supabase.ts b/discord/server/lib/supabase.ts index b86cbfbe..9d23d9fb 100644 --- a/discord/server/lib/supabase.ts +++ b/discord/server/lib/supabase.ts @@ -47,7 +47,8 @@ export function getSupabaseClient(): SupabaseClient | null { if (supabaseClient) return supabaseClient; const url = process.env.SUPABASE_URL; - const key = process.env.SUPABASE_ANON_KEY; + const key = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY; if (!url || !key) return null; try { @@ -61,7 +62,10 @@ export function getSupabaseClient(): SupabaseClient | null { } export function isSupabaseConfigured(): boolean { - return !!process.env.SUPABASE_URL && !!process.env.SUPABASE_ANON_KEY; + return ( + !!process.env.SUPABASE_URL && + !!(process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY) + ); } // ============================================================================ diff --git a/grain/scripts/backfill.ts b/grain/scripts/backfill.ts index 4287a1e4..d5ef2d16 100644 --- a/grain/scripts/backfill.ts +++ b/grain/scripts/backfill.ts @@ -1,122 +1,116 @@ #!/usr/bin/env bun /** - * Backfill script — fetches all Grain recordings not yet in grain_meeting_recordings - * and upserts them into Supabase. + * Backfill — indexes Grain recordings into grain_meeting_recordings (v2 API), + * with participants and the AI summary, the same row the webhook writes. + * Existing rows are updated. * - * Usage: bun run scripts/backfill.ts + * Usage: GRAIN_TOKEN=… SUPABASE_URL=… SUPABASE_SERVICE_ROLE_KEY=… \ + * bun run scripts/backfill.ts [--since 2026-09-30] */ - import { createClient } from "@supabase/supabase-js"; +import { + GRAIN_API_VERSION, + GRAIN_BASE_URL, + INDEX_INCLUDE, +} from "../server/constants.ts"; +import { type GrainRecordingV2, toDetails } from "../server/lib/types.ts"; const GRAIN_TOKEN = process.env.GRAIN_TOKEN; const SUPABASE_URL = process.env.SUPABASE_URL; -const SUPABASE_ANON_KEY = process.env.SUPABASE_ANON_KEY; +const SUPABASE_KEY = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY; -if (!GRAIN_TOKEN || !SUPABASE_URL || !SUPABASE_ANON_KEY) { +if (!GRAIN_TOKEN || !SUPABASE_URL || !SUPABASE_KEY) { console.error( - "Missing env vars: GRAIN_TOKEN, SUPABASE_URL, SUPABASE_ANON_KEY", + "Missing env vars: GRAIN_TOKEN, SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY", ); process.exit(1); } -const supabase = createClient(SUPABASE_URL, SUPABASE_ANON_KEY); - -interface GrainRecording { - id: string; - title?: string; - url?: string; - start_datetime?: string; - end_datetime?: string; - public_thumbnail_url?: string; - owners?: string[]; - tags?: { name?: string; value?: string }[] | string[]; - intelligence_notes_md?: string; -} +const sinceArg = process.argv.indexOf("--since"); +const since = sinceArg > 0 ? process.argv[sinceArg + 1] : undefined; -function normalizeTags(tags: GrainRecording["tags"]): string[] { - if (!tags || tags.length === 0) return []; - if (typeof tags[0] === "string") return tags as string[]; - return (tags as { name?: string; value?: string }[]) - .map((t) => t.name ?? t.value ?? "") - .filter(Boolean); -} +const supabase = createClient(SUPABASE_URL, SUPABASE_KEY, { + auth: { persistSession: false, autoRefreshToken: false }, +}); async function fetchPage( cursor?: string, -): Promise<{ recordings: GrainRecording[]; cursor?: string }> { - const url = new URL("https://api.grain.com/_/public-api/recordings"); - url.searchParams.set("limit", "100"); - if (cursor) url.searchParams.set("cursor", cursor); - - const res = await fetch(url.toString(), { - headers: { Authorization: `Bearer ${GRAIN_TOKEN}` }, +): Promise<{ recordings: GrainRecordingV2[]; cursor: string | null }> { + const res = await fetch(`${GRAIN_BASE_URL}/recordings`, { + method: "POST", + headers: { + Authorization: `Bearer ${GRAIN_TOKEN}`, + "Public-Api-Version": GRAIN_API_VERSION, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + ...(cursor ? { cursor } : {}), + filter: since ? { after_datetime: since } : {}, + include: INDEX_INCLUDE, + }), }); if (!res.ok) throw new Error(`Grain API error: ${res.status} ${await res.text()}`); return res.json(); } -async function upsertBatch(recordings: GrainRecording[]) { - const rows = recordings.map((r) => ({ - id: r.id, - title: r.title ?? null, - url: r.url ?? null, - start_datetime: r.start_datetime ?? null, - end_datetime: r.end_datetime ?? null, - public_thumbnail_url: r.public_thumbnail_url ?? null, - owners: r.owners ?? [], - tags: normalizeTags(r.tags), - participants: [], - participants_text: null, - intelligence_notes_md: r.intelligence_notes_md ?? null, - user_id: null, - webhook_type: "recording_added", - raw_payload: r, - indexed_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - })); - - const { error, count } = await supabase +async function upsertBatch(recordings: GrainRecordingV2[], userId: string) { + const now = new Date().toISOString(); + const rows = recordings.map((r) => { + const d = toDetails(r); + const participants = d.participants ?? []; + return { + id: d.id, + title: d.title ?? null, + url: d.url ?? null, + start_datetime: d.start_datetime ?? null, + end_datetime: d.end_datetime ?? null, + public_thumbnail_url: d.public_thumbnail_url ?? null, + owners: d.owners ?? [], + tags: d.tags ?? [], + participants, + participants_text: participants + .map((p) => [p.name, p.email].filter(Boolean).join(" ")) + .join(", "), + intelligence_notes_md: d.intelligence_notes_md ?? null, + user_id: userId, + webhook_type: "backfill", + raw_payload: { type: "backfill", user_id: userId, data: r }, + indexed_at: now, + updated_at: now, + }; + }); + const { error } = await supabase .from("grain_meeting_recordings") - .upsert(rows, { onConflict: "id", ignoreDuplicates: true }) - .select("id", { count: "exact", head: true }); - + .upsert(rows, { onConflict: "id" }); if (error) throw new Error(`Supabase upsert error: ${error.message}`); - return rows.length; } async function main() { - console.log("Starting Grain recordings backfill..."); + const who = await fetch(`${GRAIN_BASE_URL}/token-info`, { + headers: { + Authorization: `Bearer ${GRAIN_TOKEN}`, + "Public-Api-Version": GRAIN_API_VERSION, + }, + }).then((r) => r.json() as Promise<{ user_id: string; name: string }>); + console.log(`Backfilling as ${who.name}${since ? ` since ${since}` : ""}…`); let cursor: string | undefined; - let page = 0; let total = 0; - let skipped = 0; - - do { - page++; - const { recordings, cursor: nextCursor } = await fetchPage(cursor); - + for (let page = 1; ; page++) { + const { recordings, cursor: next } = await fetchPage(cursor); if (recordings.length === 0) break; - - const oldest = recordings.at(-1)?.start_datetime; - const newest = recordings[0]?.start_datetime; + await upsertBatch(recordings, who.user_id); + total += recordings.length; console.log( - `Page ${page}: ${recordings.length} recordings [${oldest} → ${newest}]`, + `Page ${page}: ${recordings.length} [${recordings.at(-1)?.start_datetime} → ${recordings[0]?.start_datetime}]`, ); - - const inserted = await upsertBatch(recordings); - total += recordings.length; - - cursor = nextCursor; - - // Small delay to avoid rate limiting - if (cursor) await new Promise((r) => setTimeout(r, 200)); - } while (cursor); - - console.log( - `\nDone. Fetched ${total} recordings (${skipped} skipped as existing).`, - ); + if (!next) break; + cursor = next; + await new Promise((r) => setTimeout(r, 200)); + } + console.log(`Done. Indexed ${total} recordings.`); } main().catch((err) => { diff --git a/grain/server/constants.ts b/grain/server/constants.ts index 817c0b54..276eacee 100644 --- a/grain/server/constants.ts +++ b/grain/server/constants.ts @@ -1,14 +1,22 @@ /** - * Grain API constants - * @see https://grainhq.notion.site/Grain-Personal-API-877184aa82b54c77a875083c1b560de9 + * Grain Public API v2 constants. + * v1 (`/_/public-api/...` without `/v2`) was retired — it answers 410 Gone. + * @see https://developers.grain.com/endpoints.html */ -export const GRAIN_BASE_URL = "https://api.grain.com"; +export const GRAIN_BASE_URL = "https://api.grain.com/_/public-api/v2"; +export const GRAIN_API_VERSION = "2026-10-01"; -export const GRAIN_RECORDINGS_ENDPOINT = "/_/public-api/recordings"; -export const GRAIN_VIEWS_ENDPOINT = "/_/public-api/views"; -export const GRAIN_HOOKS_ENDPOINT = "/_/public-api/hooks"; +export const GRAIN_RECORDINGS_ENDPOINT = "/recordings"; +export const GRAIN_HOOKS_ENDPOINT = "/hooks"; -export const GRAIN_HOOKS_VERSION = 2; +/** Hook types the index listens to. */ +export const INDEXED_HOOK_TYPES = [ + "recording_added", + "recording_updated", +] as const; + +/** What every indexed recording is fetched with. */ +export const INDEX_INCLUDE = { participants: true, ai_summary: true } as const; export const DEFAULT_PAGE_SIZE = 50; export const MAX_PAGE_SIZE = 100; diff --git a/grain/server/db/queries.ts b/grain/server/db/queries.ts index 5a89f1f2..dd8e99f7 100644 --- a/grain/server/db/queries.ts +++ b/grain/server/db/queries.ts @@ -39,34 +39,30 @@ function buildParticipantsText(participants: RecordingParticipant[]): string { export interface EnrichedWebhookData { payload: WebhookPayload; - details: RecordingDetails | null; + /** Fetched from the Grain API — never taken from the (unsigned) webhook body. */ + details: RecordingDetails; } -function toDbRow(data: EnrichedWebhookData): GrainRecordingRow { - const { payload, details } = data; - const participants = details?.participants ?? []; - +function toDbRow({ payload, details }: EnrichedWebhookData): GrainRecordingRow { + const participants = details.participants ?? []; + const now = new Date().toISOString(); return { - id: payload.data.id, - title: details?.title ?? payload.data.title ?? null, - url: details?.url ?? payload.data.url ?? null, - start_datetime: - details?.start_datetime ?? payload.data.start_datetime ?? null, - end_datetime: details?.end_datetime ?? payload.data.end_datetime ?? null, - public_thumbnail_url: - details?.public_thumbnail_url ?? - payload.data.public_thumbnail_url ?? - null, - owners: details?.owners ?? [], - tags: details?.tags ?? [], + id: details.id, + title: details.title ?? null, + url: details.url ?? null, + start_datetime: details.start_datetime ?? null, + end_datetime: details.end_datetime ?? null, + public_thumbnail_url: details.public_thumbnail_url ?? null, + owners: details.owners ?? [], + tags: details.tags ?? [], participants, participants_text: buildParticipantsText(participants), - intelligence_notes_md: details?.intelligence_notes_md ?? null, + intelligence_notes_md: details.intelligence_notes_md ?? null, user_id: payload.user_id ?? null, webhook_type: payload.type, raw_payload: payload, - indexed_at: new Date().toISOString(), - updated_at: new Date().toISOString(), + indexed_at: now, + updated_at: now, }; } diff --git a/grain/server/lib/grain-client.ts b/grain/server/lib/grain-client.ts index 413c7653..db301288 100644 --- a/grain/server/lib/grain-client.ts +++ b/grain/server/lib/grain-client.ts @@ -1,21 +1,21 @@ import { + GRAIN_API_VERSION, GRAIN_BASE_URL, - GRAIN_RECORDINGS_ENDPOINT, - GRAIN_VIEWS_ENDPOINT, GRAIN_HOOKS_ENDPOINT, - GRAIN_HOOKS_VERSION, + GRAIN_RECORDINGS_ENDPOINT, } from "../constants.ts"; -import type { - ListRecordingsParams, - ListRecordingsResponse, - RecordingDetails, - CreateHookParams, - CreateHookResponse, - ListHooksResponse, - ListViewsResponse, - TranscriptEntry, +import { + type GrainRecordingV2, + type Hook, + type HookType, + type ListHooksResponse, + type ListRecordingsParams, + type ListRecordingsResponse, + type RecordingDetails, + type RecordingsInclude, + type TranscriptEntry, + toDetails, } from "./types.ts"; - export interface GrainClientConfig { apiKey: string; } @@ -60,52 +60,38 @@ export class GrainAPIError extends Error { } } +/** v2 wants full date-times; the tools accept plain dates ("2025-01-01") too. */ +export function isoDateTime(value: string): string { + const d = new Date(value); + return Number.isNaN(d.getTime()) ? value : d.toISOString(); +} + export class GrainClient { private apiKey: string; - private baseUrl: string; constructor(config: GrainClientConfig) { this.apiKey = config.apiKey; - this.baseUrl = GRAIN_BASE_URL; } - private getHeaders(): Record { + private headers(accept = "application/json"): Record { return { Authorization: `Bearer ${this.apiKey}`, + "Public-Api-Version": GRAIN_API_VERSION, "Content-Type": "application/json", - Accept: "application/json", + Accept: accept, }; } private async request( - method: "GET" | "POST" | "DELETE", + method: "POST" | "DELETE", endpoint: string, - options?: { - params?: Record; - body?: Record; - }, + body?: unknown, ): Promise { - const url = new URL(`${this.baseUrl}${endpoint}`); - - if (options?.params) { - for (const [key, value] of Object.entries(options.params)) { - if (value !== undefined && value !== null) { - url.searchParams.append(key, value); - } - } - } - - const fetchOptions: RequestInit = { + const response = await fetch(`${GRAIN_BASE_URL}${endpoint}`, { method, - headers: this.getHeaders(), - }; - - if (options?.body && method !== "GET") { - fetchOptions.body = JSON.stringify(options.body); - } - - const response = await fetch(url.toString(), fetchOptions); - + headers: this.headers(), + body: body === undefined ? undefined : JSON.stringify(body), + }); if (!response.ok) { const errorText = await response.text().catch(() => "Unknown error"); throw new GrainAPIError( @@ -115,64 +101,49 @@ export class GrainClient { endpoint, ); } - return (await response.json()) as T; } + /** POST /v2/recordings — filters go in the body; `after_datetime` = on/after. */ async listRecordings( params?: ListRecordingsParams, ): Promise { - const queryParams: Record = {}; - - if (params?.cursor) queryParams.cursor = params.cursor; - if (params?.start_date) queryParams.start_date = params.start_date; - if (params?.end_date) queryParams.end_date = params.end_date; - if (params?.title) queryParams.title = params.title; - if (params?.attendance) queryParams.attendance = params.attendance; - if (params?.include_highlights) queryParams.include_highlights = "true"; - if (params?.include_participants) queryParams.include_participants = "true"; - if (params?.include_calendar_id) queryParams.include_calendar_id = "true"; - + const filter: Record = {}; + if (params?.start_date) + filter.after_datetime = isoDateTime(params.start_date); + if (params?.end_date) filter.before_datetime = isoDateTime(params.end_date); + if (params?.title) filter.title_search = params.title; + if (params?.attendance) filter.attendance = params.attendance; return this.request( - "GET", + "POST", GRAIN_RECORDINGS_ENDPOINT, - { params: queryParams }, + { + ...(params?.cursor ? { cursor: params.cursor } : {}), + filter, + include: { + highlights: Boolean(params?.include_highlights), + participants: Boolean(params?.include_participants), + }, + }, ); } + /** POST /v2/recordings/:id, normalized to the MCP's stable shape. */ async getRecording( recordingId: string, - options?: { - include_highlights?: boolean; - include_participants?: boolean; - include_owners?: boolean; - transcript_format?: "json" | "vtt"; - intelligence_notes_format?: "json" | "md" | "text"; - }, + include: RecordingsInclude = { participants: true, ai_summary: true }, ): Promise { - const params: Record = {}; - - if (options?.include_highlights) params.include_highlights = "true"; - if (options?.include_participants !== false) - params.include_participants = "true"; - if (options?.include_owners !== false) params.include_owners = "true"; - if (options?.transcript_format) - params.transcript_format = options.transcript_format; - if (options?.intelligence_notes_format) - params.intelligence_notes_format = options.intelligence_notes_format; - - return this.request( - "GET", + const recording = await this.request( + "POST", `${GRAIN_RECORDINGS_ENDPOINT}/${recordingId}`, - { params }, + { include }, ); + return toDetails(recording); } /** - * GET /_/public-api/recordings/:id/transcript → JSON - * GET /_/public-api/recordings/:id/transcript.txt → plain text - * GET /_/public-api/recordings/:id/transcript.vtt → WebVTT (paid seat) - * GET /_/public-api/recordings/:id/transcript.srt → SRT (paid seat) + * GET /v2/recordings/:id/transcript → JSON segments + * GET /v2/recordings/:id/transcript.txt → plain text (also .vtt / .srt) */ async getTranscript( recordingId: string, @@ -180,14 +151,11 @@ export class GrainClient { ): Promise { const suffix = format === "json" ? "" : `.${format}`; const endpoint = `${GRAIN_RECORDINGS_ENDPOINT}/${recordingId}/transcript${suffix}`; - - const url = new URL(`${this.baseUrl}${endpoint}`); - const headers = this.getHeaders(); - if (format !== "json") { - headers["Accept"] = "text/plain"; - } - const response = await fetch(url.toString(), { headers }); - + const response = await fetch(`${GRAIN_BASE_URL}${endpoint}`, { + headers: this.headers( + format === "json" ? "application/json" : "text/plain", + ), + }); if (!response.ok) { const errorText = await response.text().catch(() => "Unknown error"); throw new GrainAPIError( @@ -197,55 +165,43 @@ export class GrainClient { endpoint, ); } - - if (format === "json") { - return (await response.json()) as TranscriptEntry[]; - } - return await response.text(); + if (format !== "json") return await response.text(); + const segments = (await response.json()) as { + speaker: string; + text: string; + start?: number; + end?: number; + }[]; + return segments.map((s) => ({ + speaker: s.speaker, + text: s.text, + start_time: s.start, + end_time: s.end, + })); } - /** - * @see https://grainhq.notion.site/Grain-Personal-API (GET /_/public-api/views) - */ - async listViews(): Promise { - return this.request("GET", GRAIN_VIEWS_ENDPOINT, { - params: { type_filter: "recordings" }, - }); - } - - /** - * @see https://grainhq.notion.site/Grain-Personal-API (GET /_/public-api/hooks) - */ + /** POST /v2/hooks (list). */ async listHooks(): Promise { - return this.request("GET", GRAIN_HOOKS_ENDPOINT); + return this.request("POST", GRAIN_HOOKS_ENDPOINT, {}); } /** - * @see https://grainhq.notion.site/Grain-Personal-API (POST /_/public-api/hooks) - * Grain performs a reachability test on `hook_url`. The endpoint must respond 2xx. + * POST /v2/hooks/create. Grain sends a reachability POST to `hookUrl` + * first; it must answer 2xx or the hook is not created. */ async createHook( hookUrl: string, - viewId: string, - actions?: CreateHookParams["actions"], - ): Promise { - const body: Record = { - version: GRAIN_HOOKS_VERSION, + hookType: HookType, + include?: RecordingsInclude, + ): Promise { + return this.request("POST", `${GRAIN_HOOKS_ENDPOINT}/create`, { hook_url: hookUrl, - view_id: viewId, - }; - if (actions) { - body.actions = actions; - } - - return this.request("POST", GRAIN_HOOKS_ENDPOINT, { - body, + hook_type: hookType, + ...(include ? { include } : {}), }); } - /** - * @see https://grainhq.notion.site/Grain-Personal-API (DELETE /_/public-api/hooks/:id) - */ + /** DELETE /v2/hooks/:id */ async deleteHook(hookId: string): Promise { await this.request<{ success: boolean }>( "DELETE", diff --git a/grain/server/lib/supabase-client.ts b/grain/server/lib/supabase-client.ts index 69865e0d..3cc9c39b 100644 --- a/grain/server/lib/supabase-client.ts +++ b/grain/server/lib/supabase-client.ts @@ -8,11 +8,14 @@ export function getSupabaseClient(): SupabaseClient { } const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + // Server-side key: the tables have RLS on and no anon access. The anon key + // stays as a fallback only for local setups that still use it. + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { throw new Error( - "SUPABASE_URL and SUPABASE_ANON_KEY are required to use indexed recording search.", + "SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY are required to use indexed recording search.", ); } diff --git a/grain/server/lib/types.ts b/grain/server/lib/types.ts index 3cb7027f..a6aca2ae 100644 --- a/grain/server/lib/types.ts +++ b/grain/server/lib/types.ts @@ -1,19 +1,59 @@ import { z } from "zod"; /** - * Grain Public API types - * @see https://grainhq.notion.site/Grain-Personal-API + * Grain Public API v2 types + * @see https://developers.grain.com/endpoints.html */ -// -- Recordings (list) -- +// -- v2 wire shapes -- -export interface Recording { +export interface RecordingParticipant { + id?: string; + email: string | null; + name: string | null; + scope: string; + confirmed_attendee?: boolean; + observed_join_time?: string | null; + observed_leave_time?: string | null; +} + +export interface RecordingHighlight { + id: string; + recording_id: string; + text: string; + transcript: string; + timestamp: number; + duration: number; + created_datetime: string; + url: string; + thumbnail_url: string; +} + +/** A Recording as v2 returns it (fields beyond `include` may be absent). */ +export interface GrainRecordingV2 { id: string; title: string; url: string; start_datetime: string; end_datetime: string; - public_thumbnail_url: string | null; + duration_ms?: number; + thumbnail_url: string | null; + source?: string; + media_type?: string; + tags?: string[]; + recorders?: { id: string; name: string; email: string | null }[]; + participants?: RecordingParticipant[]; + highlights?: RecordingHighlight[]; + ai_summary?: { text: string } | null; + ai_template_sections?: { title: string; data: unknown }[]; + [key: string]: unknown; +} + +export interface RecordingsInclude { + participants?: boolean; + highlights?: boolean; + ai_summary?: boolean; + ai_template_sections?: { format?: "json" | "markdown" | "text" }; } export interface ListRecordingsParams { @@ -24,44 +64,34 @@ export interface ListRecordingsParams { attendance?: "hosted" | "attended"; include_highlights?: boolean; include_participants?: boolean; - include_calendar_id?: boolean; } export interface ListRecordingsResponse { cursor: string | null; - recordings: Recording[]; + recordings: GrainRecordingV2[]; } -// -- Recording details (single) -- - -export interface RecordingParticipant { - email: string; - name: string; - scope: string; -} +// -- The shape this MCP exposes (stable across the v1 → v2 move) -- -export interface RecordingHighlight { +export interface Recording { id: string; - recording_id: string; - text: string; - transcript: string; - timestamp: number; - duration: number; - created_datetime: string; + title: string; url: string; - thumbnail_url: string; + start_datetime: string; + end_datetime: string; + public_thumbnail_url: string | null; } export interface RecordingDetails extends Recording { participants?: RecordingParticipant[]; + /** Recorder emails (v1 called them owners). */ owners?: string[]; tags?: string[]; highlights?: RecordingHighlight[]; transcript_json?: TranscriptEntry[]; - transcript_vtt?: string; + /** v2's AI summary, in markdown (v1 called it intelligence notes). */ intelligence_notes_md?: string; intelligence_notes_json?: IntelligenceNoteSection[]; - intelligence_notes_text?: string; } export interface TranscriptEntry { @@ -76,91 +106,83 @@ export interface IntelligenceNoteSection { body: string; } -// -- Views -- - -export interface GrainView { - id: string; - name: string; -} - -export interface ListViewsResponse { - views: GrainView[]; - cursor?: string; +/** v2 Recording → the MCP's stable shape. Extra v2 fields pass through. */ +export function toDetails(r: GrainRecordingV2): RecordingDetails { + const owners = (r.recorders ?? []) + .map((x) => x.email) + .filter((e): e is string => Boolean(e)); + return { + ...r, + id: r.id, + title: r.title, + url: r.url, + start_datetime: r.start_datetime, + end_datetime: r.end_datetime, + public_thumbnail_url: r.thumbnail_url ?? null, + owners, + tags: r.tags ?? [], + participants: r.participants, + highlights: r.highlights, + intelligence_notes_md: r.ai_summary?.text ?? undefined, + intelligence_notes_json: r.ai_template_sections?.map((s) => ({ + title: s.title, + body: typeof s.data === "string" ? s.data : JSON.stringify(s.data), + })), + }; } -// -- Hooks (REST Hooks) -- +// -- Hooks (v2) -- -export type HookAction = "added" | "updated" | "removed"; - -export interface CreateHookParams { - version: 2; - hook_url: string; - view_id: string; - actions?: HookAction[]; -} +export type HookType = + | "recording_added" + | "recording_updated" + | "recording_deleted" + | "highlight_added" + | "highlight_updated" + | "highlight_deleted" + | "story_added" + | "story_updated" + | "story_deleted" + | "upload_added" + | "upload_updated"; export interface Hook { id: string; + enabled: boolean; hook_url: string; - view_id: string; + hook_type: HookType; + include: Record; inserted_at: string; } -export interface CreateHookResponse extends Hook {} - export interface ListHooksResponse { hooks: Hook[]; } -// -- Webhook payloads (inbound from Grain) -- - -export type WebhookEventType = - | "recording_added" - | "recording_updated" - | "recording_removed" - | "highlight_added" - | "highlight_updated" - | "highlight_removed" - | "story_added" - | "story_updated" - | "story_removed"; +// -- Webhook payloads (inbound from Grain, v2 envelope) -- -export interface WebhookRecordingData { - id: string; - title?: string; - url?: string; - start_datetime?: string; - end_datetime?: string; - public_thumbnail_url?: string | null; -} +export type WebhookEventType = HookType; export interface WebhookPayload { type: WebhookEventType; user_id?: string; - data: WebhookRecordingData; + data: { id: string; [key: string]: unknown }; } export const WebhookPayloadSchema = z.object({ type: z.enum([ "recording_added", "recording_updated", - "recording_removed", + "recording_deleted", "highlight_added", "highlight_updated", - "highlight_removed", + "highlight_deleted", "story_added", "story_updated", - "story_removed", + "story_deleted", + "upload_added", + "upload_updated", ]), user_id: z.string().optional(), - data: z - .object({ - id: z.string(), - title: z.string().optional(), - url: z.string().optional(), - start_datetime: z.string().optional(), - end_datetime: z.string().optional(), - public_thumbnail_url: z.string().nullable().optional(), - }) - .passthrough(), + data: z.object({ id: z.string() }).passthrough(), }); diff --git a/grain/server/main.ts b/grain/server/main.ts index 6d18565c..04d93b3f 100644 --- a/grain/server/main.ts +++ b/grain/server/main.ts @@ -3,6 +3,7 @@ import { serve } from "@decocms/mcps-shared/serve"; import { tools } from "./tools/index.ts"; import { GrainClient } from "./lib/grain-client.ts"; +import { INDEX_INCLUDE, INDEXED_HOOK_TYPES } from "./constants.ts"; import { getGrainApiKey } from "./lib/env.ts"; import { WebhookPayloadSchema } from "./lib/types.ts"; import type { RecordingDetails } from "./lib/types.ts"; @@ -52,52 +53,44 @@ const runtime = withRuntime({ const { hooks } = await cachedGrainClient.listHooks(); + // Our hooks pointing somewhere else (old mesh URL, dev tunnel): drop them. const staleHooks = hooks.filter( (h) => h.hook_url.includes(WEBHOOK_PUBLIC_PATH) && h.hook_url !== webhookUrl, ); - if (staleHooks.length > 0) { - await Promise.all( - staleHooks.map((stale) => - cachedGrainClient!.deleteHook(stale.id).then(() => - console.log("[GRAIN_MCP] Deleted stale webhook", { - hookId: stale.id, - url: stale.hook_url, - }), - ), + await Promise.all( + staleHooks.map((stale) => + cachedGrainClient!.deleteHook(stale.id).then(() => + console.log("[GRAIN_MCP] Deleted stale webhook", { + hookId: stale.id, + url: stale.hook_url, + }), ), - ); - } - - if (hooks.some((h) => h.hook_url === webhookUrl)) { - console.log("[GRAIN_MCP] Webhook already registered"); - return; - } + ), + ); - const { views } = await cachedGrainClient.listViews(); - if (views.length === 0) { - console.error( - "[GRAIN_MCP] No views found. Create at least one recordings view in Grain.", + // v2 hooks are one per type; make sure each indexed type has one. + for (const hookType of INDEXED_HOOK_TYPES) { + if ( + hooks.some( + (h) => + h.hook_url === webhookUrl && + h.hook_type === hookType && + h.enabled, + ) + ) + continue; + const hook = await cachedGrainClient.createHook( + webhookUrl, + hookType, + INDEX_INCLUDE, ); - return; + console.log("[GRAIN_MCP] Hook created", { + hookId: hook.id, + hookType, + }); } - - const teamView = views.find((v) => - v.name.toLowerCase().includes("team"), - ); - const selectedView = teamView ?? views[0]; - const viewId = selectedView.id; - console.log("[GRAIN_MCP] Using view for hook", { - viewId, - viewName: selectedView.name, - }); - - const hook = await cachedGrainClient.createHook(webhookUrl, viewId, [ - "added", - "updated", - ]); - console.log("[GRAIN_MCP] Hook created", { hookId: hook.id }); } catch (error) { console.error("[GRAIN_MCP] Error setting up webhook", error); } @@ -107,29 +100,36 @@ const runtime = withRuntime({ tools, }); -async function enrichRecording( +/** + * The client for webhook deliveries: the one from the last configuration, or + * GRAIN_API_KEY after a restart (onChange only runs when the connection is + * saved, so an in-memory client alone loses every webhook after a redeploy). + */ +function webhookClient(): GrainClient | null { + if (cachedGrainClient) return cachedGrainClient; + const key = process.env.GRAIN_API_KEY; + return key ? (cachedGrainClient = new GrainClient({ apiKey: key })) : null; +} + +/** + * Re-reads the recording from Grain. Webhooks are not signed, so the payload + * only names *which* recording changed; what gets indexed comes from the API. + */ +async function fetchRecording( recordingId: string, ): Promise { - if (!cachedGrainClient) { - console.warn("[GRAIN_MCP] No cached client, skipping enrichment"); + const client = webhookClient(); + if (!client) { + console.warn("[GRAIN_MCP] No Grain client (set GRAIN_API_KEY), skipping"); return null; } - try { - return await cachedGrainClient.getRecording(recordingId, { - include_highlights: true, - include_participants: true, - include_owners: true, - intelligence_notes_format: "md", - }); + return await client.getRecording(recordingId, INDEX_INCLUDE); } catch (err) { - console.warn( - "[GRAIN_MCP] Failed to enrich recording, indexing basic data", - { - recordingId, - error: err instanceof Error ? err.message : String(err), - }, - ); + console.warn("[GRAIN_MCP] Failed to fetch recording, not indexing", { + recordingId, + error: err instanceof Error ? err.message : String(err), + }); return null; } } @@ -169,13 +169,13 @@ async function handleWebhookPost( payload.type === "recording_updated" ) { try { - const details = await enrichRecording(payload.data.id); + const details = await fetchRecording(payload.data.id); + if (!details) return new Response("ok", { status: 200 }); await indexRecording({ payload, details }); console.log("[GRAIN_MCP] Recording indexed", { id: payload.data.id, - enriched: details !== null, participants: details?.participants?.length ?? 0, hasTags: (details?.tags?.length ?? 0) > 0, hasNotes: !!details?.intelligence_notes_md, diff --git a/grain/server/tools/get-recording.ts b/grain/server/tools/get-recording.ts index 25ce0f77..5a5cadbf 100644 --- a/grain/server/tools/get-recording.ts +++ b/grain/server/tools/get-recording.ts @@ -74,13 +74,16 @@ export const createGetRecordingTool = (env: Env) => try { const client = new GrainClient({ apiKey: getGrainApiKey(env) }); - const recording = await client.getRecording(context.recordingId, { - include_highlights: context.include_highlights, - include_participants: true, - include_owners: true, - transcript_format: context.include_transcript ? "json" : undefined, - intelligence_notes_format: context.include_notes ? "md" : undefined, - }); + const [recording, transcript] = await Promise.all([ + client.getRecording(context.recordingId, { + participants: true, + highlights: Boolean(context.include_highlights), + ai_summary: Boolean(context.include_notes), + }), + context.include_transcript + ? client.getTranscript(context.recordingId, "json") + : undefined, + ]); return { id: recording.id, @@ -101,7 +104,7 @@ export const createGetRecordingTool = (env: Env) => duration: h.duration, url: h.url, })), - transcript_json: recording.transcript_json, + transcript_json: Array.isArray(transcript) ? transcript : undefined, intelligence_notes_md: recording.intelligence_notes_md, }; } catch (error) { diff --git a/grain/server/tools/get-summary.ts b/grain/server/tools/get-summary.ts index fc75306f..844a4859 100644 --- a/grain/server/tools/get-summary.ts +++ b/grain/server/tools/get-summary.ts @@ -33,16 +33,18 @@ export const createGetSummaryTool = (env: Env) => const client = new GrainClient({ apiKey: getGrainApiKey(env) }); const format = context.format ?? "md"; - const recording = await client.getRecording(context.recordingId, { - include_participants: false, - include_owners: false, - intelligence_notes_format: format, - }); + // v2: the summary is `ai_summary` (markdown); structured notes are + // the AI template sections. + const recording = await client.getRecording( + context.recordingId, + format === "json" + ? { ai_template_sections: { format: "json" } } + : { ai_summary: true }, + ); let notes: string | { title: string; body: string }[] | undefined; - if (format === "md") notes = recording.intelligence_notes_md; - else if (format === "json") notes = recording.intelligence_notes_json; - else notes = recording.intelligence_notes_text; + if (format === "json") notes = recording.intelligence_notes_json; + else notes = recording.intelligence_notes_md; return { id: recording.id, diff --git a/grain/server/tools/get-transcript.ts b/grain/server/tools/get-transcript.ts index 50b1b45f..bb8c041d 100644 --- a/grain/server/tools/get-transcript.ts +++ b/grain/server/tools/get-transcript.ts @@ -43,10 +43,7 @@ export const createGetTranscriptTool = (env: Env) => const format = context.format ?? "txt"; const [recording, transcript] = await Promise.all([ - client.getRecording(context.recordingId, { - include_participants: false, - include_owners: false, - }), + client.getRecording(context.recordingId, {}), client.getTranscript(context.recordingId, format), ]); diff --git a/grain/server/tools/list-recordings.ts b/grain/server/tools/list-recordings.ts index 6241e832..dc370a61 100644 --- a/grain/server/tools/list-recordings.ts +++ b/grain/server/tools/list-recordings.ts @@ -2,6 +2,7 @@ import { createPrivateTool } from "@decocms/runtime/tools"; import { getGrainApiKey } from "../lib/env.ts"; import { z } from "zod"; import { GrainClient, GrainAPIError } from "../lib/grain-client.ts"; +import { toDetails } from "../lib/types.ts"; import type { Env } from "../types/env.ts"; export const createListRecordingsTool = (env: Env) => @@ -75,7 +76,7 @@ export const createListRecordingsTool = (env: Env) => return { // eslint-disable-next-line @typescript-eslint/no-explicit-any - recordings: response.recordings as any, + recordings: response.recordings.map(toDetails) as any, cursor: response.cursor, }; } catch (error) { diff --git a/grain/server/triggers/publisher.ts b/grain/server/triggers/publisher.ts index c7142d72..98a26458 100644 --- a/grain/server/triggers/publisher.ts +++ b/grain/server/triggers/publisher.ts @@ -4,7 +4,7 @@ import type { RecordingDetails, WebhookPayload } from "../lib/types.ts"; export function publishRecordingTrigger( connectionId: string, webhookType: "recording_added" | "recording_updated", - details: RecordingDetails | null, + details: RecordingDetails, payload: WebhookPayload, ): void { const triggerType = @@ -14,19 +14,18 @@ export function publishRecordingTrigger( triggers.notify(connectionId, triggerType, { recording_id: payload.data.id, - title: details?.title ?? payload.data.title ?? null, - url: details?.url ?? payload.data.url ?? null, - start_datetime: - details?.start_datetime ?? payload.data.start_datetime ?? null, - end_datetime: details?.end_datetime ?? payload.data.end_datetime ?? null, - owners: details?.owners ?? [], - tags: details?.tags ?? [], - participants: (details?.participants ?? []).map((p) => ({ + title: details.title ?? null, + url: details.url ?? null, + start_datetime: details.start_datetime ?? null, + end_datetime: details.end_datetime ?? null, + owners: details.owners ?? [], + tags: details.tags ?? [], + participants: (details.participants ?? []).map((p) => ({ name: p.name, email: p.email, scope: p.scope, })), - has_summary: !!details?.intelligence_notes_md, + has_summary: !!details.intelligence_notes_md, indexed_at: new Date().toISOString(), }); } diff --git a/registry/server/tools/registry-binding.ts b/registry/server/tools/registry-binding.ts index c4c35c57..a65f0d9a 100644 --- a/registry/server/tools/registry-binding.ts +++ b/registry/server/tools/registry-binding.ts @@ -275,7 +275,9 @@ export const createListRegistryTool = (_env: Env) => try { // Get configuration from environment const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? + process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { throw new Error( @@ -355,7 +357,9 @@ export const createGetRegistryTool = (_env: Env) => // Get configuration from environment const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? + process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { throw new Error( @@ -407,7 +411,9 @@ export const createVersionsRegistryTool = (_env: Env) => try { // Get configuration from environment const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? + process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { throw new Error( @@ -478,7 +484,9 @@ export const createFiltersRegistryTool = (_env: Env) => try { // Get configuration from environment const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? + process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { throw new Error( diff --git a/slack-mcp/server/lib/supabase-client.ts b/slack-mcp/server/lib/supabase-client.ts index 0403f8d6..7f643d5a 100644 --- a/slack-mcp/server/lib/supabase-client.ts +++ b/slack-mcp/server/lib/supabase-client.ts @@ -44,7 +44,8 @@ export function getSupabaseClient(): SupabaseClient | null { } const supabaseUrl = process.env.SUPABASE_URL; - const supabaseKey = process.env.SUPABASE_ANON_KEY; + const supabaseKey = + process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY; if (!supabaseUrl || !supabaseKey) { return null; @@ -70,7 +71,10 @@ export function getSupabaseClient(): SupabaseClient | null { * Check if Supabase is configured and available */ export function isSupabaseConfigured(): boolean { - return !!process.env.SUPABASE_URL && !!process.env.SUPABASE_ANON_KEY; + return ( + !!process.env.SUPABASE_URL && + !!(process.env.SUPABASE_SERVICE_ROLE_KEY ?? process.env.SUPABASE_ANON_KEY) + ); } /**