diff --git a/apps/roam/src/utils/createReifiedBlock.ts b/apps/roam/src/utils/createReifiedBlock.ts index 642ddb12a..ad78975f7 100644 --- a/apps/roam/src/utils/createReifiedBlock.ts +++ b/apps/roam/src/utils/createReifiedBlock.ts @@ -102,6 +102,36 @@ export const countReifiedRelations = async (): Promise => { return (r[0] || [0])[0] as number; }; +export type ReifiedRelationData = { + sourceUid: string; + destinationUid: string; + hasSchema: string; + importedFromRid?: string; +}; + +export type ReifiedRelationDataWithRelId = ReifiedRelationData & { + relationId: string; +}; + +export const getReifiedRelations = async (): Promise< + ReifiedRelationDataWithRelId[] +> => { + const pageUid = getExistingRelationPageUid(); + if (pageUid === undefined) return []; + const r = await window.roamAlphaAPI.data.async.q( + `[:find ?ruid ?rdata :where + [?p :block/uid "${pageUid}"] + [?p :block/children ?c] + [?c :block/uid ?ruid] + [?c :block/props ?pr] + [(get ?pr :${DISCOURSE_GRAPH_PROP_NAME}) ?rdata] ]`, + ); + return r.map((x) => ({ + relationId: x[0] as string, + ...(x[1] as ReifiedRelationData), + })); +}; + export const createReifiedRelation = async ({ sourceUid, relationBlockUid, diff --git a/apps/roam/src/utils/publishNodesToGroups.ts b/apps/roam/src/utils/publishNodesToGroups.ts index b873fc2be..f3fe7c477 100644 --- a/apps/roam/src/utils/publishNodesToGroups.ts +++ b/apps/roam/src/utils/publishNodesToGroups.ts @@ -1,7 +1,27 @@ -import { CrossAppNode } from "@repo/database/crossAppContracts"; +import { + CrossAppNode, + CrossAppRelation, + CrossAppRelationTripleSchema, +} from "@repo/database/crossAppContracts"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { getAvailableGroupIds } from "@repo/database/lib/groups"; import { nodeUidsWithTypeToCrossApp } from "./roamToCrossAppConverters"; +import { + reifiedRelationToCrossApp, + relationTripleSchemaToCrossApp, + nodeSchemaToCrossApp, +} from "./roamToCrossAppConverters"; +import getDiscourseRelations from "./getDiscourseRelations"; +import { getReifiedRelations } from "./createReifiedBlock"; +import { + crossAppNodeSchemaToDbConcept, + crossAppRelationToDbConcept, + crossAppRelationTripleSchemaToDbConcept, +} from "@repo/database/lib/crossAppConverters"; +import { ensurePartialSpaceAccess } from "@repo/database/lib/groups"; +import { isIgnorableUpsertError } from "@repo/database/lib/contextFunctions"; +import getDiscourseNodes from "./getDiscourseNodes"; +import { difference } from "@repo/utils/setOperations"; export type NodeUidWithType = { uid: string; @@ -10,14 +30,175 @@ export type NodeUidWithType = { type PublishNodesResult = { publishedNodeUids: string[]; + publishedRelationUids: string[]; + publishedNodeSchemaUids: string[]; + publishedRelationTripleSchemaUids: string[]; skippedUnsyncedUids: string[]; okGroupIds: string[]; failedGroupIds: string[]; }; -// 23505 = unique_violation: the grant already exists, which counts as success. -const isIgnorableUpsertError = (error: { code?: string } | null): boolean => - !error || error.code === "23505"; +const getAllPublishedIdsByGroup = async ( + client: DGSupabaseClient, + spaceId: number, + groupIds: string[], +): Promise>> => { + const response = await client + .from("ResourceAccess") + .select("account_uid, source_local_id") + .eq("space_id", spaceId) + .in("account_uid", groupIds); + if (response.error) throw response.error; + const publishedIdsByGroupId = Object.fromEntries( + groupIds.map((gid) => [gid, new Set()]), + ); + response.data.forEach(({ account_uid, source_local_id }) => { + publishedIdsByGroupId[account_uid].add(source_local_id); + }); + + return publishedIdsByGroupId; +}; + +const getSpaceIdAndUrlsByGroupId = async ( + client: DGSupabaseClient, + groupIds: string[], +): Promise<{ + spaceUrlById: Record; + spaceIdsByGroupId: Record>; +}> => { + const response = await client + .from("SpaceAccess") + .select("account_uid, space_id") + .in("account_uid", groupIds); + if (response.error) throw response.error; + const spaceIds = response.data.map((r) => r.space_id); + const response2 = await client + .from("Space") + .select("id, url") + .in("id", spaceIds); + if (response2.error) throw response2.error; + const spaceUrlById = Object.fromEntries( + response2.data.map(({ id, url }) => [id, url]), + ); + const spaceIdsByGroupId = Object.fromEntries( + groupIds.map((gid) => [gid, new Set()]), + ); + response.data.forEach(({ account_uid, space_id }) => { + spaceIdsByGroupId[account_uid].add(space_id); + }); + return { + spaceUrlById, + spaceIdsByGroupId, + }; +}; + +// Use readImportedSourceIdentity from eng-1859 when it's merged. +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const importedFromSpaceId = (nodeId: string): number | undefined => undefined; + +export const gatherCorrespondingRelations = async ({ + client, + spaceId, + groupIds, + syncedUids, + forNodeIds, +}: { + client: DGSupabaseClient; + spaceId: number; + groupIds: string[]; + syncedUids: Set; + forNodeIds?: Set; +}): Promise<{ + relations: CrossAppRelation[]; + relationTripleSchemas: CrossAppRelationTripleSchema[]; + relevantRelationIdsPerGroupId: Record; +}> => { + const allRelationsSchemas = getDiscourseRelations(); + const allRelationSchemasById = Object.fromEntries( + allRelationsSchemas.map((s) => [s.id, s]), + ); + // Should we even handle non-reified relations? Assuming not. + // I need a way to know if a relation is imported, see importedFromSpaceId + const allRelations = await getReifiedRelations(); + const spaceIdOfNodes: Record = {}; + const isImportedFrom = (nodeLocalId: string): number => { + let cached = spaceIdOfNodes[nodeLocalId]; + if (cached === undefined) { + cached = spaceIdOfNodes[nodeLocalId] = + importedFromSpaceId(nodeLocalId) || spaceId; + } + return cached === spaceId ? 0 : cached; + }; + const relations = + forNodeIds !== undefined + ? allRelations.filter( + (r) => + r.importedFromRid === undefined && + (forNodeIds.has(r.sourceUid) || forNodeIds.has(r.destinationUid)), + ) + : allRelations.filter((r) => r.importedFromRid === undefined); + const { spaceIdsByGroupId, spaceUrlById } = await getSpaceIdAndUrlsByGroupId( + client, + groupIds, + ); + const isImportedFromSpaceUri = (uid: string) => + spaceUrlById[isImportedFrom(uid) || 0]; + const publishedIdsByGroup = await getAllPublishedIdsByGroup( + client, + spaceId, + groupIds, + ); + // calculate separately to avoid case of a relation between nodes published to or from different groups + const relevantRelationIdsPerGroupId = Object.fromEntries( + groupIds.map((groupId) => { + const groupSpaceIds = spaceIdsByGroupId[groupId]; + const publishedIds = publishedIdsByGroup[groupId]; + return [ + groupId, + relations + .filter( + (r) => + (publishedIds.has(r.sourceUid) || + groupSpaceIds.has(isImportedFrom(r.sourceUid) || 0)) && + (publishedIds.has(r.destinationUid) || + groupSpaceIds.has(isImportedFrom(r.destinationUid) || 0)), + ) + .map((r) => r.relationId), + ]; + }), + ); + const allRelevantRelationIds = new Set( + Object.values(relevantRelationIdsPerGroupId).flat(), + ); + let allRelevantRelations = relations.filter((r) => + allRelevantRelationIds.has(r.relationId), + ); + const relationSchemaIds = new Set( + allRelevantRelations + .map((r) => r.hasSchema) + // filter out deleted schemas + .filter((id) => id in allRelationSchemasById), + ); + allRelevantRelations = allRelevantRelations.filter((r) => + relationSchemaIds.has(r.hasSchema), + ); + const missingRelationSchemaTriples = allRelationsSchemas.filter( + (r) => relationSchemaIds.has(r.id) && !syncedUids.has(r.id), + ); + const missingRelations = allRelevantRelations.filter( + (r) => !syncedUids.has(r.relationId), + ); + + return { + relations: missingRelations + .map((r) => reifiedRelationToCrossApp(r, isImportedFromSpaceUri)) + .filter((r) => r !== null), + relationTripleSchemas: missingRelationSchemaTriples + .map((rs3) => relationTripleSchemaToCrossApp(rs3)) + .filter((rs3) => rs3 !== null), + relevantRelationIdsPerGroupId, + }; +}; const onlyStrings = (values: (string | null)[]): string[] => values.filter((value): value is string => typeof value === "string"); @@ -43,6 +224,9 @@ export const publishNodesToGroups = async ({ }): Promise => { const result: PublishNodesResult = { publishedNodeUids: [], + publishedRelationUids: [], + publishedNodeSchemaUids: [], + publishedRelationTripleSchemaUids: [], skippedUnsyncedUids: [], okGroupIds: [], failedGroupIds: [], @@ -59,73 +243,113 @@ export const publishNodesToGroups = async ({ ); if (targetGroupIds.length === 0) return result; - const uids = [...new Set(nodes.map((node) => node.localId))]; + const nodeUids = [...new Set(nodes.map((node) => node.localId))]; const syncedRes = await client .from("my_concepts") .select("source_local_id") - .eq("space_id", spaceId) - .eq("is_schema", false) - .in("source_local_id", uids); + .eq("space_id", spaceId); if (syncedRes.error) throw syncedRes.error; const syncedUids = new Set( onlyStrings((syncedRes.data ?? []).map((row) => row.source_local_id)), ); - result.skippedUnsyncedUids = uids.filter((uid) => !syncedUids.has(uid)); - const syncedNodeUids = uids.filter((uid) => syncedUids.has(uid)); + result.skippedUnsyncedUids = nodeUids.filter((uid) => !syncedUids.has(uid)); + const syncedNodeUids = nodeUids.filter((uid) => syncedUids.has(uid)); if (syncedNodeUids.length === 0) return result; - // Required dependency: the node-type schema concept, when it is synced too. - const types = [ - ...new Set( - nodes - .filter((node) => syncedUids.has(node.localId)) - .map((node) => node.nodeType), + const nodeSchemaUids = new Set(nodes.map((node) => node.nodeType)); + const missingNodeSchemaUids = difference( + nodeSchemaUids, + new Set(syncedNodeUids), + ); + const missingNodeSchemas = getDiscourseNodes() + .filter((s) => missingNodeSchemaUids.has(s.type)) + .map((s) => nodeSchemaToCrossApp(s)) + .filter((s) => s !== null); + + const resourceIds = [...nodeUids, ...nodeSchemaUids]; + + const { existing: existingSpaceAccess, missing: missingSpaceAccess } = + await ensurePartialSpaceAccess({ + client, + groupIds, + spaceId, + }); + if (missingSpaceAccess && Object.keys(missingSpaceAccess).length) { + result.failedGroupIds = groupIds.filter( + (id) => !(id in existingSpaceAccess), + ); + groupIds = Object.keys(existingSpaceAccess); + } + if (groupIds.length === 0) return result; + + const { + relations: missingRelations, + relationTripleSchemas: missingRelationTripleSchemas, + relevantRelationIdsPerGroupId, + } = await gatherCorrespondingRelations({ + client, + spaceId, + groupIds: targetGroupIds, + syncedUids, + forNodeIds: new Set(syncedNodeUids), + }); + const upsertConcepts = [ + ...missingNodeSchemas.map((s) => crossAppNodeSchemaToDbConcept(s)), + ...missingRelationTripleSchemas.map((rs3) => + crossAppRelationTripleSchemaToDbConcept(rs3), ), - ]; - const schemaRes = await client - .from("my_concepts") - .select("source_local_id") - .eq("space_id", spaceId) - .eq("is_schema", true) - .in("source_local_id", types); - if (schemaRes.error) throw schemaRes.error; - const syncedSchemaIds = onlyStrings( - (schemaRes.data ?? []).map((row) => row.source_local_id), - ); - - const resourceIds = [...syncedNodeUids, ...syncedSchemaIds]; - - for (const groupId of targetGroupIds) { - // Existing reader/editor access is broader than partial, so leave it intact. - const spaceAccessRes = await client - .from("SpaceAccess") - .upsert( - { account_uid: groupId, space_id: spaceId, permissions: "partial" }, - { ignoreDuplicates: true }, - ); - if (!isIgnorableUpsertError(spaceAccessRes.error)) { - result.failedGroupIds.push(groupId); - continue; - } + ...missingRelations.map((r) => crossAppRelationToDbConcept(r)), + ].filter((r) => r !== undefined); - const grantRes = await client.from("ResourceAccess").upsert( - resourceIds.map((sourceLocalId) => ({ + const resourceAccesses = []; + for (const groupId of groupIds) { + const groupRelationIds = new Set(relevantRelationIdsPerGroupId[groupId]); + const groupMissingRelations = missingRelations.filter((r) => + groupRelationIds.has(r.localId), + ); + const groupMissingRelationIds = groupMissingRelations.map((r) => r.localId); + const groupSchemaIds = new Set( + groupMissingRelations.map((r) => r.relationType), + ); + const groupMissingSchemaIds = missingRelationTripleSchemas + .filter((rs3) => groupSchemaIds.has(rs3.localId)) + .map((rs3) => rs3.localId); + const groupMissingIds = [ + ...groupMissingRelationIds, + ...groupMissingSchemaIds, + ]; + resourceAccesses.push( + ...groupMissingIds.map((sourceLocalId) => ({ + account_uid: groupId, + source_local_id: sourceLocalId, + space_id: spaceId, + })), + ); + resourceAccesses.push( + ...resourceIds.map((sourceLocalId) => ({ account_uid: groupId, source_local_id: sourceLocalId, space_id: spaceId, })), - { ignoreDuplicates: true }, ); - if (!isIgnorableUpsertError(grantRes.error)) { - result.failedGroupIds.push(groupId); - continue; - } - - result.okGroupIds.push(groupId); } + const response = await client.rpc("upsert_concepts", { + v_space_id: spaceId, + data: upsertConcepts, + }); + if (response.error) throw response.error; + result.publishedRelationTripleSchemaUids = missingRelationTripleSchemas.map( + (s) => s.localId, + ); + result.publishedRelationUids = missingRelations.map((s) => s.localId); + const grantRes = await client + .from("ResourceAccess") + .upsert(resourceAccesses, { ignoreDuplicates: true }); + if (!isIgnorableUpsertError(grantRes.error)) throw grantRes.error; + result.publishedNodeUids = result.okGroupIds.length > 0 ? syncedNodeUids : []; return result; }; diff --git a/apps/roam/src/utils/roamToCrossAppConverters.ts b/apps/roam/src/utils/roamToCrossAppConverters.ts index bc7b9b77d..7cc8b6ba7 100644 --- a/apps/roam/src/utils/roamToCrossAppConverters.ts +++ b/apps/roam/src/utils/roamToCrossAppConverters.ts @@ -1,13 +1,21 @@ -import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import type { + CrossAppNode, + CrossAppNodeSchema, + CrossAppRelation, + CrossAppRelationTripleSchema, +} from "@repo/database/crossAppContracts"; import type { RoamFullContentNode } from "./convertRoamNodeToFullContent"; import type { DiscourseNode } from "./getDiscourseNodes"; import type { TreeNode, ViewType } from "roamjs-components/types"; import type { NodeUidWithType } from "~/utils/publishNodesToGroups"; import type { Json } from "@repo/database/dbTypes"; +import type { ReifiedRelationDataWithRelId } from "./createReifiedBlock"; +import type { DiscourseRelation } from "./getDiscourseRelations"; import { toMarkdown } from "./pageToMarkdown"; import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; import getPageViewType from "roamjs-components/queries/getPageViewType"; import { contentTypes } from "@repo/content-model"; +import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; const FULL_MARKDOWN_OPTS = { refs: true, @@ -116,3 +124,82 @@ export const nodeUidsWithTypeToCrossApp = async ( }); return results; }; + +export const reifiedRelationToCrossApp = ( + r: ReifiedRelationDataWithRelId, + isImportedFromSpaceUri: (nodeUid: string) => string | undefined, +): CrossAppRelation | null => { + const sourceSpaceUri = isImportedFromSpaceUri(r.sourceUid); + const destinationSpaceUri = isImportedFromSpaceUri(r.destinationUid); + const sourceId = + sourceSpaceUri === undefined + ? r.sourceUid + : spaceUriAndLocalIdToRid(sourceSpaceUri, r.sourceUid); + const destinationId = + destinationSpaceUri === undefined + ? r.destinationUid + : spaceUriAndLocalIdToRid(destinationSpaceUri, r.destinationUid); + const relData = window.roamAlphaAPI.pull( + "[:create/time :edit/time {:create/user [:user/uid]}]", + `[:block/uid "${r.relationId}"]`, + ) as Record; + if (relData == undefined || !relData[":create/user"]) return null; + const userUid = (relData[":create/user"] as Record)[ + ":user/uid" + ]; + + return { + localId: r.relationId, + relationType: r.hasSchema, + source: sourceId, + destination: destinationId, + authorId: userUid, + createdAt: new Date(relData[":create/time"] as number), + modifiedAt: new Date(relData[":edit/time"] as number), + }; +}; + +export const relationTripleSchemaToCrossApp = ( + r: DiscourseRelation, +): CrossAppRelationTripleSchema | null => { + const relData = window.roamAlphaAPI.pull( + "[:create/time :edit/time {:create/user [:user/uid]}]", + `[:block/uid "${r.id}"]`, + ) as Record; + if (!relData) return null; + const userUid = (relData[":create/user"] as Record)[ + ":user/uid" + ]; + + return { + localId: r.id, + sourceType: r.source, + destinationType: r.destination, + label: r.label, + complement: r.complement, + authorId: userUid, + createdAt: new Date(relData[":create/time"] as number), + modifiedAt: new Date(relData[":edit/time"] as number), + }; +}; + +export const nodeSchemaToCrossApp = ( + s: DiscourseNode, +): CrossAppNodeSchema | null => { + const relData = window.roamAlphaAPI.pull( + "[:create/time :edit/time {:create/user [:user/uid]}]", + `[:block/uid "${s.type}"]`, + ) as unknown as { + ":create/time": number; + ":edit/time": number; + ":create/user": { ":user/uid": string }; + }; + if (!relData) return null; + const userUid = relData[":create/user"][":user/uid"]; + return { + localId: s.type, + label: s.text, + authorId: userUid, + createdAt: new Date(relData[":create/time"]), + }; +}; diff --git a/packages/database/src/inputTypes.ts b/packages/database/src/inputTypes.ts index 5ff4b76a8..64f69c635 100644 --- a/packages/database/src/inputTypes.ts +++ b/packages/database/src/inputTypes.ts @@ -6,7 +6,7 @@ export type LocalAccountDataInput = Partial< export type LocalDocumentDataInput = Partial< Omit< Database["public"]["CompositeTypes"]["document_local_input"], - "author_inline" + "author_inline" | "contents" > & { author_inline: LocalAccountDataInput } >; export type LocalContentDataInput = Partial< diff --git a/packages/database/src/lib/contextFunctions.ts b/packages/database/src/lib/contextFunctions.ts index b6213a0ef..1eda4f3ad 100644 --- a/packages/database/src/lib/contextFunctions.ts +++ b/packages/database/src/lib/contextFunctions.ts @@ -47,6 +47,11 @@ export const asPostgrestFailure = ( export class FatalError extends Error {} +// 23505 = unique_violation: the grant already exists, which counts as success. +export const isIgnorableUpsertError = ( + error: { code?: string } | null, +): boolean => !error || error.code === "23505"; + export const spaceValidator = (space: SpaceCreationInput): string | null => { if (!space || typeof space !== "object") return "Invalid request body: expected a JSON object."; diff --git a/packages/database/src/lib/groups.ts b/packages/database/src/lib/groups.ts index 6a9b13f89..10eef6e6e 100644 --- a/packages/database/src/lib/groups.ts +++ b/packages/database/src/lib/groups.ts @@ -1,4 +1,6 @@ import type { DGSupabaseClient } from "./client"; +import type { Tables, Enums } from "../dbTypes"; +import { isIgnorableUpsertError } from "./contextFunctions"; export type MyGroup = { id: string; @@ -47,3 +49,57 @@ export const getMyGroups = async ( name: row.my_groups.name ?? row.group_id, })); }; + +type SpaceAccessPermissions = Enums<"SpaceAccessPermissions">; + +export const ensurePartialSpaceAccess = async ({ + client, + groupIds, + spaceId, +}: { + client: DGSupabaseClient; + groupIds: string[]; + spaceId: number; +}): Promise<{ + existing: Record; + missing?: Record; +}> => { + const existingAccessResult = await client + .from("SpaceAccess") + .select() + .eq("space_id", spaceId) + .in("group_id", groupIds); + const existingAccessByGroupId = existingAccessResult.data + ? Object.fromEntries( + existingAccessResult.data.map((sa) => [sa.account_uid, sa.permissions]), + ) + : {}; + const missingAccess: Tables<"SpaceAccess">[] = []; + for (const groupId of groupIds) { + if (existingAccessByGroupId[groupId] === undefined) { + missingAccess.push({ + space_id: spaceId, + permissions: "partial", + account_uid: groupId, + }); + } + } + if (missingAccess.length > 0) { + const upsertAccessResult = await client + .from("SpaceAccess") + .upsert(missingAccess, { ignoreDuplicates: true }); + if (!isIgnorableUpsertError(upsertAccessResult.error)) { + // allow partial results + return { + existing: existingAccessByGroupId, + missing: Object.fromEntries( + missingAccess.map((a) => [a.space_id, a.permissions]), + ), + }; + } + } + missingAccess.forEach((a) => { + existingAccessByGroupId[a.account_uid] = "partial"; + }); + return { existing: existingAccessByGroupId }; +};