From 1c2dfd9b262b732da2ee5dc3790ec3910b68fd98 Mon Sep 17 00:00:00 2001 From: Bilal Mahmoud <7252775+indietyp@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:34:26 +0200 Subject: [PATCH 1/2] feat: add atlas response documents and authorization --- libs/@local/graph/atlas/src/api/mod.rs | 3 + libs/@local/graph/atlas/src/api/problem.rs | 836 ++++++++ libs/@local/graph/atlas/src/lib.rs | 5 + .../atlas/src/serve/authorization/actor.rs | 136 ++ .../src/serve/authorization/authority.rs | 248 +++ .../atlas/src/serve/authorization/error.rs | 37 + .../atlas/src/serve/authorization/mod.rs | 55 + .../atlas/src/serve/authorization/scope.rs | 222 ++ .../atlas/src/serve/authorization/tests.rs | 65 + .../atlas/src/serve/authorization/token.rs | 216 ++ .../@local/graph/atlas/src/serve/delta/mod.rs | 14 +- .../graph/atlas/src/serve/delta/tests.rs | 1896 +++++++++++++++++ .../src/serve/document/codec/cbor/mod.rs | 174 ++ .../src/serve/document/codec/cbor/tests.rs | 421 ++++ .../src/serve/document/codec/column/mod.rs | 63 + .../src/serve/document/codec/column/tests.rs | 317 +++ .../src/serve/document/codec/envelope.rs | 375 ++++ .../atlas/src/serve/document/codec/mod.rs | 72 + .../graph/atlas/src/serve/document/current.rs | 58 + .../src/serve/document/edges/codec/mod.rs | 114 + .../src/serve/document/edges/codec/tests.rs | 535 +++++ .../atlas/src/serve/document/edges/mod.rs | 283 +++ .../atlas/src/serve/document/edges/tests.rs | 850 ++++++++ .../graph/atlas/src/serve/document/limits.rs | 10 + .../src/serve/document/locate/codec/mod.rs | 173 ++ .../src/serve/document/locate/codec/tests.rs | 875 ++++++++ .../atlas/src/serve/document/locate/mod.rs | 255 +++ .../src/serve/document/locate/subgraph.rs | 726 +++++++ .../atlas/src/serve/document/locate/tests.rs | 1708 +++++++++++++++ .../src/serve/document/locate/trailer.rs | 168 ++ .../atlas/src/serve/document/manifest/mod.rs | 140 ++ .../src/serve/document/manifest/tests.rs | 608 ++++++ .../graph/atlas/src/serve/document/masks.rs | 45 + .../graph/atlas/src/serve/document/mod.rs | 77 + .../src/serve/document/tile/codec/mod.rs | 127 ++ .../src/serve/document/tile/codec/tests.rs | 695 ++++++ .../atlas/src/serve/document/tile/mod.rs | 264 +++ .../atlas/src/serve/document/tile/tests.rs | 1028 +++++++++ .../src/serve/document/translate/codec/mod.rs | 60 + .../serve/document/translate/codec/tests.rs | 102 + .../atlas/src/serve/document/translate/mod.rs | 169 ++ .../src/serve/document/translate/tests.rs | 256 +++ libs/@local/graph/atlas/src/serve/mod.rs | 2 + 43 files changed, 14478 insertions(+), 5 deletions(-) create mode 100644 libs/@local/graph/atlas/src/api/mod.rs create mode 100644 libs/@local/graph/atlas/src/api/problem.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/actor.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/authority.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/error.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/scope.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/authorization/token.rs create mode 100644 libs/@local/graph/atlas/src/serve/delta/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/codec/cbor/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/codec/cbor/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/codec/column/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/codec/column/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/codec/envelope.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/codec/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/current.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/edges/codec/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/edges/codec/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/edges/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/edges/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/limits.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/locate/codec/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/locate/codec/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/locate/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/locate/subgraph.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/locate/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/locate/trailer.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/manifest/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/manifest/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/masks.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/tile/codec/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/tile/codec/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/tile/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/tile/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/translate/codec/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/translate/codec/tests.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/translate/mod.rs create mode 100644 libs/@local/graph/atlas/src/serve/document/translate/tests.rs diff --git a/libs/@local/graph/atlas/src/api/mod.rs b/libs/@local/graph/atlas/src/api/mod.rs new file mode 100644 index 00000000000..9771b62320d --- /dev/null +++ b/libs/@local/graph/atlas/src/api/mod.rs @@ -0,0 +1,3 @@ +//! The problem-details vocabulary the read API answers with. The routes land with the API. + +pub(crate) mod problem; diff --git a/libs/@local/graph/atlas/src/api/problem.rs b/libs/@local/graph/atlas/src/api/problem.rs new file mode 100644 index 00000000000..9ce7f9fd803 --- /dev/null +++ b/libs/@local/graph/atlas/src/api/problem.rs @@ -0,0 +1,836 @@ +//! RFC 9457 problem documents, the error surface of every handler. +//! +//! The `type` member carries Surface v1's stable root-relative URIs, the body serializes as +//! `application/problem+json`, and the shared rejections - foreign generation, foreign variant - +//! live here beside the document they produce. Requests that fail before a handler runs - malformed +//! bodies, wrong content types, unparsable tile addresses - route through `super::extract`'s +//! wrappers and answer problem documents too. The router's own rejections (an unmatched route, a +//! wrong method) never reach this module: axum answers them with a bare status. A manifest body +//! the server could not buffer answers plain text, because that route extracts its body as raw +//! bytes and axum refuses an oversize or interrupted one before the handler runs. + +use alloc::borrow::Cow; +use core::{num::NonZero, task}; + +use aide::{OperationOutput, generate::GenContext, openapi}; +use axum::{ + Json, + http::{self, StatusCode, header}, + response::{IntoResponse, Response}, +}; +use error_stack::Report; +use futures::TryFutureExt as _; +use hash_graph_postgres_store::store::postgres::query::SelectCompilerError; +use hash_graph_store::filter::ParameterConversionError; +use hash_middleware::{ + authentication::{AuthenticationRejection, request::AuthenticationError}, + rate_limit::{RateLimitRejection, TooManyRequests}, +}; + +use crate::serve::{ + document::{ + EdgesDocumentError, LocateDocumentError, TileDocumentError, TranslateDocumentError, + VARIANTS, + }, + hydrate::visibility::VisibilityProofError, + runtime::registry::ObserveError, +}; + +/// The `type` member of one problem document: Surface v1's stable root-relative URIs. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +pub(super) enum ProblemType { + /// A producer bug surfacing as a 500: the assembly panicked or its worker vanished. + #[serde(rename = "/problems/atlas/internal")] + InternalError, + /// The route names a generation this process does not serve. + #[serde(rename = "/problems/atlas/unknown-generation")] + UnknownGeneration, + /// The route names a variant outside the manifest's list. + #[serde(rename = "/problems/atlas/unknown-variant")] + UnknownVariant, + /// A tile coordinate outside the zoom range or off its grid. + #[serde(rename = "/problems/atlas/invalid-coordinate")] + InvalidCoordinate, + /// A generation path segment that is not a sha256 generation id. + #[serde(rename = "/problems/atlas/invalid-generation")] + InvalidGeneration, + /// An edges body listing more tiles than the manifest's cap. + #[serde(rename = "/problems/atlas/too-many-tiles")] + TooManyTiles, + /// A tile body carrying more `coloredTypeIds` than the manifest's cap. + #[serde(rename = "/problems/atlas/too-many-types")] + TooManyTypes, + /// A translate body listing more entity ids than the manifest's cap. + #[serde(rename = "/problems/atlas/too-many-entity-ids")] + TooManyEntityIds, + /// A locate source id that does not name a visible node. + /// + /// Nonexistent, denied, and unparsable answer identically (missing = denied). + #[serde(rename = "/problems/atlas/unknown-entity")] + UnknownEntity, + /// A locate body that does not name exactly one source: `entityId` XOR `row`. + #[serde(rename = "/problems/atlas/invalid-source")] + InvalidSource, + /// A required JSON body without a `Content-Type` header. + #[serde(rename = "/problems/atlas/missing-body")] + MissingBody, + /// A request body that is not the operation's JSON. + /// + /// Wrong content type, syntax error, shape mismatch, and oversize all use this type. + #[serde(rename = "/problems/atlas/invalid-body")] + InvalidBody, + /// A caller the authentication middleware could not resolve. + /// + /// The status and detail are the middleware's own client-safe reading of the failure. + #[serde(rename = "/problems/atlas/unauthenticated")] + Unauthenticated, + /// A request whose authority token is absent, malformed, foreign, or stale. + #[serde(rename = "/problems/atlas/unauthorized")] + Unauthorized, + /// Resolving the caller's scope failed. The process cannot say what they may see. + #[serde(rename = "/problems/atlas/visibility-unavailable")] + VisibilityUnavailable, + /// The caller is over its request budget, and `Retry-After` states when it admits again. + #[serde(rename = "/problems/atlas/too-many-requests")] + TooManyRequests, +} + +/// Serializes the problem's `status` member as its integer form. +/// +/// # Errors +/// +/// Returns [`serde::Serializer::Error`] if the serializer cannot write the status code. +#[expect( + clippy::trivially_copy_pass_by_ref, + reason = "serde's serialize_with contract passes fields by reference" +)] +fn status_as_u16( + status: &StatusCode, + serializer: S, +) -> Result { + serializer.serialize_u16(status.as_u16()) +} + +/// One RFC 9457 problem document. +#[derive(Debug, serde::Serialize, schemars::JsonSchema)] +pub(crate) struct Problem<'content> { + r#type: ProblemType, + title: Cow<'content, str>, + #[serde(serialize_with = "status_as_u16")] + #[schemars(with = "u16")] + status: StatusCode, + detail: Cow<'content, str>, +} + +impl<'content> Problem<'content> { + /// Builds a problem document with `status`'s canonical reason phrase as its title. + pub(super) fn new( + status: StatusCode, + r#type: ProblemType, + detail: impl Into>, + ) -> Self { + Self { + r#type, + title: Cow::Borrowed(status.canonical_reason().unwrap_or("error")), + status, + detail: detail.into(), + } + } + + /// Logs a whole [`error_stack::Report`] and returns a 500 Internal Server Error problem. + pub(super) fn internal(error: &Report, detail: impl Into>) -> Self { + let detail = detail.into(); + tracing::error!(?error, "{detail}"); + + Self::internal_response(detail) + } + + /// Logs a message and returns a 500 Internal Server Error problem. + pub(super) fn internal_message( + message: impl core::fmt::Display, + detail: impl Into>, + ) -> Self { + let detail = detail.into(); + tracing::error!(error = %message, "{detail}"); + + Self::internal_response(detail) + } + + /// Builds an internal-error response, exposing `detail` only in debug builds. + fn internal_response(detail: Cow<'content, str>) -> Self { + Self::new( + StatusCode::INTERNAL_SERVER_ERROR, + ProblemType::InternalError, + if cfg!(debug_assertions) { + detail + } else { + Cow::Borrowed("internal server error") + }, + ) + } +} + +/// Builds the uniform refusal for a source that does not name a visible node. +/// +/// Nonexistent, inaccessible, unparsable, and out-of-range values are indistinguishable by design: +/// missing equals denied, and an id that cannot name an entity is an entity that does not exist. +pub(super) fn unknown_entity() -> Problem<'static> { + Problem::new( + StatusCode::NOT_FOUND, + ProblemType::UnknownEntity, + "the source does not name a visible node", + ) +} + +impl From for Problem<'static> { + fn from(error: ObserveError) -> Self { + match error { + ObserveError::Unavailable(generation) => Self::new( + StatusCode::NOT_FOUND, + ProblemType::UnknownGeneration, + format!( + "generation {generation} is not served. Re-read /v1/atlas/current and retry" + ), + ), + ObserveError::Empty | ObserveError::Closed => Self::new( + StatusCode::SERVICE_UNAVAILABLE, + ProblemType::VisibilityUnavailable, + "no generation is ready to serve requests", + ), + } + } +} + +impl From> for Problem<'static> { + /// Maps an [`EdgesDocumentError`] to a client-facing problem or a logged internal one. + /// + /// A request-shaped failure (too many tiles, an invalid zoom or coordinate) becomes its + /// matching client-facing problem. A display or hydration failure becomes a logged `internal` + /// problem instead, since those describe server-side data rather than the caller's request. + fn from(error: Report) -> Self { + match error.current_context() { + EdgesDocumentError::Tiles { .. } => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::TooManyTiles, + error.to_string(), + ), + EdgesDocumentError::Zoom { .. } | EdgesDocumentError::Coordinate { .. } => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::InvalidCoordinate, + error.to_string(), + ), + // a delivered edge requires a display payload, independently of the request's fields. + EdgesDocumentError::Display => Self::internal( + &error, + "the edges assembly could not read a delivered edge's captured data", + ), + // hydration failures describe store availability or data, independently of request + // validity. + EdgesDocumentError::Hydrate(_) => Self::internal(&error, "the detail hydration failed"), + } + } +} + +impl From> for Problem<'static> { + /// Maps a [`LocateDocumentError`] to a client-facing problem or a logged internal one. + /// + /// A request-shaped failure (too many types, an unknown source) becomes its matching + /// client-facing problem. A display or hydration failure becomes a logged `internal` problem + /// instead, since those describe server-side data rather than the caller's request. + fn from(error: Report) -> Self { + match error.current_context() { + LocateDocumentError::Types { .. } => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::TooManyTypes, + error.to_string(), + ), + LocateDocumentError::UnknownEntity => unknown_entity(), + // a delivered row requires captured data, independently of the request's fields. + LocateDocumentError::Node { .. } + | LocateDocumentError::NodeDisplay { .. } + | LocateDocumentError::LinkDisplay { .. } => Self::internal( + &error, + "the locate assembly could not read a delivered row's captured data", + ), + // hydration failures describe store availability or data, independently of request + // validity. + LocateDocumentError::Hydrate(_) => { + Self::internal(&error, "the detail hydration failed") + } + } + } +} + +impl From> for Problem<'static> { + /// Maps a [`TileDocumentError`] to a client-facing problem or a logged internal one. + /// + /// A request-shaped failure (too many colored types, an out-of-range zoom or coordinate) + /// becomes its matching client-facing problem. A missing position or display payload becomes a + /// logged `internal` problem instead, since those describe server-side data rather than the + /// caller's request. + fn from(error: Report) -> Self { + match error.current_context() { + TileDocumentError::Types { .. } => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::TooManyTypes, + error.to_string(), + ), + TileDocumentError::Zoom { .. } | TileDocumentError::Coordinate { .. } => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::InvalidCoordinate, + error.to_string(), + ), + // a delivered row requires a position or display payload, independently of the + // request's fields. + TileDocumentError::Position { .. } | TileDocumentError::Display { .. } => { + Self::internal( + &error, + "the tile assembly could not read a delivered row's captured data", + ) + } + } + } +} + +impl From> for Problem<'static> { + fn from(error: Report) -> Self { + match error.current_context() { + TranslateDocumentError::Ids { .. } => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::TooManyEntityIds, + error.to_string(), + ), + } + } +} + +impl From> for Problem<'static> { + /// Exposes caller-authored filter errors, but keeps policy and store errors private. + /// + /// Invalid caller filters answer `invalid-body` with the compiler or conversion error as + /// detail. Policy-filter compilation failures answer a sanitized `internal` problem. Store + /// failures answer `visibility-unavailable`. Both private failure paths log the whole report. + fn from(error: Report) -> Self { + match error.current_context() { + VisibilityProofError::Filter => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::InvalidBody, + error.downcast_ref::().map_or_else( + || "the filter document does not compile".to_owned(), + |source| format!("the filter document does not compile: {source}"), + ), + ), + VisibilityProofError::Convert => Self::new( + StatusCode::BAD_REQUEST, + ProblemType::InvalidBody, + error + .downcast_ref::() + .map_or_else( + || "a filter parameter does not match its path's type".to_owned(), + |source| { + format!("a filter parameter does not match its path's type: {source}") + }, + ), + ), + VisibilityProofError::PolicyFilter => { + Self::internal(&error, "compiling the policy filter failed") + } + VisibilityProofError::Connect + | VisibilityProofError::Policies + | VisibilityProofError::Document + | VisibilityProofError::Query + | VisibilityProofError::ComputeView => { + tracing::error!(?error, "resolving the caller's visibility failed"); + + Problem::new( + StatusCode::SERVICE_UNAVAILABLE, + ProblemType::VisibilityUnavailable, + "the caller's scope could not be resolved", + ) + } + } + } +} + +/// Carries an authentication failure as this crate's problem document. +/// +/// The status and detail are [`AuthenticationError`]'s own client-safe readings. This crate +/// never restates the middleware's status map. +impl From for Problem<'static> { + fn from(error: AuthenticationError) -> Self { + Self::new( + error.status_code(), + ProblemType::Unauthenticated, + error.kind().client_message(), + ) + } +} + +impl From<&AuthenticationError> for Problem<'static> { + fn from(error: &AuthenticationError) -> Self { + Self::new( + error.status_code(), + ProblemType::Unauthenticated, + error.kind().client_message(), + ) + } +} + +impl IntoResponse for Problem<'_> { + fn into_response(self) -> Response { + ( + self.status, + [(header::CONTENT_TYPE, "application/problem+json")], + Json(self), + ) + .into_response() + } +} + +impl OperationOutput for Problem<'_> { + type Inner = Self; + + fn operation_response( + ctx: &mut GenContext, + _operation: &mut openapi::Operation, + ) -> Option { + let json_schema = ctx.schema.subschema_for::>(); + let mut response = openapi::Response { + description: "an RFC 9457 problem document".into(), + ..Default::default() + }; + response.content.insert( + "application/problem+json".into(), + openapi::MediaType { + schema: Some(openapi::SchemaObject { + json_schema, + example: None, + external_docs: None, + }), + ..Default::default() + }, + ); + + Some(response) + } + + /// Answers with a single default response rather than one per status code. + /// + /// A [`Problem`] document carries its own status. No route fixes one in advance. + fn inferred_responses( + ctx: &mut GenContext, + operation: &mut openapi::Operation, + ) -> Vec<(Option, openapi::Response)> { + let response = Self::operation_response(ctx, operation) + .unwrap_or_else(|| unreachable!("`operation_response` answers every operation")); + + vec![(None, response)] + } +} + +/// A [`Problem`] paired with an optional `Retry-After` delay. +/// +/// It converts from the domain rejection types axum middleware and extractors refuse a request +/// with. +pub(crate) struct ProblemResponse<'content> { + problem: Problem<'content>, + retry_after: Option>, +} + +impl<'content, T> From for ProblemResponse<'content> +where + T: Into>, +{ + fn from(problem: T) -> Self { + Self { + problem: problem.into(), + retry_after: None, + } + } +} + +impl From for ProblemResponse<'static> { + fn from(TooManyRequests { retry_after }: TooManyRequests) -> Self { + Self { + problem: Problem::new( + StatusCode::TOO_MANY_REQUESTS, + ProblemType::TooManyRequests, + "rate limit exceeded", + ), + retry_after: Some(retry_after), + } + } +} + +impl From for ProblemResponse<'static> { + fn from(error: RateLimitRejection) -> Self { + match error { + RateLimitRejection::TooManyRequests(too_many_requests) => too_many_requests.into(), + RateLimitRejection::InternalError => Problem::new( + StatusCode::INTERNAL_SERVER_ERROR, + ProblemType::InternalError, + "internal server error", + ) + .into(), + } + } +} + +impl From for ProblemResponse<'static> { + /// Converts an authentication rejection into its problem response. + /// + /// A resolved failure uses [`Problem`]'s own [`AuthenticationError`] conversion. A + /// misconfigured extractor, used on a route without the authentication middleware, answers as + /// a logged `internal` problem. + fn from(error: AuthenticationRejection) -> Self { + match error { + AuthenticationRejection::Authentication { + ref report, + metrics: _, + recorded: _, + } => Problem::from(report.current_context()).into(), + AuthenticationRejection::Misconfigured => Problem::internal_message( + "`Actor` extracted on a route without the authentication middleware", + "the caller's authentication was never resolved", + ) + .into(), + } + } +} + +impl IntoResponse for ProblemResponse<'_> { + fn into_response(self) -> Response { + let mut response = self.problem.into_response(); + if let Some(retry_after) = self.retry_after { + response + .headers_mut() + .insert(header::RETRY_AFTER, retry_after.get().into()); + } + response + } +} + +/// A [`tower::Layer`] mapping response errors into [`ProblemResponse`]. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IntoProblemLayer; + +impl tower::Layer for IntoProblemLayer { + type Service = IntoProblemService; + + fn layer(&self, inner: S) -> Self::Service { + IntoProblemService { inner } + } +} + +/// A service adapter for fallible responses with problem-document errors. +/// +/// An `Err` response from a successful service call becomes a [`ProblemResponse`]. Errors of the +/// service itself pass through unchanged, as do successful response values. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IntoProblemService { + inner: S, +} + +impl tower::Service> for IntoProblemService +where + S: tower::Service, Response = Result>, + U: Into>, +{ + type Error = S::Error; + type Response = Result>; + + type Future = impl Future>; + + fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> task::Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: http::Request) -> Self::Future { + self.inner + .call(req) + .map_ok(|result| result.map_err(Into::into)) + } +} + +/// Refuses a request that presents no acceptable authority token. +/// +/// One uniform answer covers every cause (an absent header, a malformed encoding, a failed tag, +/// a stale issue time, or an actor mismatch). A caller learns only that the server refused its +/// presentation and nothing about why. +pub(super) fn unauthorized() -> Problem<'static> { + Problem::new( + StatusCode::UNAUTHORIZED, + ProblemType::Unauthorized, + "the request presents no acceptable authority token. Re-fetch the manifest presenting the \ + held token to renew, or without one to bootstrap afresh", + ) +} + +/// Rejects a route naming a variant this generation does not serve. +/// +/// # Errors +/// +/// Returns a 404 [`Problem`] with [`ProblemType::UnknownVariant`] when `variant` is absent from +/// [`VARIANTS`]. +pub(super) fn reject_variant(variant: &str) -> Result<(), Problem<'static>> { + if VARIANTS.contains(&variant) { + return Ok(()); + } + + Err(Problem::new( + StatusCode::NOT_FOUND, + ProblemType::UnknownVariant, + format!("variant {variant} is not served. The manifest lists {VARIANTS:?}"), + )) +} + +/// Shared assertions over a conversion into an internal problem. +/// +/// Each checks the single log event such a conversion emits and the sanitized response it answers +/// with. The locate and edges document tests assert through them too. +#[cfg(test)] +pub(crate) mod tests { + use alloc::collections::BTreeMap; + use core::{fmt, panic::AssertUnwindSafe}; + use std::{io, sync::mpsc}; + + use axum::{ + body::to_bytes, + http::{StatusCode, header}, + response::IntoResponse as _, + }; + use error_stack::Report; + use hash_graph_postgres_store::store::postgres::query::SelectCompilerError; + use tracing::{ + Dispatch, Event, Level, Subscriber, + field::{Field, Visit}, + span::Id, + }; + use tracing_subscriber::{ + Layer, Registry, + layer::{Context, SubscriberExt as _}, + registry::LookupSpan, + }; + + use super::{Problem, ProblemType}; + use crate::{offload, serve::hydrate::visibility::VisibilityProofError}; + + /// One captured tracing event's fields, by name. + /// + /// The map holds each value's [`Debug`](core::fmt::Debug) rendering. + #[derive(Default)] + struct Fields(BTreeMap); + + impl Visit for Fields { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + self.0.insert(field.name().to_owned(), format!("{value:?}")); + } + } + + /// A subscriber layer that forwards every event to [`assert_internal_diagnostic`]. + /// + /// It sends the event's level, fields and enclosing span chain over a channel, for assertion + /// once the traced work has run. + struct Diagnostics(mpsc::Sender<(Level, Fields, Vec)>); + + impl LookupSpan<'lookup>> Layer for Diagnostics { + /// Forwards one event's captured fields to the waiting assertion. + /// + /// # Panics + /// + /// Panics where the assertion has already dropped its end of the channel. + fn on_event(&self, event: &Event<'_>, context: Context<'_, S>) { + let mut fields = Fields::default(); + event.record(&mut fields); + let scope = context.event_scope(event).map_or_else(Vec::new, |scope| { + scope.from_root().map(|span| span.id()).collect() + }); + self.0 + .send((*event.metadata().level(), fields, scope)) + .expect("should retain the diagnostic receiver"); + } + } + + /// Checks one offloaded error event and its sanitized HTTP response. + /// + /// `expected_fragments` are substrings that must each appear somewhere in the log's `error` + /// field, rendered through whichever of [`Problem::internal`] (full [`core::fmt::Debug`], + /// including a wrapped [`error_stack::Report`]'s retained source text and attachments, not + /// only its top context's message) or [`Problem::internal_message`] ([`core::fmt::Display`]) + /// produced the [`Problem`] under test. `detail` supplies the log's `message` field and the + /// debug-build response detail. + /// + /// # Panics + /// + /// Panics if worker setup, `produce` or response collection fails. Also panics if the log or + /// response differs from the expected values. + #[track_caller] + pub(crate) fn assert_internal_diagnostic( + produce: impl FnOnce() -> Problem<'static> + Send + 'static, + expected_fragments: &[&str], + detail: &'static str, + ) { + let (events, received) = mpsc::channel(); + let dispatch = Dispatch::new(Registry::default().with(Diagnostics(events))); + let worker_dispatch = dispatch.clone(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .spawn_handler(move |thread| { + let dispatch = worker_dispatch.clone(); + std::thread::spawn(move || { + tracing::dispatcher::with_default(&dispatch, || thread.run()); + }); + Ok(()) + }) + .build() + .expect("should build the diagnostic worker"); + let (problem, request_id) = tracing::dispatcher::with_default(&dispatch, || { + let request = tracing::info_span!("request"); + let id = request.id().expect("should enable the request span"); + let handle = + pool.install(|| request.in_scope(|| offload::run(AssertUnwindSafe(produce)))); + ( + futures::executor::block_on(handle).expect("should finish the problem conversion"), + id, + ) + }); + let events: Vec<_> = received.try_iter().collect(); + assert_eq!(events.len(), 1, "should report the failure exactly once"); + let (level, fields, scope) = &events[0]; + assert_eq!(*level, Level::ERROR); + assert_eq!(scope, &[request_id]); + assert_eq!( + fields.0.len(), + 2, + "should log exactly the error and the fixed detail: {:?}", + fields.0 + ); + assert_eq!( + fields.0.get("message"), + Some(&detail.to_owned()), + "should log the exact fixed detail as the message" + ); + let logged_error = fields.0.get("error").expect("should log an `error` field"); + for fragment in expected_fragments { + assert!( + logged_error.contains(fragment), + "the logged error {logged_error:?} should contain {fragment:?}" + ); + } + + let response = problem.into_response(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/problem+json" + ); + let bytes = futures::executor::block_on(to_bytes(response.into_body(), usize::MAX)) + .expect("should read the problem body"); + let body: serde_json::Value = + serde_json::from_slice(&bytes).expect("should parse the problem body"); + assert_eq!( + body, + serde_json::json!({ + "type": "/problems/atlas/internal", + "title": "Internal Server Error", + "status": 500, + "detail": if cfg!(debug_assertions) { detail } else { "internal server error" }, + }) + ); + } + + #[test] + fn type_root_relative_uri() { + let problem = Problem::new( + StatusCode::NOT_FOUND, + ProblemType::UnknownGeneration, + "re-bootstrap via /v1/atlas/current", + ); + let document = serde_json::to_value(&problem).expect("problem documents serialize"); + + assert_eq!(document["type"], "/problems/atlas/unknown-generation"); + assert_eq!(document["status"], 404); + } + + /// Logs an ordinary error through [`Display`](core::fmt::Display) and redacts it. + /// + /// The response omits the error text even in debug builds. + #[test] + fn internal_source_redacted() { + assert_internal_diagnostic( + || { + Problem::internal_message( + "private-store-message: private-property-value", + "the detail hydration failed", + ) + }, + &["private-store-message", "private-property-value"], + "the detail hydration failed", + ); + } + + /// Keeps the source error and attachment in one diagnostic, and neither in the response. + #[test] + fn internal_report_attachments() { + let report = + Report::new(io::Error::other("private-store-message")).attach("private-property-value"); + assert_internal_diagnostic( + move || Problem::internal(&report, "the response failed to encode"), + &["private-store-message", "private-property-value"], + "the response failed to encode", + ); + } + + /// A caller-authored filter that does not compile answers `invalid-body` at `400`. + /// + /// The detail is the store compiler's own message, which for an embedding path names the + /// binary-quantized representation it refuses to search. + #[test] + fn filter_invalid_body() { + let error = Report::new(SelectCompilerError::UnsupportedEmbeddingPath) + .change_context(VisibilityProofError::Filter); + let document = + serde_json::to_value(Problem::from(error)).expect("should serialize the problem"); + assert_eq!(document["status"], 400); + assert_eq!(document["type"], "/problems/atlas/invalid-body"); + assert!( + document["detail"] + .as_str() + .expect("should contain detail") + .contains("binary-quantized") + ); + } + + /// A policy filter that does not compile answers the sanitized `internal` problem. + /// + /// The log keeps the whole report, compiler message and attachment alike, while the response + /// carries none of it. The caller's own filter is not at fault, and nothing about the + /// deployment's policies reaches them. + #[test] + fn policy_filter_redacted() { + let error = Report::new(SelectCompilerError::UnsupportedEmbeddingPath) + .change_context(VisibilityProofError::PolicyFilter) + .attach("private-policy-detail"); + assert_internal_diagnostic( + move || error.into(), + &["binary-quantized", "private-policy-detail"], + "compiling the policy filter failed", + ); + } + + #[test] + fn visibility_store_unavailable() { + for context in [ + VisibilityProofError::Connect, + VisibilityProofError::Policies, + VisibilityProofError::Document, + VisibilityProofError::Query, + VisibilityProofError::ComputeView, + ] { + let document = serde_json::to_value(Problem::from(Report::new(context))) + .expect("should serialize the problem"); + assert_eq!(document["status"], 503); + assert_eq!(document["type"], "/problems/atlas/visibility-unavailable"); + } + } +} diff --git a/libs/@local/graph/atlas/src/lib.rs b/libs/@local/graph/atlas/src/lib.rs index 8fab98f0e2f..10145e290dd 100644 --- a/libs/@local/graph/atlas/src/lib.rs +++ b/libs/@local/graph/atlas/src/lib.rs @@ -168,6 +168,11 @@ extern crate alloc; mod allocator; +#[expect( + dead_code, + reason = "the read API that consumes the serving layer lands above this PR in the stack" +)] +pub(crate) mod api; #[cfg(feature = "bench")] pub mod bench; pub(crate) mod bitset; diff --git a/libs/@local/graph/atlas/src/serve/authorization/actor.rs b/libs/@local/graph/atlas/src/serve/authorization/actor.rs new file mode 100644 index 00000000000..34648507b78 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/actor.rs @@ -0,0 +1,136 @@ +//! The actor identity in the authority token's byte layout. +//! +//! [`ActorId`] has no fixed enum representation. [`ArchivedActorId`] supplies a fixed token +//! representation: a validated kind discriminant beside the actor's 16 UUID bytes. +#![expect(clippy::empty_enums, reason = "zerocopy uses them in the derive")] + +use core::ops::Deref; + +use type_system::principal::actor::{ActorEntityUuid, ActorId, AiId, MachineId, UserId}; +use uuid::Uuid; + +/// The byte-level form of an [`ActorEntityUuid`]. +#[derive( + Debug, + Copy, + Clone, + zerocopy::ByteEq, + zerocopy::ByteHash, + zerocopy::IntoBytes, + zerocopy::FromBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, +)] +#[repr(transparent)] +pub(crate) struct ArchivedActorEntityUuid([u8; 16]); + +impl From for ArchivedActorEntityUuid { + #[inline] + fn from(actor: ActorEntityUuid) -> Self { + Self(Uuid::from(actor).into_bytes()) + } +} + +impl Deref for ArchivedActorEntityUuid { + type Target = ActorEntityUuid; + + #[inline] + fn deref(&self) -> &Self::Target { + const { + assert!(size_of::() == size_of::()); + assert!(align_of::() == align_of::()); + } + + let ptr = &raw const *self; + + // SAFETY: Transparent structs preserve their field's layout. `Self` and the target chain + // `ActorEntityUuid(EntityUuid)`, `EntityUuid(Uuid)`, `Uuid([u8; 16])` are transparent over + // the same initialized bytes, and these types admit every 16-byte pattern. The pointer + // comes from `&self`, remains aligned for the target and retains that shared borrow's + // lifetime without mutation. Therefore it is sound to borrow these bytes as + // `ActorEntityUuid`. + unsafe { &*ptr.cast::() } + } +} + +/// The principal kind of an [`ArchivedActorId`]. +/// +/// Decoding refuses a discriminant outside this enum. Every UUID byte pattern is structurally +/// valid. Authentication establishes integrity separately: changing a kind to another valid +/// discriminant still produces a structurally valid identity. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::TryFromBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, +)] +#[repr(u8)] +pub(crate) enum ArchivedActorType { + /// A human principal, decoding to [`ActorId::User`]. + User, + /// A machine principal, decoding to [`ActorId::Machine`]. + Machine, + /// An AI principal, decoding to [`ActorId::Ai`]. + Ai, +} + +/// The byte-level form of an [`ActorId`]. +/// +/// The [`ActorId`] kinds share one UUID domain. The archived form preserves the kind and UUID and +/// reconstructs the variant on conversion. Equality compares both: the same UUID under a different +/// kind is a different principal. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::TryFromBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, +)] +#[repr(C)] +pub(crate) struct ArchivedActorId { + /// Which principal kind [`id`](Self::id) names. + pub r#type: ArchivedActorType, + /// The principal's UUID, in the shared actor identity domain. + pub id: ArchivedActorEntityUuid, +} + +impl From for ArchivedActorId { + fn from(value: ActorId) -> Self { + match value { + ActorId::User(uuid) => Self { + r#type: ArchivedActorType::User, + id: ArchivedActorEntityUuid::from(ActorEntityUuid::new(uuid)), + }, + ActorId::Machine(uuid) => Self { + r#type: ArchivedActorType::Machine, + id: ArchivedActorEntityUuid::from(ActorEntityUuid::new(uuid)), + }, + ActorId::Ai(uuid) => Self { + r#type: ArchivedActorType::Ai, + id: ArchivedActorEntityUuid::from(ActorEntityUuid::new(uuid)), + }, + } + } +} + +impl From for ActorId { + fn from(value: ArchivedActorId) -> Self { + match value.r#type { + ArchivedActorType::User => Self::User(UserId::new(*value.id)), + ArchivedActorType::Machine => Self::Machine(MachineId::new(*value.id)), + ArchivedActorType::Ai => Self::Ai(AiId::new(*value.id)), + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/authorization/authority.rs b/libs/@local/graph/atlas/src/serve/authorization/authority.rs new file mode 100644 index 00000000000..11fd7f15c70 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/authority.rs @@ -0,0 +1,248 @@ +//! Sealing and verification of the [delivery context](super) retained by a token. +//! +//! [`Authority`] derives a sealing key at construction and issues an [`EncryptedToken`] over a +//! [`Scope`]. Verification authenticates the envelope before comparing its actor, generation and +//! delta lifetime with the request. [`ScopeLease::current`] checks expiry. + +use core::time::Duration; +use std::{sync::nonpoison::Mutex, time::SystemTime}; + +use chacha20poly1305::{ + AeadCore, AeadInPlace as _, KeyInit as _, KeySizeUser, Tag, XChaCha20Poly1305, XNonce, +}; +use error_stack::Report; +use hkdf::Hkdf; +use rand::TryCryptoRng; +use sha2::{Sha256, digest::typenum::Unsigned}; +use type_system::principal::actor::ActorId; +use zerocopy::{IntoBytes as _, TryFromBytes as _}; + +use super::{ + error::AuthorityError, + scope::{Scope, ScopeLease}, + token::{EncryptedToken, MessageVersion, Token, TokenHeader, UnixEpochSeconds}, +}; +use crate::{ + integrity::SecretHexBytes, + morton::Zoom, + serve::{delta::epoch::Epoch, visibility::cache::FilterDigest}, +}; + +/// The authenticated cipher protecting the scope and clear header. +type Encryption = XChaCha20Poly1305; + +/// The HKDF info string for authorization key derivation. +/// +/// This label must change when the envelope's interpretation changes. Distinct labels separate +/// layout versions in key derivation. +const LABEL: &[u8] = b"atlas.authorization.v2"; +/// The nonce width the envelope's header reserves. +pub(crate) const NONCE_BYTES: usize = <::NonceSize as Unsigned>::USIZE; +/// The sealing key's width, the number of bytes HKDF expands to. +pub(crate) const KEY_BYTES: usize = <::KeySize as Unsigned>::USIZE; +/// The authentication tag's width, the envelope's trailer. +pub(crate) const TAG_BYTES: usize = <::TagSize as Unsigned>::USIZE; + +/// The actor, filter identity and delivery offset to seal into a token. +/// +/// The generation and delta reference come from the supplied [`Epoch`]. These remaining fields must +/// describe the resolved delivery decision: issuance authenticates the supplied values without +/// recomputing permissions. +#[expect( + clippy::min_ident_chars, + reason = "`k` is the delivery-cut offset's name throughout the density contract" +)] +pub(crate) struct Issue { + /// The principal bound to the token. + pub actor: ActorId, + /// The request filter's identity, absent for an unfiltered scope. + pub filter: Option, + /// The admitted zoom offset for the delivery cut. + pub k: Zoom, +} + +/// An issuer and verifier sharing one sealing key and token-expiration window. +/// +/// # Key and nonce requirements +/// +/// XChaCha20-Poly1305 requires a secret key and non-repeating nonces under that key. [`Self::new`] +/// derives the key with HKDF-SHA256 from a deployment secret and a random 64-bit salt. Each issue +/// draws a 192-bit nonce from `R`, with access serialized within this authority. Confidentiality +/// and authenticity depend on the cipher's security assumptions, secret key material and a +/// correctly seeded, protected random source. +/// +/// Random draws provide probabilistic separation, not uniqueness. The authority checks neither salt +/// nor nonce collisions. The [`TryCryptoRng`] marker alone does not establish correct seeding or +/// protect against duplicated generator state. Keep the secret unpredictable and avoid cloning or +/// restoring a generator's state for independent issuers. +/// +/// The derived key is not persisted. A new authority draws another salt, including after restart. +/// Equal secret, salt and label derive equal keys, and a repeated salt can therefore reproduce an +/// earlier key. Authorities constructed independently from the same deployment secret are not +/// configured to share tokens. +pub(crate) struct Authority { + key: SecretHexBytes, + expiration: Duration, + rng: Mutex, +} + +impl Authority { + /// Derives a sealing key and sets the issue-time window for its tokens. + /// + /// Each construction salts `secret` with a newly drawn `u64`. The timestamp recorded in each + /// token starts its `expiration` window. A zero duration admits no lease as current. The [key + /// and nonce requirements](Self#key-and-nonce-requirements) apply to `secret` and `rng`. + /// + /// # Errors + /// + /// Returns `rng`'s error if drawing the salt fails. + #[expect( + clippy::big_endian_bytes, + reason = "HKDF salts are byte strings, and a fixed order keeps the derivation \ + host-independent" + )] + pub(crate) fn new(secret: &[u8], expiration: Duration, mut rng: R) -> Result + where + R: TryCryptoRng, + { + let salt = rng.try_next_u64()?; + let mut key = SecretHexBytes::zeroed(); + + Hkdf::::new(Some(&salt.to_be_bytes()), secret) + .expand(LABEL, key.as_mut()) + .expect("the cipher's key size stays within HKDF-SHA256's expansion bound"); + + Ok(Self { + key, + expiration, + rng: Mutex::new(rng), + }) + } + + /// Seals the supplied delivery decision over `epoch`. + /// + /// The token records `epoch`'s generation and delta reference. The [`Issue`] fields must + /// describe an authorized decision for this epoch. This operation makes no permission query. + /// + /// # Warning + /// + /// [`UnixEpochSeconds`] discards sub-second precision from `now` and maps pre-epoch times to + /// the Unix epoch. That encoded time starts the expiration window. For post-epoch issuance, + /// truncation can shorten the usable window by less than one second. + /// + /// # Errors + /// + /// Returns `rng`'s error if drawing the nonce fails. + #[expect( + clippy::min_ident_chars, + reason = "`k` is the delivery-cut offset's name throughout the density contract" + )] + pub(crate) fn issue( + &self, + epoch: &Epoch, + now: SystemTime, + Issue { actor, filter, k }: Issue, + ) -> Result + where + R: TryCryptoRng, + { + let scope = Scope { + generation: epoch.generation(), + delta: epoch.reference(), + actor: actor.into(), + filter: filter.into(), + k, + }; + + let mut nonce = [0_u8; NONCE_BYTES]; + self.rng.lock().try_fill_bytes(&mut nonce)?; + + let header = TokenHeader { + version: MessageVersion::V1, + issued_at: UnixEpochSeconds::new(now), + nonce, + }; + + let mut blob = [0_u8; Token::BYTES]; + header + .write_to_prefix(&mut blob) + .unwrap_or_else(|_err| unreachable!("header is included in the payload")); + scope + .write_to_prefix(&mut blob[TokenHeader::BYTES..]) + .unwrap_or_else(|_err| unreachable!("scope is included in the payload")); + + // the fixed-size header and scope fit the cipher's message and associated-data bounds. + let sealed_tag = Encryption::new(self.key.as_bytes().into()) + .encrypt_in_place_detached( + XNonce::from_slice(&nonce), + header.as_bytes(), + &mut blob[size_of::()..size_of::() + size_of::()], + ) + .unwrap_or_else(|_error| { + unreachable!("XChaCha20-Poly1305 encryption is infallible for in-memory payloads") + }); + + sealed_tag + .write_to_suffix(&mut blob) + .unwrap_or_else(|_err| unreachable!("trailer only contains tag")); + + Ok(EncryptedToken::new_unchecked(blob)) + } + + /// Opens a presented token and checks that its scope binds this request. + /// + /// Verification checks the token's byte layout before AEAD authentication with this authority's + /// key, then validates the decrypted scope's representation and the timestamp's host + /// representability. Every request-binding comparison follows authentication: actor, generation + /// and delta lifetime, in that order. The delta revision takes no part in these comparisons. + /// The result retains the sealed filter and offset. + /// + /// The returned [`ScopeLease`] performs the expiry check separately. Successful decryption does + /// not resolve visibility or record token use. + /// + /// # Errors + /// + /// Returns [`AuthorityError`] for an invalid envelope, failed authentication or a + /// request-binding mismatch. Representation checks and authentication precede the actor, + /// generation and delta comparisons. + pub(crate) fn decrypt( + &self, + actor: ActorId, + epoch: &Epoch, + blob: &EncryptedToken, + ) -> Result> { + let token = Token::try_ref_from_bytes(blob.as_bytes()) + .map_err(|_err| Report::new(AuthorityError::Envelope))?; + + let mut plaintext = token.ciphertext; + XChaCha20Poly1305::new(self.key.as_bytes().into()) + .decrypt_in_place_detached( + &XNonce::from(token.header.nonce), + token.header.as_bytes(), + &mut plaintext, + &Tag::from(token.trailer.tag), + ) + .map_err(|_error| Report::new(AuthorityError::Decryption))?; + + let scope = Scope::try_read_from_bytes(&plaintext) + .map_err(|_err| Report::new(AuthorityError::Envelope))?; + + let issued_at = token + .header + .issued_at + .to_system_time() + .ok_or_else(|| Report::new(AuthorityError::Envelope))?; + + if scope.actor != actor.into() { + return Err(Report::new(AuthorityError::Actor)); + } + if scope.generation != epoch.generation() { + return Err(Report::new(AuthorityError::Generation)); + } + if scope.delta.id != epoch.reference().id { + return Err(Report::new(AuthorityError::Delta)); + } + + Ok(ScopeLease::new(scope, issued_at, self.expiration)) + } +} diff --git a/libs/@local/graph/atlas/src/serve/authorization/error.rs b/libs/@local/graph/atlas/src/serve/authorization/error.rs new file mode 100644 index 00000000000..1071369b696 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/error.rs @@ -0,0 +1,37 @@ +/// A refusal to honour a presented authority. +/// +/// [`Authority::decrypt`](super::authority::Authority::decrypt) and +/// [`ScopeLease::current`](super::scope::ScopeLease::current) retain the failed check as a typed +/// error. Formatting reports only the check, without token contents. The [API +/// extractors](crate::api) log this distinction and map token failures to one +/// unauthorized response. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum AuthorityError { + /// The token or scope is malformed, or its issue time exceeds the host's representable range. + Envelope, + /// Authenticated decryption rejects the token under this authority's key. + Decryption, + /// The scope belongs to a different actor than the one presenting it. + Actor, + /// The scope names a different generation. + Generation, + /// The check time precedes issuance or the token has reached the expiration boundary. + Expired, + /// The scope names a different [delta lifetime](crate::serve::delta::DeltaId). + Delta, +} + +impl core::fmt::Display for AuthorityError { + fn fmt(&self, fmt: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Envelope => fmt.write_str("invalid authority envelope"), + Self::Decryption => fmt.write_str("failed to decrypt authority"), + Self::Actor => fmt.write_str("authority actor mismatch"), + Self::Generation => fmt.write_str("authority generation mismatch"), + Self::Expired => fmt.write_str("authority has expired"), + Self::Delta => fmt.write_str("authority delta mismatch"), + } + } +} + +impl core::error::Error for AuthorityError {} diff --git a/libs/@local/graph/atlas/src/serve/authorization/mod.rs b/libs/@local/graph/atlas/src/serve/authorization/mod.rs new file mode 100644 index 00000000000..053099a0d92 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/mod.rs @@ -0,0 +1,55 @@ +//! Actor-bound tokens retaining a generation's delivery context between requests. +//! +//! A manifest resolves an authenticated actor's visibility and seals a [`Scope`](scope::Scope). The +//! scope contains the generation and [delta lifetime](crate::serve::delta::DeltaId), the actor, the +//! filter digest and the delivery offset. Keeping this context in an authenticated token lets later +//! requests reuse it without trusting client-supplied fields. The token carries neither the visible +//! row set nor the filter document. +//! +//! # Admission and renewal +//! +//! [`Authority::decrypt`](authority::Authority::decrypt) authenticates the token and requires the +//! independently authenticated actor, generation and delta lifetime to match. The sealed delta +//! reference records a revision, but admission compares only its lifetime identifier. Later +//! publications within that lifetime do not invalidate the token. Possession alone does not +//! identify an actor, and issuance itself performs no permission-store query. +//! +//! Data admission requires a [`CurrentScope`](scope::CurrentScope), obtained by checking the +//! token's issue-time window. Renewal may instead take a +//! [`ContinuityScope`](scope::ContinuityScope), retaining only the filter identity and delivery +//! offset even after expiry. Both paths still require successful authentication and actor, +//! generation and delta-lifetime comparisons. Renewal resolves the wanted visibility before issuing +//! another token. +//! +//! Visibility remains a separate decision through +//! [`ScopeResolver`](crate::serve::visibility::resolver::ScopeResolver). Both manifest and data +//! requests may reuse a cached result, including a soft-stale entry before its hard expiry. A +//! retained generation can reuse an eligible cached scope but cannot resolve a missing or expired +//! one. A valid token therefore establishes neither current store permissions nor the availability +//! of a reusable scope. +//! +//! # Security boundary +//! +//! [`Authority`](authority::Authority) uses XChaCha20-Poly1305 to encrypt the scope and +//! authenticate it together with the clear header. Its [key and nonce +//! requirements](authority::Authority#key-and-nonce-requirements) are part of this protection. The +//! header exposes the layout version, issue time and nonce. Hexadecimal presentation adds no +//! secrecy. +//! +//! Tokens are reusable within their admission conditions. Verification records no use and provides +//! no single-use or per-token revocation mechanism. The scope binds no route, variant or request +//! body beyond the stored filter identity and delivery offset. Token expiry is a wall-clock check +//! at admission, separate from the visibility cache's age and the generation's retention interval. +//! +//! [`token`] defines the envelope, [`scope`] the verified views, and [`actor`] the principal's byte +//! representation. The [API extractors](crate::api) own HTTP rejection and renewal +//! behavior. + +mod actor; +pub(crate) mod authority; +mod error; +pub(crate) mod scope; +pub(crate) mod token; + +#[cfg(test)] +mod tests; diff --git a/libs/@local/graph/atlas/src/serve/authorization/scope.rs b/libs/@local/graph/atlas/src/serve/authorization/scope.rs new file mode 100644 index 00000000000..7efc40e121f --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/scope.rs @@ -0,0 +1,222 @@ +//! The sealed delivery decision and the views available after verification. +//! +//! [`Scope`] records the [authorization model](super)'s delivery context. Verification returns a +//! [`ScopeLease`] that retains the issue time. Consuming a lease selects exactly one view: +//! +//! - [`CurrentScope`] retains the whole decision only after an expiry check succeeds at the +//! supplied time. +//! - [`ContinuityScope`] retains the filter identity and delivery offset without checking expiry. +//! +//! The distinction prevents renewal context from satisfying an API that requires a current scope. +//! It does not make a returned scope expire automatically. +#![expect(clippy::empty_enums, reason = "zerocopy uses them in the derive")] + +use core::{ops::Deref, time::Duration}; +use std::time::SystemTime; + +use error_stack::Report; + +use super::{actor::ArchivedActorId, error::AuthorityError}; +use crate::{ + file::generation::GenerationId, + morton::Zoom, + serve::{delta::DeltaReference, visibility::cache::FilterDigest}, +}; + +/// A scope's request filter, by identity. +/// +/// The discriminant distinguishes absence from a digest that happens to be all zero. Conversion +/// from [`Option`] writes zeros for the absent payload, but decoding accepts any +/// payload bytes under the absent discriminant. An invalid discriminant is a malformed +/// representation. Every variant has the same width: filter presence never changes the envelope's +/// length. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, + zerocopy::TryFromBytes, +)] +#[repr(u8)] +pub(crate) enum ScopeFilter { + /// No filter, with payload bytes that issuance writes as zeros. + Absent([u8; 32]), + /// The scope names the filter with this digest. + Present(FilterDigest), +} + +impl ScopeFilter { + /// Returns the filter's digest, absent when the scope names no filter. + pub(crate) const fn digest(self) -> Option { + match self { + Self::Present(digest) => Some(digest), + Self::Absent(_) => None, + } + } +} + +impl From> for ScopeFilter { + fn from(filter: Option) -> Self { + filter.map_or(Self::Absent([0; 32]), Self::Present) + } +} + +/// The delivery context sealed into an authority token. +/// +/// Authentication covers every field, while admission compares the generation, actor and [delta +/// lifetime](crate::serve::delta::DeltaId). A different delta identifier rejects an earlier token +/// even under the same generation. Random identifier generation permits a reopen to draw the same +/// identifier. The recorded revision takes no part in admission: the token can apply to later +/// publications of the same lifetime. +/// +/// The fixed representation validates the actor and filter discriminants and the [`Zoom`] domain on +/// decoding. Other byte fields admit every bit pattern. These structural checks establish a +/// readable value, while authentication and request-binding checks establish its authority. +/// Visibility resolution determines the rows an actor may receive. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, + zerocopy::TryFromBytes, +)] +#[expect( + clippy::min_ident_chars, + reason = "`k` is the delivery-cut offset's name throughout the density contract" +)] +#[repr(C)] +pub(crate) struct Scope { + /// The generation of the resolved delivery decision. + pub generation: GenerationId, + /// The issuing publication's delta reference, compared only by lifetime at admission. + pub delta: DeltaReference, + + /// The principal bound to the token. + pub actor: ArchivedActorId, + /// The request filter's identity, absent for an unfiltered scope. + pub filter: ScopeFilter, + /// The admitted zoom offset for the delivery cut. + pub k: Zoom, +} + +/// A decrypted [`Scope`] pending an admission or renewal decision. +/// +/// Verification establishes that the token passed AEAD authentication and matches the request's +/// actor, generation and delta lifetime. [`current`](Self::current) checks expiry. +/// [`continuity`](Self::continuity) retains only the filter identity and delivery offset, even for +/// an expired lease. +pub(crate) struct ScopeLease { + scope: Scope, + issued_at: SystemTime, + expiration: Duration, +} + +impl ScopeLease { + /// Holds a verified `scope` against the issuer's `expiration` window. + pub(super) const fn new(scope: Scope, issued_at: SystemTime, expiration: Duration) -> Self { + Self { + scope, + issued_at, + expiration, + } + } + + /// Resolves the lease into the whole decision, when it is current at `now`. + /// + /// Accepts exactly when the issue time is at or before `now` and the elapsed duration is + /// strictly less than the expiration window. Equality with the expiry boundary and a zero + /// window both fail the check. + /// + /// This check uses the supplied wall-clock time. A clock moving before the issue time causes + /// rejection. Other backwards adjustments can extend apparent validity or make a previously + /// expired token current again. + /// + /// # Errors + /// + /// Returns [`AuthorityError`] when `now` lies outside the issue window. + pub(crate) fn current(self, now: SystemTime) -> Result> { + if self.issued_at > now || now.saturating_duration_since(self.issued_at) >= self.expiration + { + return Err(Report::new(AuthorityError::Expired)); + } + + Ok(CurrentScope(self.scope)) + } + + /// Reduces the lease to the part that survives expiry. + /// + /// Retains the filter identity and delivery offset without checking the issue time. The result + /// carries continuity for resolving a renewed view, with no actor or generation fields that + /// could authorize data delivery. + pub(crate) const fn continuity(self) -> ContinuityScope { + ContinuityScope { + filter: self.scope.filter.digest(), + k: self.scope.k, + } + } +} + +/// The filter identity and delivery offset retained for renewal. +#[expect( + clippy::min_ident_chars, + reason = "`k` is the delivery-cut offset's name throughout the density contract" +)] +pub(crate) struct ContinuityScope { + /// The request filter's identity, absent for an unfiltered scope. + pub filter: Option, + /// The delivery cut's zoom offset sealed in the verified lease. + pub k: Zoom, +} + +/// A [`Scope`] whose issue-time check succeeded at admission. +/// +/// Only [`ScopeLease::current`] constructs this type, after checking expiry at the supplied time. +/// The result records that check without enforcing freshness as time advances. Later requests still +/// require admission and visibility resolution. +pub(crate) struct CurrentScope(Scope); + +impl Deref for CurrentScope { + type Target = Scope; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg(test)] +mod tests { + use core::time::Duration; + use std::time::SystemTime; + + use zerocopy::TryFromBytes as _; + + use super::{Scope, ScopeLease}; + use crate::{morton::Zoom, serve::visibility::cache::FilterDigest}; + + #[test] + fn continuity_filter_digest() { + let offset = Zoom::new(3).expect("should fit the zoom domain"); + for filter in [None, Some(FilterDigest::of(b"filter bytes"))] { + let mut scope = Scope::try_read_from_bytes(&[0_u8; size_of::()]) + .expect("should decode a scope with zero fields"); + scope.filter = filter.into(); + scope.k = offset; + + let continuity = + ScopeLease::new(scope, SystemTime::UNIX_EPOCH, Duration::ZERO).continuity(); + + assert_eq!(continuity.filter, filter); + assert_eq!(continuity.k, offset); + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/authorization/tests.rs b/libs/@local/graph/atlas/src/serve/authorization/tests.rs new file mode 100644 index 00000000000..3f3e6589a9d --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/tests.rs @@ -0,0 +1,65 @@ +use core::{mem::offset_of, time::Duration}; +use std::time::SystemTime; + +use zerocopy::{FromBytes as _, IntoBytes as _, TryFromBytes as _}; + +use super::{ + scope::{Scope, ScopeFilter}, + token::UnixEpochSeconds, +}; +use crate::{morton::Zoom, serve::visibility::cache::FilterDigest}; + +#[test] +fn filter_digest_roundtrip() { + let digest = FilterDigest::of(b"filter bytes"); + let filter = ScopeFilter::from(Some(digest)); + let decoded = ScopeFilter::try_read_from_bytes(filter.as_bytes()) + .expect("the written filter should decode"); + assert_eq!(decoded.digest(), Some(digest)); +} + +#[test] +fn timestamp_seconds() { + let issued_at = SystemTime::UNIX_EPOCH + Duration::from_millis(1_000_900); + let encoded = UnixEpochSeconds::new(issued_at); + assert_eq!( + encoded.to_system_time(), + Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_000)) + ); +} + +#[test] +fn timestamp_before_epoch() { + let time = SystemTime::UNIX_EPOCH - Duration::from_nanos(1); + assert_eq!( + UnixEpochSeconds::new(time).to_system_time(), + Some(SystemTime::UNIX_EPOCH) + ); +} + +// Unix and Windows system times cannot represent the full unsigned seconds range. +#[cfg(any(unix, windows))] +#[test] +fn timestamp_decoded_overflow() { + let encoded = UnixEpochSeconds::read_from_bytes(&[u8::MAX; size_of::()]) + .expect("the seconds should decode independently of the host time range"); + assert_eq!(encoded.to_system_time(), None); +} + +#[test] +fn scope_zoom_bytes() { + let mut bytes = [0_u8; size_of::()]; + assert!( + Scope::try_read_from_bytes(&bytes).is_ok(), + "zero fields should form a valid root scope before changing the zoom" + ); + + for byte in u8::MIN..=u8::MAX { + bytes[offset_of!(Scope, k)] = byte; + assert_eq!( + Scope::try_read_from_bytes(&bytes).ok().map(|scope| scope.k), + Zoom::new(byte), + "scope decoding should enforce the zoom domain for byte {byte}" + ); + } +} diff --git a/libs/@local/graph/atlas/src/serve/authorization/token.rs b/libs/@local/graph/atlas/src/serve/authorization/token.rs new file mode 100644 index 00000000000..ed9049d6704 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/authorization/token.rs @@ -0,0 +1,216 @@ +//! The authority token's byte envelope. +//! +//! A token contains a clear header, encrypted [`Scope`] and AEAD tag at fixed offsets, in that +//! order. The nonce selects the cipher's per-message state. The whole header is associated data, +//! authenticating the version and issue time along with the nonce. The expiry check interprets the +//! issue time after successful decryption. +//! +//! [`Authority`](super::authority::Authority) owns sealing and verification. [`EncryptedToken`] +//! supplies the hexadecimal presentation of the whole envelope, including the clear header. +//! Encryption covers only the scope. +#![expect(clippy::empty_enums, reason = "zerocopy uses them in the derive")] + +use core::{fmt, str::FromStr, time::Duration}; +use std::time::SystemTime; + +use zerocopy::{LE, U64}; + +use super::{ + authority::{NONCE_BYTES, TAG_BYTES}, + scope::Scope, +}; +use crate::integrity::{HexBytes, ParseHexError}; + +/// The token's envelope layout version. +/// +/// Decoding accepts only the discriminants this enum names. An unsupported version is an +/// [`AuthorityError::Envelope`](super::error::AuthorityError::Envelope). Starting at 1 prevents a +/// zero-filled buffer from parsing as a valid header. The version check validates the +/// discriminator, not the origin of the remaining bytes. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, + zerocopy::TryFromBytes, +)] +#[repr(u8)] +pub(super) enum MessageVersion { + /// The layout this module defines: [`TokenHeader`], [`Scope`], [`TokenTrailer`]. + V1 = 1, +} + +/// A point in time as whole seconds since the Unix epoch, little-endian. +/// +/// Encoding the instant as a fixed-width count makes decoding host-independent. Conversion +/// truncates towards the epoch and maps earlier times to the epoch. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::FromBytes, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, +)] +#[repr(transparent)] +pub(super) struct UnixEpochSeconds(U64); + +impl UnixEpochSeconds { + /// Encodes `time` as whole seconds since the Unix epoch. + /// + /// # Warning + /// + /// A `time` before the epoch encodes as the epoch. Conversion discards sub-second precision. + pub(super) fn new(time: SystemTime) -> Self { + let duration = time.saturating_duration_since(SystemTime::UNIX_EPOCH); + + Self(U64::new(duration.as_secs())) + } + + /// Converts the timestamp into a host-representable instant. + /// + /// Returns [`None`] when the seconds exceed this platform's [`SystemTime`] range. + pub(super) fn to_system_time(self) -> Option { + SystemTime::UNIX_EPOCH.checked_add(Duration::from_secs(self.0.get())) + } +} + +/// The token's clear prefix, and the AEAD's associated data. +/// +/// [`TokenTrailer`]'s tag authenticates these bytes together with the ciphertext. The header fields +/// remain readable without the authority key. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, + zerocopy::TryFromBytes, +)] +#[repr(C)] +pub(super) struct TokenHeader { + /// The layout the remaining bytes follow. + pub version: MessageVersion, + /// The scope's issue time, the start of its expiration window. + pub issued_at: UnixEpochSeconds, + /// The per-token nonce for sealing this envelope. + pub nonce: [u8; NONCE_BYTES], +} + +impl TokenHeader { + /// The header's width, the offset at which the ciphertext begins. + pub(super) const BYTES: usize = size_of::(); +} + +/// The AEAD tag authenticating the ciphertext and clear header. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, + zerocopy::TryFromBytes, +)] +#[repr(C)] +pub(super) struct TokenTrailer { + /// The AEAD tag over the ciphertext and the clear header. + pub tag: [u8; TAG_BYTES], +} + +/// A fixed-width authority envelope with a clear header and encrypted scope. +/// +/// For this layout, every token has the same width because the ciphertext is exactly [`Scope`]'s +/// size and [`ScopeFilter`](super::scope::ScopeFilter) pads its absent form. Filtered and +/// unfiltered scopes therefore have indistinguishable token lengths. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, + zerocopy::TryFromBytes, +)] +#[repr(C)] +pub(super) struct Token { + /// The clear prefix the tag authenticates. + pub header: TokenHeader, + /// The sealed [`Scope`]. + pub ciphertext: [u8; size_of::()], + /// The tag over the ciphertext and the header. + pub trailer: TokenTrailer, +} + +impl Token { + /// The token's total width, the length every presentation decodes to. + pub(super) const BYTES: usize = size_of::(); +} + +/// A fixed-width authority presentation in lowercase hexadecimal. +/// +/// [`core::fmt::Display`] writes the entire envelope, including its clear header. Parsing accepts +/// only exactly twice [`Token::BYTES`] lowercase hexadecimal characters and validates only this +/// encoding. Uppercase letters, prefixes and surrounding whitespace cause [`ParseHexError`]. +/// [`Authority::decrypt`](super::authority::Authority::decrypt) authenticates the envelope and +/// validates its scope. +/// +/// [`core::fmt::Debug`] also exposes the presentation. Treat either rendering as the reusable +/// token, even though neither reveals the encrypted scope directly. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + zerocopy::FromBytes, + zerocopy::IntoBytes, + zerocopy::Immutable, + zerocopy::Unaligned, + zerocopy::KnownLayout, +)] +#[repr(transparent)] +pub(crate) struct EncryptedToken(HexBytes<{ Token::BYTES }>); + +impl EncryptedToken { + /// Constructs a presentable token from sealed envelope bytes. + /// + /// `bytes` must contain a complete [`Token`] envelope sealed and authenticated by an + /// [`Authority`](super::authority::Authority). + // this unchecked invariant affects token validity, not memory safety. + pub(super) const fn new_unchecked(bytes: [u8; Token::BYTES]) -> Self { + Self(HexBytes::new(bytes)) + } +} + +impl fmt::Display for EncryptedToken { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, fmt) + } +} + +impl FromStr for EncryptedToken { + type Err = ParseHexError; + + fn from_str(value: &str) -> Result { + value.parse::>().map(Self) + } +} diff --git a/libs/@local/graph/atlas/src/serve/delta/mod.rs b/libs/@local/graph/atlas/src/serve/delta/mod.rs index c043f162c57..82d5726d128 100644 --- a/libs/@local/graph/atlas/src/serve/delta/mod.rs +++ b/libs/@local/graph/atlas/src/serve/delta/mod.rs @@ -21,17 +21,21 @@ mod projector; mod task; pub(crate) mod topology; +pub(crate) use self::{ + feed::DeltaFeedTaskOptions, + placement::{DeltaPlacementTaskOptions, EmbeddingWorkflow}, + task::{DeltaReader, DeltaTask, DeltaTaskError, DeltaTaskOptions}, +}; + +#[cfg(test)] +mod tests; + use alloc::sync::Arc; use hashql_core::id::Id as _; use rand::TryCryptoRng; use zerocopy::{NativeEndian, U64}; -pub(crate) use self::{ - feed::DeltaFeedTaskOptions, - placement::{DeltaPlacementTaskOptions, EmbeddingWorkflow}, - task::{DeltaReader, DeltaTask, DeltaTaskError, DeltaTaskOptions}, -}; use self::{ layout::{LayoutDelta, provider::NaiveLayoutProvider}, overlay::{ diff --git a/libs/@local/graph/atlas/src/serve/delta/tests.rs b/libs/@local/graph/atlas/src/serve/delta/tests.rs new file mode 100644 index 00000000000..bb702043637 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/delta/tests.rs @@ -0,0 +1,1896 @@ +use alloc::sync::Arc; +use core::{assert_matches, cell::RefCell, iter, ptr}; + +use arc_swap::Guard; +use error_stack::Report; +use hashql_core::id::Id as _; +use rand::{SeedableRng as _, rngs::StdRng}; +use type_system::{ + knowledge::entity::EntityId, + principal::actor::{ActorId, ActorType}, +}; +use uuid::Uuid; + +use super::{ + Delta, DeltaRevision, + epoch::Epoch, + overlay::{DeltaIdentityProvider, NaiveIdentityProvider}, + topology::provider::NaiveTopologyProvider, +}; +use crate::{ + bitset::CompressedBitSet, + dataset::auxiliary::{Label, OwnedIcon, OwnedLegend}, + identity::{EdgeRowId, NodeRowId, OntologyRowId}, + math::{Bounds2, Log2, Vec2}, + morton::{Depth, MortonCell, Zoom}, + postgres::id::{ArchivedEntityId, ArchivedOntologyTypeUuid}, + salt::{ + fit::prepare::IdentityProvider as _, + lod::stage::{LodConfig, WIRE_FRAME}, + }, + serve::{ + document::{ + LocateDocument, LocateDocumentError, LocateDocumentOptions, LocateLimits, LocateSource, + }, + hydrate::{HydrateError, LocateProperties, LocateRequest, LocateResolver}, + membership::OntologySelection, + scene::Scene, + schedule::{BucketSchedule, DeliveredNodes, DeliverySchedule, ScopeSchedule, ViewSchedule}, + tests::fixture::{EDGES, ENDPOINTS, NODES, TYPES, TamperFixture, secret}, + visibility::{VisibilityActor, VisibilityMask}, + walk::Walk, + world::{World, topology::TopologyProvider as _}, + }, +}; + +/// Opens a synthetic world under `name` and allocates a delta over it. +/// +/// # Panics +/// +/// Panics on failure during generation publication or world opening. +fn fixture(name: &str) -> (TamperFixture, Delta) { + let fixture = TamperFixture::publish(name); + let world = World::open(fixture.generation().clone(), &secret()) + .expect("should open the synthetic world"); + let delta = Delta::new(Arc::new(world), StdRng::seed_from_u64(17)) + .expect("should allocate a delta identity"); + (fixture, delta) +} + +/// Builds an entity id from a fixed web and `seed`, for compact literal test identities. +fn entity(seed: u128) -> ArchivedEntityId { + ArchivedEntityId { + web_id: Uuid::from_u128(1).into(), + entity_uuid: Uuid::from_u128(seed).into(), + } +} + +/// Builds a legend carrying `label` under a fixed ontology row. +fn legend(label: &str) -> OwnedLegend { + OwnedLegend::new(OntologyRowId::MIN, Label::new(label)) +} + +/// Snapshots `delta` into an epoch usable for read-side queries. +fn epoch(delta: &Delta) -> Epoch { + Epoch::from(Guard::from_inner(Arc::new(delta.clone()))) +} + +/// Counts allocated edge rows, including withdrawn and unbound rows. +/// +/// # Panics +/// +/// Panics if `epoch` does not belong to `world`. +fn edge_count(world: &World, epoch: &Epoch) -> usize { + epoch + .topology(&world.topology) + .bind(NaiveTopologyProvider::from_ref(&world.topology)) + .provide_edge_count() +} + +/// Round-trips every base row and rejects rows outside the captured domain. +/// +/// Rejected rows include the first unallocated row and the largest representable wire id. +#[test] +fn decode_base() { + let (_fixture, delta) = fixture("decode-base"); + let index = &delta.world.layout.index; + let captured = epoch(&delta); + for row in (0..NODES).map(NodeRowId::new) { + assert_eq!( + index.decode(&captured, index.encode(row)), + Some(row), + "should invert each allocated base row" + ); + } + for row in [NodeRowId::new(NODES), NodeRowId::from_u32(u32::MAX)] { + assert_eq!( + index.decode(&captured, index.encode(row)), + None, + "should reject rows outside the captured allocation domain" + ); + } +} + +/// Tracks decoded row visibility at each capture. +/// +/// It is absent before allocation (except a base row, present from the start), present once +/// live, absent again after withdrawal, and present after revival. The id's own encoding never +/// changes throughout. A wire id past the final capture's domain still decodes to absent. +#[test] +fn decode_captures() { + let (_fixture, mut delta) = fixture("decode-captures"); + let world = Arc::clone(&delta.world); + let index = &world.layout.index; + let rows = [NodeRowId::MIN, NodeRowId::new(NODES)]; + let wires = rows.map(|row| index.encode(row)); + let identities = [ + index + .identity + .key_of(rows[0]) + .expect("should resolve the base node"), + entity(905), + ]; + let before = epoch(&delta); + + delta.revision.increment_by(1); + for identity in identities { + assert_eq!( + delta.update_node(identity, legend("live"), Vec2::ZERO), + Some(true), + "should activate the base and added nodes" + ); + } + let live = epoch(&delta); + delta.revision.increment_by(1); + for identity in identities { + assert!(delta.withdraw(identity), "should withdraw the node"); + } + let withdrawn = epoch(&delta); + delta.revision.increment_by(1); + for identity in identities { + assert_eq!( + delta.update_node(identity, legend("revived"), Vec2::ZERO), + Some(true), + "should revive the existing row" + ); + } + let revived = epoch(&delta); + drop(delta); + + for ((row, wire), earlier) in rows.into_iter().zip(wires).zip([Some(rows[0]), None]) { + assert_eq!( + index.decode(&before, wire), + earlier, + "should preserve pre-allocation capture" + ); + assert_eq!( + index.decode(&live, wire), + Some(row), + "should decode the live capture" + ); + assert_eq!( + index.decode(&withdrawn, wire), + None, + "should reject the withdrawn capture" + ); + assert_eq!( + index.decode(&revived, wire), + Some(row), + "should decode the revived row" + ); + assert_eq!( + index.encode(row), + wire, + "should preserve the row's encoded identity" + ); + } + let outside = index.encode(NodeRowId::new(NODES + 1)); + assert_eq!( + index.decode(&revived, outside), + None, + "should reject the next unallocated row" + ); +} + +/// Panics when decoding a wire id against an epoch from another world. +#[test] +#[should_panic(expected = "index must belong to the epoch's world")] +fn decode_foreign_world() { + let (_left_files, left) = fixture("decode-foreign-left"); + let (_right_files, right) = fixture("decode-foreign-right"); + let index = &left.world.layout.index; + let _row = index.decode(&epoch(&right), index.encode(NodeRowId::MIN)); +} + +/// Retains the resolved actor on full and partial masks. +/// +/// Both user and machine principals preserve their identity and instance-administrator status. +#[test] +fn mask_actor() { + for (seed, kind, instance_admin) in [(1, ActorType::User, false), (2, ActorType::Machine, true)] + { + let actor = VisibilityActor { + id: ActorId::new(Uuid::from_u128(seed), kind), + instance_admin, + }; + for mask in [ + VisibilityMask::full(actor), + VisibilityMask::partial( + actor, + CompressedBitSet::default(), + CompressedBitSet::default(), + ), + ] { + let bound = mask.actor(); + assert_eq!(bound.id, actor.id, "should retain the bound actor identity"); + assert_eq!( + bound.instance_admin, actor.instance_admin, + "should retain the bound privilege" + ); + } + } +} + +/// Preserves a node's row and first coordinates across insertion and revival. +#[test] +fn node_added_revival() { + let (_fixture, mut delta) = fixture("delta-node-added-revival"); + let world = Arc::clone(&delta.world); + let before = epoch(&delta); + let entity = entity(100); + let position = Vec2::new(0.25, -0.5); + delta.revision = DeltaRevision::new(1); + assert_eq!( + delta.update_node(entity, legend("first"), position), + Some(true) + ); + let row = delta.node_row(entity).expect("should allocate a node row"); + let first = epoch(&delta); + let first_payload = world + .layout + .index + .payload(&first, row) + .expect("should borrow the added node legend"); + assert!(world.layout.index.payload(&before, row).is_none()); + assert_eq!(row, NodeRowId::new(NODES)); + assert_eq!(delta.world.layout.position(&first, row), Some(position)); + let count = usize::try_from(NODES).expect("should fit the fixture count") + 1; + assert_eq!( + first + .topology(&delta.world.topology) + .bind(NaiveTopologyProvider::from_ref(&delta.world.topology)) + .provide_node_count(), + count + ); + assert_eq!(delta.world.layout.node_count(&first), count); + assert_eq!( + delta.update_node(entity, legend("first"), Vec2::ZERO), + Some(false) + ); + + delta.revision = DeltaRevision::new(2); + assert!(delta.withdraw(entity), "should withdraw the placed node"); + assert!( + !delta.withdraw(entity), + "should leave a repeated withdrawal unchanged" + ); + assert_eq!(delta.node_row(entity), Some(row)); + let withdrawn = epoch(&delta); + assert_eq!(delta.world.layout.position(&withdrawn, row), None); + + delta.revision = DeltaRevision::new(3); + assert_eq!( + delta.update_node(entity, legend("latest"), Vec2::ZERO), + Some(true) + ); + let revived = epoch(&delta); + assert_eq!(delta.world.layout.position(&revived, row), Some(position)); + assert_eq!(delta.world.layout.position(&withdrawn, row), None); + assert_eq!(delta.world.layout.position(&first, row), Some(position)); + assert_eq!( + world + .layout + .index + .payload(&revived, row) + .expect("should borrow the revived node legend") + .label(), + "latest" + ); + assert!(world.layout.index.payload(&withdrawn, row).is_none()); + assert_eq!(first_payload.label(), "first"); +} + +/// Uses full identities to distinguish equal entity UUIDs in different webs. +#[test] +fn withdraw_other_web() { + let (_fixture, mut delta) = fixture("delta-withdraw-other-web"); + let left = entity(101); + let right = ArchivedEntityId { + web_id: Uuid::from_u128(2).into(), + ..left + }; + assert_eq!( + delta.update_node(left, legend("left"), Vec2::ZERO), + Some(true) + ); + assert_eq!( + delta.update_node(right, legend("right"), Vec2::splat(0.5)), + Some(true) + ); + let left_row = delta.node_row(left).expect("should resolve the left node"); + let right_row = delta + .node_row(right) + .expect("should resolve the right node"); + delta.revision.increment_by(1); + assert!( + delta.withdraw(left), + "should withdraw only the addressed identity" + ); + assert!( + !delta.withdraw(entity(999)), + "should ignore an unknown identity" + ); + let epoch = epoch(&delta); + assert_eq!(delta.world.layout.position(&epoch, left_row), None); + assert_eq!( + delta.world.layout.position(&epoch, right_row), + Some(Vec2::splat(0.5)) + ); +} + +/// Restores incident edges on node revival unless an edge has its own withdrawal. +#[test] +fn endpoints_node_and_edge_withdrawals() { + let (_fixture, mut delta) = fixture("delta-endpoint-withdrawals"); + let node = ENDPOINTS[0][1]; + let entity = delta + .world + .layout + .index + .identity + .key_of(node) + .expect("should resolve the fitted node"); + let edge = EdgeRowId::MIN; + let edge_entity = delta + .world + .topology + .identity + .key_of(edge) + .expect("should resolve the fitted edge"); + let held_legend = delta + .world + .layout + .index + .identity + .payload_of_row(node) + .expect("should read the fitted legend") + .to_owned(); + let before = epoch(&delta); + assert_eq!( + delta.world.topology.endpoints(&before, edge), + Some(ENDPOINTS[0]) + ); + + delta.revision.increment_by(1); + assert!( + delta.withdraw(entity), + "should withdraw the fitted endpoint" + ); + let hidden = epoch(&delta); + assert_eq!(delta.world.topology.endpoints(&hidden, edge), None); + assert_eq!( + delta.world.topology.endpoints(&hidden, EdgeRowId::new(1)), + None + ); + assert_eq!( + delta + .world + .topology + .incoming(&hidden, node) + .collect::>(), + [] + ); + assert_eq!( + delta + .world + .topology + .outgoing(&hidden, ENDPOINTS[0][0]) + .collect::>(), + [] + ); + assert_eq!( + delta.world.topology.endpoints(&before, edge), + Some(ENDPOINTS[0]) + ); + assert_eq!( + edge_count(&delta.world, &hidden), + usize::try_from(EDGES).expect("should fit the fixture count") + ); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(entity, held_legend.clone(), Vec2::ZERO), + Some(true) + ); + let revived = epoch(&delta); + assert_eq!( + delta.world.topology.endpoints(&revived, edge), + Some(ENDPOINTS[0]) + ); + assert_eq!( + delta.world.layout.position(&revived, node), + delta.world.layout.position(&before, node) + ); + + delta.revision.increment_by(1); + assert!(delta.withdraw(entity), "should withdraw the endpoint again"); + assert!( + delta.withdraw(edge_entity), + "should record the edge's own withdrawal" + ); + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(entity, held_legend, Vec2::ZERO), + Some(true) + ); + let independent = epoch(&delta); + assert_eq!(delta.world.topology.endpoints(&independent, edge), None); + assert_eq!( + delta + .world + .topology + .endpoints(&independent, EdgeRowId::new(1)), + Some(ENDPOINTS[1]) + ); +} + +/// Keeps added edges unbound until endpoint rows resolve, then retains their first pair. +#[test] +fn edge_unbound_revival() { + let (_fixture, mut delta) = fixture("delta-edge-unbound-revival"); + let entity = entity(102); + let edge = EdgeRowId::new(EDGES); + delta.revision.increment_by(1); + assert_eq!(delta.update_edge(entity, legend("edge"), None), Some(true)); + let unbound = epoch(&delta); + assert_eq!( + edge_count(&delta.world, &unbound), + usize::try_from(EDGES).expect("should fit the fixture count") + 1 + ); + assert_eq!(delta.world.topology.endpoints(&unbound, edge), None); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_edge(entity, legend("edge"), Some(ENDPOINTS[0])), + Some(true) + ); + let bound = epoch(&delta); + assert_eq!( + delta.world.topology.endpoints(&bound, edge), + Some(ENDPOINTS[0]) + ); + assert_eq!(delta.world.topology.endpoints(&unbound, edge), None); + assert_eq!( + delta + .world + .topology + .incoming(&bound, ENDPOINTS[0][1]) + .collect::>(), + [EdgeRowId::MIN, edge] + ); + + delta.revision.increment_by(1); + assert!(delta.withdraw(entity), "should withdraw the added edge"); + assert!( + !delta.withdraw(entity), + "should leave a repeated edge withdrawal unchanged" + ); + let hidden = epoch(&delta); + assert_eq!(delta.world.topology.endpoints(&hidden, edge), None); + delta.revision.increment_by(1); + assert_eq!( + delta.update_edge(entity, legend("edge"), Some(ENDPOINTS[1])), + Some(true) + ); + let revived = epoch(&delta); + assert_eq!( + delta.world.topology.endpoints(&revived, edge), + Some(ENDPOINTS[0]) + ); + assert_eq!( + edge_count(&delta.world, &revived), + usize::try_from(EDGES).expect("should fit the fixture count") + 1 + ); +} + +/// Tracks an added self-loop with its endpoint node's visibility. +#[test] +fn endpoints_added_self_loop() { + let (_fixture, mut delta) = fixture("delta-added-self-loop"); + let node_entity = entity(103); + let edge_entity = entity(104); + assert_eq!( + delta.update_node(node_entity, legend("node"), Vec2::ZERO), + Some(true) + ); + let row = delta + .node_row(node_entity) + .expect("should resolve the node"); + assert_eq!( + delta.update_edge(edge_entity, legend("edge"), Some([row, row])), + Some(true) + ); + let edge = EdgeRowId::new(EDGES); + let live = epoch(&delta); + assert_eq!( + delta + .world + .topology + .incoming(&live, row) + .collect::>(), + [edge] + ); + assert_eq!( + delta + .world + .topology + .outgoing(&live, row) + .collect::>(), + [edge] + ); + delta.revision.increment_by(1); + assert!( + delta.withdraw(node_entity), + "should hide the self-loop's endpoint" + ); + let hidden = epoch(&delta); + assert_eq!(delta.world.topology.endpoints(&hidden, edge), None); + assert_eq!( + delta + .world + .topology + .incoming(&hidden, row) + .collect::>(), + [] + ); + assert_eq!( + delta + .world + .topology + .outgoing(&hidden, row) + .collect::>(), + [] + ); +} + +/// Preserves fitted and added ontology-row identities across icon replacement. +#[test] +fn ontology_icon_replacement() { + let (_fixture, mut delta) = fixture("delta-ontology-icon-replacement"); + let world = Arc::clone(&delta.world); + let before = epoch(&delta); + delta.revision.increment_by(1); + let row = OntologyRowId::MIN; + let fitted = delta + .world + .ontology + .identity + .key_of(row) + .expect("should resolve the fitted type"); + let icon = delta + .world + .ontology + .identity + .payload_of_row(row) + .expect("should read the fitted icon") + .to_owned(); + assert_eq!(delta.register_ontology(fitted, icon), Some((row, false))); + assert_eq!( + delta.register_ontology(fitted, OwnedIcon::from("changed")), + Some((row, true)) + ); + let added: ArchivedOntologyTypeUuid = Uuid::from_u128(101).into(); + let added_row = OntologyRowId::new(TYPES); + assert_eq!( + delta.register_ontology(added, OwnedIcon::from("added")), + Some((added_row, true)) + ); + assert_eq!( + delta.register_ontology(added, OwnedIcon::from("added")), + Some((added_row, false)) + ); + let identities = DeltaIdentityProvider::from_parts( + &delta.ontology, + NaiveIdentityProvider::from_ref(&delta.world.ontology.identity), + ); + assert_eq!( + identities + .payload_of_row(row) + .expect("should read the replacement") + .as_ref(), + "changed" + ); + assert_eq!( + identities.count(), + usize::try_from(TYPES).expect("should fit the fixture count") + 1 + ); + + let captured = epoch(&delta); + let rows = [row, added_row]; + let held = rows.map(|row| { + world + .ontology + .payload(&captured, row) + .expect("should borrow the captured icon") + }); + assert!(world.ontology.payload(&before, added_row).is_none()); + assert!(world.ontology.icon(&before, added_row).is_none()); + delta.revision.increment_by(1); + for (key, row) in [fitted, added].into_iter().zip(rows) { + assert_eq!( + delta.register_ontology(key, OwnedIcon::from("latest")), + Some((row, true)) + ); + } + let replaced = epoch(&delta); + drop(delta); + for ((row, icon), expected) in rows.into_iter().zip(held).zip(["changed", "added"]) { + assert_eq!(icon.as_ref(), expected); + assert!( + ptr::eq( + icon, + world + .ontology + .icon(&captured, row) + .expect("should borrow the captured icon") + ), + "should resolve the same captured payload" + ); + assert_eq!( + world + .ontology + .icon(&replaced, row) + .expect("should borrow the replaced icon") + .as_ref(), + "latest" + ); + } + assert!(world.ontology.icon(&replaced, OntologyRowId::MAX).is_none()); +} + +/// Keeps captured inherited icons stable across later replacements. +/// +/// An ontology row without its own icon inherits an ancestor's, borrowing the same captured +/// payload. A distinct direct icon on a descendant is not shadowed by that inheritance. Replacing +/// the ancestor's icon later changes what a fresh capture inherits without invalidating an +/// earlier capture's already-borrowed icon. +#[test] +fn ontology_icon_inherited() { + let (_fixture, mut delta) = fixture("delta-ontology-icon-inherited"); + let world = Arc::clone(&delta.world); + let ancestor = OntologyRowId::MIN; + let child = OntologyRowId::new(1); + delta.revision.increment_by(1); + for (row, label) in [(ancestor, "ancestor"), (child, "direct")] { + let key = world + .ontology + .identity + .key_of(row) + .expect("should resolve the base type"); + assert_eq!( + delta.register_ontology(key, OwnedIcon::from(label)), + Some((row, true)) + ); + } + let captured = epoch(&delta); + let inherited = world + .ontology + .icon(&captured, child) + .expect("should borrow the ancestor icon"); + assert_eq!(inherited.as_ref(), "ancestor"); + assert_eq!( + world + .ontology + .payload(&captured, child) + .expect("should retain the distinct direct icon") + .as_ref(), + "direct" + ); + assert!( + ptr::eq( + inherited, + world + .ontology + .payload(&captured, ancestor) + .expect("should borrow the source payload") + ), + "should borrow the ancestor's captured payload" + ); + + delta.revision.increment_by(1); + let key = world + .ontology + .identity + .key_of(ancestor) + .expect("should resolve the ancestor"); + assert_eq!( + delta.register_ontology(key, OwnedIcon::from("latest")), + Some((ancestor, true)) + ); + let replaced = epoch(&delta); + drop(delta); + assert_eq!( + world + .ontology + .icon(&replaced, child) + .expect("should borrow the replaced ancestor icon") + .as_ref(), + "latest" + ); + assert_eq!(inherited.as_ref(), "ancestor"); +} + +/// Panics when resolving an ontology icon against an epoch from another world. +#[test] +#[should_panic(expected = "ontology must belong to the epoch's world")] +fn ontology_icon_foreign_world() { + let (_left_files, left) = fixture("delta-ontology-icon-foreign-left"); + let (_right_files, right) = fixture("delta-ontology-icon-foreign-right"); + let foreign = epoch(&right); + let _icon = left.world.ontology.icon(&foreign, OntologyRowId::MIN); +} + +/// Borrows the mapped base legend without copying it. +#[test] +fn payload_base() { + let (_fixture, mut delta) = fixture("delta-payload-base"); + let world = Arc::clone(&delta.world); + let captured = epoch(&delta); + let mapped = world + .topology + .identity + .payload_of_row(EdgeRowId::MIN) + .expect("should read the base legend"); + let payload = world + .topology + .payload(&captured, EdgeRowId::MIN) + .expect("should borrow the base legend"); + + assert!( + ptr::eq(mapped, payload), + "should borrow the same mapped legend" + ); + assert!(world.topology.payload(&captured, EdgeRowId::MAX).is_none()); + + let node = NodeRowId::MIN; + let mapped_node = world + .layout + .index + .identity + .payload_of_row(node) + .expect("should read the base node legend"); + let node_payload = world + .layout + .index + .payload(&captured, node) + .expect("should borrow the base node legend"); + assert!( + ptr::eq(mapped_node, node_payload), + "should borrow the same mapped node legend" + ); + assert!( + world + .layout + .index + .payload(&captured, NodeRowId::MAX) + .is_none() + ); + + let ontology = OntologyRowId::MIN; + let mapped_icon = world + .ontology + .identity + .payload_of_row(ontology) + .expect("should read the base icon"); + let icon = world + .ontology + .payload(&captured, ontology) + .expect("should borrow the base icon"); + assert!( + ptr::eq(mapped_icon, icon), + "should borrow the same mapped icon" + ); + assert!( + world + .ontology + .payload(&captured, OntologyRowId::MAX) + .is_none() + ); + + let key = world + .layout + .index + .identity + .key_of(node) + .expect("should resolve the base node"); + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(key, legend("replacement"), Vec2::ZERO), + Some(true) + ); + let replaced = epoch(&delta); + assert_eq!( + world + .layout + .index + .payload(&replaced, node) + .expect("should borrow the replacement legend") + .label(), + "replacement" + ); + drop(delta); + assert_eq!(payload.label(), mapped.label()); + assert_eq!(node_payload.label(), mapped_node.label()); + assert_eq!(icon.as_ref(), mapped_icon.as_ref()); +} + +/// Captures retain base overrides and added legends through withdrawal and revival. +#[test] +fn payload_captures() { + let (_fixture, mut delta) = fixture("delta-payload-captures"); + let world = Arc::clone(&delta.world); + let rows = [EdgeRowId::MIN, EdgeRowId::new(EDGES)]; + let keys = [ + world + .topology + .identity + .key_of(rows[0]) + .expect("should resolve the base edge"), + entity(903), + ]; + let before = epoch(&delta); + assert!(world.topology.payload(&before, rows[1]).is_none()); + + delta.revision.increment_by(1); + for key in keys { + assert_eq!( + delta.update_edge(key, legend("first"), Some(ENDPOINTS[0])), + Some(true) + ); + } + let captured = epoch(&delta); + let held = rows.map(|row| { + world + .topology + .payload(&captured, row) + .expect("should borrow the captured legend") + }); + + delta.revision.increment_by(1); + for key in keys { + assert!(delta.withdraw(key), "should withdraw the edge"); + } + let withdrawn = epoch(&delta); + for row in rows { + assert!(world.topology.payload(&withdrawn, row).is_none()); + } + + delta.revision.increment_by(1); + for key in keys { + assert_eq!( + delta.update_edge(key, legend("revived"), Some(ENDPOINTS[0])), + Some(true) + ); + } + let revived = epoch(&delta); + drop(delta); + for (row, payload) in rows.into_iter().zip(held) { + assert_eq!(payload.label(), "first"); + assert_eq!( + world + .topology + .payload(&revived, row) + .expect("should borrow the revived legend") + .label(), + "revived" + ); + assert!(world.topology.payload(&withdrawn, row).is_none()); + } + assert!(world.topology.payload(&before, rows[1]).is_none()); +} + +/// Excludes later registrations and unknown rows from captured ontology keys. +#[test] +fn ontology_keys_captured() { + let (_fixture, mut delta) = fixture("delta-ontology-keys-captured"); + let world = Arc::clone(&delta.world); + let base = OntologyRowId::MIN; + let base_key = world + .ontology + .identity + .key_of(base) + .expect("should resolve the base type"); + let before = epoch(&delta); + assert_eq!(world.ontology.key_of(&before, base), Some(base_key)); + assert_eq!(world.ontology.key_of(&before, OntologyRowId::MAX), None); + + let added: ArchivedOntologyTypeUuid = Uuid::from_u128(904).into(); + let row = OntologyRowId::new(TYPES); + delta.revision.increment_by(1); + assert_eq!( + delta.register_ontology(added, OwnedIcon::from("added")), + Some((row, true)) + ); + let captured = epoch(&delta); + drop(delta); + + assert_eq!(world.ontology.key_of(&captured, base), Some(base_key)); + assert_eq!(world.ontology.key_of(&captured, row), Some(added)); + assert_eq!(world.ontology.key_of(&before, row), None); + assert_eq!(world.ontology.key_of(&captured, OntologyRowId::MAX), None); +} + +/// Panics when resolving an edge payload against an epoch from another world. +#[test] +#[should_panic(expected = "topology must belong to the epoch's world")] +fn payload_foreign_world() { + let (_left_files, left) = fixture("delta-payload-foreign-left"); + let (_right_files, right) = fixture("delta-payload-foreign-right"); + let _payload = left.world.topology.payload(&epoch(&right), EdgeRowId::MIN); +} + +/// Panics when resolving an ontology key against an epoch from another world. +#[test] +#[should_panic(expected = "ontology must belong to the epoch's world")] +fn ontology_keys_foreign_world() { + let (_left_files, left) = fixture("delta-ontology-foreign-left"); + let (_right_files, right) = fixture("delta-ontology-foreign-right"); + let _key = left + .world + .ontology + .key_of(&epoch(&right), OntologyRowId::MIN); +} + +/// Panics when resolving a node payload against an epoch from another world. +#[test] +#[should_panic(expected = "index must belong to the epoch's world")] +fn node_payload_foreign_world() { + let (_left_files, left) = fixture("delta-node-payload-foreign-left"); + let (_right_files, right) = fixture("delta-node-payload-foreign-right"); + let foreign = epoch(&right); + let _payload = left.world.layout.index.payload(&foreign, NodeRowId::MIN); +} + +/// Panics when resolving an ontology payload against an epoch from another world. +#[test] +#[should_panic(expected = "ontology must belong to the epoch's world")] +fn ontology_payload_foreign_world() { + let (_left_files, left) = fixture("delta-ontology-payload-foreign-left"); + let (_right_files, right) = fixture("delta-ontology-payload-foreign-right"); + let foreign = epoch(&right); + let _payload = left.world.ontology.payload(&foreign, OntologyRowId::MIN); +} + +/// Reuses a delta allocation without mutating an earlier publication. +#[test] +fn clone_from_publication() { + let (_fixture, mut source) = fixture("delta-clone-from-publication"); + let mut reusable = source.clone(); + let entity = entity(105); + assert_eq!( + source.update_node(entity, legend("added"), Vec2::ZERO), + Some(true) + ); + reusable.clone_from(&source); + let row = source + .node_row(entity) + .expect("should resolve the added node"); + let publication = epoch(&reusable); + source.revision.increment_by(1); + assert!(source.withdraw(entity), "should withdraw the mutable copy"); + reusable.clone_from(&source); + let current = epoch(&reusable); + assert_eq!(reusable.world.layout.position(¤t, row), None); + assert_eq!( + reusable.world.layout.position(&publication, row), + Some(Vec2::ZERO) + ); + assert_eq!(current.revision(), source.revision); +} + +/// Composes schedule visibility from the mask and captured placements. +#[test] +fn schedule_captured_visibility() { + let (_fixture, mut delta) = fixture("delta-schedule-captured"); + let fitted = NodeRowId::MIN; + let fitted_id = delta + .world + .layout + .index + .identity + .key_of(fitted) + .expect("should resolve the fitted identity"); + let position = delta + .world + .layout + .position(&epoch(&delta), fitted) + .expect("should resolve the fitted placement"); + let higher = entity(200); + let lower = entity(100); + assert_eq!( + delta.update_node(higher, legend("higher"), position), + Some(true) + ); + assert_eq!( + delta.update_node(lower, legend("lower"), position), + Some(true) + ); + let higher_row = delta + .node_row(higher) + .expect("should resolve the added row"); + let lower_row = delta.node_row(lower).expect("should resolve the added row"); + let mut nodes = CompressedBitSet::default(); + for row in [fitted, higher_row, lower_row] { + nodes.insert(row); + } + let captured = epoch(&delta); + let mask = VisibilityMask::partial( + VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }, + nodes, + CompressedBitSet::default(), + ); + let buckets = BucketSchedule::new(LodConfig { + span: Log2::new(0).expect("should fit the exponent domain"), + max_tile_depth: Zoom::new(1).expect("should fit the zoom domain"), + }) + .expect("should fit the key width"); + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + let leaf_zoom = Zoom::new(1).expect("should fit the zoom domain"); + let (before, _) = ScopeSchedule::of(&delta.world.layout, &captured, &mask); + let before_cut = before.cut(buckets, Zoom::MIN).expect("should bind the cut"); + assert_eq!(before_cut.total(Zoom::MIN, root).rows, [fitted]); + assert_eq!( + before_cut.total(leaf_zoom, root).rows, + [fitted, lower_row, higher_row] + ); + + delta.revision.increment_by(1); + assert!(delta.withdraw(fitted_id), "should withdraw the fitted row"); + assert!( + delta.withdraw(lower), + "should withdraw the better added priority" + ); + let hidden = epoch(&delta); + let (after, _) = ScopeSchedule::of(&delta.world.layout, &hidden, &mask); + let after_cut = after.cut(buckets, Zoom::MIN).expect("should bind the cut"); + assert_eq!(after_cut.total(Zoom::MIN, root).rows, [higher_row]); + assert_eq!(after_cut.root_delivered(), 1); + assert_eq!(after_cut.min_resolution(), Depth::MIN); + assert_eq!(after_cut.children(Zoom::MIN, root), 0); + assert_eq!(after_cut.first_zoom(fitted), None); + assert_eq!(after_cut.first_zoom(lower_row), None); + let (rebuilt, _) = ScopeSchedule::of(&delta.world.layout, &captured, &mask); + assert_eq!( + rebuilt + .cut(buckets, Zoom::MIN) + .expect("should bind the cut") + .total(leaf_zoom, root), + before_cut.total(leaf_zoom, root) + ); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(fitted_id, legend("revived"), Vec2::ZERO), + Some(true) + ); + let (revived, _) = ScopeSchedule::of(&delta.world.layout, &epoch(&delta), &mask); + assert_eq!( + revived + .cut(buckets, Zoom::MIN) + .expect("should bind the cut") + .total(leaf_zoom, root) + .rows, + [fitted, higher_row] + ); +} + +/// Keeps corpus delivery stable while scoped delivery follows withdrawal. +/// +/// A withdrawal hides a fitted row from placement without changing the corpus-wide delivery +/// schedule. Scoping to the remaining visible rows drops exactly that one row's bucket assignment +/// and reduces the total count by one. +#[test] +fn schedule_corpus_withdrawal() { + let (_fixture, mut delta) = fixture("schedule-corpus-withdrawal"); + let world = Arc::clone(&delta.world); + let schedule = DeliverySchedule::corpus(&world); + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + let zoom = world.schedule().max_tile_depth(); + let baseline = schedule.total(zoom, root); + let root_count = schedule.root_delivered(); + let resolution = schedule.min_resolution(); + let node = NodeRowId::MIN; + let identity = world + .layout + .index + .identity + .key_of(node) + .expect("should resolve the fitted identity"); + let captured = epoch(&delta); + + delta.revision.increment_by(1); + assert!(delta.withdraw(identity), "should withdraw the fitted row"); + let withdrawn = epoch(&delta); + assert!(world.layout.position(&captured, node).is_some()); + assert_eq!(world.layout.position(&withdrawn, node), None); + assert_eq!(schedule.total(zoom, root), baseline); + assert_eq!(schedule.root_delivered(), root_count); + assert_eq!(schedule.min_resolution(), resolution); + assert!(schedule.bucket_of(node).is_some()); + + let mut nodes = CompressedBitSet::default(); + for row in 0..NODES { + nodes.insert(NodeRowId::new(row)); + } + let mask = VisibilityMask::partial( + VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }, + nodes, + CompressedBitSet::default(), + ); + let (scope, _) = ScopeSchedule::of(&world.layout, &withdrawn, &mask); + let cut = scope + .cut(world.schedule(), Zoom::MIN) + .expect("should bind the scoped schedule"); + assert_eq!(cut.bucket_of(node), None); + assert_eq!(cut.total(zoom, root).rows.len() + 1, baseline.rows.len()); +} + +/// Builds a partial visibility mask admitting exactly `nodes`, with no edges admitted. +fn schedule_mask(nodes: impl IntoIterator) -> VisibilityMask { + let mut mask = CompressedBitSet::default(); + for node in nodes { + mask.insert(node); + } + VisibilityMask::partial( + VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }, + mask, + CompressedBitSet::default(), + ) +} + +/// Combines admitted base and extension positions into view bounds. +/// +/// A view's bounds extend the base corpus bounds by an added node's position when its mask admits +/// that node, narrow to only that node's point when the mask admits nothing else, and cover every +/// added and base node under a full mask. Withdrawing the node collapses a narrow view's bounds +/// to none while leaving a wider admitted view and an unrelated cached view unaffected, and +/// reviving it restores the narrow view's bounds. +#[test] +fn bounds_extension() { + let (_fixture, mut delta) = fixture("view-bounds-extension"); + let world = Arc::clone(&delta.world); + let visible = entity(100); + let hidden = entity(200); + let position = Vec2::new(-2.0, 3.0); + assert_eq!( + delta.update_node(visible, legend("visible"), position), + Some(true) + ); + assert_eq!( + delta.update_node(hidden, legend("hidden"), Vec2::new(4.0, -5.0)), + Some(true) + ); + let row = delta + .node_row(visible) + .expect("should allocate the visible row"); + let captured = epoch(&delta); + let admitted = schedule_mask((0..NODES).map(NodeRowId::new).chain([row])); + let saturated = ViewSchedule::of(Arc::clone(&world), &captured, &admitted); + let point = Bounds2::new(position, position).expect("should bound the finite point"); + let base = world.layout.base_bounds().expect("should have base points"); + assert_eq!(saturated.bounds(), Some(base.union(point))); + + let narrow = ViewSchedule::of(Arc::clone(&world), &captured, &schedule_mask([row])); + assert_eq!(narrow.bounds(), Some(point)); + let occupancy = narrow + .occupancy() + .expect("a scope should have an occupancy profile"); + assert_eq!(occupancy.distinct_keys(), 1); + let full = VisibilityMask::full(VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }); + let corpus = ViewSchedule::of(Arc::clone(&world), &captured, &full); + assert_eq!( + corpus.bounds(), + Bounds2::new(Vec2::new(-2.0, -5.0), Vec2::new(4.0, 3.0)) + ); + + delta.revision.increment_by(1); + assert!(delta.withdraw(visible)); + let withdrawn = epoch(&delta); + let current = ViewSchedule::of(Arc::clone(&world), &withdrawn, &schedule_mask([row])); + assert_eq!(current.bounds(), None); + assert_eq!( + current + .occupancy() + .expect("an empty scope should have a profile") + .occupied_cells(Depth::MIN), + 0 + ); + assert_eq!(narrow.occupancy(), Some(occupancy)); + let current = ViewSchedule::of(Arc::clone(&world), &withdrawn, &admitted); + assert_eq!(current.bounds(), Some(base)); + assert_eq!(narrow.bounds(), Some(point)); + assert_eq!(saturated.bounds(), Some(base.union(point))); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(visible, legend("revived"), Vec2::ZERO), + Some(true) + ); + let revived = epoch(&delta); + let current = ViewSchedule::of(Arc::clone(&world), &revived, &schedule_mask([row])); + assert_eq!(current.bounds(), Some(point)); +} + +/// Distinguishes captured, scoped and corpus bounds after base withdrawal. +/// +/// Withdrawing every base node collapses a view scoped to only those nodes to no bounds, while a +/// view captured before the withdrawal keeps the original base bounds and a corpus-wide view +/// after the withdrawal still recovers the recorded base bounds. +#[test] +fn bounds_base_withdrawal() { + let (_fixture, mut delta) = fixture("view-bounds-base-withdrawal"); + let world = Arc::clone(&delta.world); + let recorded = world.layout.base_bounds(); + let initial = epoch(&delta); + let mask = schedule_mask((0..NODES).map(NodeRowId::new)); + let captured = ViewSchedule::of(Arc::clone(&world), &initial, &mask); + + delta.revision.increment_by(1); + for index in 0..NODES { + let identity = world + .layout + .index + .identity + .key_of(NodeRowId::new(index)) + .expect("should resolve the base identity"); + assert!(delta.withdraw(identity)); + } + let withdrawn = epoch(&delta); + let scoped = ViewSchedule::of(Arc::clone(&world), &withdrawn, &mask); + assert_eq!(scoped.bounds(), None); + assert_eq!(captured.bounds(), recorded); + let full = VisibilityMask::full(VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }); + let corpus = ViewSchedule::of(Arc::clone(&world), &withdrawn, &full); + assert_eq!(corpus.bounds(), recorded); +} + +/// Panics when scoped bounds include a non-finite placement. +/// +/// A corpus-wide view containing the same node still resolves its bounds. +#[test] +#[should_panic(expected = "visible placements must have finite coordinates")] +fn bounds_nonfinite_placement() { + let (_fixture, mut delta) = fixture("view-bounds-nonfinite"); + let identity = entity(100); + assert_eq!( + delta.update_node(identity, legend("nonfinite"), Vec2::new(f32::NAN, 0.0)), + Some(true) + ); + let row = delta.node_row(identity).expect("should allocate the row"); + let captured = epoch(&delta); + let base = ViewSchedule::of( + Arc::clone(&delta.world), + &captured, + &schedule_mask((0..NODES).map(NodeRowId::new)), + ); + assert_eq!(base.bounds(), delta.world.layout.base_bounds()); + ViewSchedule::of(Arc::clone(&delta.world), &captured, &schedule_mask([row])); +} + +/// Checks matched [`ViewSchedule`] and [`ScopeSchedule`] cuts over selected offsets. +/// +/// At each offset, the check visits every node row through the first unallocated row. It samples +/// the root and its child cells at every served zoom. Bounds, root delivery, minimum resolution, +/// and every visited node and cell entry must agree. +/// +/// # Panics +/// +/// Panics on fixed offset, zoom, or cell construction failure, on cut failure, if the root has no +/// children, or if the schedules disagree. +#[track_caller] +fn assert_scoped_delivery(world: &Arc, epoch: &Epoch, mask: &VisibilityMask) { + let view = ViewSchedule::of(Arc::clone(world), epoch, mask); + let (combined, bounds) = ScopeSchedule::of(&world.layout, epoch, mask); + assert_eq!(view.bounds(), bounds); + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + for offset in [0, 1, 5] { + let offset = Zoom::new(offset).expect("should fit the offset"); + let actual = view.cut(offset).expect("should bind the view"); + let expected = combined + .cut(world.schedule(), offset) + .expect("should bind the combined cascade"); + assert_eq!(actual.root_delivered(), expected.root_delivered()); + assert_eq!(actual.min_resolution(), expected.min_resolution()); + for index in 0..=world.layout.node_count(epoch) { + let node = NodeRowId::from_usize(index); + assert_eq!(actual.bucket_of(node), expected.bucket_of(node)); + assert_eq!(actual.first_zoom(node), expected.first_zoom(node)); + } + for zoom in 0..=world.schedule().max_tile_depth().get() { + let zoom = Zoom::new(zoom).expect("should fit the served zoom"); + for cell in iter::once(root).chain(root.children().expect("should have root children")) + { + assert_eq!(actual.total(zoom, cell), expected.total(zoom, cell)); + assert_eq!(actual.delta(zoom, cell), expected.delta(zoom, cell)); + assert_eq!(actual.children(zoom, cell), expected.children(zoom, cell)); + } + } + } +} + +/// Orders visible extension rows and preserves captured views across changes. +/// +/// Added rows delivered under a partial mask sort after the base rows and in identity-priority +/// order among themselves, matching a matched-scope reference walk. Withdrawing one added row +/// removes it from a fresh view while an earlier captured view's totals stay unaffected, and +/// reviving it restores the original delivery. +#[test] +fn schedule_extension_visibility() { + let (_fixture, mut delta) = fixture("schedule-extension-visibility"); + let world = Arc::clone(&delta.world); + let base = NodeRowId::MIN; + let position = world + .layout + .position(&epoch(&delta), base) + .expect("should resolve the base position"); + let higher = entity(300); + let lower = entity(200); + let hidden = entity(100); + for identity in [higher, lower, hidden] { + assert_eq!( + delta.update_node(identity, legend("extension"), position), + Some(true) + ); + } + let higher_row = delta + .node_row(higher) + .expect("should allocate the higher row"); + let lower_row = delta + .node_row(lower) + .expect("should allocate the lower row"); + let hidden_row = delta + .node_row(hidden) + .expect("should allocate the hidden row"); + let captured = epoch(&delta); + let mask = schedule_mask( + (0..NODES) + .map(NodeRowId::new) + .chain([higher_row, lower_row]), + ); + assert_scoped_delivery(&world, &captured, &mask); + let before = ViewSchedule::of(Arc::clone(&world), &captured, &mask); + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + let zoom = world.schedule().max_tile_depth(); + let cut = before.cut(Zoom::MIN).expect("should bind the view"); + let baseline = cut.total(zoom, root); + assert_eq!(cut.bucket_of(hidden_row), None); + assert_eq!( + baseline.rows.len(), + usize::try_from(NODES).expect("should fit the fixture count") + 2 + ); + let at = |node| { + baseline + .rows + .iter() + .position(|&row| row == node) + .expect("should deliver the row") + }; + assert!(at(base) < at(lower_row)); + assert!(at(lower_row) < at(higher_row)); + let base_only = ViewSchedule::of( + Arc::clone(&world), + &captured, + &schedule_mask((0..NODES).map(NodeRowId::new)), + ); + assert_eq!(before.occupancy(), base_only.occupancy()); + + delta.revision.increment_by(1); + assert!(delta.withdraw(lower), "should withdraw the extension row"); + let withdrawn = epoch(&delta); + assert_scoped_delivery(&world, &withdrawn, &mask); + let after = ViewSchedule::of(Arc::clone(&world), &withdrawn, &mask); + assert_eq!( + after + .cut(Zoom::MIN) + .expect("should bind the view") + .bucket_of(lower_row), + None + ); + assert_eq!( + before + .cut(Zoom::MIN) + .expect("should bind the captured view") + .total(zoom, root), + baseline + ); + assert_scoped_delivery(&world, &captured, &mask); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(lower, legend("revived"), Vec2::ZERO), + Some(true) + ); + let revived = ViewSchedule::of(Arc::clone(&world), &epoch(&delta), &mask); + assert_eq!( + revived + .cut(Zoom::MIN) + .expect("should bind the revived view") + .total(zoom, root), + baseline + ); +} + +/// Keeps corpus delivery stable across fitted-row withdrawal and revival. +/// +/// An added row past the base domain is assigned to the corpus's deepest bucket. Withdrawing a +/// base row removes it from a scoped view while a corpus-wide view recorded afterward reproduces +/// the same totals, root delivery, minimum resolution, and per-node schedule entries as before +/// the withdrawal, and reviving the base row restores agreement between the two schedule paths. +#[test] +fn schedule_base_withdrawal_dispatch() { + let (_fixture, mut delta) = fixture("schedule-base-withdrawal-dispatch"); + let world = Arc::clone(&delta.world); + let base = NodeRowId::MIN; + let base_identity = world + .layout + .index + .identity + .key_of(base) + .expect("should resolve the base identity"); + let position = world + .layout + .position(&epoch(&delta), base) + .expect("should resolve the base position"); + let extension = entity(100); + assert_eq!( + delta.update_node(extension, legend("extension"), position), + Some(true) + ); + let extension_row = delta + .node_row(extension) + .expect("should allocate the extension row"); + let captured = epoch(&delta); + let partial = schedule_mask((0..=NODES).map(NodeRowId::new)); + let full = VisibilityMask::full(VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }); + let corpus = ViewSchedule::of(Arc::clone(&world), &captured, &full); + let baseline = corpus.cut(Zoom::MIN).expect("should bind the corpus"); + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + let zoom = world.schedule().max_tile_depth(); + assert_eq!(baseline.bucket_of(extension_row), Some(baseline.deepest())); + assert_eq!( + baseline.total(zoom, root).rows.len(), + usize::try_from(NODES).expect("should fit the fixture count") + 1 + ); + assert_scoped_delivery(&world, &captured, &partial); + + delta.revision.increment_by(1); + assert!( + delta.withdraw(base_identity), + "should withdraw the base row" + ); + let withdrawn = epoch(&delta); + assert_scoped_delivery(&world, &withdrawn, &partial); + let scoped = ViewSchedule::of(Arc::clone(&world), &withdrawn, &partial); + assert_eq!( + scoped + .cut(Zoom::MIN) + .expect("should bind the scoped view") + .bucket_of(base), + None + ); + let corpus_after = ViewSchedule::of(Arc::clone(&world), &withdrawn, &full); + let recorded = corpus_after + .cut(Zoom::MIN) + .expect("should bind the recorded view"); + assert_eq!(recorded.total(zoom, root), baseline.total(zoom, root)); + assert_eq!(recorded.root_delivered(), baseline.root_delivered()); + assert_eq!(recorded.min_resolution(), baseline.min_resolution()); + assert_eq!( + recorded.first_zoom(extension_row), + baseline.first_zoom(extension_row) + ); + assert_eq!( + recorded.children(Zoom::MIN, root), + baseline.children(Zoom::MIN, root) + ); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(base_identity, legend("revived"), Vec2::ZERO), + Some(true) + ); + assert_scoped_delivery(&world, &epoch(&delta), &partial); +} + +/// Asserts that a delivery's per-bucket run lengths sum to its total delivered row count. +/// +/// # Panics +/// +/// Panics if the run lengths do not sum to the number of delivered rows. +#[track_caller] +fn assert_partitioned(delivered: &DeliveredNodes) { + assert_eq!( + delivered.runs.iter().sum::(), + delivered.rows.len(), + "the runs should re-sum to the delivered count" + ); +} + +/// Subtracts current withdrawals from a recorded corpus schedule. +/// +/// The schedule preserves the withdrawn row, and a revival restores the recorded delivery. +#[test] +fn walk_corpus_withdrawal() { + let (_fixture, mut delta) = fixture("walk-corpus-withdrawal"); + let world = Arc::clone(&delta.world); + let walk = Walk { + schedule: DeliverySchedule::corpus(&world), + index: &world.layout.index, + }; + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + let zoom = world.schedule().max_tile_depth(); + let node = NodeRowId::MIN; + let identity = world + .layout + .index + .identity + .key_of(node) + .expect("should resolve the fitted identity"); + + let captured = epoch(&delta); + let recorded = walk.schedule.total(zoom, root); + assert_eq!(walk.total(&captured, zoom, root), recorded); + assert_eq!( + walk.delta(&captured, Zoom::MIN, root), + walk.schedule.delta(Zoom::MIN, root) + ); + + delta.revision.increment_by(1); + assert!(delta.withdraw(identity), "should withdraw the fitted row"); + let withdrawn = epoch(&delta); + assert_eq!( + walk.schedule.total(zoom, root), + recorded, + "the recorded schedule should preserve the withdrawn row" + ); + + let subtracted = walk.total(&withdrawn, zoom, root); + assert!( + !subtracted.rows.contains(&node), + "the withdrawn row should leave the delivery" + ); + assert_eq!(subtracted.rows.len() + 1, recorded.rows.len()); + assert_eq!(subtracted.first_bucket, recorded.first_bucket); + assert_eq!(subtracted.runs.len(), recorded.runs.len()); + assert_partitioned(&subtracted); + assert_eq!( + walk.total(&captured, zoom, root), + recorded, + "the earlier epoch should preserve its delivery" + ); + + let bucket = walk + .schedule + .bucket_of(node) + .expect("the recorded schedule should keep the withdrawn row"); + let index = usize::from(bucket.get() - recorded.first_bucket.get()); + assert_eq!( + subtracted.runs[index] + 1, + recorded.runs[index], + "the withdrawal should debit the row's own bucket" + ); + + delta.revision.increment_by(1); + assert_eq!( + delta.update_node(identity, legend("revived"), Vec2::ZERO), + Some(true) + ); + assert_eq!( + walk.total(&epoch(&delta), zoom, root), + recorded, + "a revival should restore the recorded delivery" + ); +} + +/// Subtracts a newer withdrawal from the scoped schedule it reads. +#[test] +fn walk_scope_withdrawal() { + let (_fixture, mut delta) = fixture("walk-scope-withdrawal"); + let world = Arc::clone(&delta.world); + let root = MortonCell::new(Depth::MIN, 0, 0).expect("should construct the root"); + let zoom = world.schedule().max_tile_depth(); + let node = NodeRowId::MIN; + let identity = world + .layout + .index + .identity + .key_of(node) + .expect("should resolve the fitted identity"); + + let captured = epoch(&delta); + let mask = schedule_mask((0..NODES).map(NodeRowId::new)); + let view = ViewSchedule::of(Arc::clone(&world), &captured, &mask); + let walk = Walk { + schedule: view.cut(Zoom::MIN).expect("should bind the view"), + index: &world.layout.index, + }; + let recorded = walk.schedule.total(zoom, root); + assert!( + recorded.rows.contains(&node), + "the captured schedule should deliver the row" + ); + assert_eq!(walk.total(&captured, zoom, root), recorded); + + delta.revision.increment_by(1); + assert!(delta.withdraw(identity), "should withdraw the fitted row"); + let subtracted = walk.total(&epoch(&delta), zoom, root); + assert!( + !subtracted.rows.contains(&node), + "the withdrawn row should leave the captured delivery" + ); + assert_eq!(subtracted.rows.len() + 1, recorded.rows.len()); + assert_partitioned(&subtracted); +} + +/// A resolver that records the delivered node identities and answers no details. +struct RecordingResolver { + nodes: RefCell>, +} + +impl LocateResolver for RecordingResolver { + fn resolve( + &self, + request: LocateRequest<'_>, + ) -> Result, Report> { + self.nodes + .borrow_mut() + .extend(request.nodes.iter().map(|node| node.identity)); + Ok(None) + } +} + +/// A scene whose view schedule predates two partners linked to fixture node 4. +/// +/// The scene captures the schedule at one revision and reads its epoch at the next. The next +/// revision places the partners, and the schedule holds no delivery zoom for them. +struct LaterEpochScene { + world: Arc, + epoch: Epoch, + mask: VisibilityMask, + schedule: ViewSchedule, + source: ArchivedEntityId, + partners: [ArchivedEntityId; 2], + rows: [NodeRowId; 2], + _fixture: TamperFixture, +} + +impl LaterEpochScene { + /// Builds a scene fixture whose schedule predates added neighbor rows. + /// + /// Opens a synthetic world under `name`, captures a full-visibility schedule before two + /// partner nodes attach to fixture node 4, then places and links them so the captured + /// schedule predates their delivery zoom while their placement is already visible. + /// + /// # Panics + /// + /// Panics on fixture publication or world opening failure, a missing source row, rejected + /// partner or link insertion, row-domain exhaustion, zero-offset cut failure, or a failed + /// relationship assertion. + #[track_caller] + fn new(name: &str) -> Self { + let (files, mut delta) = fixture(name); + let world = Arc::clone(&delta.world); + let source_row = NodeRowId::new(4); + let source = world + .layout + .index + .identity + .key_of(source_row) + .expect("should resolve the fitted identity"); + let captured = epoch(&delta); + let actor = VisibilityActor { + id: ActorId::new(Uuid::nil(), ActorType::Machine), + instance_admin: false, + }; + let mask = VisibilityMask::full(actor); + let schedule = ViewSchedule::of(Arc::clone(&world), &captured, &mask); + + delta.revision.increment_by(1); + let partners = [entity(400), entity(401)]; + let links = [entity(410), entity(411)]; + let positions = [Vec2::new(0.25, 0.0), Vec2::new(0.5, 0.0)]; + let mut rows = [NodeRowId::MIN; 2]; + for (index, ((partner, link), position)) in + partners.into_iter().zip(links).zip(positions).enumerate() + { + assert_eq!( + delta.update_node(partner, legend("partner"), position), + Some(true) + ); + let row = delta + .node_row(partner) + .expect("should allocate the partner"); + assert_eq!( + delta.update_edge(link, legend("link"), Some([source_row, row])), + Some(true) + ); + rows[index] = row; + } + let later = epoch(&delta); + let delivery = schedule + .cut(Zoom::MIN) + .expect("should bind the zero offset"); + for row in rows { + assert!( + later.contains_node(row), + "the partner should have a placement" + ); + assert_eq!( + delivery.first_zoom(row), + None, + "the captured schedule should hold no delivery zoom for the partner" + ); + } + + Self { + world, + epoch: later, + mask, + schedule, + source, + partners, + rows, + _fixture: files, + } + } + + /// Builds a locate document capped at `edges` incident edges. + /// + /// The document uses the fixed source and this scene's captured world, epoch, mask and + /// schedule. + /// + /// # Errors + /// + /// Returns [`LocateDocumentError`] if document construction fails. + /// + /// # Panics + /// + /// Panics if the captured schedule cannot bind its zero-offset cut. + #[track_caller] + fn locate<'scene>( + &'scene self, + edges: u32, + resolver: &RecordingResolver, + ) -> Result, Report> { + let scene = Scene { + world: &self.world, + epoch: &self.epoch, + mask: &self.mask, + schedule: &self.schedule, + delivery: self + .schedule + .cut(Zoom::MIN) + .expect("should bind the zero offset"), + }; + LocateDocument::new( + scene, + LocateSource::Key(EntityId::from(self.source)), + &LocateDocumentOptions { + types: OntologySelection::new(&[]), + limits: LocateLimits { edges, .. }, + resolver, + }, + ) + } +} + +/// Skips partner ranking while a locate remains within capacity. +/// +/// Partners without a scheduled delivery zoom remain deliverable. A zero-capacity locate delivers +/// the source alone, also without ranking. +#[test] +fn locate_partner_after_schedule_complete() { + let scene = LaterEpochScene::new("delta-locate-partner-after-schedule-complete"); + let mut expected: Vec<_> = iter::once(scene.source).chain(scene.partners).collect(); + expected.sort_unstable(); + + for capacity in [2, 512] { + let resolver = RecordingResolver { + nodes: RefCell::new(Vec::new()), + }; + let _document = scene + .locate(capacity, &resolver) + .expect("should deliver the complete incident set without a delivery zoom"); + let mut delivered = resolver.nodes.borrow().clone(); + assert_eq!( + delivered[0], scene.source, + "should deliver the source first" + ); + delivered.sort_unstable(); + assert_eq!( + delivered, expected, + "should deliver both added partners at capacity {capacity}" + ); + } + + let resolver = RecordingResolver { + nodes: RefCell::new(Vec::new()), + }; + let _document = scene + .locate(0, &resolver) + .expect("should deliver an empty edge set at zero capacity without ranking"); + assert_eq!( + *resolver.nodes.borrow(), + [scene.source], + "should deliver the source alone at zero capacity" + ); +} + +/// Rejects an unrankable partner before hydrating a truncated locate. +/// +/// Ranking requires a delivery zoom for the partner in the schedule. +#[test] +fn locate_partner_after_schedule_truncated() { + let scene = LaterEpochScene::new("delta-locate-partner-after-schedule-truncated"); + let resolver = RecordingResolver { + nodes: RefCell::new(Vec::new()), + }; + let Err(report) = scene.locate(1, &resolver) else { + panic!("should refuse to rank a partner without a delivery zoom"); + }; + assert_matches!( + report.current_context(), + LocateDocumentError::Node { row } if scene.rows.contains(row), + ); + assert!( + resolver.nodes.borrow().is_empty(), + "should refuse before hydration" + ); +} + +/// Normalizes geometry against fitted rather than already-normalized bounds. +#[test] +fn normalize_fitted_frame() { + let (fixture, delta) = fixture("delta-normalize-fitted-frame"); + let bounds = fixture + .generation() + .repository() + .metadata + .evidence + .lod + .world; + let positions = [bounds.min(), bounds.centre(), bounds.max()]; + let normalized = delta.world.bounds().normalize_into(WIRE_FRAME, &positions); + assert_eq!( + normalized, + [Vec2::splat(-1.0), Vec2::ZERO, Vec2::splat(1.0)] + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/codec/cbor/mod.rs b/libs/@local/graph/atlas/src/serve/document/codec/cbor/mod.rs new file mode 100644 index 00000000000..1df83d72ed0 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/codec/cbor/mod.rs @@ -0,0 +1,174 @@ +#![expect( + clippy::big_endian_bytes, + reason = "CBOR arguments are network byte order (RFC 8949 section 3)" +)] + +#[cfg(test)] +mod tests; + +use core::{ + alloc::Allocator, + fmt::{self, Display}, +}; + +/// A [`fmt::Write`] adapter that appends UTF-8 bytes directly onto the wrapped buffer. +/// +/// It lets [`CborWriter::collect_text`] format a value without an intermediate [`String`]. +struct TextWriter<'bytes, A: Allocator>(&'bytes mut Vec); + +impl fmt::Write for TextWriter<'_, A> { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.0.extend_from_slice(s.as_bytes()); + Ok(()) + } +} + +/// Definite-length CBOR with shortest integers and source-width floats. +#[derive(Debug)] +pub(crate) struct CborWriter<'bytes, A: Allocator> { + bytes: &'bytes mut Vec, +} + +impl<'bytes, A: Allocator> CborWriter<'bytes, A> { + /// The head of a single-precision float. + const HEAD_F32: u8 = 0xFA; + /// The head of a double-precision float. + const HEAD_F64: u8 = 0xFB; + /// Major type 4: array. + const MAJOR_ARRAY: u8 = 4; + /// Major type 2: byte string. + const MAJOR_BYTES: u8 = 2; + /// Major type 5: map. + const MAJOR_MAP: u8 = 5; + /// Major type 1: negative integer. + const MAJOR_NEGATIVE: u8 = 1; + /// Major type 3: text string. + const MAJOR_TEXT: u8 = 3; + /// Major type 0: unsigned integer. + const MAJOR_UINT: u8 = 0; + /// The simple value `false`. + const SIMPLE_FALSE: u8 = 0xF4; + /// The simple value `null`. + const SIMPLE_NULL: u8 = 0xF6; + /// The simple value `true`. + const SIMPLE_TRUE: u8 = 0xF5; + + /// Opens a writer appending to `bytes`. + #[must_use] + pub(crate) const fn over(bytes: &'bytes mut Vec) -> Self { + Self { bytes } + } + + /// Emits an unsigned integer. + pub(crate) fn uint(&mut self, value: u64) { + self.head(Self::MAJOR_UINT, value); + } + + /// Emits a signed integer. + /// + /// Major type 0 for non-negative values, major type 1 otherwise, shortest form either way. + pub(crate) fn int(&mut self, value: i64) { + match u64::try_from(value) { + Ok(value) => self.head(Self::MAJOR_UINT, value), + // Major type 1 encodes the argument -1 - value. Complementing the two's-complement + // bits computes that argument without negating. For value in i64::MIN..=-1 the + // complement is in 0..=i64::MAX. Arithmetic negation overflows at i64::MIN instead. + Err(_) => self.head(Self::MAJOR_NEGATIVE, !(value.cast_unsigned())), + } + } + + /// Emits a boolean. + pub(crate) fn boolean(&mut self, value: bool) { + self.bytes.push(if value { + Self::SIMPLE_TRUE + } else { + Self::SIMPLE_FALSE + }); + } + + /// Emits `null`. + pub(crate) fn null(&mut self) { + self.bytes.push(Self::SIMPLE_NULL); + } + + /// Emits a float at single precision, including exactly representable small values. + pub(crate) fn f32(&mut self, value: f32) { + self.bytes.push(Self::HEAD_F32); + self.bytes.extend_from_slice(&value.to_be_bytes()); + } + + /// Emits a float at double precision, including exactly representable small values. + pub(crate) fn f64(&mut self, value: f64) { + self.bytes.push(Self::HEAD_F64); + self.bytes.extend_from_slice(&value.to_be_bytes()); + } + + /// Emits a byte string. + pub(crate) fn bytes(&mut self, value: &[u8]) { + self.head(Self::MAJOR_BYTES, value.len() as u64); + self.bytes.extend_from_slice(value); + } + + /// Emits a text string. + pub(crate) fn text(&mut self, value: &str) { + self.head(Self::MAJOR_TEXT, value.len() as u64); + self.bytes.extend_from_slice(value.as_bytes()); + } + + /// Formats `value` as one definite-length text string. + /// + /// The display implementation runs once. Existing buffer contents remain unchanged. + /// + /// # Panics + /// + /// This panics if [`Display::fmt`] returns an error. + pub(crate) fn collect_text(&mut self, value: impl Display) { + let start = self.bytes.len(); + fmt::write(&mut TextWriter(self.bytes), format_args!("{value}")) + .expect("formatting into a byte buffer should succeed"); + let end = self.bytes.len(); + + self.head(Self::MAJOR_TEXT, (end - start) as u64); + let header = self.bytes.len() - end; + self.bytes[start..].rotate_right(header); + } + + /// Emits an array head that the caller follows with `length` items. + pub(crate) fn array(&mut self, length: u64) { + self.head(Self::MAJOR_ARRAY, length); + } + + /// Emits a map head before `length` key-value pairs in ascending key order. + pub(crate) fn map(&mut self, length: u64) { + self.head(Self::MAJOR_MAP, length); + } + + /// Emits one head in shortest form. + #[expect( + clippy::cast_possible_truncation, + reason = "each arm's range check proves the narrowing lossless" + )] + fn head(&mut self, major: u8, argument: u64) { + let ty = major << 5; + match argument { + // Additional information 0..24: the argument is inline. + 0..0x18 => self.bytes.push(ty | argument as u8), + // 24 through 27: one, two, four, or eight argument bytes. + 0x18..=0xFF => self.bytes.extend_from_slice(&[ty | 0x18, argument as u8]), + 0x100..=0xFFFF => { + self.bytes.push(ty | 0x19); + self.bytes + .extend_from_slice(&(argument as u16).to_be_bytes()); + } + 0x1_0000..=0xFFFF_FFFF => { + self.bytes.push(ty | 0x1A); + self.bytes + .extend_from_slice(&(argument as u32).to_be_bytes()); + } + _ => { + self.bytes.push(ty | 0x1B); + self.bytes.extend_from_slice(&argument.to_be_bytes()); + } + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/codec/cbor/tests.rs b/libs/@local/graph/atlas/src/serve/document/codec/cbor/tests.rs new file mode 100644 index 00000000000..2206b88e6b6 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/codec/cbor/tests.rs @@ -0,0 +1,421 @@ +use core::{ + cell::Cell, + fmt::{self, Display}, +}; + +use super::CborWriter; + +/// Asserts `actual` equals the expected CBOR bytes for one unlabeled case. +/// +/// # Panics +/// +/// Panics if `actual` differs from `expected`. +#[track_caller] +fn assert_encoded(actual: &[u8], expected: &[u8]) { + assert_eq!(actual, expected, "should encode to the expected CBOR bytes"); +} + +/// Asserts `actual` equals the expected CBOR bytes, naming `case` in the failure message. +/// +/// # Panics +/// +/// Panics if `actual` differs from `expected`. +#[track_caller] +fn assert_case(actual: &[u8], expected: &[u8], case: impl Display) { + assert_eq!( + actual, expected, + "{case} should encode to the expected CBOR bytes" + ); +} + +/// A [`Display`] value that counts its own `fmt` calls. +/// +/// The count asserts that `collect_text` formats its argument exactly once. +struct CountedDisplay<'calls> { + calls: &'calls Cell, +} + +impl Display for CountedDisplay<'_> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.calls.set(self.calls.get() + 1); + fmt.write_str("counted") + } +} + +/// A [`Display`] value that writes its text across three separate `write_str` calls. +/// +/// Each call writes one multi-byte character. `collect_text` therefore meets a value that is both +/// multi-chunk and multi-byte per character. +struct ChunkedText; + +impl Display for ChunkedText { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.write_str("\u{65e5}")?; + fmt.write_str("\u{672c}")?; + fmt.write_str("\u{8a9e}") + } +} + +/// A [`Display`] value whose formatting always fails. +/// +/// It drives `collect_text`'s panic on a formatter error. +struct FailingDisplay; + +impl Display for FailingDisplay { + fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result { + Err(fmt::Error) + } +} + +/// Unsigned integers switch CBOR argument width at each boundary. +/// +/// The width is inline through 23, one extra byte through 255, two through 0xFFFF, four through +/// `u32::MAX`, and eight beyond it. +#[test] +fn uint_head_boundaries() { + let cases: &[(u64, &[u8])] = &[ + (0, &[0x00]), + (23, &[0x17]), + (24, &[0x18, 0x18]), + (255, &[0x18, 0xFF]), + (256, &[0x19, 0x01, 0x00]), + (0xFFFF, &[0x19, 0xFF, 0xFF]), + (0x0001_0000, &[0x1A, 0x00, 0x01, 0x00, 0x00]), + (0xFFFF_FFFF, &[0x1A, 0xFF, 0xFF, 0xFF, 0xFF]), + ( + 0x0001_0000_0000, + &[0x1B, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00], + ), + ]; + for &(value, expected) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).uint(value); + assert_case(&buffer, expected, format_args!("uint({value})")); + } +} + +/// `u64::MAX` encodes as the widest unsigned integer form. +#[test] +fn uint_max() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).uint(u64::MAX); + assert_encoded( + &buffer, + &[0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], + ); +} + +/// A non-negative signed integer encodes exactly as its unsigned value would. +#[test] +fn int_positive_dispatch() { + let cases: &[(i64, &[u8])] = &[(0, &[0x00]), (24, &[0x18, 0x18])]; + for &(value, expected) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).int(value); + assert_case(&buffer, expected, format_args!("int({value})")); + } +} + +/// Negative integers encode the argument `-1 - value` and switch width at each boundary. +/// +/// The boundaries match the unsigned ones at the equivalent magnitudes. +#[test] +fn int_negative_boundaries() { + let cases: &[(i64, &[u8])] = &[ + (-1, &[0x20]), + (-24, &[0x37]), + (-25, &[0x38, 0x18]), + (-256, &[0x38, 0xFF]), + (-257, &[0x39, 0x01, 0x00]), + (-0x0001_0000, &[0x39, 0xFF, 0xFF]), + (-0x0001_0001, &[0x3A, 0x00, 0x01, 0x00, 0x00]), + (-0x0001_0000_0000, &[0x3A, 0xFF, 0xFF, 0xFF, 0xFF]), + ( + -0x0001_0000_0001, + &[0x3B, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00], + ), + ]; + for &(value, expected) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).int(value); + assert_case(&buffer, expected, format_args!("int({value})")); + } +} + +/// `i64::MIN` encodes as the widest negative form. +/// +/// Its argument is `i64::MAX`, since `-1 - i64::MIN` equals `i64::MAX`. +#[test] +fn int_min() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).int(i64::MIN); + assert_encoded( + &buffer, + &[0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], + ); +} + +/// `i64::MAX` encodes exactly as the equal-valued unsigned integer would. +#[test] +fn int_max() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).int(i64::MAX); + assert_encoded( + &buffer, + &[0x1B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], + ); +} + +/// `true` encodes as its one-byte CBOR simple value. +#[test] +fn boolean_true() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).boolean(true); + assert_encoded(&buffer, &[0xF5]); +} + +/// `false` encodes as its one-byte CBOR simple value. +#[test] +fn boolean_false() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).boolean(false); + assert_encoded(&buffer, &[0xF4]); +} + +/// `null` encodes as its one-byte CBOR simple value. +#[test] +fn null_value() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).null(); + assert_encoded(&buffer, &[0xF6]); +} + +/// `1.0_f32` encodes as its exact single-precision bit pattern. +/// +/// The writer keeps the source width even where an exact half-precision form exists. +#[test] +fn f32_exact_small() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).f32(1.0); + assert_encoded(&buffer, &[0xFA, 0x3F, 0x80, 0x00, 0x00]); +} + +/// Positive and negative `f32` zero encode with different sign bits. +/// +/// Neither collapses into the other's representation. +#[test] +fn f32_signed_zero() { + let mut positive = Vec::new(); + let mut negative = Vec::new(); + CborWriter::over(&mut positive).f32(0.0); + CborWriter::over(&mut negative).f32(-0.0); + + assert_encoded(&positive, &[0xFA, 0x00, 0x00, 0x00, 0x00]); + assert_encoded(&negative, &[0xFA, 0x80, 0x00, 0x00, 0x00]); +} + +/// `1.0_f64` encodes as its exact double-precision bit pattern. +#[test] +fn f64_exact_small() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).f64(1.0); + assert_encoded( + &buffer, + &[0xFB, 0x3F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + ); +} + +/// Positive and negative `f64` zero encode with different sign bits. +/// +/// Neither collapses into the other's representation. +#[test] +fn f64_signed_zero() { + let mut positive = Vec::new(); + let mut negative = Vec::new(); + CborWriter::over(&mut positive).f64(0.0); + CborWriter::over(&mut negative).f64(-0.0); + + assert_encoded( + &positive, + &[0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + ); + assert_encoded( + &negative, + &[0xFB, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + ); +} + +/// An empty byte string encodes as a zero-length CBOR byte-string head with no payload. +#[test] +fn bytes_empty() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).bytes(&[]); + assert_encoded(&buffer, &[0x40]); +} + +/// A byte string encodes as its length head followed by its raw bytes. +#[test] +fn bytes_nonempty() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).bytes(&[0x01, 0x02, 0x03]); + assert_encoded(&buffer, &[0x43, 0x01, 0x02, 0x03]); +} + +/// An ASCII string encodes as its byte-length head followed by its UTF-8 bytes. +#[test] +fn text_ascii() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).text("hi"); + assert_encoded(&buffer, &[0x62, b'h', b'i']); +} + +/// A text string's length head counts its encoded UTF-8 bytes, not its character count. +#[test] +fn text_utf8_byte_lengths() { + let cases: &[(&str, &[u8])] = &[ + ("\u{e9}", &[0x62, 0xC3, 0xA9]), + ("\u{65e5}", &[0x63, 0xE6, 0x97, 0xA5]), + ("a\u{20ac}b", &[0x65, b'a', 0xE2, 0x82, 0xAC, b'b']), + ]; + for &(value, expected) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).text(value); + assert_case(&buffer, expected, format_args!("text({value:?})")); + } +} + +/// An array header encodes its element count and no element bytes. +/// +/// The count uses the same argument-width rules as an unsigned integer. +#[test] +fn array_header() { + let cases: &[(u64, &[u8])] = &[(0, &[0x80]), (3, &[0x83]), (24, &[0x98, 0x18])]; + for &(length, expected) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).array(length); + assert_case(&buffer, expected, format_args!("array({length})")); + } +} + +/// A map header encodes its pair count and no pair bytes. +/// +/// The count uses the same argument-width rules as an unsigned integer. +#[test] +fn map_header() { + let cases: &[(u64, &[u8])] = &[(0, &[0xA0]), (2, &[0xA2]), (24, &[0xB8, 0x18])]; + for &(length, expected) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).map(length); + assert_case(&buffer, expected, format_args!("map({length})")); + } +} + +/// Successive writer calls append their encodings back to back in call order. +/// +/// One value kind does not disturb another's bytes. +#[test] +fn sequential_writes() { + let mut buffer = Vec::new(); + let mut writer = CborWriter::over(&mut buffer); + writer.uint(1); + writer.boolean(true); + writer.text("ok"); + writer.null(); + + assert_encoded(&buffer, &[0x01, 0xF5, 0x62, b'o', b'k', 0xF6]); +} + +/// `collect_text` appends after the buffer's existing content instead of replacing it. +#[test] +fn collect_text_prefix() { + let mut buffer = vec![0xDE, 0xAD, 0xBE, 0xEF]; + let mut writer = CborWriter::over(&mut buffer); + writer.collect_text("hi"); + writer.uint(7); + + assert_encoded(&buffer, &[0xDE, 0xAD, 0xBE, 0xEF, 0x62, b'h', b'i', 0x07]); +} + +/// `collect_text` over a plain ASCII `&str` produces the same encoding `text` would. +#[test] +fn collect_text_ascii() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).collect_text("hi"); + assert_encoded(&buffer, &[0x62, b'h', b'i']); +} + +/// `collect_text` over an empty string produces a zero-length text head with no payload. +#[test] +fn collect_text_empty() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).collect_text(""); + assert_encoded(&buffer, &[0x60]); +} + +/// `collect_text`'s length head switches argument width at the `text` boundaries. +/// +/// Both count the same byte lengths, not character counts. +#[test] +fn collect_text_length_boundaries() { + let cases: &[(usize, &[u8])] = &[ + (23, &[0x77]), + (24, &[0x78, 0x18]), + (255, &[0x78, 0xFF]), + (256, &[0x79, 0x01, 0x00]), + (0xFFFF, &[0x79, 0xFF, 0xFF]), + (0x0001_0000, &[0x7A, 0x00, 0x01, 0x00, 0x00]), + ]; + for &(len, head) in cases { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).collect_text("a".repeat(len)); + + let mut expected = head.to_vec(); + expected.resize(head.len() + len, b'a'); + + assert_case(&buffer, &expected, format_args!("collect_text(len={len})")); + } +} + +/// Chunked formatting produces one CBOR text value. +/// +/// A multi-chunk [`Display`] value encodes as one contiguous [`CborWriter::text`] call would. +/// +/// The head counts the value's total UTF-8 bytes rather than its chunks or its characters. +#[test] +fn collect_text_chunked_unicode() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).collect_text(ChunkedText); + assert_encoded( + &buffer, + &[0x69, 0xE6, 0x97, 0xA5, 0xE6, 0x9C, 0xAC, 0xE8, 0xAA, 0x9E], + ); +} + +/// Text collection formats its argument exactly once. +/// +/// [`CborWriter::collect_text`] calls [`Display::fmt`] once, not once per internal write. +#[test] +fn collect_text_format_once() { + let calls = Cell::new(0_u32); + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).collect_text(CountedDisplay { calls: &calls }); + + assert_eq!( + calls.get(), + 1, + "Display::fmt should run exactly once per collect_text call" + ); + assert_encoded(&buffer, &[0x67, b'c', b'o', b'u', b'n', b't', b'e', b'd']); +} + +/// Text collection propagates a formatting error as a panic. +/// +/// [`CborWriter::collect_text`] panics if the argument's [`Display::fmt`] returns an error. +/// +/// Formatting into a byte buffer should never itself fail. +#[test] +#[should_panic(expected = "formatting into a byte buffer should succeed")] +fn collect_text_fmt_error() { + let mut buffer = Vec::new(); + CborWriter::over(&mut buffer).collect_text(FailingDisplay); +} diff --git a/libs/@local/graph/atlas/src/serve/document/codec/column/mod.rs b/libs/@local/graph/atlas/src/serve/document/codec/column/mod.rs new file mode 100644 index 00000000000..2c144bdeb49 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/codec/column/mod.rs @@ -0,0 +1,63 @@ +#[cfg(test)] +mod tests; + +use alloc::alloc::Allocator; + +use hashql_core::id::{Id, IdSlice, bit_vec::BitMatrix}; +use zerocopy::IntoBytes as _; + +use crate::{math::Vec2, postgres::id::ArchivedEntityId, serve::codec::EncodedRowId}; + +/// Little-endian columns in delivered-slot order. +pub(crate) struct ColumnWriter<'bytes, A: Allocator> { + bytes: &'bytes mut Vec, +} + +#[expect( + clippy::little_endian_bytes, + reason = "document columns use little-endian values" +)] +impl<'bytes, A: Allocator> ColumnWriter<'bytes, A> { + /// Opens a writer appending to `bytes`. + pub(crate) const fn over(bytes: &'bytes mut Vec) -> Self { + Self { bytes } + } + + /// Emits each row id as a little-endian column, in slot order. + pub(crate) fn rows(&mut self, values: &IdSlice>) { + self.bytes.reserve(size_of_val(values.as_raw())); + for &value in values { + self.bytes.extend_from_slice(&value.get().to_le_bytes()); + } + } + + /// Emits each position as an interleaved little-endian x/y column pair, in slot order. + pub(crate) fn positions(&mut self, values: &IdSlice) { + self.bytes.reserve(size_of_val(values.as_raw())); + for value in values { + self.bytes.extend_from_slice(&value.x().to_le_bytes()); + self.bytes.extend_from_slice(&value.y().to_le_bytes()); + } + } + + /// Emits each entity identity as a fixed-width byte column, in slot order. + pub(crate) fn identities(&mut self, values: &IdSlice) { + self.bytes.extend_from_slice(values.as_raw().as_bytes()); + } + + /// Emits each row in ceil(columns / 8) bytes, least-significant bit first. + pub(crate) fn masks(&mut self, values: &BitMatrix) { + let stride = values.col_domain_size().div_ceil(8); + self.bytes.reserve(values.row_domain_size() * stride); + for row in values.rows() { + self.bytes.extend( + values + .row(row) + .words() + .iter() + .flat_map(|word| word.to_le_bytes()) + .take(stride), + ); + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/codec/column/tests.rs b/libs/@local/graph/atlas/src/serve/document/codec/column/tests.rs new file mode 100644 index 00000000000..be4c1a2ce6b --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/codec/column/tests.rs @@ -0,0 +1,317 @@ +#![expect( + clippy::little_endian_bytes, + reason = "the tests build the writer's little-endian expectations" +)] + +use hashql_core::id::{Id as _, IdSlice, bit_vec::BitMatrix}; + +use super::ColumnWriter; +use crate::{ + identity::NodeRowId, + math::Vec2, + postgres::id::{ArchivedEntityId, ArchivedEntityUuid, ArchivedWebId}, + serve::{codec::EncodedRowId, document::TileSlot, membership::SelectionSlot}, +}; + +/// Derives the expected mask bytes from inserted `(row, col)` coordinates, not from the writer. +/// +/// Coordinates in `set` must lie inside the `rows` by `cols` grid. The byte index is `row * +/// ceil(cols / 8) + (col >> 3)` and the bit index is `col & 7`. An out-of-grid coordinate can still +/// address an allocated byte, setting a bit above the column count or in another row. +/// +/// # Panics +/// +/// Panics when the computed byte index is outside the allocated buffer. In overflow-checked builds, +/// also panics when the buffer-length or byte-index arithmetic overflows. +fn expected_mask_bytes(rows: usize, cols: usize, set: &[(usize, usize)]) -> Vec { + let stride = cols.div_ceil(8); + let mut expected = vec![0_u8; rows * stride]; + for &(row, col) in set { + expected[row * stride + (col >> 3)] |= 1 << (col & 7); + } + expected +} + +/// Encodes one matrix and compares it against the bytes its inserted coordinates imply. +/// +/// # Panics +/// +/// Panics when the encoded length or bytes differ from what `set` implies. Also propagates the +/// [`expected_mask_bytes`] panic conditions for the supplied `set`, independently of the matrix +/// contents. +#[track_caller] +fn assert_masks(matrix: &BitMatrix, set: &[(usize, usize)], case: &str) { + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).masks(matrix); + + let expected = expected_mask_bytes(matrix.row_domain_size(), matrix.col_domain_size(), set); + assert_eq!( + buffer.len(), + matrix.row_domain_size() * matrix.col_domain_size().div_ceil(8), + "{case} should emit ceil(columns / 8) bytes per row" + ); + assert_eq!( + buffer, expected, + "{case} should emit the bytes its inserted coordinates imply" + ); +} + +/// Row ids leave as four little-endian bytes each, in slot order. +/// +/// `0x0102_0304` is byte-asymmetric, where a palindromic value reads the same in either order. +#[test] +fn rows_little_endian_values() { + let values = [ + EncodedRowId::::new_unchecked(0x0000_0000), + EncodedRowId::new_unchecked(0x0000_0001), + EncodedRowId::new_unchecked(0x0102_0304), + EncodedRowId::new_unchecked(0xFFFF_FFFF), + ]; + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).rows(IdSlice::::from_raw(&values)); + + assert_eq!( + buffer, + [ + 0x00, 0x00, 0x00, 0x00, // + 0x01, 0x00, 0x00, 0x00, // + 0x04, 0x03, 0x02, 0x01, // + 0xFF, 0xFF, 0xFF, 0xFF, + ], + "should encode each row id as four little-endian bytes in slot order" + ); +} + +/// Row ids append after the buffer's existing content instead of replacing it. +#[test] +fn rows_prefix() { + let values = [EncodedRowId::::new_unchecked(0x1122_3344)]; + let mut buffer = vec![0xAA, 0xBB, 0xCC]; + + ColumnWriter::over(&mut buffer).rows(IdSlice::::from_raw(&values)); + + assert_eq!( + buffer, + [0xAA, 0xBB, 0xCC, 0x44, 0x33, 0x22, 0x11], + "should keep the existing prefix and append the row bytes after it" + ); +} + +/// Positions leave as each vector's `x` then `y`, little-endian and bit-identical to the input. +/// +/// The expectation restates each value's IEEE-754 bit pattern. `-0.0` and the smallest positive +/// subnormal are the values an incidental arithmetic pass would corrupt: adding zero turns +/// `-0.0` into `0.0`, and a flush-to-zero step turns the subnormal into `0.0`. +#[test] +fn positions_little_endian_pairs() { + let xs: [f32; 4] = [0.0, 1.5, f32::MIN, f32::from_bits(1)]; + let ys: [f32; 4] = [-0.0, -123.456, f32::MAX, f32::INFINITY]; + let values: Vec = xs.iter().zip(&ys).map(|(&x, &y)| Vec2::new(x, y)).collect(); + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).positions(IdSlice::::from_raw(&values)); + + let mut expected = Vec::new(); + for (&x, &y) in xs.iter().zip(&ys) { + expected.extend_from_slice(&x.to_bits().to_le_bytes()); + expected.extend_from_slice(&y.to_bits().to_le_bytes()); + } + + assert_eq!( + buffer, expected, + "should encode each vector's x then y as untouched little-endian f32 bytes" + ); +} + +/// Positions append after the buffer's existing content instead of replacing it. +#[test] +fn positions_prefix() { + let values = [Vec2::new(2.0, -4.0)]; + let mut buffer = vec![0x77]; + + ColumnWriter::over(&mut buffer).positions(IdSlice::::from_raw(&values)); + + assert_eq!( + buffer, + [0x77, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x80, 0xC0], + "should keep the prefix byte and append the vector's little-endian x then y bytes" + ); +} + +/// An identity leaves as its exact 32 bytes: the web id's 16, then the entity uuid's 16. +#[test] +fn identities_exact_32_bytes() { + let values = [ + ArchivedEntityId { + web_id: ArchivedWebId::from_bytes([0x01; 16]), + entity_uuid: ArchivedEntityUuid::from_bytes([0x02; 16]), + }, + ArchivedEntityId { + web_id: ArchivedWebId::from_bytes([0xAA; 16]), + entity_uuid: ArchivedEntityUuid::from_bytes([0xBB; 16]), + }, + ]; + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).identities(IdSlice::::from_raw(&values)); + + let mut expected = Vec::new(); + expected.extend_from_slice(&[0x01; 16]); + expected.extend_from_slice(&[0x02; 16]); + expected.extend_from_slice(&[0xAA; 16]); + expected.extend_from_slice(&[0xBB; 16]); + + assert_eq!( + buffer.len(), + 64, + "should emit exactly 32 bytes per identity" + ); + assert_eq!( + buffer, expected, + "should emit the web id's bytes then the entity uuid's bytes, per identity" + ); +} + +/// Identities append after the buffer's existing content instead of replacing it. +#[test] +fn identities_prefix() { + let values = [ArchivedEntityId { + web_id: ArchivedWebId::from_bytes([0x11; 16]), + entity_uuid: ArchivedEntityUuid::from_bytes([0x22; 16]), + }]; + let mut buffer = vec![0xEE, 0xEE]; + + ColumnWriter::over(&mut buffer).identities(IdSlice::::from_raw(&values)); + + let mut expected = vec![0xEE, 0xEE]; + expected.extend_from_slice(&[0x11; 16]); + expected.extend_from_slice(&[0x22; 16]); + + assert_eq!( + buffer, expected, + "should keep the prefix and append the identity's 32 bytes after it" + ); +} + +/// A matrix with no rows emits no bytes, whatever its column count. +#[test] +fn masks_zero_rows() { + let matrix = BitMatrix::::new(0, 5); + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).masks(&matrix); + + assert!( + buffer.is_empty(), + "should emit no bytes for a zero-row matrix" + ); +} + +/// A matrix with no columns emits no bytes per row, whatever its row count. +#[test] +fn masks_zero_columns() { + let matrix = BitMatrix::::new(3, 0); + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).masks(&matrix); + + assert!( + buffer.is_empty(), + "should emit no bytes per row when the column domain is empty" + ); +} + +/// Each row occupies `ceil(columns / 8)` bytes, at widths around the byte and word boundaries. +#[test] +fn masks_stride_widths() { + for (cols, stride, bit) in [ + (1_usize, 1_usize, 0_usize), + (8, 1, 7), + (9, 2, 8), + (63, 8, 62), + (64, 8, 63), + (65, 9, 64), + ] { + let mut matrix = BitMatrix::::new(1, cols); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(bit)); + + assert_eq!( + cols.div_ceil(8), + stride, + "the fixture's own stride for {cols} columns should be {stride}" + ); + assert_masks(&matrix, &[(0, bit)], &format!("{cols} columns")); + } +} + +/// Within one byte, column 0 is the least-significant bit. +/// +/// Columns 0, 1 and 4 pack least-significant-bit first as `0b0001_0011`. A most-significant-bit +/// first writer emits `0b1100_1000` from the same three columns. +#[test] +fn masks_lsb_first() { + let mut matrix = BitMatrix::::new(1, 8); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(0)); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(1)); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(4)); + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).masks(&matrix); + + assert_eq!( + buffer, + [0b0001_0011], + "columns 0, 1 and 4 should pack least-significant-bit first" + ); +} + +/// Rows leave in matrix row order, each with its own pattern. +#[test] +fn masks_row_order() { + let mut matrix = BitMatrix::::new(3, 8); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(0)); + matrix.insert(TileSlot::from_usize(1), SelectionSlot::from_usize(3)); + matrix.insert(TileSlot::from_usize(2), SelectionSlot::from_usize(7)); + + let mut buffer = Vec::new(); + ColumnWriter::over(&mut buffer).masks(&matrix); + + assert_eq!( + buffer, + [0b0000_0001, 0b0000_1000, 0b1000_0000], + "should emit the three rows in matrix order, each with its own pattern" + ); +} + +/// Per-row patterns spanning the 64-bit word boundary at column 64 encode independently. +#[test] +fn masks_word_boundary_patterns() { + let mut matrix = BitMatrix::::new(2, 65); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(63)); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(64)); + matrix.insert(TileSlot::from_usize(1), SelectionSlot::from_usize(0)); + matrix.insert(TileSlot::from_usize(1), SelectionSlot::from_usize(64)); + + assert_masks( + &matrix, + &[(0, 63), (0, 64), (1, 0), (1, 64)], + "two rows across the word boundary", + ); +} + +/// Masks append after the buffer's existing content instead of replacing it. +#[test] +fn masks_prefix() { + let mut matrix = BitMatrix::::new(1, 8); + matrix.insert(TileSlot::from_usize(0), SelectionSlot::from_usize(2)); + let mut buffer = vec![0x99]; + + ColumnWriter::over(&mut buffer).masks(&matrix); + + assert_eq!( + buffer, + [0x99, 0b0000_0100], + "should keep the prefix byte and append the row's mask byte after it" + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/codec/envelope.rs b/libs/@local/graph/atlas/src/serve/document/codec/envelope.rs new file mode 100644 index 00000000000..112aff8bce4 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/codec/envelope.rs @@ -0,0 +1,375 @@ +use alloc::alloc::Allocator; + +use error_stack::Report; +use zerocopy::{IntoBytes as _, LE, U16, U32}; + +use super::{Kind, WIRE_VERSION}; + +/// The 16-byte envelope prefix. +#[derive(zerocopy::IntoBytes, zerocopy::Immutable)] +#[repr(C)] +struct Prefix { + kind: Kind, + version: U16, + flags: U16, + slots: U16, + reserved: U16, +} + +/// A recorded slot's byte extent within the envelope directory. +#[derive(zerocopy::IntoBytes, zerocopy::Immutable)] +#[repr(C)] +struct Entry { + start: U32, + end: U32, +} + +/// The prefix's encoded byte width. +const PREFIX: usize = size_of::(); +/// One directory entry's encoded byte width. +const ENTRY: usize = size_of::(); +/// The completed binary envelope's media type. +const BINARY_CONTENT_TYPE: &str = "application/vnd.hash.saltile-v1"; + +/// Response metadata from a completed document writer. +#[derive(Debug)] +pub(crate) struct Envelope { + content_type: &'static str, +} + +impl Envelope { + /// Returns the response's `Content-Type` value. + pub(crate) const fn content_type(&self) -> &'static str { + self.content_type + } + + /// Replaces `bytes` with JSON and returns completion after serialization succeeds. + /// + /// # Errors + /// + /// Returns the serializer's error if `value` cannot be represented as JSON. The buffer may + /// contain a partial document on failure. + pub(crate) fn encode_json( + value: &impl serde::Serialize, + bytes: &mut Vec, + ) -> Result> { + bytes.clear(); + serde_json::to_writer(bytes, value).map_err(Report::new)?; + Ok(Self { + content_type: "application/json", + }) + } +} + +/// A writer for one binary envelope. +/// +/// The layout is a fixed-size directory of slot extents, followed by each slot's bytes in +/// declaration order. +#[derive(Debug)] +pub(crate) struct EnvelopeWriter<'bytes, A: Allocator> { + bytes: &'bytes mut Vec, + slots: u16, + recorded: u16, +} + +impl<'bytes, A: Allocator> EnvelopeWriter<'bytes, A> { + /// Initializes the envelope prefix and its empty directory. + /// + /// This clears the buffer first. The prefix therefore begins at byte zero whatever the buffer + /// held before, and every recorded extent is an offset into this envelope alone. + pub(crate) fn new(kind: Kind, slots: u16, bytes: &'bytes mut Vec) -> Self { + let prefix = Prefix { + kind, + version: U16::new(WIRE_VERSION), + flags: U16::ZERO, + slots: U16::new(slots), + reserved: U16::ZERO, + }; + + bytes.clear(); + bytes.extend_from_slice(prefix.as_bytes()); + bytes.resize(PREFIX + ENTRY * slots as usize, 0); + + Self { + bytes, + slots, + recorded: 0, + } + } + + /// Reserves `additional` bytes of spare capacity in the backing buffer. + pub(crate) fn reserve(&mut self, additional: usize) { + self.bytes.reserve(additional); + } + + /// Runs `write` and records its output as the next directory slot. + /// + /// The writer pads the slot's bytes to an 8-byte boundary. + /// + /// # Panics + /// + /// Panics if every declared slot is already recorded, if `write` leaves the buffer shorter + /// than it started, or, through [`record`](Self::record), if the slot's extent does not fit + /// u32. + pub(crate) fn slot(&mut self, write: impl FnOnce(&mut Vec)) { + assert!( + self.recorded < self.slots, + "the envelope declares {} slots, all recorded", + self.slots, + ); + + let start = self.bytes.len(); + write(self.bytes); + let end = self.bytes.len(); + assert!(end >= start, "a slot writer must only append"); + + self.bytes.resize(end.next_multiple_of(8), 0); + + self.record(start, end); + } + + /// Records the next slot as empty, without writing any bytes. + /// + /// # Panics + /// + /// Panics if every declared slot is already recorded, or on the first slot: slot 0 (`HEAD`) + /// must always be present. + pub(crate) fn skip(&mut self) { + assert!( + self.recorded < self.slots, + "the envelope declares {} slots, all recorded", + self.slots, + ); + assert!(self.recorded > 0, "slot 0 (HEAD) is always present"); + + self.recorded += 1; + } + + /// Completes the envelope as the binary media type, once every declared slot is recorded. + /// + /// # Panics + /// + /// Panics unless every declared slot has been recorded through [`slot`](Self::slot) or + /// [`skip`](Self::skip). + pub(crate) fn finish(self) -> Envelope { + assert_eq!( + self.recorded, self.slots, + "the envelope declares {} slots", + self.slots, + ); + Envelope { + content_type: BINARY_CONTENT_TYPE, + } + } + + /// Appends a trailer through `write`, then completes the envelope. + /// + /// The trailer follows the last recorded slot, and the envelope reports the binary media + /// type. + /// + /// # Panics + /// + /// Panics unless every declared slot has been recorded through [`slot`](Self::slot) or + /// [`skip`](Self::skip). + pub(crate) fn finish_with_trailer(self, write: impl FnOnce(&mut Vec)) -> Envelope { + assert_eq!( + self.recorded, self.slots, + "the envelope declares {} slots", + self.slots, + ); + + write(self.bytes); + Envelope { + content_type: BINARY_CONTENT_TYPE, + } + } + + /// Backfills the directory entry for the most recently written slot. + /// + /// # Panics + /// + /// Panics where `start` or `end` exceeds [`u32::MAX`]. The directory stores both as u32, and + /// an envelope whose slots reach 4 GiB therefore refuses to complete rather than recording a + /// wrapped offset a decoder would follow into the wrong bytes. + fn record(&mut self, start: usize, end: usize) { + let entry = Entry { + start: U32::new( + u32::try_from(start).expect("directory offsets fit u32: payloads end below 4 GiB"), + ), + end: U32::new( + u32::try_from(end).expect("directory offsets fit u32: payloads end below 4 GiB"), + ), + }; + + let at = PREFIX + ENTRY * self.recorded as usize; + self.bytes[at..at + ENTRY].copy_from_slice(zerocopy::IntoBytes::as_bytes(&entry)); + self.recorded += 1; + } +} + +#[cfg(test)] +mod tests { + use alloc::alloc::Global; + + use serde::{ + Serialize, Serializer, + ser::{Error as _, SerializeSeq as _}, + }; + + use super::{Envelope, EnvelopeWriter}; + use crate::serve::document::codec::{Kind, WIRE_VERSION}; + + #[test] + fn binary_content_type() { + let expected = format!("application/vnd.hash.saltile-v{WIRE_VERSION}"); + for kind in [Kind::TILE, Kind::EDGES, Kind::LOCATE] { + for trailer in [false, true] { + let mut bytes = Vec::new(); + let mut writer = EnvelopeWriter::new(kind, 1, &mut bytes); + writer.slot(|bytes| bytes.push(0xA0)); + let envelope = if trailer { + writer.finish_with_trailer(|bytes| bytes.push(0xA0)) + } else { + writer.finish() + }; + assert_eq!( + envelope.content_type(), + expected, + "both finalizers should report the binary envelope version for every kind", + ); + } + } + } + + /// Writing into a buffer that already holds unrelated bytes clears them. + /// + /// The prefix therefore begins at byte zero. The writer records each slot's write extent - a + /// written slot, an empty slot, a skipped slot and a fourth written slot - at its own + /// directory offset, and writes each slot's bytes at the extent recorded for it. The trailer + /// follows the fourth slot's padding, and no directory entry records it. + #[expect( + clippy::little_endian_bytes, + reason = "the test decodes the envelope directory" + )] + #[test] + fn buffer_reuse() { + let mut bytes = Vec::new_in(&Global); + bytes.resize(128, 0xDD); + let mut writer = EnvelopeWriter::new(Kind::EDGES, 4, &mut bytes); + writer.slot(|bytes| bytes.push(0xA0)); + writer.slot(|_| {}); + writer.skip(); + writer.slot(|bytes| bytes.extend_from_slice(&[1, 2, 3])); + let _completed = writer.finish_with_trailer(|bytes| bytes.push(0xF6)); + + assert_eq!(&bytes[..16], b"SALTILEE\x01\x00\x00\x00\x04\x00\x00\x00"); + for (slot, expected) in [(48, 49), (56, 56), (0, 0), (56, 59)] + .into_iter() + .enumerate() + { + let at = 16 + slot * 8; + let start = u32::from_le_bytes( + bytes[at..at + 4] + .try_into() + .expect("should contain a start offset"), + ); + let end = u32::from_le_bytes( + bytes[at + 4..at + 8] + .try_into() + .expect("should contain an end offset"), + ); + assert_eq!( + (start, end), + expected, + "directory slot {slot} should retain its extent" + ); + } + assert_eq!( + &bytes[48..], + &[0xA0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0xF6] + ); + } + + /// Encoding a JSON value replaces a buffer's unrelated bytes with the value alone. + /// + /// The resulting envelope reports the `application/json` content type. + #[test] + fn json_buffer_reuse() { + let mut bytes = Vec::new_in(&Global); + bytes.extend_from_slice(b"previous document"); + let envelope = + Envelope::encode_json(&serde_json::json!({"nodes": {}, "edges": {}}), &mut bytes) + .expect("should complete JSON serialization"); + assert_eq!( + envelope.content_type(), + "application/json", + "should report the media type of the completed JSON document", + ); + assert_eq!( + serde_json::from_slice::(&bytes) + .expect("should parse one complete document"), + serde_json::json!({"nodes": {}, "edges": {}}), + ); + } + + /// A value whose [`Serialize`] impl writes one sequence element and then fails. + /// + /// It asserts that a mid-serialization error leaves its partial output in the buffer. + struct PartialValue; + + impl Serialize for PartialValue { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(2))?; + sequence.serialize_element(&1)?; + Err(S::Error::custom("incomplete sequence")) + } + } + + /// Encoding a value that fails partway returns the serializer's error. + /// + /// The buffer holds exactly the bytes written before the failure, neither cleared nor + /// completed. + #[test] + fn json_partial_value() { + let mut bytes = Vec::from(b"previous document".as_slice()); + let error = Envelope::encode_json(&PartialValue, &mut bytes) + .expect_err("should return a serialization error rather than completion"); + assert_eq!(error.current_context().to_string(), "incomplete sequence"); + assert_eq!( + bytes, b"[1", + "should retain the serializer's partial output" + ); + } + + /// Finishing a writer without filling every declared slot panics. + /// + /// The panic message names the declared slot count. + #[test] + #[should_panic(expected = "the envelope declares 4 slots")] + fn finish_incomplete() { + let mut bytes = Vec::new(); + let writer = EnvelopeWriter::new(Kind::EDGES, 4, &mut bytes); + let _completed = writer.finish(); + } + + /// Finishing with a trailer, without filling every other declared slot, panics. + /// + /// The panic message names the declared slot count. This is the same guard + /// [`finish_incomplete`] exercises without a trailer. + #[test] + #[should_panic(expected = "the envelope declares 4 slots")] + fn trailer_incomplete() { + let mut bytes = Vec::new(); + let writer = EnvelopeWriter::new(Kind::EDGES, 4, &mut bytes); + let _completed = writer.finish_with_trailer(|bytes| bytes.push(0xA0)); + } + + /// Skipping slot 0 (the always-present HEAD slot) panics rather than silently omitting it. + #[test] + #[should_panic(expected = "slot 0 (HEAD) is always present")] + fn head_absent() { + let mut bytes = Vec::new(); + let mut writer = EnvelopeWriter::new(Kind::EDGES, 4, &mut bytes); + writer.skip(); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/codec/mod.rs b/libs/@local/graph/atlas/src/serve/document/codec/mod.rs new file mode 100644 index 00000000000..ba7f20dd2d5 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/codec/mod.rs @@ -0,0 +1,72 @@ +mod cbor; +mod column; +mod envelope; + +use alloc::alloc::Allocator; + +pub(super) use self::column::ColumnWriter; +pub(crate) use self::{ + cbor::CborWriter, + envelope::{Envelope, EnvelopeWriter}, +}; + +/// The envelope wire version, prefix field and media-type suffix. +/// +/// The media type `application/vnd.hash.saltile-v1` must agree with this value. The version applies +/// to every [`Kind`]. +pub(crate) const WIRE_VERSION: u16 = 1; + +/// The envelope's 8-byte magic, identifying which document a binary envelope carries. +#[derive(Debug, Copy, Clone, PartialEq, Eq, zerocopy::IntoBytes, zerocopy::Immutable)] +#[repr(transparent)] +pub(crate) struct Kind([u8; 8]); + +impl Kind { + /// The edges document's magic. + pub(crate) const EDGES: Self = Self(*b"SALTILEE"); + /// The locate document's magic. + pub(crate) const LOCATE: Self = Self(*b"SALTILEL"); + /// The tile document's magic. + pub(crate) const TILE: Self = Self(*b"SALTILET"); +} + +/// A tile delivery mode, `HEAD` key 3. +/// +/// Requests carry the mode as the JSON strings `"delta"` and `"total"`. Delta is the default. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, serde::Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "lowercase")] +pub(crate) enum Mode { + /// The tile's own additions - the points its zoom's cut admits that no ancestor delivered. + #[default] + Delta, + /// Every ancestor's delivery accumulated into this tile, which renders alone. + Total, +} + +impl Mode { + /// Returns the mode's wire code. + #[must_use] + pub(crate) const fn code(self) -> u64 { + match self { + Self::Delta => 0, + Self::Total => 1, + } + } +} + +/// Emits one detail array: text entries, `null` for an empty entry. +pub(super) fn encode_details( + cbor: &mut CborWriter<'_, A>, + entries: impl ExactSizeIterator>, +) { + cbor.array(entries.len() as u64); + for entry in entries { + let entry = entry.as_ref(); + + if entry.is_empty() { + cbor.null(); + } else { + cbor.text(entry); + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/current.rs b/libs/@local/graph/atlas/src/serve/document/current.rs new file mode 100644 index 00000000000..11a91c62273 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/current.rs @@ -0,0 +1,58 @@ +use alloc::alloc::Allocator; + +use error_stack::Report; + +use super::{Document, codec::Envelope}; +use crate::file::generation::GenerationId; + +/// The active generation captured for one request. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +pub(crate) struct CurrentDocument { + generation: GenerationId, +} + +impl CurrentDocument { + /// Captures `generation` as the active-generation document. + pub(crate) const fn new(generation: GenerationId) -> Self { + Self { generation } + } +} + +impl Document for CurrentDocument { + type Error = Report; + + fn encode(&self, buffer: &mut Vec) -> Result { + Envelope::encode_json(self, buffer) + } +} + +#[cfg(test)] +mod tests { + use alloc::alloc::Global; + + use super::CurrentDocument; + use crate::{file::generation::GenerationId, serve::document::Document as _}; + + /// Encoding a [`CurrentDocument`] replaces the buffer's bytes. + /// + /// What remains is the generation's lowercase-hexadecimal JSON encoding alone, not an appended + /// copy of it. + #[test] + fn json_generation() { + let generation: GenerationId = + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + .parse() + .expect("should parse the generation identity"); + let document = CurrentDocument::new(generation); + let mut bytes = Vec::new_in(&Global); + bytes.extend_from_slice(b"previous document"); + let _completed = document + .encode(&mut bytes) + .expect("should complete the current-generation response"); + assert_eq!( + bytes, + br#"{"generation":"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}"#, + "should encode the supplied generation in canonical hexadecimal", + ); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/edges/codec/mod.rs b/libs/@local/graph/atlas/src/serve/document/edges/codec/mod.rs new file mode 100644 index 00000000000..dc1a09202b5 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/edges/codec/mod.rs @@ -0,0 +1,114 @@ +#[cfg(test)] +mod tests; + +use alloc::alloc::Allocator; + +use hashql_core::id::Id as _; +use zerocopy::IntoBytes as _; + +use super::{EdgesDocument, EdgesTrailer}; +use crate::{ + file::generation::GenerationId, + identity::NodeRowId, + postgres::id::ArchivedEntityId, + serve::{ + codec::EncodedRowId, + document::codec::{ + CborWriter, ColumnWriter, Envelope, EnvelopeWriter, Kind, encode_details, + }, + }, +}; + +/// One edges response in writable form. +#[derive(Debug)] +pub(crate) struct EdgesResponse<'doc> { + /// `HEAD` key 0: the generation identity, echoing the route. + pub generation: GenerationId, + /// `HEAD` key 1: the variant index, echoing the route. + pub variant: u64, + pub document: &'doc EdgesDocument<'doc>, +} + +impl EdgesResponse<'_> { + /// Reserve allowance for the `HEAD` payload and the slot padding. + /// + /// `HEAD` is `map(5)` with one-byte uint keys. Its payload is a 34-byte generation echo, + /// two uints of at most nine encoded bytes, and two one-byte booleans, reaching 60 bytes at + /// the ceiling, and the four slots pad to 8-byte boundaries for at most 28 more. The + /// trailer is not counted, because its extent is store-shaped text, unknowable before + /// hydration. + const HEAD_AND_PADDING: usize = 96; + /// Bytes per delivered edge across the three columns: source, target, identity. + const ROW_SIZE: usize = size_of::>() + + size_of::>() + + size_of::(); + + /// Encodes the response as one `SALTILEE` envelope. + /// + /// # Panics + /// + /// This panics when a directory offset exceeds `u32::MAX`. + pub(crate) fn encode_into(&self, buffer: &mut Vec) -> Envelope { + let count = self.document.ids.len(); + let mut envelope = EnvelopeWriter::new(Kind::EDGES, 4, buffer); + + envelope.reserve(Self::HEAD_AND_PADDING + count * Self::ROW_SIZE); + envelope.slot(|buf| self.encode_head(buf, count as u64)); + envelope.slot(|buf| ColumnWriter::over(buf).rows(&self.document.sources)); + envelope.slot(|buf| ColumnWriter::over(buf).rows(&self.document.targets)); + envelope.slot(|buf| ColumnWriter::over(buf).identities(&self.document.ids)); + + match &self.document.trailer { + Some(trailer) => envelope.finish_with_trailer(|buf| Self::encode_trailer(buf, trailer)), + None => envelope.finish(), + } + } + + /// Encodes the `HEAD` map: keys 0 through 4. + fn encode_head(&self, buf: &mut Vec, count: u64) { + let mut cbor = CborWriter::over(buf); + cbor.map(5); + + cbor.uint(0); + cbor.bytes(self.generation.as_bytes()); + + cbor.uint(1); + cbor.uint(self.variant); + + cbor.uint(2); + cbor.uint(count); + + cbor.uint(3); + cbor.boolean(self.document.complete); + + cbor.uint(4); + cbor.boolean(self.document.trailer.is_some()); + } + + /// Encodes the edges trailer map. + /// + /// The map holds the interned representative type URLs, the per-edge labels, and each edge's + /// representative-type index into that interned list. + fn encode_trailer(buf: &mut Vec, trailer: &EdgesTrailer<'_>) { + let mut cbor = CborWriter::over(buf); + cbor.map(3); + + cbor.uint(0); + cbor.array(trailer.representative_type_urls_interner.len() as u64); + for url in trailer.representative_type_urls_interner.entries() { + cbor.collect_text(url); + } + + cbor.uint(1); + encode_details(&mut cbor, trailer.labels.iter()); + + cbor.uint(2); + cbor.array(trailer.representative_type_urls.len() as u64); + for &entry in &trailer.representative_type_urls { + match entry { + Some(index) => cbor.uint(index.as_u64()), + None => cbor.null(), + } + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/edges/codec/tests.rs b/libs/@local/graph/atlas/src/serve/document/edges/codec/tests.rs new file mode 100644 index 00000000000..f1563e2416a --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/edges/codec/tests.rs @@ -0,0 +1,535 @@ +#![expect( + clippy::little_endian_bytes, + reason = "the tests read the envelope directory and build its little-endian columns" +)] + +use alloc::alloc::Global; + +use hashql_core::id::IdVec; +use type_system::ontology::VersionedUrl; + +use super::{ + super::{EdgeSlot, EdgesDocument, EdgesTrailer}, + EdgesResponse, +}; +use crate::{ + dataset::auxiliary::Label, + file::generation::GenerationId, + identity::NodeRowId, + integrity::Sha256Digest, + postgres::id::{ArchivedEntityId, ArchivedEntityUuid, ArchivedWebId}, + serve::{codec::EncodedRowId, document::Document as _, intern::InternTable}, +}; + +/// The envelope prefix width. +const PREFIX: usize = 16; +/// The width of one directory entry. +const ENTRY: usize = 8; +/// The slot count of a `SALTILEE` envelope. +const SLOTS: usize = 4; + +/// The one-byte CBOR encoding of `true`. +const CBOR_TRUE: u8 = 0xF5; +/// The one-byte CBOR encoding of `false`. +const CBOR_FALSE: u8 = 0xF4; +/// The one-byte CBOR encoding of `null`. +const CBOR_NULL: u8 = 0xF6; + +/// Encodes one CBOR head in shortest form (RFC 8949 section 3). +/// +/// # Panics +/// +/// Panics above 255. No expectation in this module reaches a two-byte argument. +fn cbor_head(major: u8, argument: u64) -> Vec { + let ty = major << 5; + match argument { + 0..0x18 => vec![ty | u8::try_from(argument).expect("should fit the inline argument")], + 0x18..=0xFF => vec![ + ty | 0x18, + u8::try_from(argument).expect("should fit the one-byte argument"), + ], + _ => panic!("should stay below a two-byte CBOR argument"), + } +} + +/// Encodes `value` as a CBOR unsigned integer. +/// +/// # Panics +/// +/// Panics above 255, the head's limit. +fn cbor_uint(value: u64) -> Vec { + cbor_head(0, value) +} + +/// Encodes `value` as a CBOR byte string. +/// +/// # Panics +/// +/// Panics above 255 bytes, the head's limit. +fn cbor_bytes(value: &[u8]) -> Vec { + let mut bytes = cbor_head(2, value.len() as u64); + bytes.extend_from_slice(value); + bytes +} + +/// Encodes `value` as a CBOR text string. +/// +/// # Panics +/// +/// Panics above 255 bytes of UTF-8, the head's limit. +fn cbor_text(value: &str) -> Vec { + let mut bytes = cbor_head(3, value.len() as u64); + bytes.extend_from_slice(value.as_bytes()); + bytes +} + +/// Encodes `value` as its one-byte CBOR boolean. +fn cbor_bool(value: bool) -> Vec { + vec![if value { CBOR_TRUE } else { CBOR_FALSE }] +} + +/// Concatenates one definite-length array from its items. +/// +/// # Panics +/// +/// Panics above 255 items, the head's limit. +fn cbor_array(items: impl IntoIterator>) -> Vec { + let items: Vec<_> = items.into_iter().collect(); + let mut bytes = cbor_head(4, items.len() as u64); + for item in items { + bytes.extend(item); + } + bytes +} + +/// Concatenates one definite-length map from its unsigned keys and encoded values. +/// +/// # Panics +/// +/// Panics above 255 pairs, and above a key of 255: both go through the one-byte head. +fn cbor_map(pairs: impl IntoIterator)>) -> Vec { + let pairs: Vec<_> = pairs.into_iter().collect(); + let mut bytes = cbor_head(5, pairs.len() as u64); + for (key, value) in pairs { + bytes.extend(cbor_uint(key)); + bytes.extend(value); + } + bytes +} + +/// Reads one directory entry, never a slot's payload. +/// +/// # Panics +/// +/// Panics if the computed entry slices lie outside `buffer`. With overflow checking, computing +/// `PREFIX + ENTRY * slot` or either slice's end can also panic on usize overflow. +/// +/// # Warning +/// +/// Without overflow checking, overflowing offset arithmetic wraps. If the resulting slices lie +/// within `buffer`, the helper reads those bytes even when the mathematical directory offset lies +/// beyond the buffer. +fn slot_range(buffer: &[u8], slot: usize) -> (usize, usize) { + let at = PREFIX + ENTRY * slot; + let start = u32::from_le_bytes( + buffer[at..at + 4] + .try_into() + .expect("should contain a start offset"), + ); + let end = u32::from_le_bytes( + buffer[at + 4..at + 8] + .try_into() + .expect("should contain an end offset"), + ); + (start as usize, end as usize) +} + +/// Reads the payload bytes of `slot` through its directory entry. +/// +/// # Panics +/// +/// Panics under [`slot_range`]'s conditions for the entry, its computed offset included, and where +/// the extent the entry records is not a range within `buffer`. An unwritten slot's `(0, 0)` entry +/// records a readable empty range, and yields an empty slice. +fn slot_bytes(buffer: &[u8], slot: usize) -> &[u8] { + let (start, end) = slot_range(buffer, slot); + &buffer[start..end] +} + +/// Returns the bytes the writer appended past the last slot's padding. +/// +/// A directory entry records its slot's unpadded end. The writer then pads to the next multiple +/// of eight before writing the trailer, which puts the trailer at the aligned end rather than at +/// the recorded one. Edges declares four slots and skips none, and the identity column is the +/// last slot it writes. +/// +/// # Panics +/// +/// Panics where the last slot's directory entry is unreadable, and where the padded end that +/// entry records lies past `buffer`. Rounding the recorded `u32` end to a multiple of eight uses +/// `usize`. On a 32-bit target with overflow checking, an end greater than `u32::MAX - 7` panics +/// during rounding. +/// +/// # Warning +/// +/// Without overflow checking on a 32-bit target, an overflowing rounded end becomes zero and this +/// returns the entire buffer. +fn trailer_bytes(buffer: &[u8]) -> &[u8] { + let (_, end) = slot_range(buffer, SLOTS - 1); + &buffer[end.next_multiple_of(8)..] +} + +/// Builds a generation identity from a repeated byte, beside the digest bytes it echoes. +fn generation_of(byte: u8) -> ([u8; size_of::()], GenerationId) { + let bytes = [byte; size_of::()]; + ( + bytes, + GenerationId::from_digest(Sha256Digest::from_bytes_unchecked(bytes)), + ) +} + +/// Builds an encoded row id from a raw `value`, bypassing the usual encoding path. +fn row(value: u32) -> EncodedRowId { + EncodedRowId::new_unchecked(value) +} + +/// Builds a synthetic identity from repeated `web` and `entity` bytes. +fn identity(web: u8, entity: u8) -> ArchivedEntityId { + ArchivedEntityId { + web_id: ArchivedWebId::from_bytes([web; 16]), + entity_uuid: ArchivedEntityUuid::from_bytes([entity; 16]), + } +} + +/// Assembles one `SALTILEE` envelope from its parts, independent of [`EnvelopeWriter`]. +/// +/// [`EnvelopeWriter`]: crate::serve::document::codec::EnvelopeWriter +/// +/// Padding to the next eight-byte boundary follows each present slot's payload. A `None` slot +/// keeps the directory's initial zero entry. +/// +/// # Panics +/// +/// Panics above a u16 slot count, and where a payload offset does not fit u32. +fn build_envelope(slots: &[Option>], trailer: Option<&[u8]>) -> Vec { + let mut buffer = Vec::new(); + buffer.extend_from_slice(b"SALTILEE"); + buffer.extend_from_slice(&1_u16.to_le_bytes()); + buffer.extend_from_slice(&0_u16.to_le_bytes()); + buffer.extend_from_slice( + &u16::try_from(slots.len()) + .expect("should fit the fixture's slot count") + .to_le_bytes(), + ); + buffer.extend_from_slice(&0_u16.to_le_bytes()); + + let mut directory = vec![0_u8; slots.len() * ENTRY]; + buffer.resize(PREFIX + directory.len(), 0); + + for (index, slot) in slots.iter().enumerate() { + let Some(payload) = slot else { continue }; + + let start = buffer.len(); + buffer.extend_from_slice(payload); + let end = buffer.len(); + buffer.resize(end.next_multiple_of(8), 0); + + let at = index * ENTRY; + directory[at..at + 4].copy_from_slice( + &u32::try_from(start) + .expect("should fit the fixture's offsets") + .to_le_bytes(), + ); + directory[at + 4..at + ENTRY].copy_from_slice( + &u32::try_from(end) + .expect("should fit the fixture's offsets") + .to_le_bytes(), + ); + } + + buffer[PREFIX..PREFIX + directory.len()].copy_from_slice(&directory); + if let Some(trailer) = trailer { + buffer.extend_from_slice(trailer); + } + buffer +} + +/// Builds a complete delivery of zero edges at minimal detail. +fn minimal_document(generation: GenerationId) -> EdgesDocument<'static> { + EdgesDocument { + generation, + ids: IdVec::::from_raw(Vec::new()), + sources: IdVec::>::from_raw(Vec::new()), + targets: IdVec::>::from_raw(Vec::new()), + trailer: None, + complete: true, + } +} + +/// Builds the `HEAD` map of a document with `count` edges. +/// +/// # Panics +/// +/// Panics on a `digest` above 255 bytes, or a `variant` or `count` above 255: each is a one-byte +/// CBOR argument here. +fn expected_head( + digest: &[u8], + variant: u64, + count: u64, + complete: bool, + trailer: bool, +) -> Vec { + cbor_map([ + (0, cbor_bytes(digest)), + (1, cbor_uint(variant)), + (2, cbor_uint(count)), + (3, cbor_bool(complete)), + (4, cbor_bool(trailer)), + ]) +} + +/// A zero-edge minimal document replaces a longer prepopulated buffer with the whole envelope. +/// +/// The one case compared byte for byte. Every other test decodes the directory and checks one +/// region. +#[test] +fn document_encode_minimal_empty() { + let (digest, generation) = generation_of(0xAB); + let document = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + buffer.resize(512, 0xDD); + let _completed = document + .encode(&mut buffer) + .expect("should complete the minimal edges response"); + + let expected = build_envelope( + &[ + Some(expected_head(&digest, 0, 0, true, false)), + Some(Vec::new()), + Some(Vec::new()), + Some(Vec::new()), + ], + None, + ); + + assert_eq!( + buffer.as_slice(), + expected.as_slice(), + "should replace the buffer with the prefix, directory, padded head and empty data slots" + ); +} + +/// The columns keep the document's own row order, and the head reports the truncation. +#[test] +fn document_encode_ordered_columns_truncated() { + let (digest, generation) = generation_of(0x01); + let document = EdgesDocument { + ids: IdVec::from_raw(vec![ + identity(0x30, 0x31), + identity(0x10, 0x11), + identity(0x20, 0x21), + ]), + sources: IdVec::from_raw(vec![row(300), row(100), row(200)]), + targets: IdVec::from_raw(vec![row(301), row(101), row(201)]), + complete: false, + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the truncated edges response"); + + let expected_sources: Vec = [300_u32, 100, 200] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(); + let expected_targets: Vec = [301_u32, 101, 201] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(); + let mut expected_ids = Vec::new(); + for (web, entity) in [(0x30_u8, 0x31_u8), (0x10, 0x11), (0x20, 0x21)] { + expected_ids.extend_from_slice(&[web; 16]); + expected_ids.extend_from_slice(&[entity; 16]); + } + + assert_eq!( + slot_bytes(&buffer, 0), + expected_head(&digest, 0, 3, false, false).as_slice(), + "should report three edges and a truncated delivery in the head" + ); + assert_eq!( + slot_bytes(&buffer, 1), + expected_sources.as_slice(), + "should encode sources in the document's own row order" + ); + assert_eq!( + slot_bytes(&buffer, 2), + expected_targets.as_slice(), + "should encode targets in the document's own row order" + ); + assert_eq!( + slot_bytes(&buffer, 3), + expected_ids.as_slice(), + "should encode identities in the document's own row order" + ); +} + +/// The response echoes its own `variant` field rather than a fixed value. +/// +/// [`Document::encode`](crate::serve::document::Document::encode) always passes zero. A nonzero +/// variant needs the response built directly. +#[test] +fn response_variant_nonzero() { + let (digest, generation) = generation_of(0x02); + let document = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + let _completed = EdgesResponse { + generation: document.generation, + variant: 9, + document: &document, + } + .encode_into(&mut buffer); + + assert_eq!( + slot_bytes(&buffer, 0), + expected_head(&digest, 9, 0, true, false).as_slice(), + "should echo variant 9 at head key 1 and the generation at head key 0" + ); +} + +/// A document without a trailer reports it in the head and appends nothing after the slots. +#[test] +fn trailer_absent() { + let (digest, generation) = generation_of(0x03); + let document = EdgesDocument { + ids: IdVec::from_raw(vec![identity(0x01, 0x02)]), + sources: IdVec::from_raw(vec![row(1)]), + targets: IdVec::from_raw(vec![row(2)]), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the minimal-detail edges response"); + + assert_eq!( + slot_bytes(&buffer, 0), + expected_head(&digest, 0, 1, true, false).as_slice(), + "should report no trailer at head key 4" + ); + assert!( + trailer_bytes(&buffer).is_empty(), + "should append no bytes past the last slot's padding" + ); +} + +/// The auxiliary trailer's label and type-index arrays stay aligned to the edge slots. +/// +/// The expected indexes are the literals 0 and 1 because the fixture interns alpha before beta. +#[test] +fn trailer_auxiliary_aligned() { + let (digest, generation) = generation_of(0x04); + let mut interner = InternTable::new(); + let alpha = interner.intern( + "https://example.com/types/alpha/v/1" + .parse::() + .expect("should parse the fixture type URL"), + ); + let beta = interner.intern( + "https://example.com/types/beta/v/1" + .parse::() + .expect("should parse the fixture type URL"), + ); + + let document = EdgesDocument { + ids: IdVec::from_raw(vec![ + identity(0x01, 0x01), + identity(0x02, 0x02), + identity(0x03, 0x03), + ]), + sources: IdVec::from_raw(vec![row(0), row(1), row(2)]), + targets: IdVec::from_raw(vec![row(3), row(4), row(5)]), + trailer: Some(EdgesTrailer { + labels: IdVec::::from_raw(vec![ + Label::new("north"), + Label::EMPTY, + Label::new("south"), + ]), + representative_type_urls: IdVec::from_raw(vec![Some(alpha), None, Some(beta)]), + representative_type_urls_interner: interner, + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the auxiliary edges response"); + + let expected = cbor_map([ + ( + 0, + cbor_array([ + cbor_text("https://example.com/types/alpha/v/1"), + cbor_text("https://example.com/types/beta/v/1"), + ]), + ), + ( + 1, + cbor_array([cbor_text("north"), vec![CBOR_NULL], cbor_text("south")]), + ), + (2, cbor_array([cbor_uint(0), vec![CBOR_NULL], cbor_uint(1)])), + ]); + + assert_eq!( + trailer_bytes(&buffer), + expected.as_slice(), + "should carry the URL table and the label and index arrays aligned to the edge slots" + ); + assert_eq!( + slot_bytes(&buffer, 0), + expected_head(&digest, 0, 3, true, true).as_slice(), + "should report the present trailer at head key 4" + ); +} + +/// An auxiliary trailer over zero edges keeps every array present but empty. +#[test] +fn trailer_auxiliary_empty() { + let (digest, generation) = generation_of(0x05); + let document = EdgesDocument { + trailer: Some(EdgesTrailer { + labels: IdVec::::from_raw(Vec::new()), + representative_type_urls: IdVec::from_raw(Vec::new()), + representative_type_urls_interner: InternTable::new(), + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the empty auxiliary edges response"); + + let expected = cbor_map([ + (0, cbor_array([])), + (1, cbor_array([])), + (2, cbor_array([])), + ]); + + assert_eq!( + trailer_bytes(&buffer), + expected.as_slice(), + "should keep the URL table and the label and index arrays present but empty" + ); + assert_eq!( + slot_bytes(&buffer, 0), + expected_head(&digest, 0, 0, true, true).as_slice(), + "should report the trailer as present though every array inside it is empty" + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/edges/mod.rs b/libs/@local/graph/atlas/src/serve/document/edges/mod.rs new file mode 100644 index 00000000000..be3d6520e61 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/edges/mod.rs @@ -0,0 +1,283 @@ +use alloc::alloc::Allocator; +use core::{error::Error, fmt}; + +use error_stack::Report; +use hashql_core::id::IdVec; +use type_system::ontology::{VersionedUrl, id::OntologyTypeUuid}; + +use super::{Document, codec::Envelope}; +use crate::{ + bitset::CompressedBitSet, + dataset::auxiliary::Label, + file::generation::GenerationId, + identity::NodeRowId, + math::Log2, + morton::{MortonCell, MortonTile, Zoom}, + postgres::id::ArchivedEntityId, + serve::{ + codec::EncodedRowId, + hydrate::{HydrateError, TypeUrlResolver}, + intern::{InternTable, TableIndex}, + neighbourhood::{DeliveredEdge, Neighbourhood}, + scene::Scene, + }, +}; + +mod codec; +#[cfg(test)] +mod tests; + +hashql_core::id::newtype! { + /// A reference to one delivered edge by its slot in this document's edge order. + pub(crate) struct EdgeSlot(u32) +} + +/// A failure to assemble an edges document. +#[derive(Debug)] +pub(crate) enum EdgesDocumentError { + /// The request lists more tiles than the configured limit. + Tiles { count: usize, maximum: u32 }, + /// A tile's zoom exceeds the generation's maximum served zoom. + Zoom { zoom: Zoom, maximum: Zoom }, + /// A tile lies outside its zoom's grid. + Coordinate { tile: MortonTile }, + /// An admitted edge has no captured display payload for its auxiliary detail. + Display, + /// Type-URL resolution failed at the recorded [`HydrateError`] stage. + Hydrate(HydrateError), +} + +impl fmt::Display for EdgesDocumentError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Tiles { count, maximum } => write!( + fmt, + "the request lists {count} tiles, exceeding the limit of {maximum}" + ), + Self::Zoom { zoom, maximum } => write!( + fmt, + "tile zoom {zoom} exceeds the maximum served zoom {maximum}" + ), + Self::Coordinate { + tile: MortonTile { z, x, y }, + } => { + write!(fmt, "tile {}/{x}/{y} lies outside its zoom's grid", z.get()) + } + Self::Display => fmt.write_str("an admitted edge has no display payload"), + Self::Hydrate(error) => write!(fmt, "edge type URL resolution failed: {error}"), + } + } +} + +impl Error for EdgesDocumentError {} + +/// Whether an edges document carries only its columns or also the auxiliary trailer. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum EdgesDocumentDetailLevel { + /// Columns alone: source, target and identity per edge. + Minimal, + /// Columns plus the trailer: label and representative type URL per edge. + Auxiliary, +} + +/// Caps on one edges request: the tiles it may name and the edges it may induce. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +pub(crate) struct EdgesLimits { + /// The most tiles one request may list, 256 by default. + pub tiles: u32 = 256, + /// The most edges one response may induce, 0x4000 by default. + pub edges: u32 = 0x4000, +} + +/// Delivery detail, limits and a representative-type URL resolver for one edges request. +pub(crate) struct EdgesDocumentOptions { + pub detail: EdgesDocumentDetailLevel, + pub limits: EdgesLimits, + pub resolver: R, +} + +/// The auxiliary detail for one edges document. +/// +/// It holds each edge's label and representative type URL, interned once across the whole +/// response. +#[derive(Debug)] +pub(crate) struct EdgesTrailer<'details> { + labels: IdVec, + representative_type_urls: IdVec>>, + representative_type_urls_interner: InternTable, +} + +impl<'details> EdgesTrailer<'details> { + /// Builds the trailer for `edges`. + /// + /// `resolver` resolves each edge's representative ontology type. + /// + /// # Errors + /// + /// Returns [`EdgesDocumentError::Display`] when an edge has no captured display payload, and + /// [`EdgesDocumentError::Hydrate`] when `resolver` fails to resolve a representative type URL. + fn new( + Scene { world, epoch, .. }: Scene<'details>, + edges: impl IntoIterator, + resolver: &impl TypeUrlResolver, + ) -> Result> { + let edges = edges.into_iter(); + let mut labels = IdVec::with_capacity(edges.len()); + + let mut ontology_type_uuid_interner = InternTable::::new(); + let mut representative_type_urls_interner = InternTable::new(); + + let mut dispatch: IdVec>> = + IdVec::with_capacity(edges.len()); + + for edge in edges { + let legend = world + .topology + .payload(epoch, edge.row.unwrap()) + .ok_or_else(|| Report::new(EdgesDocumentError::Display))?; + labels.push(legend.label()); + dispatch.push( + world + .ontology + .key_of(epoch, legend.representative_ontology()) + .map(|uuid| ontology_type_uuid_interner.intern(*uuid)), + ); + } + + let mut mapping: IdVec, Option>> = + IdVec::with_capacity(ontology_type_uuid_interner.len()); + if !ontology_type_uuid_interner.is_empty() { + let pairs = resolver + .resolve(ontology_type_uuid_interner.entries().iter().copied()) + .map_err(|report| { + let error = EdgesDocumentError::Hydrate(*report.current_context()); + report.change_context(error) + })?; + + for (uuid, url) in pairs { + if let Some(slot) = ontology_type_uuid_interner.index_of(&uuid) { + mapping.insert(slot, representative_type_urls_interner.intern(url)); + } + } + } + + let representative_type_urls = dispatch + .into_iter() + .map(|slot| mapping.lookup(slot?).copied()) + .collect(); + + Ok(Self { + labels, + representative_type_urls, + representative_type_urls_interner, + }) + } +} + +/// The identity and endpoint columns of delivered edges, with an optional auxiliary trailer. +#[derive(Debug)] +pub(crate) struct EdgesDocument<'details> { + generation: GenerationId, + + ids: IdVec, + sources: IdVec>, + targets: IdVec>, + + trailer: Option>, + + complete: bool, +} + +impl<'details> EdgesDocument<'details> { + /// Assembles the edges induced by the nodes delivered in `tiles`. + /// + /// # Errors + /// + /// Returns [`EdgesDocumentError`] for an invalid tile count or address, unavailable captured + /// display data, or failed type-URL hydration. + pub(crate) fn new( + scene @ Scene { + world, delivery, .. + }: Scene<'details>, + tiles: &[MortonTile], + EdgesDocumentOptions { + detail, + limits, + resolver, + }: &EdgesDocumentOptions, + ) -> Result> + where + R: TypeUrlResolver, + { + if tiles.len() > limits.tiles as usize { + return Err(Report::new(EdgesDocumentError::Tiles { + count: tiles.len(), + maximum: limits.tiles, + })); + } + + let maximum = world.schedule().max_tile_depth(); + + let mut delivered = CompressedBitSet::new(); + for &tile in tiles { + let zoom = tile.z.zoom(Log2::ZERO); + if zoom > maximum { + return Err(Report::new(EdgesDocumentError::Zoom { zoom, maximum })); + } + + let cell = MortonCell::from_tile(tile) + .ok_or_else(|| Report::new(EdgesDocumentError::Coordinate { tile }))?; + for node in delivery.total(zoom, cell).rows { + delivered.insert(node); + } + } + + let induced = Neighbourhood { provider: scene }.induced(&delivered, limits.edges as usize); + + let length = induced.edges.len(); + let trailer = match detail { + EdgesDocumentDetailLevel::Minimal => None, + EdgesDocumentDetailLevel::Auxiliary => Some(EdgesTrailer::new( + scene, + induced.edges.iter().copied(), + resolver, + )?), + }; + + let mut this = Self { + generation: world.generation().id(), + ids: IdVec::with_capacity(length), + sources: IdVec::with_capacity(length), + targets: IdVec::with_capacity(length), + trailer, + complete: induced.complete, + }; + + for DeliveredEdge { + endpoints: [source, target], + identity, + .. + } in induced.edges + { + this.ids.push(identity); + this.sources.push(world.layout.index.encode(source)); + this.targets.push(world.layout.index.encode(target)); + } + + Ok(this) + } +} + +impl Document for EdgesDocument<'_> { + type Error = !; + + fn encode(&self, buffer: &mut Vec) -> Result { + Ok(self::codec::EdgesResponse { + generation: self.generation, + variant: 0, + document: self, + } + .encode_into(buffer)) + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/edges/tests.rs b/libs/@local/graph/atlas/src/serve/document/edges/tests.rs new file mode 100644 index 00000000000..19685567128 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/edges/tests.rs @@ -0,0 +1,850 @@ +use alloc::sync::Arc; +use core::{assert_matches, cell::RefCell, iter}; +use std::io; + +use arc_swap::Guard; +use camino::Utf8Path; +use error_stack::Report; +use hashql_core::id::Id as _; +use rand::{SeedableRng as _, rngs::StdRng}; +use type_system::{ + ontology::{VersionedUrl, id::OntologyTypeUuid}, + principal::actor::{ActorId, ActorType}, +}; +use uuid::Uuid; + +use super::{ + EdgeSlot, EdgesDocument, EdgesDocumentDetailLevel, EdgesDocumentError, EdgesDocumentOptions, + EdgesLimits, EdgesTrailer, +}; +use crate::{ + api::problem::{Problem, tests::assert_internal_diagnostic}, + bitset::CompressedBitSet, + dataset::auxiliary::{Label, OwnedLegend}, + file::generation::Generation, + identity::{EdgeRowId, NodeRowId, OntologyRowId}, + morton::{Depth, MortonTile, Zoom}, + postgres::id::{ArchivedEntityId, ArchivedOntologyTypeUuid}, + salt::{ + fit::prepare::identity::IdentityTable, + lod::{key, stage::WIRE_FRAME}, + }, + serve::{ + codec::EncodedRowId, + delta::{Delta, epoch::Epoch}, + hydrate::{HydrateError, TypeUrlResolver}, + scene::Scene, + schedule::ViewSchedule, + tests::fixture::{EDGE_SEED, EDGES, ENDPOINTS, NODES, TamperFixture, secret}, + visibility::{VisibilityActor, VisibilityMask}, + world::World, + }, +}; + +/// A [`TypeUrlResolver`] whose every request fails. +/// +/// Its failure carries a store message and an attachment. A test asserts on the internal-problem +/// mapping and the redaction a resolver failure produces. +struct FailingResolver; + +impl TypeUrlResolver for FailingResolver { + fn resolve( + &self, + types: impl IntoIterator, + ) -> Result, Report> + { + let _ = types; + Err::, _>( + Report::new(io::Error::other("private-store-message")) + .change_context(HydrateError::Query) + .attach("private-property-value"), + ) + } +} + +/// A [`TypeUrlResolver`] fixture answering a fixed set of type URLs. +/// +/// It records every requested batch. A test asserts which types the document asked for. +struct FakeResolver { + answers: Vec<(OntologyTypeUuid, VersionedUrl)>, + asked: RefCell>>, +} + +impl FakeResolver { + /// Constructs a resolver that returns exactly `answers` for every request. + fn new(answers: Vec<(OntologyTypeUuid, VersionedUrl)>) -> Self { + Self { + answers, + asked: RefCell::new(Vec::new()), + } + } + + /// Builds a deterministic fixture URL for `name`. + /// + /// # Panics + /// + /// Panics where `name` leaves the interpolated URL unparseable. + fn url(name: &str) -> VersionedUrl { + format!("https://example.com/types/{name}/v/1") + .parse() + .expect("should parse the fixture URL") + } + + /// Asserts that this resolver was called exactly once, for the types in `expected`. + /// + /// That call's batch names exactly the distinct types in `expected`, in any order. + /// + /// # Panics + /// + /// Panics if the recorded calls differ from that expectation or the call log is borrowed + /// mutably. + #[track_caller] + fn assert_query(&self, expected: &[OntologyTypeUuid]) { + let asked = self.asked.borrow(); + assert_eq!(asked.len(), 1, "should call the resolver exactly once"); + assert_eq!( + asked[0].len(), + expected.len(), + "should query each distinct type once" + ); + for uuid in expected { + assert!( + asked[0].contains(uuid), + "should query the expected type {uuid:?}" + ); + } + } +} + +impl TypeUrlResolver for FakeResolver { + fn resolve( + &self, + types: impl IntoIterator, + ) -> Result, Report> + { + self.asked.borrow_mut().push(types.into_iter().collect()); + Ok(self.answers.clone()) + } +} + +/// Reads each edge's representative type URL out of `trailer`, in slot order. +/// +/// A slot is [`None`] where the trailer recorded no representative type. +fn resolved_urls(trailer: &EdgesTrailer<'_>) -> Vec> { + trailer + .representative_type_urls + .iter() + .map(|slot| { + slot.map(|index| trailer.representative_type_urls_interner.entries()[index].clone()) + }) + .collect() +} + +/// An opened synthetic generation, shared by every test below. +/// +/// It carries the captured epoch, actor, visibility mask and delivery schedule that `scene()` +/// assembles into one scene. +struct Fixture { + world: Arc, + epoch: Epoch, + actor: VisibilityActor, + mask: VisibilityMask, + schedule: ViewSchedule, + _files: TamperFixture, +} + +impl Fixture { + /// Opens `generation`, published from `files`, with a fresh delta identity. + /// + /// The mask grants one fixed test actor full visibility. + /// + /// # Panics + /// + /// Panics if [`World::open`] fails to open or validate the serving artifacts. + fn from_generation(files: TamperFixture, generation: Generation) -> Self { + let world = Arc::new( + World::open(generation, &secret()).expect("should open the synthetic generation"), + ); + let delta = Delta::new(Arc::clone(&world), StdRng::seed_from_u64(17)) + .expect("should allocate a delta identity"); + let epoch = Epoch::from(Guard::from_inner(Arc::new(delta))); + let actor = VisibilityActor { + id: ActorId::new(Uuid::from_u128(1), ActorType::User), + instance_admin: false, + }; + let mask = VisibilityMask::full(actor); + let schedule = ViewSchedule::of(Arc::clone(&world), &epoch, &mask); + Self { + world, + epoch, + actor, + mask, + schedule, + _files: files, + } + } + + /// Publishes and opens the named synthetic generation unmodified. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails, and for the conditions + /// [`Self::from_generation`] states. + fn new(name: &str) -> Self { + let files = TamperFixture::publish(name); + let generation = files.generation().clone(); + Self::from_generation(files, generation) + } + + /// Reopens `path` for writing, clearing its read-only staged-artifact permission first. + /// + /// # Panics + /// + /// Panics on any of three filesystem failures against `path`: an unreadable metadata entry, + /// a permission change the process may not make, and a truncating reopen that fails. + #[expect( + clippy::permissions_set_readonly_false, + reason = "the staged copy is this test's own scratch file" + )] + fn recreate_writable(path: &Utf8Path) -> std::fs::File { + let mut permissions = std::fs::metadata(path) + .expect("should read the staged artifact's metadata") + .permissions(); + permissions.set_readonly(false); + std::fs::set_permissions(path, permissions).expect("should set the permissions"); + std::fs::File::create(path).expect("should rewrite the staged artifact") + } + + /// Builds a synthetic identity for edge `row`, derived from the fixture's own edge seed. + /// + /// # Panics + /// + /// Panics above a row of 255, which does not convert to u8. A row whose sum with + /// [`EDGE_SEED`] leaves u8 overflows instead: a checked build panics there, an unchecked one + /// wraps. + fn edge_identity(row: u64) -> ArchivedEntityId { + let seed = EDGE_SEED + u8::try_from(row).expect("should fit the fixture row in u8"); + ArchivedEntityId { + web_id: Uuid::from_bytes([seed; 16]).into(), + entity_uuid: Uuid::from_bytes([seed ^ 0xFF; 16]).into(), + } + } + + /// Publishes the named synthetic generation, then rewrites its edge-identity table. + /// + /// The rewritten table holds [`Self::edge_identity`] values and per-row ontology legends that + /// distinguish row 0 from the rest. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails and where the rewritten table does + /// not write. It also panics for the conditions [`Self::recreate_writable`] and + /// [`Self::from_generation`] state. + fn varied(name: &str) -> Self { + let files = TamperFixture::publish(name); + let name = files.generation().repository().files.edge_identities.name(); + let generation = files.tamper(&name, |path| { + let mut table = IdentityTable::::new(); + for row in 0..EDGES { + table.push(Self::edge_identity(row)); + } + let legends: Vec<_> = (0..EDGES) + .map(|row| { + let ontology = OntologyRowId::new(if row == 0 { 0 } else { 2 }); + OwnedLegend::new(ontology, Label::new(&format!("edge-{row}"))) + }) + .collect(); + let mut file = Self::recreate_writable(path); + let _digest = table + .write_into(legends.iter().map(AsRef::as_ref), &mut file) + .expect("should write the varied edge identities"); + }); + Self::from_generation(files, generation) + } + + /// Narrows this fixture's visibility mask and schedule to exactly the given rows. + /// + /// A later `scene()` call sees only those node and edge rows. + fn restrict( + &mut self, + nodes: impl IntoIterator, + edges: impl IntoIterator, + ) { + self.mask = VisibilityMask::partial( + self.actor, + CompressedBitSet::from_rows(nodes.into_iter().map(NodeRowId::new)), + CompressedBitSet::from_rows(edges.into_iter().map(EdgeRowId::new)), + ); + self.schedule = ViewSchedule::of(Arc::clone(&self.world), &self.epoch, &self.mask); + } + + /// Assembles this fixture's world, epoch, mask and schedule into one [`Scene`]. + /// + /// Its delivery is the schedule's cut at the zero offset. + fn scene(&self) -> Scene<'_> { + Scene { + world: &self.world, + epoch: &self.epoch, + mask: &self.mask, + schedule: &self.schedule, + delivery: self + .schedule + .cut(Zoom::MIN) + .expect("should bind the zero offset"), + } + } + + /// Builds a [`MortonTile`] at zoom `z` and coordinates `(x, y)`. + /// + /// # Panics + /// + /// Panics where `z` exceeds [`Depth::MAX`], the bound its depth carries. + fn tile(z: u8, x: u32, y: u32) -> MortonTile { + MortonTile { + z: Depth::new(z), + x, + y, + } + } + + /// Returns the deepest-cut tile that contains `node`'s captured position. + /// + /// # Panics + /// + /// Panics where the fixture's epoch captured no position for `node`. + fn tile_of(&self, node: NodeRowId) -> MortonTile { + let position = self + .world + .layout + .position(&self.epoch, node) + .expect("should resolve the fixture node's position"); + // The deepest served cut includes every bucket, including the scoped catch-all. + let depth = Depth::from_zoom(self.world.schedule().max_tile_depth()); + key::keys(&[position], WIRE_FRAME)[0].tile(depth) + } + + /// Collects every deepest-cut tile the fixture's nodes occupy, sorted and deduplicated. + fn all_tiles(&self) -> Vec { + let mut tiles: Vec<_> = (0..NODES) + .map(|row| self.tile_of(NodeRowId::new(row))) + .collect(); + tiles.sort_unstable_by_key(|tile| (tile.z.get(), tile.x, tile.y)); + tiles.dedup(); + tiles + } + + /// Returns the identity the fixture generation assigned to `edge`. + /// + /// # Panics + /// + /// Panics where that generation holds no identity for `edge`. + fn identity_of(&self, edge: EdgeRowId) -> ArchivedEntityId { + self.world + .topology + .key_of(&self.epoch, edge) + .expect("should resolve the fixture edge's identity") + } + + /// Encodes the wire row id for `node`. + /// + /// # Panics + /// + /// Panics under [`NodeIndex::encode`](crate::serve::world::NodeIndex::encode)'s row-range and + /// cache-miss conditions. + /// + /// # Warning + /// + /// On a 32-bit target, a node outside the wire range can instead return a cached row's + /// encoding. + fn encode(&self, node: NodeRowId) -> EncodedRowId { + self.world.layout.index.encode(node) + } + + /// Returns `edge`'s captured display label. + /// + /// # Panics + /// + /// Panics where the fixture captured no display payload for `edge`. + fn label_of(&self, edge: EdgeRowId) -> &Label { + self.world + .topology + .payload(&self.epoch, edge) + .expect("should resolve the fixture edge's display payload") + .label() + } + + /// Returns the ontology type uuid the fixture generation assigned to ontology row `row`. + /// + /// # Panics + /// + /// Panics where that generation holds no ontology row `row`. + fn ontology_uuid(&self, row: u64) -> ArchivedOntologyTypeUuid { + self.world + .ontology + .key_of(&self.epoch, OntologyRowId::new(row)) + .expect("should resolve the fixture ontology row") + } + + /// Asserts that `document` is complete and lists exactly the edges in `rows`. + /// + /// The identities, sources and targets appear in the order `rows` gives them. + /// + /// # Panics + /// + /// Panics where `document` departs from that expectation, and where a row in `rows` reaches + /// past the fixture's [`ENDPOINTS`], which it indexes. + #[track_caller] + fn assert_edges(&self, document: &EdgesDocument<'_>, rows: &[u64]) { + assert!(document.complete, "should deliver the complete edge set"); + // The fixture's ascending seed bytes make row order equal to identity order. + let expected_ids: Vec<_> = rows + .iter() + .map(|&row| self.identity_of(EdgeRowId::new(row))) + .collect(); + assert_eq!( + document.ids.iter().copied().collect::>(), + expected_ids, + "should list the expected edge identities in order" + ); + for (column, endpoint) in [(&document.sources, 0), (&document.targets, 1)] { + let expected: Vec<_> = rows + .iter() + .map(|&row| { + let row = usize::try_from(row).expect("should fit the fixture row in usize"); + self.encode(ENDPOINTS[row][endpoint]) + }) + .collect(); + assert_eq!( + column.iter().copied().collect::>(), + expected, + "should encode endpoint {endpoint} of each edge" + ); + } + } +} + +/// More tiles than the configured limit refuses with the exceeded count and the limit. +#[test] +fn tiles_count_over_limit() { + let fixture = Fixture::new("edges-tiles-count-over-limit"); + let tile = Fixture::tile(0, 0, 0); + let Err(report) = EdgesDocument::new( + fixture.scene(), + &[tile, tile], + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { tiles: 1, .. }, + resolver: FailingResolver, + }, + ) else { + panic!("should refuse a tile list past the limit"); + }; + assert_matches!( + report.current_context(), + EdgesDocumentError::Tiles { + count: 2, + maximum: 1 + }, + ); +} + +#[test] +fn tiles_zoom_over_maximum() { + let fixture = Fixture::new("edges-tiles-zoom-over-maximum"); + let maximum = fixture.world.schedule().max_tile_depth(); + let tile = Fixture::tile(maximum.get() + 1, 0, 0); + let Err(report) = EdgesDocument::new( + fixture.scene(), + &[tile], + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: FailingResolver, + }, + ) else { + panic!("should refuse a tile zoom past the schedule maximum"); + }; + assert_matches!( + report.current_context(), + EdgesDocumentError::Zoom { maximum: actual, .. } if *actual == maximum, + ); +} + +#[test] +fn tiles_coordinate_outside_grid() { + let fixture = Fixture::new("edges-tiles-coordinate-outside-grid"); + for tile in [Fixture::tile(1, 2, 0), Fixture::tile(1, 0, 2)] { + let Err(report) = EdgesDocument::new( + fixture.scene(), + &[tile], + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: FailingResolver, + }, + ) else { + panic!("should refuse a coordinate outside its zoom's grid"); + }; + assert_matches!( + report.current_context(), + EdgesDocumentError::Coordinate { tile: refused } if *refused == tile, + ); + } +} + +#[test] +fn trailer_minimal() { + let fixture = Fixture::new("edges-trailer-minimal"); + let resolver = FakeResolver::new(Vec::new()); + let document = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Minimal, + limits: EdgesLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with terminal-depth tiles"); + assert_eq!( + document.ids.len(), + ENDPOINTS.len(), + "should deliver every edge" + ); + assert!(document.trailer.is_none(), "should omit the trailer"); + assert!(resolver.asked.borrow().is_empty(), "should skip resolution"); +} + +#[test] +fn trailer_empty() { + let fixture = Fixture::new("edges-trailer-empty"); + let resolver = FakeResolver::new(Vec::new()); + let document = EdgesDocument::new( + fixture.scene(), + &[], + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with no tiles"); + fixture.assert_edges(&document, &[]); + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + assert!(trailer.labels.is_empty(), "should have no labels"); + assert!( + trailer.representative_type_urls.is_empty(), + "should have no URL slots" + ); + assert!( + trailer.representative_type_urls_interner.is_empty(), + "should have no URLs" + ); + assert!( + resolver.asked.borrow().is_empty(), + "should skip resolution even for an empty query" + ); +} + +/// A minimal-detail request over every terminal-depth tile delivers every edge in row order. +#[test] +fn constructor_slot_alignment() { + let fixture = Fixture::new("edges-constructor-slot-alignment"); + let document = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Minimal, + limits: EdgesLimits { .. }, + resolver: FailingResolver, + }, + ) + .expect("should construct with terminal-depth tiles"); + fixture.assert_edges(&document, &(0..EDGES).collect::>()); +} + +#[test] +fn capacity_boundaries() { + let fixture = Fixture::new("edges-capacity-boundaries"); + let tiles = fixture.all_tiles(); + let total = ENDPOINTS.len(); + for capacity in [0, total - 1, total, total + 5] { + let document = EdgesDocument::new( + fixture.scene(), + &tiles, + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Minimal, + limits: EdgesLimits { + edges: u32::try_from(capacity).expect("should fit the capacity in u32"), + .. + }, + resolver: FailingResolver, + }, + ) + .expect("should construct with terminal-depth tiles"); + assert_eq!( + document.ids.len(), + capacity.min(total), + "should respect capacity {capacity}" + ); + assert_eq!( + document.complete, + capacity >= total, + "should report completeness at capacity {capacity}" + ); + } +} + +/// Duplicate and permuted tile lists deliver the same edges as their canonical form. +/// +/// A self-loop tile listed twice delivers its edge exactly once, and a reciprocal pair of tiles +/// delivers the same edges in either listed order. +#[test] +fn tiles_duplicate_and_permuted() { + let fixture = Fixture::new("edges-tiles-duplicate-and-permuted"); + // Node 2 has a self-loop. Nodes 5 and 7 form the reciprocal pair. + let looped = fixture.tile_of(NodeRowId::new(2)); + let source = fixture.tile_of(NodeRowId::new(5)); + let target = fixture.tile_of(NodeRowId::new(7)); + assert_ne!(source, target, "should exercise distinct tiles"); + let options = EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Minimal, + limits: EdgesLimits { .. }, + resolver: FailingResolver, + }; + for tiles in [vec![looped], vec![looped, looped]] { + let document = EdgesDocument::new(fixture.scene(), &tiles, &options) + .expect("should construct the self-loop document"); + fixture.assert_edges(&document, &[2]); + } + for tiles in [[source, target], [target, source]] { + let document = EdgesDocument::new(fixture.scene(), &tiles, &options) + .expect("should construct the reciprocal-pair document"); + fixture.assert_edges(&document, &[3, 4]); + } +} + +#[test] +fn trailer_distinct_borrowed_labels() { + let fixture = Fixture::varied("edges-trailer-distinct-borrowed-labels"); + let document = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: FakeResolver::new(Vec::new()), + }, + ) + .expect("should construct with terminal-depth tiles"); + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + assert_eq!( + trailer.labels.len(), + ENDPOINTS.len(), + "should label every edge" + ); + for row in 0..EDGES { + let slot = EdgeSlot::from_usize(usize::try_from(row).expect("should fit the row in usize")); + assert_eq!( + trailer.labels[slot].as_ref(), + format!("edge-{row}"), + "should carry edge {row}'s text" + ); + assert!( + core::ptr::eq(trailer.labels[slot], fixture.label_of(EdgeRowId::new(row))), + "should borrow edge {row}'s label from the captured scene" + ); + } +} + +#[test] +fn dispatch_dedup_two_types() { + let fixture = Fixture::varied("edges-dispatch-dedup-two-types"); + let first = *fixture.ontology_uuid(0); + let shared = *fixture.ontology_uuid(2); + let unsolicited = *fixture.ontology_uuid(1); + let first_url = FakeResolver::url("first"); + let shared_url = FakeResolver::url("shared"); + let resolver = FakeResolver::new(vec![ + (shared, shared_url.clone()), + (unsolicited, FakeResolver::url("unsolicited")), + (first, first_url.clone()), + ]); + let document = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with terminal-depth tiles"); + resolver.assert_query(&[first, shared]); + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + let mut expected = vec![Some(shared_url); ENDPOINTS.len()]; + expected[0] = Some(first_url); + assert_eq!( + resolved_urls(&trailer), + expected, + "should resolve each edge's own type" + ); + assert_eq!( + trailer.representative_type_urls_interner.len(), + 2, + "should omit the unsolicited answer" + ); +} + +#[test] +fn dispatch_unresolved_type() { + let fixture = Fixture::varied("edges-dispatch-unresolved-type"); + let first = *fixture.ontology_uuid(0); + let shared = *fixture.ontology_uuid(2); + let shared_url = FakeResolver::url("shared"); + let resolver = FakeResolver::new(vec![(shared, shared_url.clone())]); + let document = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with terminal-depth tiles"); + resolver.assert_query(&[first, shared]); + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + let mut expected = vec![Some(shared_url); ENDPOINTS.len()]; + expected[0] = None; + assert_eq!( + resolved_urls(&trailer), + expected, + "should leave the unanswered type's edge unresolved" + ); +} + +/// Each edge resolves to its own type's URL whatever order the answers arrive in. +#[test] +fn dispatch_reversed_answers() { + let fixture = Fixture::varied("edges-dispatch-reversed-answers"); + let first = *fixture.ontology_uuid(0); + let shared = *fixture.ontology_uuid(2); + let first_url = FakeResolver::url("first"); + let shared_url = FakeResolver::url("shared"); + let forward = vec![(first, first_url.clone()), (shared, shared_url.clone())]; + let reversed = vec![(shared, shared_url.clone()), (first, first_url.clone())]; + let mut expected = vec![Some(shared_url); ENDPOINTS.len()]; + expected[0] = Some(first_url); + for answers in [forward, reversed] { + let resolver = FakeResolver::new(answers); + let document = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with terminal-depth tiles"); + resolver.assert_query(&[first, shared]); + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + assert_eq!( + resolved_urls(&trailer), + expected, + "should resolve each edge independently of answer order" + ); + } +} + +/// The log records the whole report while the response redacts it. +/// +/// The logged [`Debug`](core::fmt::Debug) rendering includes the underlying store error's text and +/// the private attachment added below [`HydrateError::Query`]. Outside debug builds the response +/// carries the fixed detail instead of either. +#[test] +fn resolver_failure() { + let fixture = Fixture::new("edges-resolver-failure"); + let Err(report) = EdgesDocument::new( + fixture.scene(), + &fixture.all_tiles(), + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Auxiliary, + limits: EdgesLimits { .. }, + resolver: FailingResolver, + }, + ) else { + panic!("should propagate the resolver's failure"); + }; + assert_matches!( + report.current_context(), + EdgesDocumentError::Hydrate(HydrateError::Query) + ); + assert_matches!( + report.downcast_ref::(), + Some(HydrateError::Query) + ); + assert!( + format!("{report:#}").contains("private-store-message"), + "should retain the supplied source text in the report" + ); + assert!( + format!("{report:?}").contains("private-property-value"), + "should retain the supplied attachment in the report" + ); + assert_internal_diagnostic( + move || Problem::from(report), + &["private-store-message", "private-property-value"], + "the detail hydration failed", + ); +} + +/// Hiding either endpoint of an edge, or the edge itself, removes it from the document. +/// +/// The cases hide node 3, node 6, and edge 5, the only edge touching those nodes. All three +/// deliver every other edge. +#[test] +fn scene_hidden_rows() { + let mut fixture = Fixture::new("edges-scene-hidden-rows"); + let tiles = fixture.all_tiles(); + // Edge 5 is the only edge touching nodes 3 or 6. + let cases: [(Vec, Vec); 3] = [ + ( + (0..NODES).filter(|&row| row != 3).collect(), + (0..EDGES).collect(), + ), + ( + (0..NODES).filter(|&row| row != 6).collect(), + (0..EDGES).collect(), + ), + ( + (0..NODES).collect(), + (0..EDGES).filter(|&row| row != 5).collect(), + ), + ]; + for (nodes, edges) in cases { + fixture.restrict(nodes, edges); + let document = EdgesDocument::new( + fixture.scene(), + &tiles, + &EdgesDocumentOptions { + detail: EdgesDocumentDetailLevel::Minimal, + limits: EdgesLimits { .. }, + resolver: FailingResolver, + }, + ) + .expect("should construct with a restricted scene"); + fixture.assert_edges(&document, &[0, 1, 2, 3, 4]); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/limits.rs b/libs/@local/graph/atlas/src/serve/document/limits.rs new file mode 100644 index 00000000000..1920d8d45fb --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/limits.rs @@ -0,0 +1,10 @@ +use super::{EdgesLimits, LocateLimits, TileLimits, TranslateLimits}; + +/// Route limits shared by request assembly and bootstrap metadata. +#[derive(Copy, Clone, serde::Serialize, schemars::JsonSchema)] +pub(crate) struct DocumentLimits { + pub tile: TileLimits = TileLimits { .. }, + pub edges: EdgesLimits = EdgesLimits { .. }, + pub locate: LocateLimits = LocateLimits { .. }, + pub translate: TranslateLimits = TranslateLimits { .. }, +} diff --git a/libs/@local/graph/atlas/src/serve/document/locate/codec/mod.rs b/libs/@local/graph/atlas/src/serve/document/locate/codec/mod.rs new file mode 100644 index 00000000000..61533e86b5a --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/locate/codec/mod.rs @@ -0,0 +1,173 @@ +#[cfg(test)] +mod tests; + +use alloc::alloc::Allocator; + +use hashql_core::id::Id as _; +use zerocopy::IntoBytes as _; + +use super::{ + LocateDocument, + trailer::{LocateTrailer, PropertyMap}, +}; +use crate::serve::{ + document::codec::{CborWriter, ColumnWriter, Envelope, EnvelopeWriter, Kind, encode_details}, + hydrate::scalar::ScalarValue, +}; + +/// One locate response in writable form. +pub(crate) struct LocateResponse<'doc> { + pub variant: u64, + pub document: &'doc LocateDocument<'doc>, +} + +impl LocateResponse<'_> { + /// Encodes the response as one `SALTILEL` envelope. + /// + /// # Panics + /// + /// This panics when a directory offset exceeds `u32::MAX`. + pub(crate) fn encode_into(&self, buffer: &mut Vec) -> Envelope { + let mut envelope = EnvelopeWriter::new(Kind::LOCATE, 7, buffer); + + envelope.slot(|bytes| self.encode_head(bytes)); + envelope.slot(|bytes| ColumnWriter::over(bytes).positions(&self.document.positions)); + envelope.slot(|bytes| ColumnWriter::over(bytes).rows(&self.document.ids)); + + match &self.document.type_masks { + Some(masks) => envelope.slot(|bytes| ColumnWriter::over(bytes).masks(&masks.bits)), + None => envelope.skip(), + } + + envelope.slot(|bytes| ColumnWriter::over(bytes).rows(&self.document.edge_sources)); + envelope.slot(|bytes| ColumnWriter::over(bytes).rows(&self.document.edge_targets)); + envelope.slot(|bytes| ColumnWriter::over(bytes).identities(&self.document.edge_ids)); + + envelope.finish_with_trailer(|bytes| Self::encode_trailer(bytes, &self.document.trailer)) + } + + /// Encodes the `HEAD` map: keys 0 through 9. + fn encode_head(&self, bytes: &mut Vec) { + let document = self.document; + let mut cbor = CborWriter::over(bytes); + cbor.map(10); + + cbor.uint(0); + cbor.bytes(document.generation.as_bytes()); + + cbor.uint(1); + cbor.uint(self.variant); + + cbor.uint(2); + cbor.uint(document.ids.len() as u64); + + cbor.uint(3); + cbor.uint(u64::from(document.coordinate.z.get())); + + cbor.uint(4); + cbor.array(3); + cbor.uint(u64::from(document.coordinate.z.get())); + cbor.uint(u64::from(document.coordinate.x)); + cbor.uint(u64::from(document.coordinate.y)); + + cbor.uint(5); + cbor.uint(document.edge_ids.len() as u64); + + cbor.uint(6); + cbor.boolean(document.complete); + + cbor.uint(7); + cbor.bytes(document.entity_id.as_bytes()); + + cbor.uint(8); + cbor.boolean(document.trailer.type_ids_complete); + + cbor.uint(9); + cbor.boolean(document.trailer.properties_complete); + } + + /// Encodes an optional property map. + /// + /// An absent map encodes as `null`, and a present one as a map of interned base-URL index to + /// scalar value. + fn encode_properties( + cbor: &mut CborWriter<'_, A>, + properties: Option<&PropertyMap>, + ) { + let Some(PropertyMap(properties)) = properties else { + cbor.null(); + return; + }; + cbor.map(properties.len() as u64); + for (index, value) in properties { + cbor.uint(index.as_u64()); + match value { + ScalarValue::String(value) => cbor.text(value), + ScalarValue::Integer(value) => cbor.int(*value), + ScalarValue::Float(value) => cbor.f64(*value), + ScalarValue::Bool(value) => cbor.boolean(*value), + ScalarValue::Null => cbor.null(), + } + } + } + + /// Encodes the locate trailer map. + /// + /// The map holds interned type and property URLs, source labels and properties, and link + /// labels, types and properties. + fn encode_trailer(bytes: &mut Vec, trailer: &LocateTrailer<'_>) { + let mut cbor = CborWriter::over(bytes); + cbor.map(10); + + cbor.uint(0); + cbor.array(trailer.type_urls.len() as u64); + for url in trailer.type_urls.entries() { + cbor.collect_text(url); + } + + cbor.uint(1); + cbor.array(trailer.property_urls.len() as u64); + for url in trailer.property_urls.entries() { + cbor.text(url.as_str()); + } + + cbor.uint(2); + encode_details(&mut cbor, trailer.labels.iter()); + + cbor.uint(3); + cbor.array(trailer.representative_type_urls.len() as u64); + for &index in &trailer.representative_type_urls { + match index { + Some(index) => cbor.uint(index.as_u64()), + None => cbor.null(), + } + } + + cbor.uint(4); + Self::encode_properties(&mut cbor, trailer.properties.as_ref()); + + cbor.uint(5); + encode_details(&mut cbor, trailer.link_labels.iter()); + + cbor.uint(6); + cbor.array(trailer.link_type_urls.len() as u64); + for indexes in &trailer.link_type_urls { + cbor.array(indexes.len() as u64); + for &index in indexes { + cbor.uint(index.as_u64()); + } + } + + cbor.uint(7); + cbor.bytes(trailer.link_type_urls_complete.words().as_bytes()); + + cbor.uint(8); + cbor.array(trailer.link_properties.len() as u64); + for properties in &trailer.link_properties { + Self::encode_properties(&mut cbor, properties.as_ref()); + } + + cbor.uint(9); + cbor.bytes(trailer.link_properties_complete.words().as_bytes()); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/locate/codec/tests.rs b/libs/@local/graph/atlas/src/serve/document/locate/codec/tests.rs new file mode 100644 index 00000000000..5a5a89f5daa --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/locate/codec/tests.rs @@ -0,0 +1,875 @@ +#![expect( + clippy::little_endian_bytes, + reason = "the tests read the envelope directory and build its little-endian columns" +)] +#![expect( + clippy::big_endian_bytes, + reason = "CBOR arguments are network byte order (RFC 8949 section 3)" +)] + +use alloc::{alloc::Global, collections::BTreeMap}; +use core::iter; + +use hashql_core::id::{Id as _, IdVec, bit_vec::BitMatrix}; +use type_system::ontology::{VersionedUrl, id::BaseUrl}; + +use super::{ + super::{ + LocateDocument, + trailer::{LocateTrailer, PropertyMap}, + }, + LocateResponse, +}; +use crate::{ + bitset::DenseBitSlice, + dataset::auxiliary::Label, + file::generation::GenerationId, + identity::NodeRowId, + integrity::Sha256Digest, + math::Vec2, + morton::{Depth, MortonTile}, + postgres::id::{ArchivedEntityId, ArchivedEntityUuid, ArchivedWebId}, + serve::{ + codec::EncodedRowId, + document::{Document as _, masks::TypeMasks}, + hydrate::{EdgeSlot, NodeSlot, scalar::ScalarValue}, + intern::InternTable, + membership::SelectionSlot, + }, +}; + +/// The envelope prefix width. +const PREFIX: usize = 16; +/// The width of one directory entry. +const ENTRY: usize = 8; +/// The slot count of a `SALTILEL` envelope. +const SLOTS: usize = 7; +/// The directory index of the mask column. +const MASK_SLOT: usize = 3; + +/// The one-byte CBOR encoding of `true`. +const CBOR_TRUE: u8 = 0xF5; +/// The one-byte CBOR encoding of `false`. +const CBOR_FALSE: u8 = 0xF4; +/// The one-byte CBOR encoding of `null`. +const CBOR_NULL: u8 = 0xF6; +/// The head of a double-precision float. +const CBOR_HEAD_F64: u8 = 0xFB; + +/// Encodes one CBOR head in shortest form (RFC 8949 section 3). +/// +/// # Panics +/// +/// Panics above 255. No expectation in this module reaches a two-byte argument. +fn cbor_head(major: u8, argument: u64) -> Vec { + let ty = major << 5; + match argument { + 0..0x18 => vec![ty | u8::try_from(argument).expect("should fit the inline argument")], + 0x18..=0xFF => vec![ + ty | 0x18, + u8::try_from(argument).expect("should fit the one-byte argument"), + ], + _ => panic!("should stay below a two-byte CBOR argument"), + } +} + +/// Encodes `value` as a CBOR unsigned integer. +/// +/// # Panics +/// +/// Panics above 255, the head's limit. +fn cbor_uint(value: u64) -> Vec { + cbor_head(0, value) +} + +/// Encodes a signed integer: major type 0 when non-negative, major type 1 otherwise. +/// +/// The negative branch widens to `i128` instead of reusing the writer's bitwise negation. +/// +/// # Panics +/// +/// Panics outside `-256..=255`. The head carries one byte of argument, and a negative value +/// encodes `-1 - value` into it. +fn cbor_int(value: i64) -> Vec { + if value >= 0 { + cbor_head(0, value.cast_unsigned()) + } else { + let argument = -1_i128 - i128::from(value); + cbor_head( + 1, + u64::try_from(argument).expect("should fit the negated argument"), + ) + } +} + +/// Encodes `value` as a CBOR byte string. +/// +/// # Panics +/// +/// Panics above 255 bytes, the head's limit. +fn cbor_bytes(value: &[u8]) -> Vec { + let mut bytes = cbor_head(2, value.len() as u64); + bytes.extend_from_slice(value); + bytes +} + +/// Encodes `value` as a CBOR text string. +/// +/// # Panics +/// +/// Panics above 255 bytes of UTF-8, the head's limit. +fn cbor_text(value: &str) -> Vec { + let mut bytes = cbor_head(3, value.len() as u64); + bytes.extend_from_slice(value.as_bytes()); + bytes +} + +/// Encodes `value` as its one-byte CBOR boolean. +fn cbor_bool(value: bool) -> Vec { + vec![if value { CBOR_TRUE } else { CBOR_FALSE }] +} + +/// Encodes a double-precision float: the head, then the bit pattern in network byte order. +fn cbor_f64(value: f64) -> Vec { + let mut bytes = vec![CBOR_HEAD_F64]; + bytes.extend_from_slice(&value.to_bits().to_be_bytes()); + bytes +} + +/// Concatenates one definite-length array from its items. +/// +/// # Panics +/// +/// Panics above 255 items, the head's limit. +fn cbor_array(items: impl IntoIterator>) -> Vec { + let items: Vec<_> = items.into_iter().collect(); + let mut bytes = cbor_head(4, items.len() as u64); + for item in items { + bytes.extend(item); + } + bytes +} + +/// Concatenates one definite-length map from its unsigned keys and encoded values. +/// +/// # Panics +/// +/// Panics above 255 pairs, and above a key of 255: both go through the one-byte head. +fn cbor_map(pairs: impl IntoIterator)>) -> Vec { + let pairs: Vec<_> = pairs.into_iter().collect(); + let mut bytes = cbor_head(5, pairs.len() as u64); + for (key, value) in pairs { + bytes.extend(cbor_uint(key)); + bytes.extend(value); + } + bytes +} + +/// Derives a dense bit slice's expected word bytes from the bit indexes a test set. +/// +/// # Panics +/// +/// Panics where a bit selects a word past the allocated storage, which holds `domain` rounded up +/// to whole 64-bit words. A bit between `domain` and that rounding sets a padding bit instead. +fn expected_bit_words(domain: usize, set: &[usize]) -> Vec { + let mut words = vec![0_u64; domain.div_ceil(64)]; + for &bit in set { + words[bit >> 6] |= 1_u64 << (bit & 63); + } + + let mut bytes = Vec::new(); + for word in words { + bytes.extend_from_slice(&word.to_le_bytes()); + } + bytes +} + +/// Reads one directory entry, never a slot's payload. +/// +/// # Panics +/// +/// Panics if the computed entry slices lie outside `buffer`. With overflow checking, computing +/// `PREFIX + ENTRY * slot` or either slice's end can also panic on usize overflow. +/// +/// # Warning +/// +/// Without overflow checking, overflowing offset arithmetic wraps. If the resulting slices lie +/// within `buffer`, the helper reads those bytes even when the mathematical directory offset lies +/// beyond the buffer. +fn slot_range(buffer: &[u8], slot: usize) -> (usize, usize) { + let at = PREFIX + ENTRY * slot; + let start = u32::from_le_bytes( + buffer[at..at + 4] + .try_into() + .expect("should contain a start offset"), + ); + let end = u32::from_le_bytes( + buffer[at + 4..at + 8] + .try_into() + .expect("should contain an end offset"), + ); + (start as usize, end as usize) +} + +/// Reads the payload bytes of `slot` through its directory entry. +/// +/// # Panics +/// +/// Panics under [`slot_range`]'s conditions for the entry, its computed offset included, and where +/// the extent the entry records is not a range within `buffer`. An omitted column's `(0, 0)` entry +/// records a readable empty range, and yields an empty slice. +fn slot_bytes(buffer: &[u8], slot: usize) -> &[u8] { + let (start, end) = slot_range(buffer, slot); + &buffer[start..end] +} + +/// Returns the mandatory trailer: the bytes past the last slot's padding. +/// +/// A directory entry records its slot's unpadded end. The writer then pads to the next multiple +/// of eight before writing the trailer, which puts the trailer at the aligned end rather than at +/// the recorded one. Locate skips only the mask column, and the edge-identity column is the last +/// slot it writes. +/// +/// # Panics +/// +/// Panics where the last slot's directory entry is unreadable, and where the padded end that +/// entry records lies past `buffer`. Rounding the recorded `u32` end to a multiple of eight uses +/// `usize`. On a 32-bit target with overflow checking, an end greater than `u32::MAX - 7` panics +/// during rounding. +/// +/// # Warning +/// +/// Without overflow checking on a 32-bit target, an overflowing rounded end becomes zero and this +/// returns the entire buffer. +fn trailer_bytes(buffer: &[u8]) -> &[u8] { + let (_, end) = slot_range(buffer, SLOTS - 1); + &buffer[end.next_multiple_of(8)..] +} + +/// Asserts that `needle` occurs somewhere in `haystack`. +/// +/// # Panics +/// +/// Panics with `message` where it does not, and on an empty `needle`, which is not a window +/// width. +#[track_caller] +fn assert_contains(haystack: &[u8], needle: &[u8], message: &str) { + assert!( + haystack + .windows(needle.len()) + .any(|window| window == needle), + "{message}" + ); +} + +/// Builds a generation identity from a repeated byte, beside the digest bytes it echoes. +fn generation_of(byte: u8) -> ([u8; size_of::()], GenerationId) { + let bytes = [byte; size_of::()]; + + ( + bytes, + GenerationId::from_digest(Sha256Digest::from_bytes_unchecked(bytes)), + ) +} + +/// Builds a [`MortonTile`] at zoom `z` and coordinates `(x, y)`. +/// +/// # Panics +/// +/// Panics where `z` exceeds [`Depth::MAX`], the bound its depth carries. +fn tile(z: u8, x: u32, y: u32) -> MortonTile { + MortonTile { + z: Depth::new(z), + x, + y, + } +} + +/// Builds an encoded row id from a raw `value`, bypassing the usual encoding path. +fn row(value: u32) -> EncodedRowId { + EncodedRowId::new_unchecked(value) +} + +/// Builds a synthetic identity from repeated `web` and `entity` bytes. +fn identity(web: u8, entity: u8) -> ArchivedEntityId { + ArchivedEntityId { + web_id: ArchivedWebId::from_bytes([web; 16]), + entity_uuid: ArchivedEntityUuid::from_bytes([entity; 16]), + } +} + +/// Builds the 32 identity bytes `HEAD` key 7 carries for [`identity`]. +fn identity_bytes(web: u8, entity: u8) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[web; 16]); + bytes.extend_from_slice(&[entity; 16]); + bytes +} + +/// Builds a synthetic property [`BaseUrl`] distinguished by `name`. +/// +/// # Panics +/// +/// Panics where `name` leaves the interpolated URL unparseable. +fn property_url(name: &str) -> BaseUrl { + BaseUrl::new(format!("https://example.com/property/{name}/")) + .expect("should parse the fixture property URL") +} + +/// Assembles one `SALTILEL` envelope from its parts, independent of [`EnvelopeWriter`]. +/// +/// [`EnvelopeWriter`]: crate::serve::document::codec::EnvelopeWriter +/// +/// Padding to the next eight-byte boundary follows each present slot's payload. A `None` slot +/// keeps the directory's initial zero entry: what an omitted mask column leaves behind. +/// +/// # Panics +/// +/// Panics above a u16 slot count, and where a payload offset does not fit u32. +fn build_envelope(slots: &[Option>], trailer: &[u8]) -> Vec { + let mut buffer = Vec::new(); + buffer.extend_from_slice(b"SALTILEL"); + buffer.extend_from_slice(&1_u16.to_le_bytes()); + buffer.extend_from_slice(&0_u16.to_le_bytes()); + buffer.extend_from_slice( + &u16::try_from(slots.len()) + .expect("should fit the fixture's slot count") + .to_le_bytes(), + ); + buffer.extend_from_slice(&0_u16.to_le_bytes()); + + let mut directory = vec![0_u8; slots.len() * ENTRY]; + buffer.resize(PREFIX + directory.len(), 0); + + for (index, slot) in slots.iter().enumerate() { + let Some(payload) = slot else { continue }; + + let start = buffer.len(); + buffer.extend_from_slice(payload); + let end = buffer.len(); + buffer.resize(end.next_multiple_of(8), 0); + + let at = index * ENTRY; + directory[at..at + 4].copy_from_slice( + &u32::try_from(start) + .expect("should fit the fixture's offsets") + .to_le_bytes(), + ); + directory[at + 4..at + ENTRY].copy_from_slice( + &u32::try_from(end) + .expect("should fit the fixture's offsets") + .to_le_bytes(), + ); + } + + buffer[PREFIX..PREFIX + directory.len()].copy_from_slice(&directory); + buffer.extend_from_slice(trailer); + buffer +} + +/// Builds a trailer with every array present and carrying no node and no link. +fn empty_trailer() -> LocateTrailer<'static> { + LocateTrailer { + type_urls: InternTable::new(), + property_urls: InternTable::new(), + labels: IdVec::::from_raw(Vec::new()), + representative_type_urls: IdVec::from_raw(Vec::new()), + properties: None, + type_ids_complete: false, + properties_complete: false, + link_labels: IdVec::::from_raw(Vec::new()), + link_type_urls: IdVec::from_raw(Vec::new()), + link_type_urls_complete: DenseBitSlice::new_empty(0), + link_properties: IdVec::from_raw(Vec::new()), + link_properties_complete: DenseBitSlice::new_empty(0), + } +} + +/// Builds the wire bytes of [`empty_trailer`]: ten keys over empty collections and a null map. +fn expected_empty_trailer() -> Vec { + cbor_map([ + (0, cbor_array([])), + (1, cbor_array([])), + (2, cbor_array([])), + (3, cbor_array([])), + (4, vec![CBOR_NULL]), + (5, cbor_array([])), + (6, cbor_array([])), + (7, cbor_bytes(&[])), + (8, cbor_array([])), + (9, cbor_bytes(&[])), + ]) +} + +/// Builds a nonroot source alone in its ego graph, with an empty trailer. +fn minimal_document(generation: GenerationId) -> LocateDocument<'static> { + LocateDocument { + generation, + coordinate: tile(2, 1, 3), + entity_id: identity(0xCC, 0xDD), + complete: true, + positions: IdVec::::from_raw(Vec::new()), + ids: IdVec::>::from_raw(Vec::new()), + type_masks: None, + edge_ids: IdVec::::from_raw(Vec::new()), + edge_sources: IdVec::>::from_raw(Vec::new()), + edge_targets: IdVec::>::from_raw(Vec::new()), + trailer: empty_trailer(), + } +} + +/// Builds the ten-key `HEAD` map of [`minimal_document`], over the given variant and flags. +/// +/// # Panics +/// +/// Panics on a `digest` above 255 bytes, or a `variant` above 255: each is a one-byte CBOR +/// argument here. +fn expected_minimal_head( + digest: &[u8], + variant: u64, + type_ids_complete: bool, + properties_complete: bool, +) -> Vec { + cbor_map([ + (0, cbor_bytes(digest)), + (1, cbor_uint(variant)), + (2, cbor_uint(0)), + (3, cbor_uint(2)), + (4, cbor_array([cbor_uint(2), cbor_uint(1), cbor_uint(3)])), + (5, cbor_uint(0)), + (6, cbor_bool(true)), + (7, cbor_bytes(&identity_bytes(0xCC, 0xDD))), + (8, cbor_bool(type_ids_complete)), + (9, cbor_bool(properties_complete)), + ]) +} + +/// The minimal document replaces a longer prepopulated buffer with the whole envelope. +/// +/// The one case compared byte for byte. Every other test decodes the directory and checks one +/// region. +#[test] +fn response_encode_minimal_shape() { + let (digest, generation) = generation_of(0x20); + let document = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + buffer.resize(512, 0xDD); + let _completed = document + .encode(&mut buffer) + .expect("should complete the minimal locate response"); + + let expected = build_envelope( + &[ + Some(expected_minimal_head(&digest, 0, false, false)), + Some(Vec::new()), + Some(Vec::new()), + None, + Some(Vec::new()), + Some(Vec::new()), + Some(Vec::new()), + ], + &expected_empty_trailer(), + ); + + assert_eq!( + buffer.as_slice(), + expected.as_slice(), + "should replace the buffer with six present slots, one skipped and the mandatory trailer" + ); +} + +/// The response echoes its own `variant` field rather than a fixed value. +/// +/// [`Document::encode`](crate::serve::document::Document::encode) always passes zero. A nonzero +/// variant needs the response built directly. +#[test] +fn response_variant_nonzero() { + let (digest, generation) = generation_of(0x21); + let document = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + let _completed = LocateResponse { + variant: 3, + document: &document, + } + .encode_into(&mut buffer); + + assert_eq!( + slot_bytes(&buffer, 0), + expected_minimal_head(&digest, 3, false, false).as_slice(), + "should echo variant 3 at head key 1 rather than a fixed value" + ); +} + +/// An omitted mask column and a present-but-empty one differ in the directory alone. +#[test] +fn response_mask_omitted_versus_empty() { + let (_, generation) = generation_of(0x22); + let omitted = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + let _completed = omitted + .encode(&mut buffer) + .expect("should complete the response without a mask column"); + assert_eq!( + slot_range(&buffer, MASK_SLOT), + (0, 0), + "an omitted mask column should leave the directory entry at its initial (0, 0)" + ); + + let (_, generation) = generation_of(0x23); + let present = LocateDocument { + type_masks: Some(TypeMasks { + bits: BitMatrix::::new(0, 2), + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = present + .encode(&mut buffer) + .expect("should complete the response with an empty mask column"); + let (start, end) = slot_range(&buffer, MASK_SLOT); + assert_eq!( + start, end, + "a present-but-empty mask column should record a zero-length extent" + ); + assert!( + start >= PREFIX + ENTRY * SLOTS, + "the present-but-empty column should sit past the directory rather than at offset zero" + ); +} + +/// Node order and edge order stay independent, and the source's own identity crosses verbatim. +/// +/// The edge columns permute their endpoints against the node order. A writer that sorted a +/// column, or filled one column from another, disagrees with at least one expectation here. +#[test] +fn response_node_and_edge_order() { + let (_, generation) = generation_of(0x24); + let document = LocateDocument { + entity_id: identity(0x40, 0x41), + positions: IdVec::from_raw(vec![Vec2::new(1.0, 2.0), Vec2::new(3.0, 4.0)]), + ids: IdVec::from_raw(vec![row(30), row(10)]), + edge_ids: IdVec::from_raw(vec![identity(0x50, 0x51), identity(0x60, 0x61)]), + edge_sources: IdVec::from_raw(vec![row(10), row(30)]), + edge_targets: IdVec::from_raw(vec![row(10), row(10)]), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the ego-graph locate response"); + + let mut expected_positions = Vec::new(); + for value in [1.0_f32, 2.0, 3.0, 4.0] { + expected_positions.extend_from_slice(&value.to_bits().to_le_bytes()); + } + let expected_ids: Vec = [30_u32, 10] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(); + let expected_sources: Vec = [10_u32, 30] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(); + let expected_targets: Vec = [10_u32, 10] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(); + let mut expected_edge_ids = identity_bytes(0x50, 0x51); + expected_edge_ids.extend(identity_bytes(0x60, 0x61)); + + assert_eq!( + slot_bytes(&buffer, 1), + expected_positions.as_slice(), + "should keep the node positions in document order" + ); + assert_eq!( + slot_bytes(&buffer, 2), + expected_ids.as_slice(), + "should keep the node rows in document order" + ); + assert_eq!( + slot_bytes(&buffer, 4), + expected_sources.as_slice(), + "should keep the edge sources in their own order, independent of the node rows" + ); + assert_eq!( + slot_bytes(&buffer, 5), + expected_targets.as_slice(), + "should keep the edge targets in their own order, independent of the sources" + ); + assert_eq!( + slot_bytes(&buffer, 6), + expected_edge_ids.as_slice(), + "should keep the edge identities in document order" + ); + + let mut expected_source = cbor_uint(7); + expected_source.extend(cbor_bytes(&identity_bytes(0x40, 0x41))); + assert_contains( + slot_bytes(&buffer, 0), + &expected_source, + "should carry the source's own upstream identity verbatim at head key 7", + ); + + let mut expected_counts = cbor_uint(2); + expected_counts.extend(cbor_uint(2)); + assert_contains( + slot_bytes(&buffer, 0), + &expected_counts, + "should report two delivered nodes at head key 2", + ); + let mut expected_edges = cbor_uint(5); + expected_edges.extend(cbor_uint(2)); + assert_contains( + slot_bytes(&buffer, 0), + &expected_edges, + "should report two delivered edges at head key 5", + ); +} + +/// The trailer's two completeness flags reach `HEAD` keys 8 and 9 independently. +#[test] +fn response_completeness_flags() { + for (type_ids_complete, properties_complete) in + [(false, false), (true, false), (false, true), (true, true)] + { + let (digest, generation) = generation_of(0x25); + let mut document = minimal_document(generation); + document.trailer.type_ids_complete = type_ids_complete; + document.trailer.properties_complete = properties_complete; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the locate response"); + + assert_eq!( + slot_bytes(&buffer, 0), + expected_minimal_head(&digest, 0, type_ids_complete, properties_complete).as_slice(), + "should encode type_ids_complete={type_ids_complete} at head key 8 and \ + properties_complete={properties_complete} at head key 9" + ); + } +} + +/// The source's property map holds every [`ScalarValue`] variant, keyed by property index. +#[test] +fn trailer_scalar_variants() { + let (_, generation) = generation_of(0x26); + let mut property_urls = InternTable::new(); + let string = property_urls.intern(property_url("name")); + let integer = property_urls.intern(property_url("age")); + let float = property_urls.intern(property_url("weight")); + let boolean = property_urls.intern(property_url("active")); + let null = property_urls.intern(property_url("note")); + + let document = LocateDocument { + positions: IdVec::from_raw(vec![Vec2::ZERO]), + ids: IdVec::from_raw(vec![row(0)]), + trailer: LocateTrailer { + labels: IdVec::from_raw(vec![Label::new("source")]), + representative_type_urls: IdVec::from_raw(vec![None]), + properties: Some(PropertyMap(BTreeMap::from([ + (string, ScalarValue::String("Ada".to_owned())), + (integer, ScalarValue::Integer(-7)), + (float, ScalarValue::Float(0.5)), + (boolean, ScalarValue::Bool(true)), + (null, ScalarValue::Null), + ]))), + property_urls, + ..empty_trailer() + }, + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the hydrated locate response"); + + let mut expected = cbor_uint(4); + expected.extend(cbor_map([ + (string.as_u64(), cbor_text("Ada")), + (integer.as_u64(), cbor_int(-7)), + (float.as_u64(), cbor_f64(0.5)), + (boolean.as_u64(), cbor_bool(true)), + (null.as_u64(), vec![CBOR_NULL]), + ])); + + assert_contains( + trailer_bytes(&buffer), + &expected, + "should encode each scalar variant in its own CBOR shape, keyed by property index", + ); +} + +/// An unresolved source reads `null` at trailer key 4, and a resolved empty one reads a map. +#[test] +fn trailer_properties_null_versus_empty() { + let (_, generation) = generation_of(0x27); + let unresolved = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + let _completed = unresolved + .encode(&mut buffer) + .expect("should complete the unresolved locate response"); + let mut expected_null = cbor_uint(4); + expected_null.push(CBOR_NULL); + assert_contains( + trailer_bytes(&buffer), + &expected_null, + "an unresolved source should read null at trailer key 4", + ); + + let (_, generation) = generation_of(0x28); + let resolved = LocateDocument { + trailer: LocateTrailer { + properties: Some(PropertyMap(BTreeMap::new())), + ..empty_trailer() + }, + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = resolved + .encode(&mut buffer) + .expect("should complete the resolved locate response"); + let mut expected_empty = cbor_uint(4); + expected_empty.extend(cbor_map([])); + assert_contains( + trailer_bytes(&buffer), + &expected_empty, + "a resolved source with no properties should read an empty map rather than null", + ); +} + +/// A URL the source and a link both reference interns once, under one table index. +#[test] +fn trailer_url_interning_shared() { + let (_, generation) = generation_of(0x29); + let mut type_urls = InternTable::new(); + let shared = type_urls.intern( + "https://example.com/types/shared/v/1" + .parse::() + .expect("should parse the fixture type URL"), + ); + + let document = LocateDocument { + positions: IdVec::from_raw(vec![Vec2::new(1.0, 2.0)]), + ids: IdVec::from_raw(vec![row(0)]), + edge_ids: IdVec::from_raw(vec![identity(0x70, 0x71)]), + edge_sources: IdVec::from_raw(vec![row(0)]), + edge_targets: IdVec::from_raw(vec![row(1)]), + trailer: LocateTrailer { + labels: IdVec::from_raw(vec![Label::new("source")]), + representative_type_urls: IdVec::from_raw(vec![Some(shared)]), + link_labels: IdVec::from_raw(vec![Label::new("link")]), + link_type_urls: IdVec::from_raw(vec![vec![shared]]), + link_type_urls_complete: DenseBitSlice::new_empty(1), + link_properties: IdVec::from_raw(vec![None]), + link_properties_complete: DenseBitSlice::new_empty(1), + type_urls, + ..empty_trailer() + }, + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the interned locate response"); + + let expected = cbor_map([ + ( + 0, + cbor_array([cbor_text("https://example.com/types/shared/v/1")]), + ), + (1, cbor_array([])), + (2, cbor_array([cbor_text("source")])), + (3, cbor_array([cbor_uint(shared.as_u64())])), + (4, vec![CBOR_NULL]), + (5, cbor_array([cbor_text("link")])), + (6, cbor_array([cbor_array([cbor_uint(shared.as_u64())])])), + (7, cbor_bytes(&expected_bit_words(1, &[]))), + (8, cbor_array([vec![CBOR_NULL]])), + (9, cbor_bytes(&expected_bit_words(1, &[]))), + ]); + + assert_eq!( + trailer_bytes(&buffer), + expected.as_slice(), + "should intern the shared URL once and reference one table index from node and link" + ); +} + +/// Both link completeness bit arrays cross the 64-bit word boundary independently. +/// +/// Seventy edges put the boundary inside the array rather than at its edge. Their patterns +/// differ, which stops either from standing in for the other. +#[test] +fn trailer_bit_completeness_word_boundary() { + const EDGES: usize = 70; + + let (_, generation) = generation_of(0x2A); + let mut link_type_urls_complete = DenseBitSlice::::new_empty(EDGES); + link_type_urls_complete.insert(EdgeSlot::from_usize(63)); + link_type_urls_complete.insert(EdgeSlot::from_usize(64)); + + let mut link_properties_complete = DenseBitSlice::::new_empty(EDGES); + link_properties_complete.insert(EdgeSlot::from_usize(0)); + link_properties_complete.insert(EdgeSlot::from_usize(64)); + + let document = LocateDocument { + positions: IdVec::from_raw(vec![Vec2::ZERO]), + ids: IdVec::from_raw(vec![row(0)]), + edge_ids: IdVec::from_raw(vec![identity(0x01, 0x01); EDGES]), + edge_sources: IdVec::from_raw(vec![row(0); EDGES]), + edge_targets: IdVec::from_raw(vec![row(0); EDGES]), + trailer: LocateTrailer { + labels: IdVec::from_raw(vec![Label::EMPTY]), + representative_type_urls: IdVec::from_raw(vec![None]), + link_labels: IdVec::from_raw(vec![Label::EMPTY; EDGES]), + link_type_urls: IdVec::from_raw(vec![Vec::new(); EDGES]), + link_type_urls_complete, + link_properties: IdVec::from_raw(iter::repeat_with(|| None).take(EDGES).collect()), + link_properties_complete, + ..empty_trailer() + }, + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the wide locate response"); + let trailer = trailer_bytes(&buffer); + + let expected_type_urls = expected_bit_words(EDGES, &[63, 64]); + let expected_properties = expected_bit_words(EDGES, &[0, 64]); + assert_eq!( + expected_type_urls.len(), + 16, + "seventy edges should occupy two eight-byte words" + ); + + let mut expected_type_urls_entry = cbor_uint(7); + expected_type_urls_entry.extend(cbor_bytes(&expected_type_urls)); + assert_contains( + trailer, + &expected_type_urls_entry, + "should carry the link-type-url completeness words with both boundary bits set", + ); + + let mut expected_properties_entry = cbor_uint(9); + expected_properties_entry.extend(cbor_bytes(&expected_properties)); + assert_contains( + trailer, + &expected_properties_entry, + "should carry the link-property completeness words with their own distinct pattern", + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/locate/mod.rs b/libs/@local/graph/atlas/src/serve/document/locate/mod.rs new file mode 100644 index 00000000000..83377fac218 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/locate/mod.rs @@ -0,0 +1,255 @@ +//! Source geometry and hydrated details for a visible node's ego graph. +//! +//! Construction resolves either source identity into the captured scene and caps its incident edges +//! before hydration. + +use alloc::alloc::Allocator; +use core::{error::Error, fmt}; + +use error_stack::Report; +use hashql_core::id::IdVec; +use type_system::knowledge::entity::EntityId; + +use self::{ + subgraph::{LocateSubgraph, SourcePoint}, + trailer::LocateTrailer, +}; +use super::{Document, codec::Envelope, masks::TypeMasks}; +use crate::{ + file::generation::GenerationId, + identity::{EdgeRowId, NodeRowId}, + math::Vec2, + morton::MortonTile, + postgres::id::ArchivedEntityId, + serve::{ + codec::EncodedRowId, + hydrate::{ + EdgeSlot, HydrateError, LocateEntity, LocateRequest, LocateResolver, LocateResponse, + NodeSlot, + }, + membership::OntologySelection, + neighbourhood::DeliveredEdge, + scene::Scene, + }, +}; + +mod codec; +mod subgraph; +mod trailer; + +#[cfg(test)] +mod tests; + +/// A node named by its upstream identity or encoded row. +#[derive(Debug, Copy, Clone)] +pub(crate) enum LocateSource { + Key(EntityId), + Row(EncodedRowId), +} + +/// Limits on requested types, incident edges and per-source and per-link detail. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LocateLimits { + /// Most requested types, including duplicates. The default is 32. + pub colored_type_ids: u32 = 32, + /// Most incident edges. The default is 512. + pub edges: u32 = 512, + /// Most scalar properties on the source. The default is 10. + pub properties: u32 = 10, + /// Most direct types per link. The default is 5. + pub link_type_ids: u32 = 5, + /// Most scalar properties per link. The default is 10. + pub link_properties: u32 = 10, +} + +/// Ontology selection, limits and a live-detail resolver for one locate request. +pub(crate) struct LocateDocumentOptions<'selection, R> { + pub types: &'selection OntologySelection, + pub limits: LocateLimits, + pub resolver: R, +} + +/// A failure to assemble a locate document. +#[derive(Debug)] +pub(crate) enum LocateDocumentError { + /// The request exceeds the configured type-count limit. + Types { count: usize, maximum: u32 }, + /// The source does not name a visible, placed node in the captured scene. + UnknownEntity, + /// A delivered node has no identity, position or delivery zoom in the captured scene. + Node { row: NodeRowId }, + /// A resolved node has no captured display payload. + NodeDisplay { row: NodeRowId }, + /// A resolved link has no captured display payload. + LinkDisplay { row: EdgeRowId }, + /// The resolver failed to read the requested details. + Hydrate(HydrateError), +} + +impl fmt::Display for LocateDocumentError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Types { count, maximum } => write!( + fmt, + "the request lists {count} colored type ids, exceeding the limit of {maximum}" + ), + Self::UnknownEntity => fmt.write_str("the source does not name a visible node"), + Self::Node { row } => write!( + fmt, + "delivered node {row} has no identity, position or delivery zoom" + ), + Self::NodeDisplay { row } => write!(fmt, "resolved node {row} has no display payload"), + Self::LinkDisplay { row } => write!(fmt, "resolved link {row} has no display payload"), + Self::Hydrate(error) => write!(fmt, "locate detail resolution failed: {error}"), + } + } +} + +impl Error for LocateDocumentError {} + +/// A source-first node set, identity-ordered edges and mandatory hydrated details. +/// +/// Partners occur once, ordered by encoded node row. Truncation selects edges by squared distance +/// to the partner, then the partner's first delivery zoom, then link identity. +pub(crate) struct LocateDocument<'details> { + generation: GenerationId, + coordinate: MortonTile, + entity_id: ArchivedEntityId, + complete: bool, + positions: IdVec, + ids: IdVec>, + type_masks: Option>, + edge_ids: IdVec, + edge_sources: IdVec>, + edge_targets: IdVec>, + trailer: LocateTrailer<'details>, +} + +impl<'details> LocateDocument<'details> { + /// Builds the visible ego graph and resolves its detail columns. + /// + /// Missing, withdrawn and hidden sources receive the same error. Draft identities also read + /// absent. Labels come from the captured scene and remain empty for entities the resolver no + /// longer serves. Property hydration uses the scene mask's actor. + /// + /// An empty type selection omits the mask column. Unknown types and nodes outside the fitted + /// position domain have zero bits. + /// + /// # Errors + /// + /// Returns [`LocateDocumentError`] for an invalid type count, unavailable source or incomplete + /// captured data, and for failed hydration. + pub(crate) fn new( + scene @ Scene { + world, epoch, mask, .. + }: Scene<'details>, + source: LocateSource, + LocateDocumentOptions { + types, + limits, + resolver, + }: &LocateDocumentOptions<'_, R>, + ) -> Result> { + if types.len() > limits.colored_type_ids as usize { + return Err(Report::new(LocateDocumentError::Types { + count: types.len(), + maximum: limits.colored_type_ids, + })); + } + + let source = SourcePoint::new(scene, source) + .ok_or_else(|| Report::new(LocateDocumentError::UnknownEntity))?; + let subgraph = LocateSubgraph::new(scene, source, limits.edges as usize)?; + + let mut positions = IdVec::with_capacity(subgraph.nodes.len()); + let mut ids = IdVec::with_capacity(subgraph.nodes.len()); + let mut nodes = IdVec::with_capacity(subgraph.nodes.len()); + + for &row in &subgraph.nodes { + nodes.push(LocateEntity::new( + world + .layout + .index + .key_of(epoch, row) + .ok_or_else(|| Report::new(LocateDocumentError::Node { row }))?, + )); + positions.push( + world + .layout + .position(epoch, row) + .ok_or_else(|| Report::new(LocateDocumentError::Node { row }))?, + ); + ids.push(world.layout.index.encode(row)); + } + + let type_masks = (!types.is_empty()) + .then(|| TypeMasks::new(scene, subgraph.nodes.iter().copied(), types)); + + let mut edge_ids = IdVec::with_capacity(subgraph.edges.len()); + let mut edge_sources = IdVec::with_capacity(subgraph.edges.len()); + let mut edge_targets = IdVec::with_capacity(subgraph.edges.len()); + + for &DeliveredEdge { + row: _, + endpoints: [source, target], + identity, + } in &subgraph.edges + { + edge_ids.push(identity); + edge_sources.push(world.layout.index.encode(source)); + edge_targets.push(world.layout.index.encode(target)); + } + + let mut links: IdVec<_, _> = edge_ids.iter().copied().map(LocateEntity::new).collect(); + let source_properties = resolver + .resolve(LocateRequest { + actor: mask.actor(), + nodes: &mut nodes, + links: &mut links, + properties: limits.properties, + link_type_ids: limits.link_type_ids, + link_properties: limits.link_properties, + }) + .map_err(|report| { + let error = LocateDocumentError::Hydrate(*report.current_context()); + report.change_context(error) + })?; + let trailer = LocateTrailer::new( + scene, + &subgraph, + types, + LocateResponse { + nodes, + links, + source_properties, + }, + )?; + + Ok(Self { + generation: world.generation().id(), + coordinate: subgraph.source.cell, + entity_id: subgraph.source.identity, + complete: subgraph.complete, + positions, + ids, + type_masks, + edge_ids, + edge_sources, + edge_targets, + trailer, + }) + } +} + +impl Document for LocateDocument<'_> { + type Error = !; + + fn encode(&self, buffer: &mut Vec) -> Result { + Ok(self::codec::LocateResponse { + variant: 0, + document: self, + } + .encode_into(buffer)) + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/locate/subgraph.rs b/libs/@local/graph/atlas/src/serve/document/locate/subgraph.rs new file mode 100644 index 00000000000..42fad3ecfba --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/locate/subgraph.rs @@ -0,0 +1,726 @@ +use alloc::collections::BinaryHeap; +use core::cmp::Ordering; + +use error_stack::Report; +use hashql_core::id::IdVec; +use type_system::knowledge::entity::EntityId; + +use super::{LocateDocumentError, LocateSource}; +use crate::{ + identity::NodeRowId, + math::{DNonNegative, Vec2}, + morton::{Depth, MortonKey, MortonTile, Zoom}, + postgres::id::ArchivedEntityId, + salt::lod::stage::WIRE_FRAME, + serve::{ + hydrate::NodeSlot, + neighbourhood::{DeliveredEdge, EdgeSet, Neighbourhood}, + scene::Scene, + visibility::Visible, + }, +}; + +/// A resolved source's row, identity, position and first delivery cell. +pub(crate) struct SourcePoint { + pub row: Visible, + pub identity: ArchivedEntityId, + pub position: Vec2, + pub cell: MortonTile, +} + +impl SourcePoint { + /// Resolves a visible source's identity, position and first delivery cell. + /// + /// Returns [`None`] for a draft key or a source without a live identity, visibility, position + /// or delivery zoom. + /// + /// # Panics + /// + /// Panics if the scene's node index does not belong to its epoch and `source` is not a draft + /// key. + pub(crate) fn new( + Scene { + world, + epoch, + mask, + delivery, + .. + }: Scene<'_>, + source: LocateSource, + ) -> Option { + let row = match source { + LocateSource::Key(EntityId { + web_id, + entity_uuid, + draft_id: None, + }) => world.layout.index.row_of( + epoch, + ArchivedEntityId { + web_id: web_id.into(), + entity_uuid: entity_uuid.into(), + }, + )?, + LocateSource::Key(_) => return None, + LocateSource::Row(row) => world.layout.index.decode(epoch, row)?, + }; + + let row = mask.visible_node(row)?; + let identity = world.layout.index.key_of(epoch, row.unwrap())?; + let position = world.layout.position(epoch, row.unwrap())?; + let zoom = delivery.first_zoom(row.unwrap())?; + let [x, y] = WIRE_FRAME.quantize(position); + + Some(Self { + row, + identity, + position, + cell: MortonKey::new(x, y).tile(Depth::from_zoom(zoom)), + }) + } +} + +/// The truncation key of an incident edge. +/// +/// The squared distance to the partner orders first. The partner's first delivery zoom breaks a +/// distance tie, and the link identity breaks a zoom tie. Link identities are distinct, which makes +/// the order total. +type NearestKey = (DNonNegative, Zoom, ArchivedEntityId); + +/// An edge ordered by its truncation key alone. +struct Keyed { + key: NearestKey, + edge: DeliveredEdge, +} + +impl PartialEq for Keyed { + /// Compares `key` alone. Edges sharing a key are interchangeable for this cap's ranking. + fn eq(&self, other: &Self) -> bool { + self.key == other.key + } +} + +impl Eq for Keyed {} + +impl PartialOrd for Keyed { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Keyed { + fn cmp(&self, other: &Self) -> Ordering { + self.key.cmp(&other.key) + } +} + +/// The edges a [`NearestCap`] holds. +enum Retained { + /// Every offered edge, unranked, while their count is at most the capacity. + Buffered(Vec), + /// The `capacity` smallest keys offered, with the largest on top. + Ranked(BinaryHeap), +} + +/// The `capacity` nearest incident edges of one source, selected while the incident set streams. +/// +/// Offered edges buffer unranked until one more than `capacity` has arrived. That offer ranks the +/// buffered edges and itself, and every later offer ranks on arrival, keeping the `capacity` +/// smallest [`NearestKey`]s. The rank function, and any error it returns, therefore runs only when +/// the selection truncates, and at zero capacity never: the response is then empty and the +/// truncation flag is the only fact recorded. +/// +/// # Complexity +/// +/// For capacity `c` and `n` offered edges, let `L = 1 + logâ‚‚(c + 1)`. Excluding the rank function's +/// work, a buffering offer costs amortized `O(1)`, with a buffer copy on reallocation. The first +/// overflow ranks the buffered edges and itself, building the heap by repeated pushes in +/// `O(cL)` work. Every later offer costs `O(L)`. At zero capacity each offer costs `O(1)` and +/// skips ranking. Total container work is `O(1 + nL)`, with at most one rank call per offered +/// edge. Edge storage is `O(min(n, c))`, in addition to fixed state and the rank closure. +struct NearestCap { + capacity: usize, + rank: F, + retained: Retained, + truncated: bool, +} + +impl NearestCap +where + F: FnMut(&DeliveredEdge) -> Result, +{ + /// Selects at most `capacity` edges under `rank`. + /// + /// Storage grows with the offered edges rather than with `capacity`, which a request may set + /// far above any degree. + const fn new(capacity: usize, rank: F) -> Self { + Self { + capacity, + rank, + retained: Retained::Buffered(Vec::new()), + truncated: false, + } + } + + /// Offers one incident edge. + /// + /// # Errors + /// + /// Returns the rank function's error for any edge ranked by this offer: the buffered edges at + /// the transition, or `edge` alone afterwards. The selection is unusable after an error. + fn offer(&mut self, edge: DeliveredEdge) -> Result<(), E> { + match &mut self.retained { + Retained::Buffered(buffer) if buffer.len() < self.capacity => { + buffer.push(edge); + Ok(()) + } + Retained::Buffered(buffer) => { + self.truncated = true; + if self.capacity == 0 { + return Ok(()); + } + + let mut kept = BinaryHeap::with_capacity(buffer.len()); + for buffered in buffer.drain(..) { + let key = (self.rank)(&buffered)?; + kept.push(Keyed { + key, + edge: buffered, + }); + } + + let key = (self.rank)(&edge)?; + Self::keep_smaller(&mut kept, Keyed { key, edge }); + self.retained = Retained::Ranked(kept); + Ok(()) + } + Retained::Ranked(kept) => { + self.truncated = true; + let key = (self.rank)(&edge)?; + Self::keep_smaller(kept, Keyed { key, edge }); + Ok(()) + } + } + } + + /// Replaces the largest kept key with `candidate` when the candidate is smaller. + fn keep_smaller(kept: &mut BinaryHeap, candidate: Keyed) { + if let Some(mut worst) = kept.peek_mut() + && candidate < *worst + { + *worst = candidate; + } + } + + /// Returns the selected edges in unspecified order. + /// + /// The set is complete when the selection dropped no offer. + fn into_set(self) -> EdgeSet { + let edges = match self.retained { + Retained::Buffered(buffer) => buffer, + Retained::Ranked(kept) => kept.into_iter().map(|keyed| keyed.edge).collect(), + }; + + EdgeSet { + complete: !self.truncated, + edges, + } + } +} + +/// A capped incident set with the source first and distinct partners in wire-id order. +pub(crate) struct LocateSubgraph { + pub source: SourcePoint, + pub nodes: IdVec, + pub edges: Vec, + pub complete: bool, +} + +impl LocateSubgraph { + /// Selects incident edges and their distinct partners around `source`. + /// + /// Truncation keeps the edges with the smallest [`NearestKey`]. The selection reads partner + /// positions and delivery zooms only when the incident set exceeds `capacity`, and never at + /// zero capacity. + /// + /// # Errors + /// + /// Returns [`LocateDocumentError::Node`] when a partner of a ranked edge has no captured + /// position or delivery zoom. + /// + /// # Panics + /// + /// Panics if the scene's topology does not belong to its epoch. + /// + /// # Complexity + /// + /// Let `d` count the edges offered by the incident iterator, `c` be the capacity, + /// `m = min(d, c)` and `L = 1 + logâ‚‚(c + 1)`. Selection takes `O(1 + dL)` container work, + /// including the one-time heap construction. The output sorts take + /// `O(1 + m(1 + logâ‚‚(m + 1)))` work, within the same total bound. Storage beyond the response + /// is `O(1 + m)`. These bounds exclude iterator costs and queries on the + /// [`World`](crate::serve::world::World) and + /// [`DeliverySchedule`](crate::serve::schedule::DeliverySchedule). The iterator + /// can inspect more underlying entries than it yields. The remaining query work consists of + /// at most one rank call per offered edge and one wire encoding per retained partner + /// occurrence before deduplication. + pub(crate) fn new( + scene @ Scene { + world, + epoch, + delivery, + .. + }: Scene<'_>, + source: SourcePoint, + capacity: usize, + ) -> Result> { + let source_row = source.row.unwrap(); + let source_position = source.position; + + let mut nearest = NearestCap::new( + capacity, + |edge: &DeliveredEdge| -> Result> { + let row = edge + .partner_of(source_row) + .expect("an incident edge should contain the source"); + + let position = world + .layout + .position(epoch, row) + .ok_or_else(|| Report::new(LocateDocumentError::Node { row }))?; + + let zoom = delivery + .first_zoom(row) + .ok_or_else(|| Report::new(LocateDocumentError::Node { row }))?; + + Ok(( + position.distance_squared_wide(source_position), + zoom, + edge.identity, + )) + }, + ); + + let neighbourhood = Neighbourhood { provider: scene }; + for edge in neighbourhood.incident(source_row) { + nearest.offer(edge)?; + } + + let EdgeSet { + complete, + mut edges, + } = nearest.into_set(); + edges.sort_unstable_by_key(|edge| edge.identity); + + let mut partners: Vec<_> = edges + .iter() + .flat_map(|edge| edge.endpoints) + .filter(|&row| row != source_row) + .map(|row| (world.layout.index.encode(row), row)) + .collect(); + partners.sort_unstable_by_key(|&(wire, _)| wire); + partners.dedup_by_key(|&mut (wire, _)| wire); + + let mut nodes = IdVec::with_capacity(partners.len() + 1); + nodes.push(source_row); + nodes.extend(partners.into_iter().map(|(_, row)| row)); + + Ok(Self { + source, + nodes, + edges, + complete, + }) + } +} + +#[cfg(test)] +mod tests { + use core::cell::Cell; + + use hashql_core::id::{Id as _, IdSlice, IdVec}; + use proptest::{collection, property_test}; + use rand::{RngExt as _, SeedableRng as _, rngs::StdRng}; + use uuid::Uuid; + + use super::{NearestCap, NearestKey, Retained}; + use crate::{ + identity::{EdgeRowId, NodeRowId}, + math::DNonNegative, + morton::Zoom, + postgres::id::ArchivedEntityId, + serve::{ + neighbourhood::{DeliveredEdge, EdgeSet}, + visibility::Visible, + }, + }; + + /// The source every synthetic edge is incident to. + const SOURCE: NodeRowId = NodeRowId::new(0); + + /// Builds a synthetic identity distinguished by `value` alone, sharing one fixed web id. + fn identity(value: u128) -> ArchivedEntityId { + ArchivedEntityId { + web_id: Uuid::from_u128(1).into(), + entity_uuid: Uuid::from_u128(value).into(), + } + } + + /// Parses `value` as a [`Zoom`] within the domain. + /// + /// # Panics + /// + /// Panics if `value` does not fit the domain. + #[track_caller] + fn zoom(value: u8) -> Zoom { + Zoom::new(value).expect("should fit the zoom domain") + } + + /// Synthetic incident edges keyed by edge row, with the source as every edge's first endpoint. + struct Incident { + edges: IdVec, + keys: IdVec, + } + + impl Incident { + /// Builds an empty incident set with no recorded edges or keys. + fn new() -> Self { + Self { + edges: IdVec::new(), + keys: IdVec::new(), + } + } + + /// Adds an edge whose key is `(distance, zoom, identity)` and returns it. + /// + /// # Panics + /// + /// Panics where [`zoom()`] cannot parse `zoom` into the [`Zoom`] domain. + fn link(&mut self, distance: usize, zoom: u8, identity: u128) -> DeliveredEdge { + let identity = self::identity(identity); + let row = self.edges.push_with(|row| DeliveredEdge { + row: Visible::new(row.as_u64()), + endpoints: [SOURCE, NodeRowId::new(row.as_u64() + 1)], + identity, + }); + self.keys.push(( + DNonNegative::from_usize(distance), + self::zoom(zoom), + identity, + )); + self.edges[row] + } + + /// Returns a rank function over the recorded keys that counts its calls. + /// + /// # Panics + /// + /// The returned function panics where an offered edge's row lies outside this set's key + /// table. In overflow-checked builds it panics where `calls` already holds `usize::MAX`, + /// which the count it records exceeds by one. + fn rank<'this>( + &'this self, + calls: &'this Cell, + ) -> impl FnMut(&DeliveredEdge) -> Result + 'this { + move |edge| { + calls.set(calls.get() + 1); + Ok(self.keys[edge.row.unwrap()]) + } + } + + /// Offers every edge in row order and returns the identity-ordered selection. + /// + /// # Panics + /// + /// Panics where the selection retains more than `capacity` after an offer, and for the + /// conditions [`Self::rank`]'s returned function states. That function runs only after the + /// offers exceed a nonzero `capacity`. + #[track_caller] + fn select(&self, capacity: usize) -> EdgeSet { + let calls = Cell::new(0); + let mut nearest = NearestCap::new(capacity, self.rank(&calls)); + for edge in self.edges.iter().copied() { + nearest.offer(edge).expect("should rank every edge"); + assert!( + retained(&nearest) <= capacity, + "should retain at most the capacity after every offer" + ); + } + let mut set = nearest.into_set(); + set.edges.sort_unstable_by_key(|edge| edge.identity); + set + } + + /// Selects this set's `capacity` smallest keys by a full sort, in identity order. + /// + /// # Panics + /// + /// Panics for the conditions [`reference()`] states, over this set's edges and keys. + fn reference(&self, capacity: usize) -> EdgeSet { + reference(&self.edges, &self.keys, capacity) + } + } + + /// Selects the `capacity` smallest-keyed edges by full sort and returns them in identity order. + /// + /// # Panics + /// + /// Panics where the sort evaluates the key function for an edge whose row lies outside + /// `keys`. The sort evaluates that function only for the edges it compares. + fn reference( + edges: &IdSlice, + keys: &IdSlice, + capacity: usize, + ) -> EdgeSet { + let mut ranked: Vec<_> = edges.iter().copied().collect(); + ranked.sort_unstable_by_key(|edge| keys[edge.row.unwrap()]); + let complete = ranked.len() <= capacity; + ranked.truncate(capacity); + ranked.sort_unstable_by_key(|edge| edge.identity); + EdgeSet { + complete, + edges: ranked, + } + } + + /// Counts the edges a selection currently holds. + fn retained(nearest: &NearestCap) -> usize { + match &nearest.retained { + Retained::Buffered(buffer) => buffer.len(), + Retained::Ranked(kept) => kept.len(), + } + } + + /// Within capacity the selection keeps every edge in offer order and ranks none. + #[test] + fn nearest_within_capacity() { + let mut incident = Incident::new(); + let expected = [ + incident.link(9, 4, 30), + incident.link(1, 0, 20), + incident.link(5, 2, 10), + ]; + let calls = Cell::new(0); + let mut nearest = NearestCap::new(3, incident.rank(&calls)); + for edge in expected { + nearest.offer(edge).expect("should buffer within capacity"); + } + let set = nearest.into_set(); + assert!(set.complete); + assert_eq!(set.edges, expected); + assert_eq!(calls.get(), 0, "should rank nothing within capacity"); + } + + /// Zero capacity records truncation without ranking or retaining anything. + #[test] + fn nearest_zero_capacity() { + let mut incident = Incident::new(); + incident.link(1, 0, 10); + incident.link(2, 0, 11); + let calls = Cell::new(0); + let mut nearest = NearestCap::new(0, incident.rank(&calls)); + for edge in incident.edges.iter().copied() { + nearest + .offer(edge) + .expect("should never rank at zero capacity"); + assert_eq!(retained(&nearest), 0); + } + let set = nearest.into_set(); + assert!(!set.complete); + assert!(set.edges.is_empty()); + assert_eq!(calls.get(), 0); + + let empty = NearestCap::new(0, incident.rank(&calls)); + assert!( + empty.into_set().complete, + "should be complete with no offer" + ); + } + + /// Past capacity the selection ranks every offered edge exactly once and keeps the nearest. + #[test] + fn nearest_ranks_once_per_edge() { + let mut incident = Incident::new(); + let far = incident.link(9, 0, 10); + let nearest_edge = incident.link(1, 0, 11); + let middle = incident.link(5, 0, 12); + let farther = incident.link(12, 0, 13); + let near = incident.link(2, 0, 14); + let calls = Cell::new(0); + let mut nearest = NearestCap::new(2, incident.rank(&calls)); + for edge in [far, nearest_edge, middle, farther, near] { + nearest.offer(edge).expect("should rank every edge"); + assert!(retained(&nearest) <= 2); + } + assert_eq!(calls.get(), 5, "should rank each offered edge once"); + let mut set = nearest.into_set(); + set.edges.sort_unstable_by_key(|edge| edge.identity); + assert!(!set.complete); + assert_eq!(set.edges, [nearest_edge, near]); + } + + /// Equal distances compare the partner's delivery zoom before the link identity. + #[test] + fn nearest_zoom_tie() { + let mut incident = Incident::new(); + let deep = incident.link(4, 3, 10); + let shallow = incident.link(4, 1, 20); + assert!(deep.identity < shallow.identity); + let set = incident.select(1); + assert!(!set.complete); + assert_eq!( + set.edges, + [shallow], + "should keep the shallower delivery zoom" + ); + } + + /// Equal distances and zooms compare the link identity. + #[test] + fn nearest_identity_tie() { + let mut incident = Incident::new(); + let later = incident.link(4, 2, 20); + let earlier = incident.link(4, 2, 10); + let set = incident.select(1); + assert!(!set.complete); + assert_eq!(set.edges, [earlier]); + assert_ne!(set.edges, [later]); + } + + /// A rank error surfaces at the transition for a buffered edge and afterwards for the arrival. + #[test] + fn nearest_rank_error() { + let mut incident = Incident::new(); + let first = incident.link(1, 0, 10); + let second = incident.link(2, 0, 11); + let third = incident.link(3, 0, 12); + + let mut failing = NearestCap::new(1, |edge: &DeliveredEdge| { + if edge.identity == first.identity { + Err(edge.row.unwrap()) + } else { + Ok(incident.keys[edge.row.unwrap()]) + } + }); + failing + .offer(first) + .expect("should buffer the first edge unranked"); + assert_eq!( + failing.offer(second), + Err(first.row.unwrap()), + "should rank the buffered edge at the transition" + ); + + let mut failing = NearestCap::new(1, |edge: &DeliveredEdge| { + if edge.identity == third.identity { + Err(edge.row.unwrap()) + } else { + Ok(incident.keys[edge.row.unwrap()]) + } + }); + failing.offer(first).expect("should buffer the first edge"); + failing + .offer(second) + .expect("should rank both edges at the transition"); + assert_eq!( + failing.offer(third), + Err(third.row.unwrap()), + "should rank an arrival after the transition" + ); + } + + /// A large incident set selects as a full sort does while holding at most the capacity. + #[test] + fn nearest_large_degree_bounded() { + const DEGREE: usize = 10_000; + const CAPACITY: usize = 8; + + let mut rng = StdRng::seed_from_u64(0x5EED); + let mut incident = Incident::new(); + for index in 0..DEGREE { + let distance = rng.random_range(0..64_usize); + let zoom = rng.random_range(0..8_u8); + let identity = u128::try_from(index).expect("should fit the edge index"); + incident.link(distance, zoom, identity); + } + + let calls = Cell::new(0); + let mut nearest = NearestCap::new(CAPACITY, incident.rank(&calls)); + for edge in incident.edges.iter().copied() { + nearest.offer(edge).expect("should rank every edge"); + assert!(retained(&nearest) <= CAPACITY); + } + assert_eq!(calls.get(), DEGREE); + let Retained::Ranked(kept) = &nearest.retained else { + panic!("should rank past the capacity"); + }; + assert!( + kept.capacity() <= 2 * CAPACITY, + "should size the heap by the capacity rather than the degree" + ); + + let mut set = nearest.into_set(); + set.edges.sort_unstable_by_key(|edge| edge.identity); + let expected = incident.reference(CAPACITY); + assert!(!set.complete); + assert_eq!(set.edges, expected.edges); + } + + /// A maximal capacity buffers the offered edges and allocates for them alone. + /// + /// The offer path must survive `usize::MAX` without an allocation sized by the capacity or a + /// `capacity + 1` computation, and the set comes back complete and unranked. + #[test] + fn nearest_max_capacity() { + let mut incident = Incident::new(); + let mut expected = [ + incident.link(3, 1, 10), + incident.link(1, 0, 11), + incident.link(2, 2, 12), + ]; + let calls = Cell::new(0); + let mut nearest = NearestCap::new(usize::MAX, incident.rank(&calls)); + let Retained::Buffered(buffer) = &nearest.retained else { + panic!("should start unranked"); + }; + assert_eq!(buffer.capacity(), 0); + for edge in expected { + nearest + .offer(edge) + .expect("should buffer under a maximal capacity"); + } + assert_eq!(retained(&nearest), expected.len()); + assert_eq!( + calls.get(), + 0, + "should rank nothing under a maximal capacity" + ); + let mut set = nearest.into_set(); + assert!(set.complete); + set.edges.sort_unstable_by_key(|edge| edge.identity); + expected.sort_unstable_by_key(|edge| edge.identity); + assert_eq!(set.edges, expected); + } + + /// The selection agrees with a full sort at every capacity. + /// + /// The index suffix makes identities unique. Only distance and zoom can tie. + #[property_test] + fn nearest_reference( + #[strategy = collection::vec((0_usize..6, 0_u8..4, 0_u8..3), 0..40)] inputs: Vec<( + usize, + u8, + u8, + )>, + #[strategy = 0_usize..45] capacity: usize, + ) { + let mut incident = Incident::new(); + for (index, (distance, zoom, key)) in inputs.into_iter().enumerate() { + let index = u128::try_from(index).expect("should fit the generated edge index"); + incident.link(distance, zoom, (u128::from(key) << 64) | index); + } + let actual = incident.select(capacity); + let expected = incident.reference(capacity); + proptest::prop_assert_eq!(actual.complete, expected.complete); + proptest::prop_assert_eq!(actual.edges, expected.edges); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/locate/tests.rs b/libs/@local/graph/atlas/src/serve/document/locate/tests.rs new file mode 100644 index 00000000000..5f06da5c134 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/locate/tests.rs @@ -0,0 +1,1708 @@ +use alloc::{collections::BTreeMap, sync::Arc}; +use core::{assert_matches, cell::RefCell, iter}; +use std::io; + +use arc_swap::Guard; +use camino::Utf8Path; +use error_stack::Report; +use hashql_core::id::{Id as _, IdVec}; +use rand::{SeedableRng as _, rngs::StdRng}; +use serde_json::json; +use type_system::{ + knowledge::entity::{EntityId, id::DraftId}, + ontology::{VersionedUrl, id::BaseUrl}, + principal::actor::{ActorId, ActorType}, +}; +use uuid::Uuid; + +use super::{ + LocateDocument, LocateDocumentError, LocateDocumentOptions, LocateLimits, LocateSource, + trailer::{LocateTrailer, PropertyMap}, +}; +use crate::{ + api::problem::{Problem, tests::assert_internal_diagnostic}, + bitset::CompressedBitSet, + dataset::auxiliary::{Label, OwnedLegend}, + file::generation::Generation, + identity::{EdgeRowId, NodeRowId, OntologyRowId}, + math::Vec2, + morton::{Depth, MortonTile, Zoom}, + postgres::id::{ArchivedEntityId, ArchivedOntologyTypeUuid}, + salt::{ + fit::prepare::identity::IdentityTable, + lod::{key, stage::WIRE_FRAME}, + }, + serve::{ + codec::EncodedRowId, + delta::{Delta, epoch::Epoch}, + hydrate::{ + EdgeSlot, HydrateError, LocateLink, LocateNode, LocateProperties, LocateRequest, + LocateResolver, NodeSlot, + scalar::{ScalarProperties, ScalarValue}, + }, + intern::TableIndex, + membership::{OntologySelection, SelectionSlot}, + scene::Scene, + schedule::ViewSchedule, + tests::fixture::{EDGE_SEED, EDGES, ENDPOINTS, NODES, TamperFixture, secret}, + visibility::{VisibilityActor, VisibilityMask}, + world::World, + }, +}; + +/// One locate request as the resolver received it. +#[derive(Debug, PartialEq, Eq)] +struct RecordedRequest { + actor_id: ActorId, + instance_admin: bool, + nodes: Vec, + links: Vec, + properties: u32, + link_type_ids: u32, + link_properties: u32, +} + +impl RecordedRequest { + /// Copies the parts of `request` a [`FakeResolver`] needs to compare against an expectation. + fn of(request: &LocateRequest<'_>) -> Self { + Self { + actor_id: request.actor.id, + instance_admin: request.actor.instance_admin, + nodes: request.nodes.iter().map(|node| node.identity).collect(), + links: request.links.iter().map(|link| link.identity).collect(), + properties: request.properties, + link_type_ids: request.link_type_ids, + link_properties: request.link_properties, + } + } +} + +/// The resolved detail a [`FakeResolver`] answers a request with. +struct Details { + nodes: IdVec>, + links: IdVec>, + source_properties: Option, +} + +/// A configured response from [`FakeResolver`]. +/// +/// Successful responses supply [`Details`] or report no resolution. Failure is the other configured +/// outcome. +enum Answer { + Response(Details), + Unresolved, + Failure, +} + +/// A synchronous resolver that answers one request and records every request it receives. +struct FakeResolver { + answer: RefCell>, + requests: RefCell>, +} + +impl FakeResolver { + /// Builds a resolver answering its one call with `response`. + fn answering(response: Details) -> Self { + Self { + answer: RefCell::new(Some(Answer::Response(response))), + requests: RefCell::new(Vec::new()), + } + } + + /// Builds a resolver failing its one call. + const fn failing() -> Self { + Self { + answer: RefCell::new(Some(Answer::Failure)), + requests: RefCell::new(Vec::new()), + } + } + + /// Constructs a resolver whose `resolve` method must never run. + const fn unexpected() -> Self { + Self { + answer: RefCell::new(None), + requests: RefCell::new(Vec::new()), + } + } + + /// Builds a deterministic fixture type URL for `name`. + /// + /// # Panics + /// + /// Panics where `name` leaves the interpolated URL unparseable. + fn url(name: &str) -> VersionedUrl { + format!("https://example.com/types/{name}/v/1") + .parse() + .expect("should parse the fixture type URL") + } + + /// Builds a deterministic fixture property [`BaseUrl`] for `name`. + /// + /// # Panics + /// + /// Panics where `name` leaves the interpolated URL unparseable. + fn property(name: &str) -> BaseUrl { + BaseUrl::new(format!("https://example.com/property/{name}/")) + .expect("should parse the fixture property URL") + } + + /// Builds the [`ScalarProperties`] a JSON `value` reduces to. + /// + /// The reduction runs with no value cap and no label property. + fn scalars(value: serde_json::Value) -> ScalarProperties { + let (properties, _truncated) = ScalarProperties::new(value, None, usize::MAX); + properties + } + + /// Builds [`Details`] where every one of `nodes` nodes and `edges` links resolves. + /// + /// Each resolved row carries empty properties and no types. + fn resolved(nodes: usize, edges: usize) -> Details { + Details { + nodes: IdVec::from_fn(nodes, |_| { + Some(LocateNode { + type_urls: Vec::new(), + }) + }), + links: IdVec::from_fn(edges, |_| { + Some(LocateLink { + type_urls: Vec::new(), + type_urls_complete: true, + properties: LocateProperties { + values: ScalarProperties::EMPTY, + complete: true, + }, + }) + }), + source_properties: Some(LocateProperties { + values: ScalarProperties::EMPTY, + complete: true, + }), + } + } + + /// Marks node `slot` unresolved in `response`. + /// + /// Accepts a slot that is already unresolved. Marking the source node also clears the source + /// properties. + /// + /// # Panics + /// + /// Panics where `slot` is not one of `response`'s node slots, and where `slot` lies outside + /// the [`NodeSlot`] domain. + fn unresolve_node(response: &mut Details, slot: usize) { + let slot = NodeSlot::from_usize(slot); + response.nodes[slot] = None; + if slot == NodeSlot::MIN { + response.source_properties = None; + } + } + + /// Marks link `slot` unresolved in `response`. + /// + /// Marking an already unresolved slot changes nothing. + /// + /// # Panics + /// + /// Panics where `slot` is not one of `response`'s link slots, and where `slot` lies outside + /// the [`EdgeSlot`] domain. + fn unresolve_link(response: &mut Details, slot: usize) { + response.links[EdgeSlot::from_usize(slot)] = None; + } + + /// Builds [`Details`] over three nodes and two links that share one type URL. + /// + /// The node types and link 0's types share the `alpha` URL, and link 1 is left unresolved. + fn shared_urls() -> Details { + let alpha = Self::url("alpha"); + let mut response = Self::resolved(3, 2); + response.nodes[NodeSlot::MIN] = Some(LocateNode { + type_urls: vec![alpha.clone(), Self::url("gamma")], + }); + response.nodes[NodeSlot::from_usize(1)] = Some(LocateNode { + type_urls: vec![alpha.clone()], + }); + response.source_properties = Some(LocateProperties { + values: Self::scalars(json!({ + "https://example.com/property/name/": "Ada", + "https://example.com/property/age/": 36, + })), + complete: true, + }); + response.links[EdgeSlot::MIN] = Some(LocateLink { + type_urls: vec![alpha, Self::url("beta")], + type_urls_complete: true, + properties: LocateProperties { + values: Self::scalars(json!({ + "https://example.com/property/name/": true, + "https://example.com/property/weight/": 0.5, + "https://example.com/property/note/": null, + })), + complete: true, + }, + }); + Self::unresolve_link(&mut response, 1); + response + } + + /// Asserts that this resolver was called exactly once, with exactly the expected request. + /// + /// # Panics + /// + /// Panics if the recorded calls differ from that expectation or the request log is borrowed + /// mutably. + #[track_caller] + fn assert_request(&self, expected: &RecordedRequest) { + let requests = self.requests.borrow(); + assert_eq!(requests.len(), 1, "should resolve exactly once"); + assert_eq!( + requests[0], *expected, + "should forward the delivered identities, the actor and the limits" + ); + } + + /// Asserts that this resolver was never called. + /// + /// # Panics + /// + /// Panics if a request was recorded or the request log is borrowed mutably. + #[track_caller] + fn assert_unasked(&self) { + assert!( + self.requests.borrow().is_empty(), + "should refuse before hydration" + ); + } +} + +impl LocateResolver for FakeResolver { + /// Records the request and answers according to the configured [`Answer`]. + /// + /// The request must arrive with every slot's details unset. The configured answer fills + /// [`Details`] slot by slot or reports no resolution on success. It can instead fail. + /// + /// # Panics + /// + /// Panics if the request has resolved slots, its lengths differ from the configured response, + /// no answer remains or either fixture cell is already borrowed. + #[expect( + clippy::panic_in_result_fn, + reason = "the fake asserts the constructor's request before answering" + )] + fn resolve( + &self, + request: LocateRequest<'_>, + ) -> Result, Report> { + self.requests + .borrow_mut() + .push(RecordedRequest::of(&request)); + assert!(request.nodes.iter().all(|node| node.details.is_none())); + assert!(request.links.iter().all(|link| link.details.is_none())); + match self.answer.borrow_mut().take() { + Some(Answer::Response(response)) => { + assert_eq!( + request.nodes.len(), + response.nodes.len(), + "should receive the expected node slots" + ); + assert_eq!( + request.links.len(), + response.links.len(), + "should receive the expected link slots" + ); + for (slot, details) in request.nodes.iter_mut().zip(response.nodes) { + slot.details = details; + } + for (slot, details) in request.links.iter_mut().zip(response.links) { + slot.details = details; + } + Ok(response.source_properties) + } + Some(Answer::Unresolved) => Ok(None), + Some(Answer::Failure) => Err(Report::new(io::Error::other("private-store-message")) + .change_context(HydrateError::Query) + .attach("private-property-value")), + None => panic!("should receive no further request"), + } + } +} + +/// An opened synthetic generation, shared by every test below. +/// +/// It carries the captured epoch, actor, visibility mask and delivery schedule that `scene()` +/// assembles into one scene. +struct Fixture { + world: Arc, + epoch: Epoch, + actor: VisibilityActor, + mask: VisibilityMask, + schedule: ViewSchedule, + _files: TamperFixture, +} + +impl Fixture { + /// Builds a user [`VisibilityActor`] distinguished by `id`, with the given instance-admin flag. + fn actor(id: u128, instance_admin: bool) -> VisibilityActor { + VisibilityActor { + id: ActorId::new(Uuid::from_u128(id), ActorType::User), + instance_admin, + } + } + + /// Opens `generation`, published from `files`, with a fresh delta identity. + /// + /// The mask grants `actor` full visibility. + /// + /// # Panics + /// + /// Panics if [`World::open`] fails to open or validate the serving artifacts. + fn from_generation( + files: TamperFixture, + generation: Generation, + actor: VisibilityActor, + ) -> Self { + let world = Arc::new( + World::open(generation, &secret()).expect("should open the synthetic generation"), + ); + let delta = Delta::new(Arc::clone(&world), StdRng::seed_from_u64(17)) + .expect("should allocate a delta identity"); + let epoch = Epoch::from(Guard::from_inner(Arc::new(delta))); + let mask = VisibilityMask::full(actor); + let schedule = ViewSchedule::of(Arc::clone(&world), &epoch, &mask); + Self { + world, + epoch, + actor, + mask, + schedule, + _files: files, + } + } + + /// Publishes and opens the named synthetic generation unmodified. + /// + /// The fixture's actor is a fixed non-admin test user. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails, and for the conditions + /// [`Self::from_generation`] states. + fn new(name: &str) -> Self { + let files = TamperFixture::publish(name); + let generation = files.generation().clone(); + Self::from_generation(files, generation, Self::actor(1, false)) + } + + /// Reopens `path` for writing, clearing its read-only staged-artifact permission first. + /// + /// # Panics + /// + /// Panics on any of three filesystem failures against `path`: an unreadable metadata entry, + /// a permission change the process may not make, and a truncating reopen that fails. + #[expect( + clippy::permissions_set_readonly_false, + reason = "the staged copy is this test's own scratch file" + )] + fn recreate_writable(path: &Utf8Path) -> std::fs::File { + let mut permissions = std::fs::metadata(path) + .expect("should read the staged artifact's metadata") + .permissions(); + permissions.set_readonly(false); + std::fs::set_permissions(path, permissions).expect("should set the permissions"); + std::fs::File::create(path).expect("should rewrite the staged artifact") + } + + /// Builds a synthetic identity from a repeated `seed` byte. + fn identity(seed: u8) -> ArchivedEntityId { + ArchivedEntityId { + web_id: Uuid::from_bytes([seed; 16]).into(), + entity_uuid: Uuid::from_bytes([seed ^ 0xFF; 16]).into(), + } + } + + /// Republishes with the label `node-{row}` on every node row. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails and where the rewritten table does + /// not write. It also panics for the conditions [`Self::recreate_writable`] and + /// [`Self::from_generation`] state. + fn varied_nodes(name: &str) -> Self { + let files = TamperFixture::publish(name); + let target = files.generation().repository().files.node_identities.name(); + let generation = files.tamper(&target, |path| { + let mut table = IdentityTable::::new(); + for row in 0..NODES { + table.push(Self::identity( + u8::try_from(row).expect("should fit the fixture row in u8"), + )); + } + let legends: Vec<_> = (0..NODES) + .map(|row| { + OwnedLegend::new( + OntologyRowId::new(row & 1), + Label::new(&format!("node-{row}")), + ) + }) + .collect(); + let mut file = Self::recreate_writable(path); + let _digest = table + .write_into(legends.iter().map(AsRef::as_ref), &mut file) + .expect("should write the varied node identities"); + }); + Self::from_generation(files, generation, Self::actor(1, false)) + } + + /// Reverses edge identities while preserving each row's label. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails and where the rewritten table does + /// not write. It also panics for the conditions [`Self::recreate_writable`] and + /// [`Self::from_generation`] state. + fn varied_links(name: &str, actor: VisibilityActor) -> Self { + let files = TamperFixture::publish(name); + let target = files.generation().repository().files.edge_identities.name(); + let generation = files.tamper(&target, |path| { + let mut table = IdentityTable::::new(); + for row in 0..EDGES { + let offset = + u8::try_from(EDGES - 1 - row).expect("should fit the fixture row in u8"); + table.push(Self::identity(EDGE_SEED + offset)); + } + let legends: Vec<_> = (0..EDGES) + .map(|row| { + OwnedLegend::new(OntologyRowId::new(2), Label::new(&format!("edge-{row}"))) + }) + .collect(); + let mut file = Self::recreate_writable(path); + let _digest = table + .write_into(legends.iter().map(AsRef::as_ref), &mut file) + .expect("should write the varied edge identities"); + }); + Self::from_generation(files, generation, actor) + } + + /// Narrows this fixture's visibility mask and schedule to exactly the given rows. + /// + /// A later `scene()` call sees only those node and edge rows. + fn restrict( + &mut self, + nodes: impl IntoIterator, + edges: impl IntoIterator, + ) { + self.mask = VisibilityMask::partial( + self.actor, + CompressedBitSet::from_rows(nodes.into_iter().map(NodeRowId::new)), + CompressedBitSet::from_rows(edges.into_iter().map(EdgeRowId::new)), + ); + self.schedule = ViewSchedule::of(Arc::clone(&self.world), &self.epoch, &self.mask); + } + + /// Assembles this fixture's world, epoch, mask and schedule into one [`Scene`]. + /// + /// Its delivery is the schedule's cut at the zero offset. + fn scene(&self) -> Scene<'_> { + Scene { + world: &self.world, + epoch: &self.epoch, + mask: &self.mask, + schedule: &self.schedule, + delivery: self + .schedule + .cut(Zoom::MIN) + .expect("should bind the zero offset"), + } + } + + /// Builds an empty type selection, admitting no ontology type. + fn no_types() -> &'static OntologySelection { + OntologySelection::new(&[]) + } + + /// Encodes the wire row id for `node`. + /// + /// # Panics + /// + /// Panics under [`NodeIndex::encode`](crate::serve::world::NodeIndex::encode)'s row-range and + /// cache-miss conditions. + /// + /// # Warning + /// + /// On a 32-bit target, a node outside the wire range can instead return a cached row's + /// encoding. + fn encode(&self, node: NodeRowId) -> EncodedRowId { + self.world.layout.index.encode(node) + } + + /// Returns the identity the fixture generation assigned to `node`. + /// + /// # Panics + /// + /// Panics where that generation holds no identity for `node`. + fn node_identity(&self, node: NodeRowId) -> ArchivedEntityId { + self.world + .layout + .index + .key_of(&self.epoch, node) + .expect("should resolve the fixture node's identity") + } + + /// Builds a [`LocateSource::Key`] naming `node`'s [`EntityId`]. + /// + /// # Panics + /// + /// Panics where the fixture generation holds no identity for `node`, the condition + /// [`Self::node_identity`] states. + fn entity_source(&self, node: NodeRowId) -> LocateSource { + LocateSource::Key(EntityId::from(self.node_identity(node))) + } + + /// Builds a [`LocateSource::Row`] naming `node`'s encoded row. + fn row_source(&self, node: NodeRowId) -> LocateSource { + LocateSource::Row(self.encode(node)) + } + + /// Returns `node`'s captured wire position. + /// + /// # Panics + /// + /// Panics where the fixture's epoch captured no position for `node`. + fn position_of(&self, node: NodeRowId) -> Vec2 { + self.world + .layout + .position(&self.epoch, node) + .expect("should resolve the fixture node's position") + } + + /// Returns the tile of the node's wire position at its first delivery zoom. + /// + /// # Panics + /// + /// Panics where the zero-offset delivery schedules no zoom for `node`, and on the captured + /// position `position_of` requires. + fn fly_to(&self, node: NodeRowId) -> MortonTile { + let zoom = self + .scene() + .delivery + .first_zoom(node) + .expect("should schedule the fixture node"); + key::keys(&[self.position_of(node)], WIRE_FRAME)[0].tile(Depth::from_zoom(zoom)) + } + + /// Returns the identity the fixture generation assigned to `edge`. + /// + /// # Panics + /// + /// Panics where that generation holds no identity for `edge`. + fn identity_of(&self, edge: EdgeRowId) -> ArchivedEntityId { + self.world + .topology + .key_of(&self.epoch, edge) + .expect("should resolve the fixture edge's identity") + } + + /// Returns `node`'s captured display label. + /// + /// # Panics + /// + /// Panics where the fixture captured no display payload for `node`. + fn node_label_of(&self, node: NodeRowId) -> &Label { + self.world + .layout + .index + .payload(&self.epoch, node) + .expect("should resolve the fixture node's display payload") + .label() + } + + /// Returns `edge`'s captured display label. + /// + /// # Panics + /// + /// Panics where the fixture captured no display payload for `edge`. + fn link_label_of(&self, edge: EdgeRowId) -> &Label { + self.world + .topology + .payload(&self.epoch, edge) + .expect("should resolve the fixture edge's display payload") + .label() + } + + /// Returns the ontology type uuid the fixture generation assigned to ontology row `row`. + /// + /// # Panics + /// + /// Panics where that generation holds no ontology row `row`. + fn ontology_uuid(&self, row: u64) -> ArchivedOntologyTypeUuid { + self.world + .ontology + .key_of(&self.epoch, OntologyRowId::new(row)) + .expect("should resolve the fixture ontology row") + } + + /// Selects the edge rows of [`ENDPOINTS`] touching `source`, in identity order. + fn incident(&self, source: NodeRowId) -> Vec { + let mut edges: Vec<_> = ENDPOINTS + .iter() + .enumerate() + .filter(|(_, endpoints)| endpoints.contains(&source)) + .map(|(row, _)| EdgeRowId::from_usize(row)) + .collect(); + edges.sort_unstable_by_key(|&edge| self.identity_of(edge)); + edges + } + + /// Lists the source followed by the distinct partners of `edges`, in encoded-row order. + /// + /// # Panics + /// + /// Panics where an edge in `edges` lies outside the fixture's endpoint table. + fn nodes_of(&self, source: NodeRowId, edges: &[EdgeRowId]) -> Vec { + let mut partners: Vec<_> = edges + .iter() + .flat_map(|&edge| ENDPOINTS[edge.as_usize()]) + .filter(|&row| row != source) + .collect(); + partners.sort_unstable_by_key(|&row| self.encode(row)); + partners.dedup(); + iter::once(source).chain(partners).collect() + } + + /// Resolves each of `indices` against `trailer`'s interned type-URL table, in order. + /// + /// # Panics + /// + /// Panics where an index in `indices` lies outside that table. + fn decode_type_urls( + trailer: &LocateTrailer<'_>, + indices: impl IntoIterator>, + ) -> Vec { + indices + .into_iter() + .map(|index| trailer.type_urls.entries()[index].clone()) + .collect() + } + + /// Resolves `properties`' interned indices against `trailer`'s property-URL table. + /// + /// The result uses the resolved [`BaseUrl`] as its key. + /// + /// # Panics + /// + /// Panics where an index in `properties` lies outside that table. + fn decode_properties( + trailer: &LocateTrailer<'_>, + properties: &PropertyMap, + ) -> BTreeMap { + properties + .0 + .iter() + .map(|(&index, value)| { + ( + trailer.property_urls.entries()[index].clone(), + value.clone(), + ) + }) + .collect() + } + + /// Asserts the source-first node columns and the identity-ordered edge columns over `edges`. + /// + /// # Panics + /// + /// Panics where an edge in `edges` lies outside the fixture's endpoint table, and for the + /// conditions [`Self::node_identity`] and [`Self::fly_to`] state. + #[track_caller] + fn assert_geometry( + &self, + document: &LocateDocument<'_>, + source: NodeRowId, + edges: &[EdgeRowId], + ) { + let nodes = self.nodes_of(source, edges); + + assert_eq!( + document.generation, + self.world.generation().id(), + "should echo the generation identity" + ); + assert_eq!( + document.entity_id, + self.node_identity(source), + "should carry the source's upstream identity" + ); + assert_eq!( + document.coordinate, + self.fly_to(source), + "should place the fly-to tile at the source's first delivery zoom" + ); + assert_eq!( + document.ids.iter().copied().collect::>(), + nodes + .iter() + .map(|&row| self.encode(row)) + .collect::>(), + "should list the source first and distinct partners by encoded row" + ); + assert_eq!( + document.positions.iter().copied().collect::>(), + nodes + .iter() + .map(|&row| self.position_of(row)) + .collect::>(), + "should align positions with the delivered node slots" + ); + assert_eq!( + document.edge_ids.iter().copied().collect::>(), + edges + .iter() + .map(|&edge| self.identity_of(edge)) + .collect::>(), + "should list edge identities in identity order" + ); + for (column, endpoint) in [(&document.edge_sources, 0), (&document.edge_targets, 1)] { + assert_eq!( + column.iter().copied().collect::>(), + edges + .iter() + .map(|&edge| self.encode(ENDPOINTS[edge.as_usize()][endpoint])) + .collect::>(), + "should encode endpoint {endpoint} of each edge" + ); + } + } + + /// Asserts that `result` failed with [`LocateDocumentError::UnknownEntity`]. + #[track_caller] + fn assert_unknown_entity(result: Result, Report>) { + let Err(report) = result else { + panic!("should refuse a source that names no visible node"); + }; + assert_matches!(report.current_context(), LocateDocumentError::UnknownEntity); + } +} + +/// Both source forms resolve to one source-first geometry and fly-to coordinate. +#[test] +fn source_forms() { + let fixture = Fixture::new("locate-source-forms"); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let mut ids = Vec::new(); + for source_form in [fixture.entity_source(source), fixture.row_source(source)] { + let resolver = FakeResolver::answering(FakeResolver::resolved(3, 2)); + let document = LocateDocument::new( + fixture.scene(), + source_form, + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct from a visible source"); + assert!(document.complete, "should deliver every incident edge"); + fixture.assert_geometry(&document, source, &edges); + ids.push(document.ids.iter().copied().collect::>()); + } + assert_eq!( + ids[0], ids[1], + "should deliver the same nodes for both forms" + ); +} + +#[test] +fn source_unknown_entity() { + let fixture = Fixture::new("locate-source-unknown-entity"); + let resolver = FakeResolver::unexpected(); + let unknown = ArchivedEntityId { + web_id: Uuid::from_u128(0xDEAD).into(), + entity_uuid: Uuid::from_u128(0xBEEF).into(), + }; + Fixture::assert_unknown_entity(LocateDocument::new( + fixture.scene(), + LocateSource::Key(EntityId::from(unknown)), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + )); + resolver.assert_unasked(); +} + +#[test] +fn source_draft_entity() { + let fixture = Fixture::new("locate-source-draft-entity"); + let resolver = FakeResolver::unexpected(); + let draft = EntityId { + draft_id: Some(DraftId::new(Uuid::from_u128(7))), + ..EntityId::from(fixture.node_identity(NodeRowId::new(1))) + }; + Fixture::assert_unknown_entity(LocateDocument::new( + fixture.scene(), + LocateSource::Key(draft), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + )); + resolver.assert_unasked(); +} + +#[test] +fn source_row_out_of_domain() { + let fixture = Fixture::new("locate-source-row-out-of-domain"); + let resolver = FakeResolver::unexpected(); + Fixture::assert_unknown_entity(LocateDocument::new( + fixture.scene(), + LocateSource::Row(fixture.encode(NodeRowId::new(NODES))), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + )); + resolver.assert_unasked(); +} + +/// Either source form of a masked node refuses under [`LocateDocumentError::UnknownEntity`]. +#[test] +fn source_hidden_by_mask() { + let mut fixture = Fixture::new("locate-source-hidden-by-mask"); + let source = NodeRowId::new(1); + fixture.restrict((0..NODES).filter(|&row| row != 1), 0..EDGES); + for source_form in [fixture.entity_source(source), fixture.row_source(source)] { + let resolver = FakeResolver::unexpected(); + Fixture::assert_unknown_entity(LocateDocument::new( + fixture.scene(), + source_form, + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + )); + resolver.assert_unasked(); + } +} + +#[test] +fn scene_hidden_partner_and_edge() { + let mut fixture = Fixture::new("locate-scene-hidden-partner-and-edge"); + let source = NodeRowId::new(1); + let cases: [(Vec, Vec); 2] = [ + ( + (0..NODES).filter(|&row| row != 0).collect(), + (0..EDGES).collect(), + ), + ( + (0..NODES).collect(), + (0..EDGES).filter(|&row| row != 0).collect(), + ), + ]; + for (nodes, edges) in cases { + fixture.restrict(nodes, edges); + let resolver = FakeResolver::answering(FakeResolver::resolved(2, 1)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct over the restricted scene"); + assert!(document.complete, "should count only visible edges"); + fixture.assert_geometry(&document, source, &[EdgeRowId::new(1)]); + } +} + +#[test] +fn edges_self_loop() { + let fixture = Fixture::new("locate-edges-self-loop"); + let source = NodeRowId::new(2); + let edges = fixture.incident(source); + let resolver = FakeResolver::answering(FakeResolver::resolved(2, 2)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct the self-loop document"); + assert!(document.complete, "should deliver both incident edges"); + fixture.assert_geometry(&document, source, &edges); + let loop_slot = EdgeSlot::from_usize( + edges + .iter() + .position(|&edge| edge == EdgeRowId::new(2)) + .expect("should include the self-loop"), + ); + assert_eq!( + ( + document.edge_sources[loop_slot], + document.edge_targets[loop_slot] + ), + (fixture.encode(source), fixture.encode(source)), + "should keep both self-loop endpoints on the source" + ); + assert_eq!( + document.ids.len(), + 2, + "should deliver the source and one partner" + ); +} + +/// Reciprocal links share one partner slot. +#[test] +fn edges_reciprocal_pair() { + let fixture = Fixture::new("locate-edges-reciprocal-pair"); + let source = NodeRowId::new(5); + let edges = fixture.incident(source); + let resolver = FakeResolver::answering(FakeResolver::resolved(2, 2)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct the reciprocal-pair document"); + assert!(document.complete, "should deliver both reciprocal links"); + fixture.assert_geometry(&document, source, &edges); + assert_eq!(document.edge_ids.len(), 2, "should keep both links"); + assert_eq!( + document.ids.len(), + 2, + "should deliver the shared partner once" + ); +} + +/// Completeness follows the incident count against the capacity, including at zero. +#[test] +fn edges_capacity_boundaries() { + let fixture = Fixture::new("locate-edges-capacity-boundaries"); + let linked = NodeRowId::new(5); + let incident = fixture.incident(linked); + for capacity in [0, 2, 3] { + let expected: &[EdgeRowId] = if capacity == 0 { &[] } else { &incident }; + let resolver = FakeResolver::answering(FakeResolver::resolved( + fixture.nodes_of(linked, expected).len(), + expected.len(), + )); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(linked), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { + edges: capacity, + .. + }, + resolver: &resolver, + }, + ) + .expect("should construct at every capacity"); + assert_eq!( + document.complete, + usize::try_from(capacity).expect("should fit the capacity in usize") >= incident.len(), + "should report completeness at capacity {capacity}" + ); + fixture.assert_geometry(&document, linked, expected); + } + + let isolated = NodeRowId::new(4); + let resolver = FakeResolver::answering(FakeResolver::resolved(1, 0)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(isolated), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { edges: 0, .. }, + resolver: &resolver, + }, + ) + .expect("should construct an isolated source at zero capacity"); + assert!( + document.complete, + "should be complete with no incident edge" + ); + fixture.assert_geometry(&document, isolated, &[]); +} + +/// Link identity breaks a tie in partner distance and delivery zoom. +#[test] +fn edges_reciprocal_identity_tie() { + let fixture = Fixture::varied_links( + "locate-edges-reciprocal-identity-tie", + Fixture::actor(1, false), + ); + let source = NodeRowId::new(5); + let incident = fixture.incident(source); + assert!( + fixture.identity_of(EdgeRowId::new(4)) < fixture.identity_of(EdgeRowId::new(3)), + "the varied fixture should order identities against rows" + ); + let retained = incident[0]; + assert_eq!(retained, EdgeRowId::new(4), "should retain row 4"); + let resolver = FakeResolver::answering(FakeResolver::resolved(2, 1)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { edges: 1, .. }, + resolver: &resolver, + }, + ) + .expect("should construct at capacity one"); + assert!(!document.complete, "should report the dropped link"); + fixture.assert_geometry(&document, source, &[retained]); +} + +/// Capacity one keeps the nearest partner, including a self-loop. +#[test] +fn edges_nearest_truncation() { + let fixture = Fixture::new("locate-edges-nearest-truncation"); + for (source, kept, nodes) in [ + (NodeRowId::new(1), EdgeRowId::new(1), 2), + (NodeRowId::new(2), EdgeRowId::new(2), 1), + ] { + let resolver = FakeResolver::answering(FakeResolver::resolved(nodes, 1)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { edges: 1, .. }, + resolver: &resolver, + }, + ) + .expect("should construct at capacity one"); + assert!(!document.complete, "should report the dropped link"); + fixture.assert_geometry(&document, source, &[kept]); + } +} + +/// Hydration uses the mask's actor and the configured limits. +#[test] +fn hydrate_request_recorded() { + let fixture = Fixture::varied_links("locate-hydrate-request-recorded", Fixture::actor(2, true)); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let nodes = fixture.nodes_of(source, &edges); + assert_ne!( + edges, + [EdgeRowId::new(0), EdgeRowId::new(1)], + "the varied fixture should order the links against their rows" + ); + let resolver = FakeResolver::answering(FakeResolver::resolved(nodes.len(), edges.len())); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { + colored_type_ids: 9, + edges: 7, + properties: 3, + link_type_ids: 2, + link_properties: 4, + }, + resolver: &resolver, + }, + ) + .expect("should construct with nondefault limits"); + fixture.assert_geometry(&document, source, &edges); + resolver.assert_request(&RecordedRequest { + actor_id: fixture.actor.id, + instance_admin: true, + nodes: nodes + .iter() + .map(|&row| fixture.node_identity(row)) + .collect(), + links: edges + .iter() + .map(|&edge| fixture.identity_of(edge)) + .collect(), + properties: 3, + link_type_ids: 2, + link_properties: 4, + }); +} + +#[test] +fn hydrate_request_empty_links() { + let fixture = Fixture::new("locate-hydrate-request-empty-links"); + let source = NodeRowId::new(4); + assert!( + fixture.incident(source).is_empty(), + "the fixture should isolate node 4" + ); + let resolver = FakeResolver::answering(FakeResolver::resolved(1, 0)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct an isolated source"); + assert!( + document.complete, + "should be complete with no incident edge" + ); + fixture.assert_geometry(&document, source, &[]); + resolver.assert_request(&RecordedRequest { + actor_id: fixture.actor.id, + instance_admin: false, + nodes: vec![fixture.node_identity(source)], + links: Vec::new(), + properties: 10, + link_type_ids: 5, + link_properties: 10, + }); +} + +/// The log records the whole report while the response redacts it. +/// +/// The logged [`Debug`](core::fmt::Debug) rendering includes the underlying store error's text and +/// the private attachment added below [`HydrateError::Query`]. Outside debug builds the response +/// carries the fixed detail instead of either. +#[test] +fn hydrate_failure() { + let fixture = Fixture::new("locate-hydrate-failure"); + let resolver = FakeResolver::failing(); + let Err(report) = LocateDocument::new( + fixture.scene(), + fixture.entity_source(NodeRowId::new(1)), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) else { + panic!("should propagate the resolver's failure"); + }; + assert_matches!( + report.current_context(), + LocateDocumentError::Hydrate(HydrateError::Query) + ); + assert_matches!( + report.downcast_ref::(), + Some(HydrateError::Query) + ); + assert!( + format!("{report:#}").contains("private-store-message"), + "should retain the supplied source text in the report" + ); + assert!( + format!("{report:?}").contains("private-property-value"), + "should retain the supplied attachment in the report" + ); + assert_internal_diagnostic( + move || Problem::from(report), + &["private-store-message", "private-property-value"], + "the detail hydration failed", + ); +} + +/// Unavailable details preserve the delivered geometry and every trailer slot. +#[test] +fn hydrate_unresolved() { + let fixture = Fixture::varied_nodes("locate-hydrate-unresolved"); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let resolver = FakeResolver { + answer: RefCell::new(Some(Answer::Unresolved)), + requests: RefCell::new(Vec::new()), + }; + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct when the store resolves no details"); + fixture.assert_geometry(&document, source, &edges); + let trailer = &document.trailer; + assert_eq!(trailer.labels.len(), 3); + assert!(trailer.labels.iter().all(|label| label.as_ref().is_empty())); + assert_eq!(trailer.representative_type_urls.len(), 3); + assert!(trailer.representative_type_urls.iter().all(Option::is_none)); + assert!(trailer.properties.is_none()); + assert!(!trailer.type_ids_complete); + assert!(!trailer.properties_complete); + assert_eq!(trailer.link_labels.len(), 2); + assert!( + trailer + .link_labels + .iter() + .all(|label| label.as_ref().is_empty()) + ); + assert_eq!(trailer.link_type_urls.len(), 2); + assert!(trailer.link_type_urls.iter().all(Vec::is_empty)); + assert_eq!(trailer.link_properties.len(), 2); + assert!(trailer.link_properties.iter().all(Option::is_none)); + assert_eq!(trailer.link_type_urls_complete.domain_size(), 2); + assert!(trailer.link_type_urls_complete.iter().next().is_none()); + assert_eq!(trailer.link_properties_complete.domain_size(), 2); + assert!(trailer.link_properties_complete.iter().next().is_none()); + assert!(trailer.type_urls.is_empty()); + assert!(trailer.property_urls.is_empty()); +} + +/// Resolved nodes borrow their captured labels and unresolved nodes read empty. +#[test] +fn labels_nodes_partial() { + let fixture = Fixture::varied_nodes("locate-labels-nodes-partial"); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let nodes = fixture.nodes_of(source, &edges); + let mut response = FakeResolver::resolved(nodes.len(), edges.len()); + FakeResolver::unresolve_node(&mut response, 2); + let resolver = FakeResolver::answering(response); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with a partially resolved response"); + fixture.assert_geometry(&document, source, &edges); + let labels = &document.trailer.labels; + assert_eq!( + labels.len(), + nodes.len(), + "should label every delivered node" + ); + for (index, &row) in nodes.iter().enumerate().take(2) { + let slot = NodeSlot::from_usize(index); + assert_eq!( + labels[slot].as_ref(), + format!("node-{}", row.as_u64()), + "should carry row {row}'s own text at slot {index}" + ); + assert!( + core::ptr::eq(labels[slot], fixture.node_label_of(row)), + "should borrow row {row}'s label from the captured scene" + ); + } + assert_eq!( + labels[NodeSlot::from_usize(2)].as_ref(), + "", + "should leave the unresolved node's label empty" + ); +} + +/// Resolved links borrow their captured labels and unresolved links read empty. +#[test] +fn labels_links_partial() { + let fixture = Fixture::varied_links("locate-labels-links-partial", Fixture::actor(1, false)); + let source = NodeRowId::new(5); + let edges = fixture.incident(source); + let mut response = FakeResolver::resolved(2, edges.len()); + FakeResolver::unresolve_link(&mut response, 0); + let resolver = FakeResolver::answering(response); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with a partially resolved response"); + fixture.assert_geometry(&document, source, &edges); + let labels = &document.trailer.link_labels; + assert_eq!( + labels.len(), + edges.len(), + "should label every delivered link" + ); + assert_eq!( + labels[EdgeSlot::from_usize(0)].as_ref(), + "", + "should leave the unresolved link's label empty" + ); + let delivered_edge = edges[1]; + assert_eq!( + labels[EdgeSlot::from_usize(1)].as_ref(), + format!("edge-{}", delivered_edge.as_u64()), + "should carry the resolved link's own text" + ); + assert!( + core::ptr::eq( + labels[EdgeSlot::from_usize(1)], + fixture.link_label_of(delivered_edge) + ), + "should borrow the resolved link's label from the captured scene" + ); +} + +/// Shared URLs preserve each slot's values and missing property maps. +#[test] +fn trailer_url_interning() { + let fixture = Fixture::new("locate-trailer-url-interning"); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let alpha = FakeResolver::url("alpha"); + let beta = FakeResolver::url("beta"); + let gamma = FakeResolver::url("gamma"); + + let resolver = FakeResolver::answering(FakeResolver::shared_urls()); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with hydrated details"); + fixture.assert_geometry(&document, source, &edges); + let trailer = &document.trailer; + + let representatives: Vec<_> = trailer + .representative_type_urls + .iter() + .map(|slot| slot.map(|index| trailer.type_urls.entries()[index].clone())) + .collect(); + assert_eq!( + representatives, + [Some(alpha.clone()), Some(alpha.clone()), None], + "should keep each node's first type at its slot" + ); + assert_eq!( + Fixture::decode_type_urls( + trailer, + trailer.link_type_urls[EdgeSlot::from_usize(0)] + .iter() + .copied() + ), + [alpha, beta], + "should preserve the resolver's link type order" + ); + assert!( + trailer.link_type_urls[EdgeSlot::from_usize(1)].is_empty(), + "should keep no types for the unresolved link" + ); + assert_eq!( + trailer.type_urls.len(), + 2, + "should intern each emitted type URL once" + ); + assert!( + !trailer.type_urls.entries().iter().any(|url| *url == gamma), + "should intern no nonrepresentative source type" + ); + + let source_properties = trailer + .properties + .as_ref() + .expect("should carry the resolved source's properties"); + assert_eq!( + Fixture::decode_properties(trailer, source_properties), + BTreeMap::from([ + (FakeResolver::property("age"), ScalarValue::Integer(36)), + ( + FakeResolver::property("name"), + ScalarValue::String("Ada".to_owned()) + ), + ]), + "should decode the source's properties by URL" + ); + let link_properties = trailer.link_properties[EdgeSlot::from_usize(0)] + .as_ref() + .expect("should carry the resolved link's properties"); + assert_eq!( + Fixture::decode_properties(trailer, link_properties), + BTreeMap::from([ + (FakeResolver::property("name"), ScalarValue::Bool(true)), + (FakeResolver::property("note"), ScalarValue::Null), + (FakeResolver::property("weight"), ScalarValue::Float(0.5)), + ]), + "should decode the link's properties by URL" + ); + assert!( + trailer.link_properties[EdgeSlot::from_usize(1)].is_none(), + "should carry no map for the unresolved link" + ); + assert_eq!( + trailer.property_urls.len(), + 4, + "should intern each distinct property URL once across source and links" + ); +} + +/// One named input to [`trailer_completeness`]. +/// +/// The case supplies the source's direct types and requested type selection, with expected +/// completeness for both types and properties. +struct CompletenessCase<'selection> { + name: &'static str, + source_types: Vec, + selection: &'selection [ArchivedOntologyTypeUuid], + expected_types_complete: bool, + properties_complete: bool, +} + +/// Complete type coverage requires a resolved source with nonempty, fully selected direct types. +#[test] +fn trailer_completeness() { + let fixture = Fixture::new("locate-trailer-completeness"); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let alpha = FakeResolver::url("alpha"); + let beta = FakeResolver::url("beta"); + let alpha_id = ArchivedOntologyTypeUuid::from_url(&alpha); + let beta_id = ArchivedOntologyTypeUuid::from_url(&beta); + let both = [alpha_id, beta_id]; + let alpha_only = [alpha_id]; + + let cases = [ + CompletenessCase { + name: "partial", + source_types: vec![alpha.clone(), beta.clone()], + selection: &alpha_only, + expected_types_complete: false, + properties_complete: false, + }, + CompletenessCase { + name: "empty", + source_types: Vec::new(), + selection: &alpha_only, + expected_types_complete: false, + properties_complete: true, + }, + CompletenessCase { + name: "complete", + source_types: vec![alpha, beta], + selection: &both, + expected_types_complete: true, + properties_complete: true, + }, + CompletenessCase { + name: "unresolved", + source_types: Vec::new(), + selection: &alpha_only, + expected_types_complete: false, + properties_complete: false, + }, + ]; + for CompletenessCase { + name: case, + source_types, + selection, + expected_types_complete, + properties_complete, + } in cases + { + let mut response = FakeResolver::resolved(3, 2); + response.nodes[NodeSlot::MIN] = Some(LocateNode { + type_urls: source_types, + }); + response.source_properties = Some(LocateProperties { + values: ScalarProperties::EMPTY, + complete: properties_complete, + }); + for (slot, link) in response.links.iter_enumerated_mut() { + let link = link.as_mut().expect("should have link details"); + link.type_urls_complete = slot == EdgeSlot::MIN; + link.properties.complete = slot != EdgeSlot::MIN; + } + if case == "unresolved" { + FakeResolver::unresolve_node(&mut response, 0); + } + let resolver = FakeResolver::answering(response); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: OntologySelection::new(selection), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct every completeness case"); + fixture.assert_geometry(&document, source, &edges); + let trailer = &document.trailer; + assert_eq!( + trailer.type_ids_complete, expected_types_complete, + "should report source type coverage for the {case} case" + ); + assert_eq!( + trailer.properties_complete, properties_complete, + "should pass the source property completeness through in the {case} case" + ); + assert_eq!( + trailer.properties.is_some(), + case != "unresolved", + "should carry a property map only for a resolved source in the {case} case" + ); + assert_eq!( + trailer.link_type_urls_complete.iter().collect::>(), + [EdgeSlot::from_usize(0)], + "should pass the link type completeness set through" + ); + assert_eq!( + trailer.link_properties_complete.iter().collect::>(), + [EdgeSlot::from_usize(1)], + "should pass the link property completeness set through" + ); + } +} + +/// Refuses a request past the configured type-count limit, under [`LocateDocumentError::Types`]. +#[test] +fn types_over_limit() { + let fixture = Fixture::new("locate-types-over-limit"); + let types = [fixture.ontology_uuid(0); 3]; + let resolver = FakeResolver::unexpected(); + let Err(report) = LocateDocument::new( + fixture.scene(), + fixture.entity_source(NodeRowId::new(1)), + &LocateDocumentOptions { + types: OntologySelection::new(&types), + limits: LocateLimits { + colored_type_ids: 2, + .. + }, + resolver: &resolver, + }, + ) else { + panic!("should refuse a type count past the limit"); + }; + assert_matches!( + report.current_context(), + LocateDocumentError::Types { + count: 3, + maximum: 2 + }, + ); + resolver.assert_unasked(); +} + +/// Admits a request at exactly the configured type-count cap. +#[test] +fn types_limit_boundary() { + let fixture = Fixture::new("locate-types-limit-boundary"); + let types = [fixture.ontology_uuid(0), fixture.ontology_uuid(1)]; + let resolver = FakeResolver::answering(FakeResolver::resolved(3, 2)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(NodeRowId::new(1)), + &LocateDocumentOptions { + types: OntologySelection::new(&types), + limits: LocateLimits { + colored_type_ids: 2, + .. + }, + resolver: &resolver, + }, + ) + .expect("should admit exactly the configured type-count cap"); + let masks = document + .type_masks + .expect("should keep a mask column for a nonempty selection"); + assert_eq!( + masks.bits.col_domain_size(), + types.len(), + "should keep one column per requested type at the exact cap" + ); + assert_eq!( + masks.bits.row_domain_size(), + 3, + "should keep one row per delivered node" + ); +} + +#[test] +fn masks_omitted_empty_selection() { + let fixture = Fixture::new("locate-masks-omitted-empty-selection"); + let resolver = FakeResolver::answering(FakeResolver::resolved(3, 2)); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(NodeRowId::new(1)), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with an empty type selection"); + assert!( + document.type_masks.is_none(), + "should omit the mask column for an empty selection" + ); +} + +/// Mask rows follow the source-first slots, and duplicate requests keep separate columns. +/// +/// Type 1 is a child of type 0. Every node matches type 0, and the odd rows match type 1. Type 2 +/// has no node members. +#[test] +fn masks_source_first_slots() { + let fixture = Fixture::new("locate-masks-source-first-slots"); + let source = NodeRowId::new(1); + let edges = fixture.incident(source); + let nodes = fixture.nodes_of(source, &edges); + let type0 = fixture.ontology_uuid(0); + let type1 = fixture.ontology_uuid(1); + let type2 = fixture.ontology_uuid(2); + let unknown = ArchivedOntologyTypeUuid::from(Uuid::from_u128(0xDEAD_BEEF)); + let requested = [type0, type1, type2, unknown, type0]; + let membership: [Option; 5] = [Some(0), Some(1), Some(2), None, Some(0)]; + + let resolver = FakeResolver::answering(FakeResolver::resolved(nodes.len(), edges.len())); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(source), + &LocateDocumentOptions { + types: OntologySelection::new(&requested), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with a nonempty type selection"); + fixture.assert_geometry(&document, source, &edges); + let masks = document + .type_masks + .expect("should keep a mask column for a nonempty selection"); + assert_eq!(masks.bits.row_domain_size(), nodes.len()); + assert_eq!(masks.bits.col_domain_size(), requested.len()); + for (index, &row) in nodes.iter().enumerate() { + let slot = NodeSlot::from_usize(index); + for (bit, &type_row) in membership.iter().enumerate() { + let expected = match type_row { + Some(0) => true, + Some(1) => row.as_u64() & 1 == 1, + _ => false, + }; + assert_eq!( + masks.bits.contains(slot, SelectionSlot::from_usize(bit)), + expected, + "row {row}'s membership at requested slot {bit} should match its own type" + ); + } + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs b/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs new file mode 100644 index 00000000000..fe580e3ee8f --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs @@ -0,0 +1,168 @@ +use alloc::collections::BTreeMap; + +use error_stack::Report; +use hashql_core::id::{Id as _, IdVec}; +use type_system::ontology::{VersionedUrl, id::BaseUrl}; + +use super::{LocateDocumentError, subgraph::LocateSubgraph}; +use crate::{ + bitset::DenseBitSlice, + dataset::auxiliary::Label, + postgres::id::ArchivedOntologyTypeUuid, + serve::{ + hydrate::{ + EdgeSlot, LocateResponse, NodeSlot, + scalar::{ScalarProperties, ScalarValue}, + }, + intern::{InternTable, TableIndex}, + membership::OntologySelection, + scene::Scene, + }, +}; + +/// Scalar properties keyed by this trailer's property table. +pub(crate) struct PropertyMap(pub BTreeMap, ScalarValue>); + +impl PropertyMap { + /// Interns `properties`'s base URLs into `table`, keying each value by its table index. + fn new(properties: ScalarProperties, table: &mut InternTable) -> Self { + Self( + properties + .into_iter() + .map(|(name, value)| (table.intern(name), value)) + .collect(), + ) + } +} + +/// Interned URLs and per-source and per-link detail for one locate document. +/// +/// The auxiliary data includes interned type and property URLs alongside the source's and every +/// link's labels, representative types and properties. +pub(crate) struct LocateTrailer<'details> { + pub type_urls: InternTable, + pub property_urls: InternTable, + pub labels: IdVec, + pub representative_type_urls: IdVec>>, + pub properties: Option, + pub type_ids_complete: bool, + pub properties_complete: bool, + pub link_labels: IdVec, + pub link_type_urls: IdVec>>, + pub link_type_urls_complete: Box>, + pub link_properties: IdVec>, + pub link_properties_complete: Box>, +} + +impl<'details> LocateTrailer<'details> { + /// Aligns captured labels with the resolved details and builds their URL tables. + /// + /// # Errors + /// + /// Returns [`LocateDocumentError`] for a resolved entity without a captured display payload. + pub(crate) fn new( + Scene { world, epoch, .. }: Scene<'details>, + subgraph: &LocateSubgraph, + types: &OntologySelection, + LocateResponse { + nodes, + links, + source_properties, + }: LocateResponse, + ) -> Result> { + let type_ids_complete = source_properties.is_some() + && nodes[NodeSlot::MIN].details.as_ref().is_some_and(|source| { + !source.type_urls.is_empty() + && source + .type_urls + .iter() + .all(|url| types.contains(ArchivedOntologyTypeUuid::from_url(url))) + }); + + let mut type_urls = InternTable::new(); + let mut labels = IdVec::with_capacity(nodes.len()); + let mut representative_type_urls = IdVec::with_capacity(nodes.len()); + for (slot, node) in nodes.into_iter_enumerated() { + let Some(details) = node.details else { + labels.push(Label::EMPTY); + representative_type_urls.push(None); + continue; + }; + + let row = subgraph.nodes[slot]; + labels.push( + world + .layout + .index + .payload(epoch, row) + .ok_or_else(|| Report::new(LocateDocumentError::NodeDisplay { row }))? + .label(), + ); + representative_type_urls.push( + details + .type_urls + .into_iter() + .next() + .map(|url| type_urls.intern(url)), + ); + } + + let mut property_urls = InternTable::new(); + let properties_complete = source_properties + .as_ref() + .is_some_and(|properties| properties.complete); + let properties = source_properties + .map(|properties| PropertyMap::new(properties.values, &mut property_urls)); + + let mut link_labels = IdVec::with_capacity(links.len()); + let mut link_type_urls = IdVec::with_capacity(links.len()); + let mut link_type_urls_complete = DenseBitSlice::new_empty(links.len()); + let mut link_properties = IdVec::with_capacity(links.len()); + let mut link_properties_complete = DenseBitSlice::new_empty(links.len()); + for (slot, link) in links.into_iter_enumerated() { + let Some(details) = link.details else { + link_labels.push(Label::EMPTY); + link_type_urls.push(Vec::new()); + link_properties.push(None); + continue; + }; + + let row = subgraph.edges[slot.as_usize()].row.unwrap(); + link_labels.push( + world + .topology + .payload(epoch, row) + .ok_or_else(|| Report::new(LocateDocumentError::LinkDisplay { row }))? + .label(), + ); + link_type_urls.push( + details + .type_urls + .into_iter() + .map(|url| type_urls.intern(url)) + .collect(), + ); + link_type_urls_complete.set(slot, details.type_urls_complete); + link_properties.push(Some(PropertyMap::new( + details.properties.values, + &mut property_urls, + ))); + link_properties_complete.set(slot, details.properties.complete); + } + + Ok(Self { + type_urls, + property_urls, + labels, + representative_type_urls, + properties, + type_ids_complete, + properties_complete, + link_labels, + link_type_urls, + link_type_urls_complete, + link_properties, + link_properties_complete, + }) + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/manifest/mod.rs b/libs/@local/graph/atlas/src/serve/document/manifest/mod.rs new file mode 100644 index 00000000000..787aa9c1dc2 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/manifest/mod.rs @@ -0,0 +1,140 @@ +//! Bootstrap metadata for a captured generation and its resolved delivery schedule. +//! +//! Route limits come from request configuration. Scope metadata comes from the bound schedule that +//! supplies document rows. + +use alloc::alloc::Allocator; +use core::fmt; + +use error_stack::Report; +use hash_graph_temporal_versioning::{DecisionTime, Timestamp}; +use serde::{Serialize, Serializer}; + +use super::{ + Document, DocumentLimits, VARIANTS, + codec::{Envelope, WIRE_VERSION}, +}; +use crate::{ + file::generation::GenerationId, + math::Log2, + morton::Zoom, + serve::{scene::Scene, visibility::cache::VisibilityLimits}, +}; + +#[cfg(test)] +mod tests; + +/// A bucket-schedule cut, rendered as `z+` for its `n`-level span. +#[derive(schemars::JsonSchema)] +#[schemars(with = "String")] +struct BucketCut(Log2); + +impl fmt::Display for BucketCut { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "z+{}", self.0.get()) + } +} + +impl Serialize for BucketCut { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +/// The corpus bucket schedule's span, cut and maximum served zoom. +#[derive(serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +struct BucketDescription { + span: u64, + cut: BucketCut, + max_zoom: Zoom, +} + +/// The resolved delivery scope's offset, cut and maximum served zoom for this request. +#[derive(serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +struct ScopeDescription { + #[serde(rename = "k")] + offset: Zoom, + cut: BucketCut, + max_zoom: Zoom, +} + +/// The document limits alongside the request's resolved authority windows. +/// +/// The windows are the refresh threshold and the hard expiry, in whole seconds. +#[derive(serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +struct ManifestLimits<'limits> { + #[serde(flatten)] + documents: &'limits DocumentLimits, + authority_refresh_seconds: u64, + authority_hard_seconds: u64, +} + +/// Generation metadata and the delivery cut resolved for one request. +#[derive(serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ManifestDocument<'limits> { + generation: GenerationId, + wire_version: u16, + variants: [&'static str; VARIANTS.len()], + bucket_schedule: BucketDescription, + scope_schedule: ScopeDescription, + limits: ManifestLimits<'limits>, + #[serde(skip_serializing_if = "Option::is_none")] + #[schemars(with = "Option")] + created_at: Option>, +} + +impl<'limits> ManifestDocument<'limits> { + /// Builds the manifest for `scene`'s bound world and delivery schedule. + /// + /// The published limits are `limits` together with the request's resolved authority windows. + pub(crate) fn new( + Scene { + world, delivery, .. + }: Scene<'_>, + limits: &'limits DocumentLimits, + VisibilityLimits { soft, hard, .. }: VisibilityLimits, + ) -> Self { + let buckets = world.schedule(); + let scoped = delivery.buckets(); + + Self { + generation: world.generation().id(), + wire_version: WIRE_VERSION, + variants: VARIANTS, + bucket_schedule: BucketDescription { + span: 1_u64 << buckets.span().get(), + cut: BucketCut(buckets.span()), + max_zoom: buckets.max_tile_depth(), + }, + scope_schedule: ScopeDescription { + offset: delivery.offset(), + cut: BucketCut(scoped.span()), + max_zoom: scoped.first_zoom(delivery.min_resolution()), + }, + limits: ManifestLimits { + documents: limits, + authority_refresh_seconds: soft.as_secs(), + authority_hard_seconds: hard.as_secs(), + }, + created_at: world + .generation() + .repository() + .metadata + .snapshot + .axes + .map(|axes| axes.decision_time), + } + } +} + +impl Document for ManifestDocument<'_> { + type Error = Report; + + fn encode(&self, buffer: &mut Vec) -> Result { + Envelope::encode_json(self, buffer) + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/manifest/tests.rs b/libs/@local/graph/atlas/src/serve/document/manifest/tests.rs new file mode 100644 index 00000000000..e3ba9fff705 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/manifest/tests.rs @@ -0,0 +1,608 @@ +use alloc::{ + alloc::{Allocator, Global}, + sync::Arc, +}; +use core::time::Duration; + +use arc_swap::Guard; +use camino::Utf8PathBuf; +use hash_graph_temporal_versioning::{DecisionTime, Timestamp, TransactionTime}; +use rand::{SeedableRng as _, rngs::StdRng}; +use serde_json::{Value, json}; +use type_system::principal::actor::{ActorId, ActorType}; +use uuid::Uuid; + +use super::{ + super::{ + Document as _, DocumentLimits, EdgesLimits, LocateLimits, TileLimits, TranslateLimits, + }, + ManifestDocument, +}; +use crate::{ + bitset::CompressedBitSet, + dataset::TemporalAxes, + file::{ + generation::{Generation, GenerationId, GenerationRoot}, + salt::metadata::SaltMetadata, + }, + identity::NodeRowId, + math::Log2, + morton::Zoom, + salt::lod::stage::LodConfig, + serve::{ + delta::{Delta, epoch::Epoch}, + scene::Scene, + schedule::ViewSchedule, + tests::fixture::{TamperFixture, secret}, + visibility::{VisibilityActor, VisibilityMask, cache::VisibilityLimits}, + world::World, + }, +}; + +/// The fixture's soft authority-refresh window, deliberately fractional to check truncation. +const AUTHORITY_REFRESH: Duration = Duration::from_millis(90_500); + +/// The fixture's hard authority-expiry window, deliberately fractional to check truncation. +const AUTHORITY_HARD: Duration = Duration::from_millis(300_250); + +/// A fixture snapshot transaction-time timestamp. +const TRANSACTION_TIME: &str = "2026-07-19T08:30:00Z"; + +/// A fixture snapshot decision-time timestamp. +const DECISION_TIME: &str = "2026-05-04T11:45:06Z"; + +/// Builds distinct, distinguishable limit values for every group the manifest publishes. +fn limits() -> DocumentLimits { + DocumentLimits { + tile: TileLimits { + colored_type_ids: 7, + }, + edges: EdgesLimits { + tiles: 11, + edges: 13, + }, + locate: LocateLimits { + colored_type_ids: 17, + edges: 19, + properties: 23, + link_type_ids: 29, + link_properties: 31, + }, + translate: TranslateLimits { entity_ids: 37 }, + } +} + +/// Builds the JSON the [`limits`] fixture should render as, with the authority window seconds. +fn expected_limits() -> Value { + json!({ + "tile": {"coloredTypeIds": 7}, + "edges": {"tiles": 11, "edges": 13}, + "locate": { + "coloredTypeIds": 17, + "edges": 19, + "properties": 23, + "linkTypeIds": 29, + "linkProperties": 31, + }, + "translate": {"entityIds": 37}, + "authorityRefreshSeconds": 90, + "authorityHardSeconds": 300, + }) +} + +/// Builds a [`VisibilityLimits`] at `bytes` capacity, under the fixture's authority windows. +fn visibility(bytes: u64) -> VisibilityLimits { + VisibilityLimits { + bytes, + soft: AUTHORITY_REFRESH, + hard: AUTHORITY_HARD, + } +} + +/// Builds the JSON the synthetic generation's bucket schedule should render as. +fn expected_bucket_schedule() -> Value { + json!({"span": 64, "cut": "z+6", "maxZoom": 18}) +} + +/// Builds the JSON an unshifted (zero-offset) scope should render as. +fn expected_unshifted_scope() -> Value { + json!({"k": 0, "cut": "z+6", "maxZoom": 0}) +} + +/// Encodes a [`ManifestDocument`] and parses its bytes back as JSON. +/// +/// The helper also asserts the encoded envelope's media type. +/// +/// # Panics +/// +/// Panics where the document does not encode into `buffer` under `limits`, and where the encoded +/// bytes do not parse as JSON. +#[track_caller] +fn manifest( + scene: Scene<'_>, + limits: &DocumentLimits, + visibility: VisibilityLimits, + buffer: &mut Vec, +) -> Value { + let envelope = ManifestDocument::new(scene, limits, visibility) + .encode(buffer) + .expect("should complete the manifest document"); + assert_eq!( + envelope.content_type(), + "application/json", + "should report the media type of the completed JSON document" + ); + + serde_json::from_slice(buffer.as_slice()).expect("should parse one complete document") +} + +/// A generation republished from `source` with an edited [`SaltMetadata`]. +/// +/// It owns a temporary [`GenerationRoot`] until dropped. +struct Republished { + root: GenerationRoot, +} + +impl Republished { + /// Copies and republishes `source` with edited repository metadata. + /// + /// Copies `source`'s published artifacts into a temporary root named for this process and + /// `name`, and applies `edit` to a clone of its repository metadata. It then seals the edited + /// copy as a new generation. The pre-clean removal drops its error, and a missing root is one + /// ordinary source of that error. Where the removal fails over a root that exists, the copy + /// proceeds into the contents left behind, whole or partial. Opening the root and creating + /// the staging must still succeed before the copy begins. + /// + /// # Panics + /// + /// Panics where the process temporary directory is not UTF-8, and where the supplied `edit` + /// panics. Every republication step that returns a result panics on failure, apart from the + /// pre-clean removal, whose error the fixture drops. + fn publish( + name: &str, + source: &Generation, + edit: impl FnOnce(&mut SaltMetadata), + ) -> (Self, Generation) { + let path = Utf8PathBuf::from_path_buf(std::env::temp_dir()) + .expect("should use a UTF-8 temp directory") + .join(format!( + "hash-graph-atlas-manifest-{}-{name}", + std::process::id() + )); + let _removed: Result<(), std::io::Error> = std::fs::remove_dir_all(&path); + + let owned = Self { + root: GenerationRoot::new(path).expect("should open the republication root"), + }; + let published = { + let staging = owned + .root + .stage() + .expect("should create the republication staging"); + for file in source.repository().files.files() { + std::fs::copy(source.path_of(&file.name), staging.path_of(&file.name)) + .expect("should copy a published artifact into the staging"); + } + + let mut repository = source.repository().clone(); + edit(&mut repository.metadata); + staging + .seal(&repository) + .expect("should seal the republished generation") + }; + let generation = owned + .root + .open(published.id()) + .expect("should open the republished generation"); + + (owned, generation) + } +} + +impl Drop for Republished { + fn drop(&mut self) { + drop(std::fs::remove_dir_all(self.root.path())); + } +} + +/// An opened synthetic generation, optionally republished with edited metadata. +/// +/// It carries the captured epoch, actor, visibility mask and delivery schedule that `scene()` +/// assembles into one scene. +struct Fixture { + world: Arc, + epoch: Epoch, + actor: VisibilityActor, + mask: VisibilityMask, + schedule: ViewSchedule, + files: TamperFixture, + _republished: Option, +} + +impl Fixture { + /// Opens `generation` with a fresh delta identity. + /// + /// `files` published `generation`, which is optionally the source of `republished`. The mask + /// grants one fixed test actor full visibility. + /// + /// # Panics + /// + /// Panics if [`World::open`] fails to open or validate the serving artifacts. + fn open( + files: TamperFixture, + generation: Generation, + republished: Option, + ) -> Self { + let world = Arc::new( + World::open(generation, &secret()).expect("should open the synthetic generation"), + ); + let delta = Delta::new(Arc::clone(&world), StdRng::seed_from_u64(17)) + .expect("should allocate a delta identity"); + let epoch = Epoch::from(Guard::from_inner(Arc::new(delta))); + let actor = VisibilityActor { + id: ActorId::new(Uuid::from_u128(1), ActorType::User), + instance_admin: false, + }; + let mask = VisibilityMask::full(actor); + let schedule = ViewSchedule::of(Arc::clone(&world), &epoch, &mask); + Self { + world, + epoch, + actor, + mask, + schedule, + files, + _republished: republished, + } + } + + /// Publishes and opens the named synthetic generation unmodified. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails, and for the conditions + /// [`Self::open`] states. + fn new(name: &str) -> Self { + let files = TamperFixture::publish(name); + let generation = files.generation().clone(); + Self::open(files, generation, None) + } + + /// Publishes the named synthetic generation, then opens a republished copy of it. + /// + /// The copy takes `edit`'s changes to its metadata before sealing. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails, where the supplied `edit` panics, + /// and for the conditions [`Republished::publish`] and [`Self::open`] state. + fn edited(name: &str, edit: impl FnOnce(&mut SaltMetadata)) -> Self { + let files = TamperFixture::publish(name); + let (republished, generation) = Republished::publish(name, files.generation(), edit); + Self::open(files, generation, Some(republished)) + } + + /// Returns the id of the generation this fixture opened (the republished one, if any). + fn generation(&self) -> GenerationId { + self.world.generation().id() + } + + /// Returns the id of the originally published generation, before any republication. + fn source_generation(&self) -> GenerationId { + self.files.generation().id() + } + + /// Narrows this fixture's visibility mask and schedule to exactly the given node rows. + /// + /// No edge remains visible. + fn restrict(&mut self, nodes: impl IntoIterator) { + self.mask = VisibilityMask::partial( + self.actor, + CompressedBitSet::from_rows(nodes.into_iter().map(NodeRowId::new)), + CompressedBitSet::from_rows(core::iter::empty()), + ); + self.schedule = ViewSchedule::of(Arc::clone(&self.world), &self.epoch, &self.mask); + } + + /// Assembles this fixture's world, epoch, mask and schedule into one [`Scene`]. + /// + /// Its delivery is the schedule's cut at `offset`. + /// + /// # Panics + /// + /// Panics where the schedule binds no cut at `offset`. + fn scene(&self, offset: Zoom) -> Scene<'_> { + Scene { + world: &self.world, + epoch: &self.epoch, + mask: &self.mask, + schedule: &self.schedule, + delivery: self + .schedule + .cut(offset) + .expect("should bind the requested density offset"), + } + } +} + +/// Every limit group publishes under its own key, with distinguishable values. +/// +/// The authority windows truncate their fractional seconds in the published integer keys, and the +/// encoder overwrites a buffer that already holds unrelated bytes. +#[test] +fn limits_grouped_distinct() { + let fixture = Fixture::new("manifest-limits-grouped-distinct"); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + buffer.extend_from_slice(b"previous document"); + + let document = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + + assert_eq!( + document, + json!({ + "generation": fixture.generation().to_string(), + "wireVersion": 1, + "variants": ["plain"], + "bucketSchedule": expected_bucket_schedule(), + "scopeSchedule": expected_unshifted_scope(), + "limits": expected_limits(), + }), + "should publish the whole bootstrap document" + ); +} + +#[test] +fn limits_cache_bytes_unpublished() { + let fixture = Fixture::new("manifest-limits-cache-bytes-unpublished"); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + + let small = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + let large = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(0x4000_0000), + &mut buffer, + ); + + assert_eq!( + small, large, + "should publish one document for either cache budget" + ); + assert_eq!( + large["limits"], + expected_limits(), + "should still publish every route cap and both authority windows" + ); +} + +/// A corpus request publishes offset zero and the unshifted cut at any requested offset. +/// +/// A corpus request is a full-visibility one. It publishes the bucket schedule as recorded. +#[test] +fn schedule_corpus_offset() { + let fixture = Fixture::new("manifest-schedule-corpus-offset"); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + let offset = Zoom::new(5).expect("should fit the key width"); + + let document = manifest( + fixture.scene(offset), + &limits, + visibility(4096), + &mut buffer, + ); + + assert_eq!( + document["scopeSchedule"], + expected_unshifted_scope(), + "should publish offset zero and the unshifted cut for a corpus request" + ); + assert_eq!( + document["bucketSchedule"], + expected_bucket_schedule(), + "should leave the recorded bucket schedule alone" + ); +} + +/// Distinct root quadrants require only one subdivision to separate these rows. +/// +/// Rows 0, 1 and 2 have coordinates (-2, -1), (-1, 2) and (0, 0) in a [-3, 3]² frame. Normalization +/// preserves their quadrants. The [first-occupant cascade](crate::salt::lod::cascade) assigns the +/// best-ranked row to depth zero and the others to depth one. A span exponent of zero gives maximum +/// zoom 1. Offset 2 changes it to max(1 - 2, 0) = 0. +#[test] +fn schedule_scoped_offset() { + let mut fixture = Fixture::edited("manifest-schedule-scoped-offset", |metadata| { + metadata.reproducibility.config.lod.span = Log2::ZERO; + }); + fixture.restrict([0, 1, 2]); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + + let unshifted = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + assert_eq!( + unshifted["scopeSchedule"], + json!({"k": 0, "cut": "z+0", "maxZoom": 1}), + "should first deliver the deepest occupied bucket at zoom one" + ); + + let offset = Zoom::new(2).expect("should fit the key width"); + let shifted = manifest( + fixture.scene(offset), + &limits, + visibility(4096), + &mut buffer, + ); + + assert_eq!( + shifted["scopeSchedule"], + json!({"k": 2, "cut": "z+2", "maxZoom": 0}), + "should publish the requested offset and the cut two subdivisions deeper" + ); + assert_eq!( + shifted["bucketSchedule"], + json!({"span": 1, "cut": "z+0", "maxZoom": 18}), + "should leave the recorded bucket schedule alone" + ); +} + +/// A view with no visible nodes still publishes a schedule. +/// +/// The bucket schedule is the recorded one and the scope is root-only and unshifted, rather than +/// empty or missing. +#[test] +fn schedule_empty_scope() { + let mut fixture = Fixture::new("manifest-schedule-empty-scope"); + fixture.restrict(core::iter::empty()); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + + let document = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + + assert_eq!( + document, + json!({ + "generation": fixture.generation().to_string(), + "wireVersion": 1, + "variants": ["plain"], + "bucketSchedule": expected_bucket_schedule(), + "scopeSchedule": expected_unshifted_scope(), + "limits": expected_limits(), + }), + "should publish the recorded schedule and a root-only scope over an empty view" + ); +} + +/// A generation with no observed temporal axes omits `createdAt` entirely. +/// +/// The key is absent rather than null or defaulted. +#[test] +fn snapshot_axes_absent() { + let fixture = Fixture::new("manifest-snapshot-axes-absent"); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + + let document = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + + assert!( + document.get("createdAt").is_none(), + "should leave out the creation time of a generation with no observed axes" + ); + assert_eq!( + document["generation"], + json!(fixture.generation().to_string()), + "should publish the generation the scene captured" + ); +} + +#[test] +fn snapshot_axes_present() { + let decision_time: Timestamp = DECISION_TIME + .parse() + .expect("should parse the fixture decision time"); + let transaction_time: Timestamp = TRANSACTION_TIME + .parse() + .expect("should parse the fixture transaction time"); + let fixture = Fixture::edited("manifest-snapshot-axes-present", |metadata| { + metadata.snapshot.axes = Some(TemporalAxes { + transaction_time, + decision_time, + }); + }); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + + let mut document = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + + let created_at = document + .as_object_mut() + .expect("should publish a JSON object") + .remove("createdAt") + .expect("should publish the creation time of an observed snapshot"); + assert_eq!( + serde_json::from_value::>(created_at) + .expect("should publish a decision-time timestamp"), + decision_time, + "should publish the snapshot's own decision time" + ); + assert_ne!( + fixture.generation(), + fixture.source_generation(), + "should carry the identity of the republished metadata" + ); + assert_eq!( + document, + json!({ + "generation": fixture.generation().to_string(), + "wireVersion": 1, + "variants": ["plain"], + "bucketSchedule": expected_bucket_schedule(), + "scopeSchedule": expected_unshifted_scope(), + "limits": expected_limits(), + }), + "should leave the rest of the document as an axesless generation publishes it" + ); +} + +/// A span exponent of 32 requires a count wider than a 32-bit integer. +#[test] +fn schedule_maximum_span() { + let fixture = Fixture::edited("manifest-schedule-maximum-span", |metadata| { + metadata.reproducibility.config.lod = LodConfig { + span: Log2::new(32).expect("should fit the exponent domain"), + max_tile_depth: Zoom::MIN, + }; + }); + let limits = limits(); + let mut buffer = Vec::new_in(&Global); + + let document = manifest( + fixture.scene(Zoom::MIN), + &limits, + visibility(4096), + &mut buffer, + ); + + assert_eq!( + document["bucketSchedule"], + json!({"span": 0x0001_0000_0000_u64, "cut": "z+32", "maxZoom": 0}), + "should publish one axis of cells per tile and the root as the deepest served zoom" + ); + assert_eq!( + document["scopeSchedule"], + json!({"k": 0, "cut": "z+32", "maxZoom": 0}), + "should publish the same cut for the corpus scope" + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/masks.rs b/libs/@local/graph/atlas/src/serve/document/masks.rs new file mode 100644 index 00000000000..88325018a86 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/masks.rs @@ -0,0 +1,45 @@ +use hashql_core::id::{Id, bit_vec::BitMatrix}; + +use crate::{ + identity::NodeRowId, + serve::{ + membership::{OntologySelection, SelectionSlot}, + scene::Scene, + }, +}; + +/// Membership over delivered slots and requested-type slots. +pub(super) struct TypeMasks { + pub bits: BitMatrix, +} + +impl TypeMasks { + /// Builds the membership matrix for `rows` against `types`. + /// + /// The matrix holds one row per delivered row, in the order `rows` supplies them. A row with + /// no position in `world`'s epoch keeps its own slot with every bit clear, and the rows after + /// it keep their slot numbers. + pub(super) fn new( + Scene { world, epoch, .. }: Scene<'_>, + rows: impl IntoIterator, + types: &OntologySelection, + ) -> Self { + let rows = rows.into_iter(); + let mut bits = BitMatrix::new(rows.len(), types.len()); + let memberships = types.resolve(&world.ontology); + + for (slot, row) in rows.enumerate() { + let Some(position) = world.layout.index.reverse(epoch, row) else { + continue; + }; + let slot = I::from_usize(slot); + for (selection, membership) in memberships.iter_enumerated() { + if membership.contains(position) { + bits.insert(slot, selection); + } + } + } + + Self { bits } + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/mod.rs b/libs/@local/graph/atlas/src/serve/document/mod.rs new file mode 100644 index 00000000000..7e7030ff0ec --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/mod.rs @@ -0,0 +1,77 @@ +//! The response body each route answers with, and its encoding into caller-supplied bytes. +//! +//! [`Document::encode`] is the contract every route's body shares: it replaces a caller-owned +//! buffer with the encoded body and returns the [`Envelope`] naming the media type it wrote. +//! Preparing a document and encoding it are separate steps, and they fail differently. A +//! request-shaped refusal, a gap in the captured data and a failed store read all surface during +//! preparation, before encoding touches the buffer. +//! +//! Preparation is scene-backed for five of the six. [`ManifestDocument`], [`TileDocument`], +//! [`EdgesDocument`], [`LocateDocument`] and [`TranslateDocument`] gather what their response +//! delivers from a captured [`Scene`](crate::serve::scene::Scene), meaning its bound world and +//! delivery schedule. [`CurrentDocument`] takes a generation id alone and reads no scene. +//! +//! A returned encoding error comes from a serde serializer: [`CurrentDocument`], +//! [`ManifestDocument`] and [`TranslateDocument`] return theirs, and a serialization that fails +//! part-way leaves its partial output in the buffer. The binary documents declare +//! `type Error = !`, which says they return no error rather than that they cannot fail. They fail +//! by panic instead. The envelope directory addresses recorded slots with u32 offsets. A body +//! recording a slot offset at or past 4 GiB therefore panics rather than record a wrapped one. +//! [`codec`] holds the envelope, column and CBOR writers they share. + +use alloc::alloc::Allocator; + +use self::codec::Envelope; + +mod codec; +mod current; +mod edges; +mod limits; +mod locate; +mod manifest; +mod masks; +mod tile; +mod translate; + +#[cfg(test)] +pub(crate) use self::tile::TileSlot; +pub(crate) use self::{ + codec::Mode, + current::CurrentDocument, + edges::{ + EdgesDocument, EdgesDocumentDetailLevel, EdgesDocumentError, EdgesDocumentOptions, + EdgesLimits, + }, + limits::DocumentLimits, + locate::{ + LocateDocument, LocateDocumentError, LocateDocumentOptions, LocateLimits, LocateSource, + }, + manifest::ManifestDocument, + tile::{ + TileDocument, TileDocumentDetailLevel, TileDocumentError, TileDocumentOptions, TileLimits, + }, + translate::{TranslateDocument, TranslateDocumentError, TranslateLimits}, +}; + +/// The fitted-variant names this generation ever serves. +pub(crate) const VARIANTS: [&str; 1] = ["plain"]; + +/// A response body that encodes itself into a caller-supplied buffer. +pub(crate) trait Document { + /// The failure [`encode`](Self::encode) can report. + type Error; + + /// Replaces `buffer` and returns its writer's completion token. + /// + /// # Errors + /// + /// Returns the implementation's encoding error. The buffer may contain a partial document on + /// failure. + /// + /// # Panics + /// + /// A binary implementation panics where the encoded slots reach the envelope directory's u32 + /// offset bound. No request-shaped refusal covers that case: the size follows the data the + /// document gathered rather than any field of the request. + fn encode(&self, buffer: &mut Vec) -> Result; +} diff --git a/libs/@local/graph/atlas/src/serve/document/tile/codec/mod.rs b/libs/@local/graph/atlas/src/serve/document/tile/codec/mod.rs new file mode 100644 index 00000000000..90973a679d1 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/tile/codec/mod.rs @@ -0,0 +1,127 @@ +#[cfg(test)] +mod tests; + +use alloc::alloc::Allocator; + +use zerocopy::IntoBytes as _; + +use super::{GlobalHead, TileDocument, TileTrailer}; +use crate::serve::document::codec::{ + CborWriter, ColumnWriter, Envelope, EnvelopeWriter, Kind, encode_details, +}; + +/// One tile response in writable form. +pub(crate) struct TileResponse<'doc> { + pub variant: u64, + pub document: &'doc TileDocument<'doc>, +} + +impl TileResponse<'_> { + /// Encodes the response as one `SALTILET` envelope. + /// + /// # Panics + /// + /// This panics when a directory offset exceeds `u32::MAX`. + pub(crate) fn encode_into(&self, buffer: &mut Vec) -> Envelope { + let mut envelope = EnvelopeWriter::new(Kind::TILE, 5, buffer); + + envelope.slot(|bytes| self.encode_head(bytes)); + envelope.slot(|bytes| ColumnWriter::over(bytes).positions(&self.document.positions)); + envelope.slot(|bytes| ColumnWriter::over(bytes).rows(&self.document.ids)); + + match &self.document.type_masks { + Some(masks) => envelope.slot(|bytes| ColumnWriter::over(bytes).masks(&masks.bits)), + None => envelope.skip(), + } + + // Slot 4 holds the reserved `MASS` column. A wireVersion-1 server marks it absent, which + // the directory's initial (0, 0) entry already records. + envelope.skip(); + match &self.document.trailer { + Some(trailer) => { + envelope.finish_with_trailer(|bytes| Self::encode_trailer(bytes, trailer)) + } + None => envelope.finish(), + } + } + + /// Encodes the `HEAD` map. + /// + /// The map runs from key 0 through key 10. It reserves key 5, and carries key 8 only for a + /// global-view document. + fn encode_head(&self, bytes: &mut Vec) { + let document = self.document; + let mut cbor = CborWriter::over(bytes); + cbor.map(9 + u64::from(document.global.is_some())); + + cbor.uint(0); + cbor.bytes(document.generation.as_bytes()); + + cbor.uint(1); + cbor.uint(self.variant); + + cbor.uint(2); + cbor.array(3); + cbor.uint(u64::from(document.coordinate.z.get())); + cbor.uint(u64::from(document.coordinate.x)); + cbor.uint(u64::from(document.coordinate.y)); + + cbor.uint(3); + cbor.uint(document.mode.code()); + + cbor.uint(4); + cbor.uint(document.ids.len() as u64); + + cbor.uint(6); + cbor.uint(u64::from(document.first_bucket.get())); + + cbor.uint(7); + cbor.array(document.runs.len() as u64); + for &count in &document.runs { + cbor.uint(count as u64); + } + + if let Some(global) = &document.global { + cbor.uint(8); + Self::encode_global(&mut cbor, global); + } + + cbor.uint(9); + cbor.uint(u64::from(document.children)); + + cbor.uint(10); + cbor.boolean(document.trailer.is_some()); + } + + /// Encodes the global-view head: visible count, optional bounds and minimum resolution. + fn encode_global(cbor: &mut CborWriter<'_, A>, global: &GlobalHead) { + cbor.map(2 + u64::from(global.bounds.is_some())); + + cbor.uint(0); + cbor.uint(global.visible); + + if let Some(bounds) = &global.bounds { + cbor.uint(1); + cbor.array(4); + cbor.f32(bounds.min().x()); + cbor.f32(bounds.min().y()); + cbor.f32(bounds.max().x()); + cbor.f32(bounds.max().y()); + } + + cbor.uint(2); + cbor.uint(global.min_resolution); + } + + /// Encodes the tile trailer map: per-point labels and icons. + fn encode_trailer(bytes: &mut Vec, trailer: &TileTrailer<'_>) { + let mut cbor = CborWriter::over(bytes); + cbor.map(2); + + cbor.uint(0); + encode_details(&mut cbor, trailer.labels.iter()); + + cbor.uint(1); + encode_details(&mut cbor, trailer.icons.iter()); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/tile/codec/tests.rs b/libs/@local/graph/atlas/src/serve/document/tile/codec/tests.rs new file mode 100644 index 00000000000..7269be57f43 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/tile/codec/tests.rs @@ -0,0 +1,695 @@ +#![expect( + clippy::little_endian_bytes, + reason = "the tests read the envelope directory and build its little-endian columns" +)] +#![expect( + clippy::big_endian_bytes, + reason = "CBOR arguments are network byte order (RFC 8949 section 3)" +)] + +use alloc::alloc::Global; + +use hashql_core::id::{IdVec, bit_vec::BitMatrix}; + +use super::{ + super::{GlobalHead, TileDocument, TileSlot, TileTrailer}, + TileResponse, +}; +use crate::{ + dataset::auxiliary::{Icon, Label}, + file::generation::GenerationId, + identity::NodeRowId, + integrity::Sha256Digest, + math::{Bounds2, Vec2}, + morton::{Depth, MortonTile}, + serve::{ + codec::EncodedRowId, + document::{Document as _, Mode, masks::TypeMasks}, + membership::SelectionSlot, + }, +}; + +/// The envelope prefix width. +const PREFIX: usize = 16; +/// The width of one directory entry. +const ENTRY: usize = 8; +/// The slot count of a `SALTILET` envelope. +const SLOTS: usize = 5; +/// The directory index of the mask column. +const MASK_SLOT: usize = 3; +/// The directory index of the reserved `MASS` slot. +const MASS_SLOT: usize = 4; + +/// The one-byte CBOR encoding of `true`. +const CBOR_TRUE: u8 = 0xF5; +/// The one-byte CBOR encoding of `false`. +const CBOR_FALSE: u8 = 0xF4; +/// The one-byte CBOR encoding of `null`. +const CBOR_NULL: u8 = 0xF6; +/// The head of a single-precision float. +const CBOR_HEAD_F32: u8 = 0xFA; + +/// Encodes one CBOR head in shortest form (RFC 8949 section 3). +/// +/// # Panics +/// +/// Panics above 255. No expectation in this module reaches a two-byte argument. +fn cbor_head(major: u8, argument: u64) -> Vec { + let ty = major << 5; + match argument { + 0..0x18 => vec![ty | u8::try_from(argument).expect("should fit the inline argument")], + 0x18..=0xFF => vec![ + ty | 0x18, + u8::try_from(argument).expect("should fit the one-byte argument"), + ], + _ => panic!("should stay below a two-byte CBOR argument"), + } +} + +/// Encodes `value` as a CBOR unsigned integer. +/// +/// # Panics +/// +/// Panics above 255, the head's limit. +fn cbor_uint(value: u64) -> Vec { + cbor_head(0, value) +} + +/// Encodes `value` as a CBOR byte string. +/// +/// # Panics +/// +/// Panics above 255 bytes, the head's limit. +fn cbor_bytes(value: &[u8]) -> Vec { + let mut bytes = cbor_head(2, value.len() as u64); + bytes.extend_from_slice(value); + bytes +} + +/// Encodes `value` as a CBOR text string. +/// +/// # Panics +/// +/// Panics above 255 bytes of UTF-8, the head's limit. +fn cbor_text(value: &str) -> Vec { + let mut bytes = cbor_head(3, value.len() as u64); + bytes.extend_from_slice(value.as_bytes()); + bytes +} + +/// Encodes `value` as its one-byte CBOR boolean. +fn cbor_bool(value: bool) -> Vec { + vec![if value { CBOR_TRUE } else { CBOR_FALSE }] +} + +/// Encodes a single-precision float: the head, then the bit pattern in network byte order. +fn cbor_f32(value: f32) -> Vec { + let mut bytes = vec![CBOR_HEAD_F32]; + bytes.extend_from_slice(&value.to_bits().to_be_bytes()); + bytes +} + +/// Concatenates one definite-length array from its items. +/// +/// # Panics +/// +/// Panics above 255 items, the head's limit. +fn cbor_array(items: impl IntoIterator>) -> Vec { + let items: Vec<_> = items.into_iter().collect(); + let mut bytes = cbor_head(4, items.len() as u64); + for item in items { + bytes.extend(item); + } + bytes +} + +/// Concatenates one definite-length map from its unsigned keys and encoded values. +/// +/// # Panics +/// +/// Panics above 255 pairs, and above a key of 255: both go through the one-byte head. +fn cbor_map(pairs: impl IntoIterator)>) -> Vec { + let pairs: Vec<_> = pairs.into_iter().collect(); + let mut bytes = cbor_head(5, pairs.len() as u64); + for (key, value) in pairs { + bytes.extend(cbor_uint(key)); + bytes.extend(value); + } + bytes +} + +/// Reads one directory entry, never a slot's payload. +/// +/// # Panics +/// +/// Panics if the computed entry slices lie outside `buffer`. With overflow checking, computing +/// `PREFIX + ENTRY * slot` or either slice's end can also panic on usize overflow. +/// +/// # Warning +/// +/// Without overflow checking, overflowing offset arithmetic wraps. If the resulting slices lie +/// within `buffer`, the helper reads those bytes even when the mathematical directory offset lies +/// beyond the buffer. +fn slot_range(buffer: &[u8], slot: usize) -> (usize, usize) { + let at = PREFIX + ENTRY * slot; + let start = u32::from_le_bytes( + buffer[at..at + 4] + .try_into() + .expect("should contain a start offset"), + ); + let end = u32::from_le_bytes( + buffer[at + 4..at + 8] + .try_into() + .expect("should contain an end offset"), + ); + (start as usize, end as usize) +} + +/// Reads the payload bytes of `slot`, through its directory entry. +/// +/// # Panics +/// +/// Panics under [`slot_range`]'s conditions for the entry, its computed offset included, and where +/// the extent the entry records is not a range within `buffer`. An omitted column's `(0, 0)` entry +/// records a readable empty range, and yields an empty slice. +fn slot_bytes(buffer: &[u8], slot: usize) -> &[u8] { + let (start, end) = slot_range(buffer, slot); + &buffer[start..end] +} + +/// Returns the bytes the writer appended past the last written slot's padding. +/// +/// The writer omits the mask column whenever a request selects no types, and it never writes the +/// reserved `MASS` slot. Either way the final directory entry stays zero, and the trailer follows +/// the last slot the writer filled, at that slot's recorded end rounded up to the next multiple of +/// eight. +/// +/// # Panics +/// +/// Panics where a directory entry is unreadable, where the directory records no slot at all, +/// since `HEAD` is always present, and where the padded end of the last recorded slot lies past +/// `buffer`. Rounding the recorded `u32` end to a multiple of eight uses `usize`. On a 32-bit +/// target with overflow checking, an end greater than `u32::MAX - 7` panics during rounding. +/// +/// # Warning +/// +/// Without overflow checking on a 32-bit target, an overflowing rounded end becomes zero and this +/// returns the entire buffer. +fn trailer_bytes(buffer: &[u8]) -> &[u8] { + let (_, end) = (0..SLOTS) + .rev() + .map(|slot| slot_range(buffer, slot)) + .find(|&extent| extent != (0, 0)) + .expect("should find the always-present HEAD slot"); + &buffer[end.next_multiple_of(8)..] +} + +/// Asserts that `needle` occurs somewhere in `haystack`. +/// +/// # Panics +/// +/// Panics with `message` where it does not, and on an empty `needle`, which is not a window +/// width. +#[track_caller] +fn assert_contains(haystack: &[u8], needle: &[u8], message: &str) { + assert!( + haystack + .windows(needle.len()) + .any(|window| window == needle), + "{message}" + ); +} + +/// Builds a generation identity from a repeated byte, beside the digest bytes it echoes. +fn generation_of(byte: u8) -> ([u8; size_of::()], GenerationId) { + let bytes = [byte; size_of::()]; + ( + bytes, + GenerationId::from_digest(Sha256Digest::from_bytes_unchecked(bytes)), + ) +} + +/// Builds a [`MortonTile`] at zoom `z` and coordinates `(x, y)`. +/// +/// # Panics +/// +/// Panics where `z` exceeds [`Depth::MAX`], the bound its depth carries. +fn tile(z: u8, x: u32, y: u32) -> MortonTile { + MortonTile { + z: Depth::new(z), + x, + y, + } +} + +/// Assembles one `SALTILET` envelope from its parts, independent of [`EnvelopeWriter`]. +/// +/// [`EnvelopeWriter`]: crate::serve::document::codec::EnvelopeWriter +/// +/// Padding to the next eight-byte boundary follows each present slot's payload. A `None` slot +/// keeps the directory's initial zero entry: what the omitted mask column and the reserved `MASS` +/// slot leave behind. +/// +/// # Panics +/// +/// Panics above a u16 slot count, and where a payload offset does not fit u32. +fn build_envelope(slots: &[Option>], trailer: Option<&[u8]>) -> Vec { + let mut buffer = Vec::new(); + buffer.extend_from_slice(b"SALTILET"); + buffer.extend_from_slice(&1_u16.to_le_bytes()); + buffer.extend_from_slice(&0_u16.to_le_bytes()); + buffer.extend_from_slice( + &u16::try_from(slots.len()) + .expect("should fit the fixture's slot count") + .to_le_bytes(), + ); + buffer.extend_from_slice(&0_u16.to_le_bytes()); + + let mut directory = vec![0_u8; slots.len() * ENTRY]; + buffer.resize(PREFIX + directory.len(), 0); + + for (index, slot) in slots.iter().enumerate() { + let Some(payload) = slot else { continue }; + + let start = buffer.len(); + buffer.extend_from_slice(payload); + let end = buffer.len(); + buffer.resize(end.next_multiple_of(8), 0); + + let at = index * ENTRY; + directory[at..at + 4].copy_from_slice( + &u32::try_from(start) + .expect("should fit the fixture's offsets") + .to_le_bytes(), + ); + directory[at + 4..at + ENTRY].copy_from_slice( + &u32::try_from(end) + .expect("should fit the fixture's offsets") + .to_le_bytes(), + ); + } + + buffer[PREFIX..PREFIX + directory.len()].copy_from_slice(&directory); + if let Some(trailer) = trailer { + buffer.extend_from_slice(trailer); + } + buffer +} + +/// Builds a nonroot delta tile with an empty row set and minimal detail. +fn minimal_document(generation: GenerationId) -> TileDocument<'static> { + TileDocument { + generation, + coordinate: tile(2, 1, 3), + mode: Mode::Delta, + first_bucket: Depth::new(2), + runs: Vec::new(), + positions: IdVec::::from_raw(Vec::new()), + ids: IdVec::>::from_raw(Vec::new()), + type_masks: None, + global: None, + children: 0, + trailer: None, + } +} + +/// Builds the nine-key `HEAD` map of [`minimal_document`], over `rows` delivered rows. +/// +/// Global metadata adds key 8 between keys 7 and 9, which the tests that carry it build for +/// themselves. +/// +/// # Panics +/// +/// Panics on a `digest` above 255 bytes, or a `variant` or `rows` above 255: each is a one-byte +/// CBOR argument here. +fn expected_minimal_head(digest: &[u8], variant: u64, rows: u64, trailer: bool) -> Vec { + cbor_map([ + (0, cbor_bytes(digest)), + (1, cbor_uint(variant)), + (2, cbor_array([cbor_uint(2), cbor_uint(1), cbor_uint(3)])), + (3, cbor_uint(0)), + (4, cbor_uint(rows)), + (6, cbor_uint(2)), + (7, cbor_array([])), + (9, cbor_uint(0)), + (10, cbor_bool(trailer)), + ]) +} + +/// The minimal document replaces a longer prepopulated buffer with the whole envelope. +/// +/// The one case compared byte for byte. Every other test decodes the directory and checks one +/// region. +#[test] +fn response_encode_minimal_shape() { + let (digest, generation) = generation_of(0x10); + let document = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + buffer.resize(512, 0xDD); + let _completed = document + .encode(&mut buffer) + .expect("should complete the minimal tile response"); + + let expected = build_envelope( + &[ + Some(expected_minimal_head(&digest, 0, 0, false)), + Some(Vec::new()), + Some(Vec::new()), + None, + None, + ], + None, + ); + + assert_eq!( + buffer.as_slice(), + expected.as_slice(), + "should replace the buffer with HEAD, the two geometry slots and two skipped entries" + ); +} + +/// The response echoes its own `variant` field rather than a fixed value. +/// +/// [`Document::encode`](crate::serve::document::Document::encode) always passes zero. A nonzero +/// variant needs the response built directly. +#[test] +fn response_variant_nonzero() { + let (digest, generation) = generation_of(0x11); + let document = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + let _completed = TileResponse { + variant: 5, + document: &document, + } + .encode_into(&mut buffer); + + assert_eq!( + slot_bytes(&buffer, 0), + expected_minimal_head(&digest, 5, 0, false).as_slice(), + "should echo variant 5 at head key 1 rather than a fixed value" + ); +} + +#[test] +fn response_geometry_columns() { + let (digest, generation) = generation_of(0x19); + let document = TileDocument { + positions: IdVec::from_raw(vec![Vec2::new(2.0, -4.0), Vec2::new(0.5, 0.0)]), + ids: IdVec::from_raw(vec![ + EncodedRowId::new_unchecked(0x0102_0304), + EncodedRowId::new_unchecked(7), + ]), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the tile response"); + + let mut expected_positions = Vec::new(); + for value in [2.0_f32, -4.0, 0.5, 0.0] { + expected_positions.extend_from_slice(&value.to_bits().to_le_bytes()); + } + + assert_eq!( + slot_bytes(&buffer, 0), + expected_minimal_head(&digest, 0, 2, false).as_slice(), + "should report two delivered rows at head key 4" + ); + assert_eq!( + slot_bytes(&buffer, 1), + expected_positions.as_slice(), + "should encode each position's x then y in slot order" + ); + assert_eq!( + slot_bytes(&buffer, 2), + [0x04, 0x03, 0x02, 0x01, 0x07, 0x00, 0x00, 0x00], + "should encode each row id little-endian in slot order" + ); +} + +/// An omitted mask column and a present-but-empty one differ in the directory alone. +/// +/// A skipped slot never records an entry, and its extent stays the envelope's initial `(0, 0)`. +/// A present-but-empty column records `(start, start)` at whatever offset the earlier slots +/// reached, which is past the directory and never zero. +#[test] +fn response_mask_omitted_versus_empty() { + let (_, generation) = generation_of(0x12); + let omitted = minimal_document(generation); + + let mut buffer = Vec::new_in(&Global); + let _completed = omitted + .encode(&mut buffer) + .expect("should complete the response without a mask column"); + assert_eq!( + slot_range(&buffer, MASK_SLOT), + (0, 0), + "an omitted mask column should leave the directory entry at its initial (0, 0)" + ); + + let (_, generation) = generation_of(0x13); + let present = TileDocument { + type_masks: Some(TypeMasks { + bits: BitMatrix::::new(0, 3), + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = present + .encode(&mut buffer) + .expect("should complete the response with an empty mask column"); + let (start, end) = slot_range(&buffer, MASK_SLOT); + assert_eq!( + start, end, + "a present-but-empty mask column should record a zero-length extent" + ); + assert!( + start >= PREFIX + ENTRY * SLOTS, + "the present-but-empty column should sit past the directory rather than at offset zero" + ); +} + +/// Slot 4 stays reserved: the writer skips `MASS` even in a response that fills every other slot. +#[test] +fn response_mass_reserved() { + let (_, generation) = generation_of(0x18); + let document = TileDocument { + positions: IdVec::from_raw(vec![Vec2::new(1.0, 2.0)]), + ids: IdVec::from_raw(vec![EncodedRowId::new_unchecked(7)]), + type_masks: Some(TypeMasks { + bits: BitMatrix::::new(1, 3), + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the response with a mask column"); + + for slot in 0..MASS_SLOT { + assert_ne!( + slot_range(&buffer, slot), + (0, 0), + "slot {slot} should record an extent" + ); + } + assert_eq!( + slot_range(&buffer, MASS_SLOT), + (0, 0), + "the reserved `MASS` slot should stay at its initial (0, 0)" + ); +} + +/// Root global metadata carries the schedule's bounds under key 1 and lifts `HEAD` to ten keys. +#[test] +fn response_global_with_bounds() { + let (_, generation) = generation_of(0x14); + let min = Vec2::new(-1.5, 2.0); + let max = Vec2::new(3.5, 6.0); + let document = TileDocument { + global: Some(GlobalHead { + visible: 7, + bounds: Some(Bounds2::new(min, max).expect("should accept ordered finite corners")), + min_resolution: 4, + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the root tile response"); + + let mut expected = cbor_uint(8); + expected.extend(cbor_map([ + (0, cbor_uint(7)), + ( + 1, + cbor_array([cbor_f32(-1.5), cbor_f32(2.0), cbor_f32(3.5), cbor_f32(6.0)]), + ), + (2, cbor_uint(4)), + ])); + + let head = slot_bytes(&buffer, 0); + assert_contains( + head, + &expected, + "should carry a three-key global map with the bounds array at head key 8", + ); + assert_eq!( + head[0], + cbor_head(5, 10)[0], + "should declare a ten-key HEAD map when global metadata is present" + ); +} + +/// Without bounds the global map keeps its two-key form: key 1 is absent rather than null. +#[test] +fn response_global_without_bounds() { + let (_, generation) = generation_of(0x15); + let document = TileDocument { + global: Some(GlobalHead { + visible: 0, + bounds: None, + min_resolution: 0, + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the bound-free root tile response"); + + let mut expected = cbor_uint(8); + expected.extend(cbor_map([(0, cbor_uint(0)), (2, cbor_uint(0))])); + + let head = slot_bytes(&buffer, 0); + assert_contains( + head, + &expected, + "should carry a two-key global map with no bounds key rather than a null placeholder", + ); + assert!( + !head.contains(&CBOR_NULL), + "should place no null anywhere in the head" + ); + assert_eq!( + head[0], + cbor_head(5, 10)[0], + "should declare a ten-key HEAD map when global metadata is present" + ); +} + +/// The run partition, the mode code and the children bitmask encode as literals. +/// +/// The expected mode codes are the literals 0 and 1 rather than a call to [`Mode::code`]. +#[test] +fn response_mode_runs_children() { + for (mode, code) in [(Mode::Delta, 0_u64), (Mode::Total, 1_u64)] { + let (digest, generation) = generation_of(0x16); + let document = TileDocument { + mode, + runs: vec![2, 0, 5], + children: 0b1011, + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the tile response"); + + let expected = cbor_map([ + (0, cbor_bytes(&digest)), + (1, cbor_uint(0)), + (2, cbor_array([cbor_uint(2), cbor_uint(1), cbor_uint(3)])), + (3, cbor_uint(code)), + (4, cbor_uint(0)), + (6, cbor_uint(2)), + (7, cbor_array([cbor_uint(2), cbor_uint(0), cbor_uint(5)])), + (9, cbor_uint(0b1011)), + (10, cbor_bool(false)), + ]); + + assert_eq!( + slot_bytes(&buffer, 0), + expected.as_slice(), + "mode {mode:?} should encode wire code {code}, the run partition [2, 0, 5] and the \ + children bitmask 0b1011" + ); + } +} + +/// The trailer's label and icon arrays null their entries independently. +/// +/// The fixture's three rows leave the row column ending off an eight-byte boundary, which puts +/// the trailer's start past both the padding and the two skipped trailing slots. +#[test] +fn response_trailer_labels_icons() { + let (digest, generation) = generation_of(0x17); + let document = TileDocument { + positions: IdVec::from_raw(vec![ + Vec2::new(1.0, 2.0), + Vec2::new(3.0, 4.0), + Vec2::new(5.0, 6.0), + ]), + ids: IdVec::from_raw(vec![ + EncodedRowId::new_unchecked(11), + EncodedRowId::new_unchecked(12), + EncodedRowId::new_unchecked(13), + ]), + trailer: Some(TileTrailer { + labels: IdVec::::from_raw(vec![ + Label::new("north"), + Label::EMPTY, + Label::new("south"), + ]), + icons: IdVec::::from_raw(vec![ + Icon::empty(), + Icon::new("marker"), + Icon::empty(), + ]), + }), + ..minimal_document(generation) + }; + + let mut buffer = Vec::new_in(&Global); + let _completed = document + .encode(&mut buffer) + .expect("should complete the auxiliary tile response"); + + let expected = cbor_map([ + ( + 0, + cbor_array([cbor_text("north"), vec![CBOR_NULL], cbor_text("south")]), + ), + ( + 1, + cbor_array([vec![CBOR_NULL], cbor_text("marker"), vec![CBOR_NULL]]), + ), + ]); + + let (_, rows_end) = slot_range(&buffer, 2); + assert_ne!( + rows_end & 7, + 0, + "the fixture's row column should end off an eight-byte boundary" + ); + assert_eq!( + trailer_bytes(&buffer), + expected.as_slice(), + "should pair each slot's label and icon independently, nulling only the empty one" + ); + assert_eq!( + slot_bytes(&buffer, 0), + expected_minimal_head(&digest, 0, 3, true).as_slice(), + "should report the present trailer at head key 10" + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/tile/mod.rs b/libs/@local/graph/atlas/src/serve/document/tile/mod.rs new file mode 100644 index 00000000000..d4bf311816e --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/tile/mod.rs @@ -0,0 +1,264 @@ +//! Aligned tile columns over a captured scene. +//! +//! Construction gathers delivery rows and optional display payloads before serialization. + +mod codec; +#[cfg(test)] +mod tests; + +use alloc::alloc::Allocator; +use core::{error::Error, fmt}; + +use error_stack::Report; +use hashql_core::id::{Id as _, IdVec}; + +use super::{ + Document, + codec::{Envelope, Mode}, + masks::TypeMasks, +}; +use crate::{ + dataset::auxiliary::{Icon, Label}, + file::generation::GenerationId, + identity::NodeRowId, + math::{Bounds2, Log2, Vec2}, + morton::{Depth, MortonCell, MortonTile, Zoom}, + serve::{codec::EncodedRowId, membership::OntologySelection, scene::Scene, walk::Walk}, +}; + +hashql_core::id::newtype! { + /// A reference to one delivered point by its slot in this document's tile order. + pub(crate) struct TileSlot(u32) +} + +/// Caps on one tile request: the requested types it may carry. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TileLimits { + /// Most requested types, including duplicates. The default is 32. + pub colored_type_ids: u32 = 32, +} + +/// Whether a tile document carries only its columns or also the auxiliary trailer. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum TileDocumentDetailLevel { + /// Columns alone: position, row id and optional type mask per point. + Minimal, + /// Columns plus the trailer: label and icon per point. + Auxiliary, +} + +/// Delivery mode, detail level, requested types and limits for one tile request. +pub(crate) struct TileDocumentOptions<'selection> { + pub mode: Mode, + pub detail: TileDocumentDetailLevel, + /// Mask bits in request order, including duplicate types. + pub types: &'selection OntologySelection, + pub limits: TileLimits, +} + +/// A failure to assemble a tile document. +#[derive(Debug)] +pub(crate) enum TileDocumentError { + /// The request exceeds the configured type-count limit. + Types { count: usize, maximum: u32 }, + /// The zoom exceeds the generation's deepest served tile. + Zoom { zoom: Zoom, maximum: Zoom }, + /// The coordinate lies outside its zoom's grid. + Coordinate { tile: MortonTile }, + /// A delivered row has no position in the captured scene. + Position { row: NodeRowId }, + /// A delivered row has no display payload in the captured scene. + Display { row: NodeRowId }, +} + +impl fmt::Display for TileDocumentError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Types { count, maximum } => write!( + fmt, + "the request lists {count} colored type ids, exceeding the limit of {maximum}" + ), + Self::Zoom { zoom, maximum } => write!( + fmt, + "tile zoom {zoom} exceeds the maximum served zoom {maximum}" + ), + Self::Coordinate { + tile: MortonTile { z, x, y }, + } => { + write!(fmt, "tile {}/{x}/{y} lies outside its zoom's grid", z.get()) + } + Self::Position { row } => write!(fmt, "delivered node {row} has no position"), + Self::Display { row } => write!(fmt, "delivered node {row} has no display payload"), + } + } +} + +impl Error for TileDocumentError {} + +/// The auxiliary detail for one tile document: each delivered point's label and icon. +pub(crate) struct TileTrailer<'details> { + labels: IdVec, + icons: IdVec, +} + +impl<'details> TileTrailer<'details> { + /// Builds an empty trailer with room for `capacity` points. + fn new(capacity: usize) -> Self { + Self { + labels: IdVec::with_capacity(capacity), + icons: IdVec::with_capacity(capacity), + } + } + + /// Appends `row`'s label and icon from the captured scene. + /// + /// # Errors + /// + /// Returns [`TileDocumentError::Display`] when `row` has no display payload in the captured + /// scene. + fn push( + &mut self, + Scene { world, epoch, .. }: Scene<'details>, + row: NodeRowId, + ) -> Result<(), Report> { + let legend = world + .layout + .index + .payload(epoch, row) + .ok_or_else(|| Report::new(TileDocumentError::Display { row }))?; + self.labels.push(legend.label()); + self.icons.push( + world + .ontology + .icon(epoch, legend.representative_ontology()) + .unwrap_or(Icon::empty()), + ); + Ok(()) + } +} + +/// Metadata of the entire post-intersection visible set. +struct GlobalHead { + visible: u64, + bounds: Option, + min_resolution: u64, +} + +/// One tile's geometry and optional details in bucket-major delivery order. +pub(crate) struct TileDocument<'details> { + generation: GenerationId, + coordinate: MortonTile, + mode: Mode, + first_bucket: Depth, + runs: Vec, + positions: IdVec, + ids: IdVec>, + type_masks: Option>, + global: Option, + children: u8, + trailer: Option>, +} + +impl<'details> TileDocument<'details> { + /// Gathers one tile from the scene's delivery schedule. + /// + /// Type masks use the generation's captured ontology memberships. Unknown types and rows + /// outside the fitted position domain have zero bits. An empty type selection omits the mask + /// column. A nonempty selection over an empty tile keeps an empty column. + /// + /// Auxiliary detail borrows each row's label and resolves the representative type's icon + /// through the captured ontology. Missing icons use an empty value. + /// + /// # Errors + /// + /// Returns [`TileDocumentError`] for invalid request bounds or a delivered row without a + /// position or display payload. + pub(crate) fn new( + scene @ Scene { + world, + epoch, + delivery, + schedule, + .. + }: Scene<'details>, + coordinate: MortonTile, + TileDocumentOptions { + mode, + detail, + types, + limits, + }: &TileDocumentOptions<'_>, + ) -> Result> { + if types.len() > limits.colored_type_ids as usize { + return Err(Report::new(TileDocumentError::Types { + count: types.len(), + maximum: limits.colored_type_ids, + })); + } + let zoom = coordinate.z.zoom(Log2::ZERO); + let maximum = world.schedule().max_tile_depth(); + if zoom > maximum { + return Err(Report::new(TileDocumentError::Zoom { zoom, maximum })); + } + let cell = MortonCell::from_tile(coordinate) + .ok_or_else(|| Report::new(TileDocumentError::Coordinate { tile: coordinate }))?; + let walk = Walk { + schedule: delivery, + index: &world.layout.index, + }; + let delivered = match mode { + Mode::Delta => walk.delta(epoch, zoom, cell), + Mode::Total => walk.total(epoch, zoom, cell), + }; + let count = delivered.rows.len(); + let type_masks = (!types.is_empty()) + .then(|| TypeMasks::new(scene, delivered.rows.iter().copied(), types)); + let global = (coordinate.z == Depth::MIN).then(|| GlobalHead { + visible: delivery.root_delivered() as u64, + bounds: schedule.bounds(), + min_resolution: u64::from(delivery.min_resolution().get()), + }); + let trailer = match detail { + TileDocumentDetailLevel::Minimal => None, + TileDocumentDetailLevel::Auxiliary => Some(TileTrailer::new(count)), + }; + let mut this = Self { + generation: world.generation().id(), + coordinate, + mode: *mode, + first_bucket: delivered.first_bucket, + runs: delivered.runs, + positions: IdVec::with_capacity(count), + ids: IdVec::with_capacity(count), + type_masks, + global, + children: delivery.children(zoom, cell), + trailer, + }; + for row in delivered.rows { + let position = world + .layout + .position(epoch, row) + .ok_or_else(|| Report::new(TileDocumentError::Position { row }))?; + this.positions.push(position); + this.ids.push(world.layout.index.encode(row)); + if let Some(trailer) = &mut this.trailer { + trailer.push(scene, row)?; + } + } + Ok(this) + } +} + +impl Document for TileDocument<'_> { + type Error = !; + + fn encode(&self, buffer: &mut Vec) -> Result { + Ok(self::codec::TileResponse { + variant: 0, + document: self, + } + .encode_into(buffer)) + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/tile/tests.rs b/libs/@local/graph/atlas/src/serve/document/tile/tests.rs new file mode 100644 index 00000000000..487e6cbfb2b --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/tile/tests.rs @@ -0,0 +1,1028 @@ +use alloc::sync::Arc; +use core::assert_matches; +use std::io; + +use arc_swap::Guard; +use camino::Utf8Path; +use error_stack::Report; +use hashql_core::id::Id as _; +use rand::{SeedableRng as _, rngs::StdRng}; +use type_system::principal::actor::{ActorId, ActorType}; +use uuid::Uuid; + +use super::{ + super::masks::TypeMasks, TileDocument, TileDocumentDetailLevel, TileDocumentError, + TileDocumentOptions, TileLimits, TileSlot, +}; +use crate::{ + api::problem::{Problem, tests::assert_internal_diagnostic}, + bitset::CompressedBitSet, + dataset::auxiliary::{Label, OwnedLegend}, + file::{generation::Generation, morton::read::MortonFile}, + identity::{BasePosition, Column, NodeRowId, OntologyRowId}, + math::Bounds2, + morton::{Depth, MortonCell, MortonKey, MortonTile, Zoom}, + postgres::id::{ArchivedEntityId, ArchivedOntologyTypeUuid}, + salt::{ + fit::prepare::identity::IdentityTable, + lod::{key, stage::WIRE_FRAME}, + }, + serve::{ + delta::{Delta, epoch::Epoch}, + document::codec::Mode, + membership::{OntologySelection, SelectionSlot}, + scene::Scene, + schedule::ViewSchedule, + tests::fixture::{NODES, TamperFixture, secret}, + visibility::{VisibilityActor, VisibilityMask}, + world::World, + }, +}; + +/// An opened synthetic generation, shared by every test below. +/// +/// It carries the captured epoch, actor, visibility mask and delivery schedule that `scene()` +/// assembles into one scene. +struct Fixture { + world: Arc, + epoch: Epoch, + actor: VisibilityActor, + mask: VisibilityMask, + schedule: ViewSchedule, + _files: TamperFixture, +} + +impl Fixture { + /// Opens `generation`, published from `files`, with a fresh delta identity. + /// + /// The mask grants one fixed test actor full visibility. + /// + /// # Panics + /// + /// Panics if [`World::open`] fails to open or validate the serving artifacts. + fn from_generation(files: TamperFixture, generation: Generation) -> Self { + let world = Arc::new( + World::open(generation, &secret()).expect("should open the synthetic generation"), + ); + let delta = Delta::new(Arc::clone(&world), StdRng::seed_from_u64(17)) + .expect("should allocate a delta identity"); + let epoch = Epoch::from(Guard::from_inner(Arc::new(delta))); + let actor = VisibilityActor { + id: ActorId::new(Uuid::from_u128(1), ActorType::User), + instance_admin: false, + }; + let mask = VisibilityMask::full(actor); + let schedule = ViewSchedule::of(Arc::clone(&world), &epoch, &mask); + Self { + world, + epoch, + actor, + mask, + schedule, + _files: files, + } + } + + /// Publishes and opens the named synthetic generation unmodified. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails, and for the conditions + /// [`Self::from_generation`] states. + fn new(name: &str) -> Self { + let files = TamperFixture::publish(name); + let generation = files.generation().clone(); + Self::from_generation(files, generation) + } + + /// Reopens `path` for writing, clearing its read-only staged-artifact permission first. + /// + /// # Panics + /// + /// Panics on any of three filesystem failures against `path`: an unreadable metadata entry, + /// a permission change the process may not make, and a truncating reopen that fails. + #[expect( + clippy::permissions_set_readonly_false, + reason = "the staged copy is this test's own scratch file" + )] + fn recreate_writable(path: &Utf8Path) -> std::fs::File { + let mut permissions = std::fs::metadata(path) + .expect("should read the staged artifact's metadata") + .permissions(); + permissions.set_readonly(false); + std::fs::set_permissions(path, permissions).expect("should set the permissions"); + std::fs::File::create(path).expect("should rewrite the staged artifact") + } + + /// Builds a synthetic identity for node `seed`, derived from the seed byte alone. + fn node_identity(seed: u8) -> ArchivedEntityId { + ArchivedEntityId { + web_id: Uuid::from_bytes([seed; 16]).into(), + entity_uuid: Uuid::from_bytes([seed ^ 0xFF; 16]).into(), + } + } + + /// Assigns distinct labels and both empty and nonempty representative icons. + /// + /// # Panics + /// + /// Panics where publishing the synthetic generation fails and where the rewritten table does + /// not write. It also panics for the conditions [`Self::recreate_writable`] and + /// [`Self::from_generation`] state. + fn varied(name: &str) -> Self { + let files = TamperFixture::publish(name); + let target = files.generation().repository().files.node_identities.name(); + let generation = files.tamper(&target, |path| { + let mut table = IdentityTable::::new(); + for row in 0..NODES { + table.push(Self::node_identity( + u8::try_from(row).expect("fixture row counts fit u8"), + )); + } + let legends: Vec<_> = (0..NODES) + .map(|row| { + let representative = OntologyRowId::new(if row == 0 { 0 } else { 2 }); + OwnedLegend::new(representative, Label::new(&format!("node-{row}"))) + }) + .collect(); + let mut file = Self::recreate_writable(path); + let _digest = table + .write_into(legends.iter().map(AsRef::as_ref), &mut file) + .expect("should write the varied node identities"); + }); + Self::from_generation(files, generation) + } + + /// Narrows this fixture's visibility mask and schedule to exactly the given node rows. + /// + /// No edge remains visible. + fn restrict(&mut self, nodes: impl IntoIterator) { + self.mask = VisibilityMask::partial( + self.actor, + CompressedBitSet::from_rows(nodes.into_iter().map(NodeRowId::new)), + CompressedBitSet::from_rows(core::iter::empty()), + ); + self.schedule = ViewSchedule::of(Arc::clone(&self.world), &self.epoch, &self.mask); + } + + /// Assembles this fixture's world, epoch, mask and schedule into one [`Scene`]. + /// + /// Its delivery is the schedule's cut at the zero offset. + fn scene(&self) -> Scene<'_> { + Scene { + world: &self.world, + epoch: &self.epoch, + mask: &self.mask, + schedule: &self.schedule, + delivery: self + .schedule + .cut(Zoom::MIN) + .expect("should bind the zero offset"), + } + } + + /// Builds a [`MortonTile`] at zoom `z` and coordinates `(x, y)`. + /// + /// # Panics + /// + /// Panics where `z` exceeds [`Depth::MAX`], the bound its depth carries. + fn tile(z: u8, x: u32, y: u32) -> MortonTile { + MortonTile { + z: Depth::new(z), + x, + y, + } + } + + /// Returns the [`MortonKey`] of node row `row`'s captured position. + /// + /// # Panics + /// + /// Panics where the fixture's epoch captured no position for `row`. + fn key_of(&self, row: u64) -> MortonKey { + let position = self + .world + .layout + .position(&self.epoch, NodeRowId::new(row)) + .expect("should resolve the fixture node's position"); + key::keys(&[position], WIRE_FRAME)[0] + } + + /// Returns the ontology type uuid the fixture generation assigned to ontology row `row`. + /// + /// # Panics + /// + /// Panics where that generation holds no ontology row `row`. + fn ontology_uuid(&self, row: u64) -> ArchivedOntologyTypeUuid { + self.world + .ontology + .key_of(&self.epoch, OntologyRowId::new(row)) + .expect("should resolve the fixture ontology row") + } + + /// Returns `row`'s captured display label. + /// + /// # Panics + /// + /// Panics where the fixture captured no display payload for `row`. + fn label_of(&self, row: NodeRowId) -> &Label { + self.world + .layout + .index + .payload(&self.epoch, row) + .expect("should resolve the fixture node's display payload") + .label() + } + + /// Returns a terminal-zoom tile none of the fixture's nodes occupy. + fn empty_terminal_tile(&self) -> MortonTile { + let depth = Depth::from_zoom(self.world.schedule().max_tile_depth()); + let occupied: Vec<_> = (0..NODES).map(|row| self.key_of(row).tile(depth)).collect(); + let mut candidate = MortonTile { + z: depth, + x: 0, + y: 0, + }; + while occupied.contains(&candidate) { + candidate.x += 1; + } + candidate + } + + /// Reads the stored bucket, Morton key and row permutation. + /// + /// # Panics + /// + /// Panics where either published artifact does not open: the Morton key file or the + /// row-of-position column. Both open on every call, after the world itself opened. + fn records(&self) -> Vec<(Depth, MortonKey, NodeRowId)> { + let generation = self.world.generation(); + let files = &generation.repository().files; + let morton: MortonFile = files + .morton + .open(generation) + .expect("should open the recorded keys"); + let rows: Column = files + .row_of_position + .open(generation) + .expect("should open the row permutation"); + + (0..morton.count()) + .map(|index| { + let position = BasePosition::from_u64(index); + ( + morton.bucket_of(position), + morton.code(position), + rows.view()[position], + ) + }) + .collect() + } + + /// Selects stored rows in bucket-major order with counts for each bucket. + /// + /// # Panics + /// + /// Panics under `records`'s conditions, which it takes by opening the stored artifacts on + /// every call. + fn delivered_in( + &self, + first: Depth, + last: Depth, + cell: MortonCell, + ) -> (Vec, Vec) { + let records = self.records(); + let rows = records + .iter() + .filter(|&&(bucket, key, _)| first <= bucket && bucket <= last && cell.contains(key)) + .map(|&(_, _, node)| node) + .collect(); + let runs = (first..=last) + .map(|bucket| { + records + .iter() + .filter(|&&(held, key, _)| held == bucket && cell.contains(key)) + .count() + }) + .collect(); + (rows, runs) + } +} + +/// Refuses a request past the configured type-count limit, under [`TileDocumentError::Types`]. +#[test] +fn types_over_limit() { + let fixture = Fixture::new("tile-types-over-limit"); + let types = [ArchivedOntologyTypeUuid::from(Uuid::nil()); 3]; + let Err(report) = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&types), + limits: TileLimits { + colored_type_ids: 2, + }, + }, + ) else { + panic!("should refuse a type count past the limit"); + }; + assert_matches!( + report.current_context(), + TileDocumentError::Types { + count: 3, + maximum: 2 + }, + ); +} + +/// Admits a request at exactly the configured type-count cap. +#[test] +fn types_limit_boundary() { + let fixture = Fixture::new("tile-types-limit-boundary"); + let types = [ArchivedOntologyTypeUuid::from(Uuid::nil()); 32]; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&types), + limits: TileLimits { .. }, + }, + ) + .expect("should admit exactly the configured type-count cap"); + let masks = document + .type_masks + .expect("should keep a mask column for a nonempty selection"); + assert_eq!( + masks.bits.col_domain_size(), + types.len(), + "should keep one column per requested type at the exact cap" + ); +} + +/// Refuses a zoom past the schedule's deepest served tile, under [`TileDocumentError::Zoom`]. +#[test] +fn zoom_over_maximum() { + let fixture = Fixture::new("tile-zoom-over-maximum"); + let maximum = fixture.world.schedule().max_tile_depth(); + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let coordinate = MortonTile { + z: Depth::new(maximum.get() + 1), + x: 0, + y: 0, + }; + let Err(report) = TileDocument::new( + fixture.scene(), + coordinate, + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) else { + panic!("should refuse a zoom past the schedule maximum"); + }; + assert_matches!( + report.current_context(), + TileDocumentError::Zoom { maximum: actual, .. } if *actual == maximum, + ); +} + +/// Refuses a coordinate outside its own zoom's grid, under [`TileDocumentError::Coordinate`]. +#[test] +fn coordinate_outside_grid() { + let fixture = Fixture::new("tile-coordinate-outside-grid"); + let empty: [ArchivedOntologyTypeUuid; 0] = []; + for coordinate in [ + MortonTile { + z: Depth::new(1), + x: 2, + y: 0, + }, + MortonTile { + z: Depth::new(1), + x: 0, + y: 2, + }, + ] { + let Err(report) = TileDocument::new( + fixture.scene(), + coordinate, + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) else { + panic!("should refuse a coordinate outside its zoom's grid"); + }; + assert_matches!( + report.current_context(), + TileDocumentError::Coordinate { tile: refused } if *refused == coordinate, + ); + } +} + +/// An occupied nonroot tile can contain earlier buckets without adding rows at its own cut. +#[test] +fn mode_delta_empty_total_full() { + let fixture = Fixture::new("tile-mode-delta-empty-total-full"); + let maximum = fixture.world.schedule().max_tile_depth(); + + let (coordinate, cut, delta_runs, total_rows, total_runs) = (0..NODES) + .flat_map(|row| { + let key = fixture.key_of(row); + (1..=maximum.get()).map(move |level| (key, level)) + }) + .find_map(|(key, level)| { + let zoom = Zoom::new(level).expect("should lie within the served zoom range"); + let coordinate = key.tile(Depth::new(level)); + let cell = MortonCell::from_tile(coordinate).expect("should resolve the cell"); + let cut = fixture.world.schedule().cut(zoom); + let (delta_rows, delta_runs) = fixture.delivered_in(cut, cut, cell); + let (total_rows, total_runs) = fixture.delivered_in(Depth::MIN, cut, cell); + (delta_rows.is_empty() && !total_rows.is_empty()) + .then_some((coordinate, cut, delta_runs, total_rows, total_runs)) + }) + .expect( + "the fixture should express a nonroot tile whose delta is empty and whose total is not", + ); + + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let selection = OntologySelection::new(&empty); + let options = |mode| TileDocumentOptions { + mode, + detail: TileDocumentDetailLevel::Minimal, + types: selection, + limits: TileLimits { .. }, + }; + + let delta_document = TileDocument::new(fixture.scene(), coordinate, &options(Mode::Delta)) + .expect("should construct the delta document"); + assert!( + delta_document.ids.is_empty(), + "should carry no rows this tile newly delivers" + ); + assert_eq!(delta_document.generation, fixture.world.generation().id()); + assert_eq!(delta_document.coordinate, coordinate); + assert_eq!(delta_document.mode, Mode::Delta); + assert_eq!(delta_document.positions.len(), delta_document.ids.len()); + assert_eq!(delta_document.first_bucket, cut); + assert_eq!(delta_document.runs, delta_runs); + assert_eq!( + delta_document.runs.iter().sum::(), + delta_document.ids.len(), + "the run partition should sum to the delivered row count" + ); + + let total_document = TileDocument::new(fixture.scene(), coordinate, &options(Mode::Total)) + .expect("should construct the total document"); + let expected_ids: Vec<_> = total_rows + .iter() + .map(|&row| fixture.world.layout.index.encode(row)) + .collect(); + let expected_positions: Vec<_> = total_rows + .iter() + .map(|&row| { + fixture + .world + .layout + .position(&fixture.epoch, row) + .expect("should resolve the fixture node's position") + }) + .collect(); + assert_eq!( + total_document.ids.iter().copied().collect::>(), + expected_ids, + "should list the cumulative rows in bucket-major order" + ); + assert_eq!( + total_document.positions.iter().copied().collect::>(), + expected_positions, + "should list each cumulative row's own position, read independently through World" + ); + assert_eq!(total_document.generation, fixture.world.generation().id()); + assert_eq!(total_document.coordinate, coordinate); + assert_eq!(total_document.mode, Mode::Total); + assert_eq!(total_document.positions.len(), total_document.ids.len()); + assert_eq!(total_document.first_bucket, Depth::MIN); + assert_eq!(total_document.runs, total_runs); + assert_eq!( + total_document.runs.iter().sum::(), + total_rows.len(), + "the run partition should sum to the delivered row count" + ); +} + +/// Root metadata describes the admitted fixture rows. +#[test] +fn global_root_metadata() { + let fixture = Fixture::new("tile-global-root-metadata"); + let records = fixture.records(); + let cut = fixture.world.schedule().cut(Zoom::MIN); + let visible = records + .iter() + .filter(|&&(bucket, ..)| bucket <= cut) + .count(); + let deepest_occupied = records + .iter() + .map(|&(bucket, ..)| bucket) + .max() + .expect("should contain base rows"); + + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct the root tile"); + + let global = document + .global + .expect("should include global metadata at the root"); + assert_eq!(global.visible, visible as u64); + assert_eq!(global.bounds, fixture.world.layout.base_bounds()); + assert_eq!(global.min_resolution, u64::from(deepest_occupied.get())); +} + +#[test] +fn global_nonroot_absent() { + let fixture = Fixture::new("tile-global-nonroot-absent"); + let coordinate = fixture.key_of(0).tile(Depth::new(1)); + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + coordinate, + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct a nonroot tile"); + assert!( + document.global.is_none(), + "should omit global metadata off the root" + ); +} + +#[test] +fn children_zero_terminal() { + let fixture = Fixture::new("tile-children-zero-terminal"); + let depth = Depth::from_zoom(fixture.world.schedule().max_tile_depth()); + let coordinate = MortonTile { + z: depth, + x: 0, + y: 0, + }; + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + coordinate, + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct at the deepest served zoom"); + assert_eq!( + document.children, 0, + "should report no children once the cut reaches the schedule's deepest bucket" + ); +} + +/// Minimal detail omits the trailer. +#[test] +fn trailer_minimal_omitted() { + let fixture = Fixture::new("tile-trailer-minimal-omitted"); + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct with the minimal detail level"); + assert!( + document.trailer.is_none(), + "should omit the trailer at minimal detail" + ); +} + +/// Auxiliary details borrow labels and preserve each row's representative icon. +#[test] +fn trailer_auxiliary_slot_alignment() { + let fixture = Fixture::varied("tile-trailer-auxiliary-slot-alignment"); + let cut = fixture.world.schedule().cut(Zoom::MIN); + let root_cell = + MortonCell::from_tile(Fixture::tile(0, 0, 0)).expect("the root cell always resolves"); + let (rows, _) = fixture.delivered_in(Depth::MIN, cut, root_cell); + assert!( + rows.len() >= 2, + "the fixture should deliver at least two rows to witness distinct values" + ); + + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Auxiliary, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct with the auxiliary detail level"); + + let expected_ids: Vec<_> = rows + .iter() + .map(|&row| fixture.world.layout.index.encode(row)) + .collect(); + let expected_positions: Vec<_> = rows + .iter() + .map(|&row| { + fixture + .world + .layout + .position(&fixture.epoch, row) + .expect("should resolve the fixture node's position") + }) + .collect(); + assert_eq!(document.ids.len(), rows.len()); + assert_eq!(document.positions.len(), rows.len()); + assert_eq!( + document.ids.iter().copied().collect::>(), + expected_ids, + "should list the delivered rows in bucket-major order" + ); + assert_eq!( + document.positions.iter().copied().collect::>(), + expected_positions, + "should list each delivered row's own position, read independently through World" + ); + + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + assert_eq!(trailer.labels.len(), rows.len()); + assert_eq!(trailer.icons.len(), rows.len()); + + let mut saw_icon = false; + let mut saw_empty_icon = false; + for (index, &row) in rows.iter().enumerate() { + let slot = TileSlot::from_usize(index); + let row_index = row.as_u64(); + + assert_eq!( + trailer.labels[slot].as_ref(), + format!("node-{row_index}"), + "should carry row {row_index}'s own text" + ); + assert!( + core::ptr::eq(trailer.labels[slot], fixture.label_of(row)), + "should borrow row {row_index}'s label from the captured scene" + ); + + let icon = trailer.icons[slot].as_ref(); + if row_index == 0 { + assert_eq!( + icon, "fixture-icon", + "row 0's representative has its own icon" + ); + saw_icon = true; + } else { + assert_eq!( + icon, "", + "row {row_index}'s representative should have no icon" + ); + saw_empty_icon = true; + } + } + assert!( + saw_icon && saw_empty_icon, + "should witness both representative icon values" + ); +} + +#[test] +fn mask_omitted_empty_selection() { + let fixture = Fixture::new("tile-mask-omitted-empty-selection"); + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct with an empty type selection"); + assert!( + document.type_masks.is_none(), + "should omit the mask column for an empty selection" + ); +} + +#[test] +fn mask_present_empty() { + let fixture = Fixture::new("tile-mask-present-empty"); + let type0 = fixture.ontology_uuid(0); + let requested = [type0]; + let coordinate = fixture.empty_terminal_tile(); + let document = TileDocument::new( + fixture.scene(), + coordinate, + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&requested), + limits: TileLimits { .. }, + }, + ) + .expect("should construct an empty tile at the deepest served zoom"); + let masks = document + .type_masks + .expect("should keep a present-but-empty column"); + assert_eq!( + masks.bits.row_domain_size(), + 0, + "should carry no rows over an empty tile" + ); + assert_eq!( + masks.bits.col_domain_size(), + requested.len(), + "should still carry one column per requested type" + ); +} + +#[test] +fn mask_unknown_uuid_zero_bits() { + let fixture = Fixture::new("tile-mask-unknown-uuid-zero-bits"); + let unknown = ArchivedOntologyTypeUuid::from(Uuid::from_u128(0xDEAD_BEEF)); + let requested = [unknown]; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&requested), + limits: TileLimits { .. }, + }, + ) + .expect("should construct with an unresolved type request"); + let masks = document.type_masks.expect("should keep a present column"); + assert_eq!( + masks.bits.col_domain_size(), + requested.len(), + "should keep one column for the unresolved type" + ); + assert!( + masks.bits.row_domain_size() > 0, + "the root cut should deliver at least one row" + ); + assert!( + masks.bits.rows().all(|slot| masks.bits.is_empty_row(slot)), + "should set no bit for a type no row carries" + ); +} + +/// Duplicate types retain separate selection slots. +/// +/// Type 1 is a child of type 0. Every node therefore matches type 0, including the odd rows +/// whose direct type is 1. Type 2 has no node members. +#[test] +fn mask_duplicate_slots() { + let fixture = Fixture::new("tile-mask-duplicate-slots"); + let cut = fixture.world.schedule().cut(Zoom::MIN); + let root_cell = + MortonCell::from_tile(Fixture::tile(0, 0, 0)).expect("the root cell always resolves"); + let (rows, _) = fixture.delivered_in(Depth::MIN, cut, root_cell); + assert!( + !rows.is_empty(), + "the root cut should deliver at least one row" + ); + + let type0 = fixture.ontology_uuid(0); + let type1 = fixture.ontology_uuid(1); + let type2 = fixture.ontology_uuid(2); + let unknown = ArchivedOntologyTypeUuid::from(Uuid::from_u128(0xDEAD_BEEF)); + let requested = [ + type0, type1, type2, unknown, type0, type1, type2, unknown, type0, + ]; + let membership: [Option; 9] = [ + Some(0), + Some(1), + Some(2), + None, + Some(0), + Some(1), + Some(2), + None, + Some(0), + ]; + + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Minimal, + types: OntologySelection::new(&requested), + limits: TileLimits { .. }, + }, + ) + .expect("should construct the root tile"); + let masks = document + .type_masks + .expect("should keep a mask column for a nonempty selection"); + + assert_eq!(masks.bits.row_domain_size(), rows.len()); + assert_eq!(masks.bits.col_domain_size(), requested.len()); + for (row_index, &row) in rows.iter().enumerate() { + let row_slot = TileSlot::from_usize(row_index); + for (bit, &type_row) in membership.iter().enumerate() { + let expected = match type_row { + Some(0) => true, + Some(1) => row.as_u64() & 1 == 1, + _ => false, + }; + assert_eq!( + masks + .bits + .contains(row_slot, SelectionSlot::from_usize(bit)), + expected, + "row {row_index}'s membership at requested slot {bit} should match its own type" + ); + } + } +} + +/// Mask slots follow the input row order. +#[test] +fn mask_permuted_delivered_order() { + let fixture = Fixture::new("tile-mask-permuted-delivered-order"); + let type1 = fixture.ontology_uuid(1); + let requested = [type1]; + let selection = OntologySelection::new(&requested); + let rows = [NodeRowId::new(3), NodeRowId::new(0), NodeRowId::new(5)]; + let masks = TypeMasks::::new(fixture.scene(), rows, selection); + let bit = SelectionSlot::from_usize(0); + assert!( + masks.bits.contains(TileSlot::from_usize(0), bit), + "row 3 should match type 1" + ); + assert!( + !masks.bits.contains(TileSlot::from_usize(1), bit), + "row 0 should not match type 1" + ); + assert!( + masks.bits.contains(TileSlot::from_usize(2), bit), + "row 5 should match type 1 at its input slot" + ); +} + +#[test] +fn mask_row_out_of_domain() { + let fixture = Fixture::new("tile-mask-row-out-of-domain"); + let type1 = fixture.ontology_uuid(1); + let requested = [type1]; + let selection = OntologySelection::new(&requested); + let rows = [NodeRowId::new(0), NodeRowId::new(NODES), NodeRowId::new(1)]; + let masks = TypeMasks::::new(fixture.scene(), rows, selection); + let bit = SelectionSlot::from_usize(0); + assert!( + !masks.bits.contains(TileSlot::from_usize(0), bit), + "row 0 should not match type 1" + ); + assert!( + !masks.bits.contains(TileSlot::from_usize(1), bit), + "a row outside the fitted position domain should keep its slot's bits clear" + ); + assert!( + masks.bits.contains(TileSlot::from_usize(2), bit), + "row 1 should match type 1 at its input slot" + ); +} + +#[test] +fn visibility_singleton_root() { + let mut fixture = Fixture::varied("tile-visibility-singleton-root"); + let position = fixture + .world + .layout + .position(&fixture.epoch, NodeRowId::new(0)) + .expect("should resolve the fixture node's position"); + fixture.restrict([0]); + + let empty: [ArchivedOntologyTypeUuid; 0] = []; + let document = TileDocument::new( + fixture.scene(), + Fixture::tile(0, 0, 0), + &TileDocumentOptions { + mode: Mode::Total, + detail: TileDocumentDetailLevel::Auxiliary, + types: OntologySelection::new(&empty), + limits: TileLimits { .. }, + }, + ) + .expect("should construct the root tile over the restricted scope"); + + let global = document + .global + .expect("should include global metadata at the root"); + assert_eq!( + global.visible, 1, + "the lone admitted row should be the whole visible set" + ); + assert_eq!( + global.bounds, + Bounds2::new(position, position), + "the tight extent of one point is that point twice" + ); + assert_eq!( + global.min_resolution, 0, + "a lone admitted row is the cascade's sole occupant, its own natural bucket the root" + ); + + assert_eq!( + document.ids.len(), + 1, + "the lone admitted row is the whole delivery" + ); + assert_eq!(document.positions.len(), 1); + let slot = TileSlot::from_usize(0); + assert_eq!( + document.ids[slot], + fixture.world.layout.index.encode(NodeRowId::new(0)), + "should deliver exactly row 0" + ); + assert_eq!( + document.positions[slot], position, + "should carry the sole visible row's own position" + ); + + let trailer = document + .trailer + .expect("should include an auxiliary trailer"); + assert_eq!(trailer.labels.len(), 1, "should carry exactly one label"); + assert_eq!(trailer.icons.len(), 1, "should carry exactly one icon"); + assert_eq!( + trailer.labels[slot].as_ref(), + "node-0", + "should carry the sole visible row's own label" + ); + assert_eq!( + trailer.icons[slot].as_ref(), + "fixture-icon", + "should carry the sole visible row's own icon" + ); +} + +/// A delivered row missing its position or display payload logs an internal problem. +/// +/// [`Problem`]'s [`TileDocumentError`] conversion records the whole report in +/// [`Debug`](core::fmt::Debug) form, including the underlying source error's text and an attachment +/// placed on top of it. Outside debug builds the response carries the fixed detail instead of +/// either. +#[test] +fn missing_row_data_internal() { + for context in [ + TileDocumentError::Position { + row: NodeRowId::new(0), + }, + TileDocumentError::Display { + row: NodeRowId::new(0), + }, + ] { + let report = Report::new(io::Error::other("private-store-message")) + .change_context(context) + .attach("private-property-value"); + assert_internal_diagnostic( + move || Problem::from(report), + &["private-store-message", "private-property-value"], + "the tile assembly could not read a delivered row's captured data", + ); + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/translate/codec/mod.rs b/libs/@local/graph/atlas/src/serve/document/translate/codec/mod.rs new file mode 100644 index 00000000000..a8e81c8472f --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/translate/codec/mod.rs @@ -0,0 +1,60 @@ +use alloc::{borrow::Cow, collections::BTreeMap}; + +use serde::{Serialize, Serializer, ser::SerializeStruct as _}; + +use super::{TranslateDocument, TranslatedEdge, TranslatedNode}; +use crate::{identity::NodeRowId, postgres::id::ArchivedEntityId, serve::codec::EncodedRowId}; + +#[cfg(test)] +mod tests; + +/// One translate response in writable form. +#[derive(serde::Serialize, schemars::JsonSchema)] +pub(super) struct TranslateResponse<'doc> { + nodes: &'doc BTreeMap, + edges: &'doc BTreeMap, +} + +impl<'doc> TranslateResponse<'doc> { + /// Borrows `document`'s node and edge maps for serialization. + pub(super) const fn new(document: &'doc TranslateDocument) -> Self { + Self { + nodes: &document.nodes, + edges: &document.edges, + } + } +} + +impl schemars::JsonSchema for TranslatedNode { + fn schema_name() -> Cow<'static, str> { + "TranslatedNode".into() + } + + /// Describes the object [`Self::serialize`] actually writes. + /// + /// The schema exposes `id`, `x` and `y` as top-level fields. + fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "properties": { + "id": generator.subschema_for::>(), + "x": generator.subschema_for::(), + "y": generator.subschema_for::(), + }, + "required": ["id", "x", "y"], + }) + } +} + +impl Serialize for TranslatedNode { + /// Writes `id`, `x` and `y` as a flat three-field struct. + fn serialize(&self, serializer: S) -> Result { + // a default derive would nest `position` instead of splitting it into its two coordinate + // fields + let mut object = serializer.serialize_struct("TranslatedNode", 3)?; + object.serialize_field("id", &self.id)?; + object.serialize_field("x", &self.position.x())?; + object.serialize_field("y", &self.position.y())?; + object.end() + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/translate/codec/tests.rs b/libs/@local/graph/atlas/src/serve/document/translate/codec/tests.rs new file mode 100644 index 00000000000..ed293e330a9 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/translate/codec/tests.rs @@ -0,0 +1,102 @@ +use alloc::{alloc::Global, collections::BTreeMap}; + +use serde_json::json; +use uuid::Uuid; + +use crate::{ + math::Vec2, + postgres::id::ArchivedEntityId, + serve::{ + codec::EncodedRowId, + document::{ + Document as _, + translate::{TranslateDocument, TranslatedEdge, TranslatedNode}, + }, + }, +}; + +/// Encoding an empty [`TranslateDocument`] replaces the buffer's bytes. +/// +/// What remains is the empty `{"nodes":{},"edges":{}}` object alone, not an appended copy of it. +#[test] +fn json_empty() { + let document = TranslateDocument { + nodes: BTreeMap::new(), + edges: BTreeMap::new(), + }; + let mut bytes = Vec::new_in(&Global); + bytes.extend_from_slice(b"previous document"); + let _completed = document + .encode(&mut bytes) + .expect("should complete the empty response"); + assert_eq!(bytes, br#"{"nodes":{},"edges":{}}"#); +} + +/// One node and one edge serialize as display-string keys over numeric fields. +/// +/// The encoder writes each archived `web_id~entity_uuid` key as its display string, and each +/// translated position or edge as its numeric fields. A later encode of an empty document into the +/// same buffer replaces its contents rather than appending to the earlier bytes. +#[test] +fn json_nodes_edges() { + let node = ArchivedEntityId { + web_id: Uuid::from_u128(1).into(), + entity_uuid: Uuid::from_u128(2).into(), + }; + let edge = ArchivedEntityId { + web_id: Uuid::from_u128(3).into(), + entity_uuid: Uuid::from_u128(2).into(), + }; + let document = TranslateDocument { + nodes: BTreeMap::from([( + node, + TranslatedNode { + id: EncodedRowId::new_unchecked(17), + position: Vec2::new(1.25, -2.5), + }, + )]), + edges: BTreeMap::from([( + edge, + TranslatedEdge { + source: EncodedRowId::new_unchecked(17), + target: EncodedRowId::new_unchecked(u32::MAX), + }, + )]), + }; + let mut bytes = Vec::new_in(&Global); + let _completed = document + .encode(&mut bytes) + .expect("should complete the geometry response"); + let actual: serde_json::Value = + serde_json::from_slice(&bytes).expect("should decode a complete translate response"); + assert_eq!( + actual, + json!({ + "nodes": { + "00000000-0000-0000-0000-000000000001~00000000-0000-0000-0000-000000000002": { + "id": 17, + "x": 1.25, + "y": -2.5, + }, + }, + "edges": { + "00000000-0000-0000-0000-000000000003~00000000-0000-0000-0000-000000000002": { + "source": 17, + "target": u32::MAX, + }, + }, + }) + ); + + let empty = TranslateDocument { + nodes: BTreeMap::new(), + edges: BTreeMap::new(), + }; + let _completed = empty + .encode(&mut bytes) + .expect("should complete the replacement response"); + assert_eq!( + bytes, br#"{"nodes":{},"edges":{}}"#, + "should replace the longer response" + ); +} diff --git a/libs/@local/graph/atlas/src/serve/document/translate/mod.rs b/libs/@local/graph/atlas/src/serve/document/translate/mod.rs new file mode 100644 index 00000000000..09345d93016 --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/translate/mod.rs @@ -0,0 +1,169 @@ +//! Entity identities for correlating graph results with map geometry. +//! +//! Hidden, unknown and draft identities produce absent keys. + +use alloc::{alloc::Allocator, borrow::Cow, collections::BTreeMap}; +use core::{error::Error, fmt}; + +use error_stack::Report; +use type_system::knowledge::entity::EntityId; + +use super::{Document, codec::Envelope}; +use crate::{ + identity::NodeRowId, + math::Vec2, + postgres::id::ArchivedEntityId, + serve::{codec::EncodedRowId, neighbourhood::NeighbourhoodProvider as _, scene::Scene}, +}; + +mod codec; +#[cfg(test)] +mod tests; + +/// Caps on one translate request: the input identities it may name. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TranslateLimits { + /// Most input identities, including duplicates. The default is 1024. + pub entity_ids: u32 = 1024, +} + +/// A failure to assemble a translate document. +#[derive(Debug)] +pub(crate) enum TranslateDocumentError { + /// The request lists more identities than the configured limit permits. + Ids { count: usize, maximum: u32 }, +} + +impl fmt::Display for TranslateDocumentError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Ids { count, maximum } => write!( + fmt, + "the request lists {count} entity ids, exceeding the limit of {maximum}" + ), + } + } +} + +impl Error for TranslateDocumentError {} + +/// One resolved node's encoded row id and position. +#[derive(Debug, Copy, Clone, PartialEq)] +pub(crate) struct TranslatedNode { + id: EncodedRowId, + position: Vec2, +} + +/// One resolved edge's encoded source and target row ids. +#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, schemars::JsonSchema)] +pub(crate) struct TranslatedEdge { + source: EncodedRowId, + target: EncodedRowId, +} + +/// Resolved map geometry keyed by entity identity. +#[derive(Debug, PartialEq)] +pub(crate) struct TranslateDocument { + nodes: BTreeMap, + edges: BTreeMap, +} + +impl TranslateDocument { + /// Resolves identities against the captured scene. + /// + /// Nodes include their wire-frame position. Links require admission of their own row and both + /// endpoint rows. Duplicate identities collapse to one entry. + /// + /// # Errors + /// + /// Returns [`TranslateDocumentError`] when the input count exceeds the limit, before reading + /// any identity. + #[tracing::instrument(level = "debug", skip_all, fields(ids))] + pub(crate) fn new( + scene @ Scene { + world, epoch, mask, .. + }: Scene<'_>, + ids: impl IntoIterator, + limits: TranslateLimits, + ) -> Result> { + let ids = ids.into_iter(); + tracing::Span::current().record("ids", ids.len()); + if ids.len() > limits.entity_ids as usize { + return Err(Report::new(TranslateDocumentError::Ids { + count: ids.len(), + maximum: limits.entity_ids, + })); + } + + let mut this = Self { + nodes: BTreeMap::new(), + edges: BTreeMap::new(), + }; + for id in ids { + let EntityId { + web_id, + entity_uuid, + draft_id: None, + } = id + else { + continue; + }; + + let key = ArchivedEntityId { + web_id: web_id.into(), + entity_uuid: entity_uuid.into(), + }; + + if let Some(row) = world.layout.index.row_of(epoch, key) { + let Some(row) = mask.visible_node(row) else { + continue; + }; + let Some(position) = world.layout.position(epoch, row.unwrap()) else { + continue; + }; + this.nodes.insert( + key, + TranslatedNode { + id: world.layout.index.encode(row.unwrap()), + position, + }, + ); + continue; + } + + if let Some(row) = world.topology.row_of(epoch, key) + && let Some(edge) = scene.provide_edge(row) + { + let [source, target] = edge.endpoints; + this.edges.insert( + key, + TranslatedEdge { + source: world.layout.index.encode(source), + target: world.layout.index.encode(target), + }, + ); + } + } + + Ok(this) + } +} + +impl schemars::JsonSchema for TranslateDocument { + fn schema_name() -> Cow<'static, str> { + "TranslateDocument".into() + } + + fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + as schemars::JsonSchema>::json_schema(generator) + } +} + +impl Document for TranslateDocument { + type Error = Report; + + fn encode(&self, buffer: &mut Vec) -> Result { + Envelope::encode_json(&self::codec::TranslateResponse::new(self), buffer) + } +} diff --git a/libs/@local/graph/atlas/src/serve/document/translate/tests.rs b/libs/@local/graph/atlas/src/serve/document/translate/tests.rs new file mode 100644 index 00000000000..aeca99c394f --- /dev/null +++ b/libs/@local/graph/atlas/src/serve/document/translate/tests.rs @@ -0,0 +1,256 @@ +use alloc::{collections::BTreeMap, sync::Arc}; +use core::assert_matches; + +use arc_swap::Guard; +use rand::{SeedableRng as _, rngs::StdRng}; +use type_system::{ + knowledge::entity::{ + EntityId, + id::{DraftId, EntityUuid}, + }, + principal::actor::{ActorId, ActorType}, +}; +use uuid::Uuid; + +use super::{ + TranslateDocument, TranslateDocumentError, TranslateLimits, TranslatedEdge, TranslatedNode, +}; +use crate::{ + bitset::CompressedBitSet, + identity::{EdgeRowId, NodeRowId}, + morton::Zoom, + postgres::id::ArchivedEntityId, + serve::{ + delta::{Delta, epoch::Epoch}, + scene::Scene, + schedule::ViewSchedule, + tests::fixture::{EDGES, ENDPOINTS, NODES, TamperFixture, secret}, + visibility::{VisibilityActor, VisibilityMask}, + world::World, + }, +}; + +/// An opened synthetic generation, shared by every test below. +/// +/// It carries the captured epoch, actor, visibility mask and delivery schedule that `scene()` +/// assembles into one scene. +struct Fixture { + world: Arc, + epoch: Epoch, + actor: VisibilityActor, + mask: VisibilityMask, + schedule: ViewSchedule, + _files: TamperFixture, +} + +impl Fixture { + /// Publishes and opens the named synthetic generation, with a fresh delta identity. + /// + /// The mask grants one fixed test actor full visibility. + /// + /// # Panics + /// + /// Panics if publishing fails or [`World::open`] cannot open or validate the serving artifacts. + fn new(name: &str) -> Self { + let files = TamperFixture::publish(name); + let world = Arc::new( + World::open(files.generation().clone(), &secret()) + .expect("should open the synthetic generation"), + ); + let delta = Delta::new(Arc::clone(&world), StdRng::seed_from_u64(17)) + .expect("should allocate a delta identity"); + let epoch = Epoch::from(Guard::from_inner(Arc::new(delta))); + let actor = VisibilityActor { + id: ActorId::new(Uuid::from_u128(1), ActorType::User), + instance_admin: false, + }; + let mask = VisibilityMask::full(actor); + let schedule = ViewSchedule::of(Arc::clone(&world), &epoch, &mask); + Self { + world, + epoch, + actor, + mask, + schedule, + _files: files, + } + } + + /// Narrows this fixture's visibility mask and schedule to exactly the given rows. + /// + /// A later `scene()` call sees only those node and edge rows. + fn restrict( + &mut self, + nodes: impl IntoIterator, + edges: impl IntoIterator, + ) { + self.mask = VisibilityMask::partial( + self.actor, + CompressedBitSet::from_rows(nodes.into_iter().map(NodeRowId::new)), + CompressedBitSet::from_rows(edges.into_iter().map(EdgeRowId::new)), + ); + self.schedule = ViewSchedule::of(Arc::clone(&self.world), &self.epoch, &self.mask); + } + + /// Assembles this fixture's world, epoch, mask and schedule into one [`Scene`]. + /// + /// Its delivery is the schedule's cut at the zero offset. + fn scene(&self) -> Scene<'_> { + Scene { + world: &self.world, + epoch: &self.epoch, + mask: &self.mask, + schedule: &self.schedule, + delivery: self + .schedule + .cut(Zoom::MIN) + .expect("should bind the zero offset"), + } + } + + /// Returns the [`EntityId`] the fixture generation assigned to node `row`. + /// + /// # Panics + /// + /// Panics where that generation holds no node row `row`. + fn node_id(&self, row: u64) -> EntityId { + EntityId::from( + self.world + .layout + .index + .key_of(&self.epoch, NodeRowId::new(row)) + .expect("should resolve the fixture node"), + ) + } + + /// Returns the [`EntityId`] the fixture generation assigned to edge `row`. + /// + /// # Panics + /// + /// Panics where that generation holds no edge row `row`. + fn edge_id(&self, row: u64) -> EntityId { + EntityId::from( + self.world + .topology + .key_of(&self.epoch, EdgeRowId::new(row)) + .expect("should resolve the fixture edge"), + ) + } +} + +/// A mixed request keeps only the plain node and edge among its ids. +/// +/// The request carries a duplicated node id, an edge id, a draft node id, a draft edge id and an +/// id with an unknown UUID. The duplicate collapses, and the document drops every id it cannot +/// resolve. +#[test] +fn identities_mixed_input() { + let fixture = Fixture::new("translate-mixed-input"); + let node_id = fixture.node_id(0); + let edge_id = fixture.edge_id(0); + let draft_node = EntityId { + draft_id: Some(DraftId::new(Uuid::from_u128(3))), + ..fixture.node_id(1) + }; + let draft_edge = EntityId { + draft_id: Some(DraftId::new(Uuid::from_u128(4))), + ..fixture.edge_id(1) + }; + let unknown = EntityId { + entity_uuid: EntityUuid::new(Uuid::nil()), + ..node_id + }; + let document = TranslateDocument::new( + fixture.scene(), + [node_id, node_id, edge_id, draft_node, draft_edge, unknown], + TranslateLimits { .. }, + ) + .expect("should construct within the identity limit"); + let node = TranslatedNode { + id: fixture.world.layout.index.encode(NodeRowId::new(0)), + position: fixture + .world + .layout + .position(&fixture.epoch, NodeRowId::new(0)) + .expect("should read the node's wire position"), + }; + let [source, target] = ENDPOINTS[0]; + assert_eq!( + document, + TranslateDocument { + nodes: BTreeMap::from([(ArchivedEntityId::from(node_id), node)]), + edges: BTreeMap::from([( + ArchivedEntityId::from(edge_id), + TranslatedEdge { + source: fixture.world.layout.index.encode(source), + target: fixture.world.layout.index.encode(target), + } + )]), + } + ); +} + +#[test] +fn identities_count_limit() { + let fixture = Fixture::new("translate-count-limit"); + let id = fixture.node_id(0); + let report = + TranslateDocument::new(fixture.scene(), [id, id], TranslateLimits { entity_ids: 1 }) + .expect_err("should count duplicates toward the limit"); + assert_matches!( + report.current_context(), + TranslateDocumentError::Ids { + count: 2, + maximum: 1 + } + ); + let boundary = + TranslateDocument::new(fixture.scene(), [id, id], TranslateLimits { entity_ids: 2 }) + .expect("should admit the exact limit"); + assert_eq!(boundary.nodes.len(), 1, "should collapse the repeated key"); + assert!(boundary.edges.is_empty()); + let empty = TranslateDocument::new(fixture.scene(), [], TranslateLimits { entity_ids: 0 }) + .expect("should admit an empty request at a zero limit"); + assert!(empty.nodes.is_empty()); + assert!(empty.edges.is_empty()); +} + +/// Hiding a row drops it from the document, along with the edge touching it. +/// +/// An untouched node or edge keeps the value it had in the unrestricted baseline. +#[test] +fn identities_masked_domains() { + let mut fixture = Fixture::new("translate-masked-domains"); + let ids = [ + fixture.node_id(0), + fixture.edge_id(0), + fixture.node_id(2), + fixture.edge_id(2), + ]; + let keys = ids.map(ArchivedEntityId::from); + let baseline = TranslateDocument::new(fixture.scene(), ids, TranslateLimits { .. }) + .expect("should construct the full scene"); + assert_eq!(baseline.nodes.len(), 2); + assert_eq!(baseline.edges.len(), 2); + + // Edge 0 joins nodes 0 and 1. Edge 2 is a self-loop at node 2. + for (hidden_node, hidden_edge, source_visible) in [ + (Some(0), None, false), + (Some(1), None, true), + (None, Some(0), true), + ] { + fixture.restrict( + (0..NODES).filter(|row| Some(*row) != hidden_node), + (0..EDGES).filter(|row| Some(*row) != hidden_edge), + ); + let document = TranslateDocument::new(fixture.scene(), ids, TranslateLimits { .. }) + .expect("should construct the restricted scene"); + assert_eq!(document.nodes.contains_key(&keys[0]), source_visible); + assert!( + !document.edges.contains_key(&keys[1]), + "should omit the hidden link or endpoint" + ); + assert_eq!(document.nodes.get(&keys[2]), baseline.nodes.get(&keys[2])); + assert_eq!(document.edges.get(&keys[3]), baseline.edges.get(&keys[3])); + } +} diff --git a/libs/@local/graph/atlas/src/serve/mod.rs b/libs/@local/graph/atlas/src/serve/mod.rs index b82d0349fae..36de5514740 100644 --- a/libs/@local/graph/atlas/src/serve/mod.rs +++ b/libs/@local/graph/atlas/src/serve/mod.rs @@ -16,9 +16,11 @@ //! request using the new token may therefore reuse a soft-stale scope while its detached refresh //! runs. +pub(crate) mod authorization; pub(crate) mod codec; pub(crate) mod delta; pub(crate) mod density; +pub(crate) mod document; pub(crate) mod hydrate; mod intern; pub(crate) mod membership; From 920e0f4c71d226d555f6878fb823617d0e84c493 Mon Sep 17 00:00:00 2001 From: Bilal Mahmoud <7252775+indietyp@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:53:07 +0200 Subject: [PATCH 2/2] feat: refine type-complete logic for trailer selection - Fix completeness to check selection emptiness, not source types - Empty selection always renders types incomplete, regardless of source - Add tests for empty selection and missing source scenarios - Clarify completeness definition in test documentation --- .../atlas/src/serve/document/locate/tests.rs | 68 ++++++++++++++++++- .../src/serve/document/locate/trailer.rs | 2 +- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/libs/@local/graph/atlas/src/serve/document/locate/tests.rs b/libs/@local/graph/atlas/src/serve/document/locate/tests.rs index 5f06da5c134..2dcd94e54d9 100644 --- a/libs/@local/graph/atlas/src/serve/document/locate/tests.rs +++ b/libs/@local/graph/atlas/src/serve/document/locate/tests.rs @@ -1466,7 +1466,7 @@ struct CompletenessCase<'selection> { properties_complete: bool, } -/// Complete type coverage requires a resolved source with nonempty, fully selected direct types. +/// A resolved source is type-complete if a nonempty selection covers every direct type. #[test] fn trailer_completeness() { let fixture = Fixture::new("locate-trailer-completeness"); @@ -1488,10 +1488,10 @@ fn trailer_completeness() { properties_complete: false, }, CompletenessCase { - name: "empty", + name: "empty-source-types", source_types: Vec::new(), selection: &alpha_only, - expected_types_complete: false, + expected_types_complete: true, properties_complete: true, }, CompletenessCase { @@ -1509,6 +1509,7 @@ fn trailer_completeness() { properties_complete: false, }, ]; + for CompletenessCase { name: case, source_types, @@ -1572,6 +1573,67 @@ fn trailer_completeness() { } } +#[test] +fn trailer_selection_empty() { + let fixture = Fixture::new("locate-trailer-selection-empty"); + for source_types in [Vec::new(), vec![FakeResolver::url("alpha")]] { + let mut response = FakeResolver::resolved(3, 2); + response.nodes[NodeSlot::MIN] = Some(LocateNode { + type_urls: source_types, + }); + let resolver = FakeResolver::answering(response); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(NodeRowId::new(1)), + &LocateDocumentOptions { + types: Fixture::no_types(), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with an empty type selection"); + assert!( + !document.trailer.type_ids_complete, + "should report incomplete types for an empty selection even with no source types" + ); + } +} + +#[test] +fn trailer_source_missing() { + let fixture = Fixture::new("locate-trailer-source-missing"); + let alpha = FakeResolver::url("alpha"); + let selection = [ArchivedOntologyTypeUuid::from_url(&alpha)]; + + for missing_node in [false, true] { + let mut response = FakeResolver::resolved(3, 2); + response.nodes[NodeSlot::MIN] = Some(LocateNode { + type_urls: vec![alpha.clone()], + }); + if missing_node { + response.nodes[NodeSlot::MIN] = None; + } else { + response.source_properties = None; + } + let resolver = FakeResolver::answering(response); + let document = LocateDocument::new( + fixture.scene(), + fixture.entity_source(NodeRowId::new(1)), + &LocateDocumentOptions { + types: OntologySelection::new(&selection), + limits: LocateLimits { .. }, + resolver: &resolver, + }, + ) + .expect("should construct with missing source details"); + assert!( + !document.trailer.type_ids_complete, + "should report incomplete types when either source query has no row (missing node: \ + {missing_node})" + ); + } +} + /// Refuses a request past the configured type-count limit, under [`LocateDocumentError::Types`]. #[test] fn types_over_limit() { diff --git a/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs b/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs index fe580e3ee8f..2fc8d9a1e3d 100644 --- a/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs +++ b/libs/@local/graph/atlas/src/serve/document/locate/trailer.rs @@ -72,7 +72,7 @@ impl<'details> LocateTrailer<'details> { ) -> Result> { let type_ids_complete = source_properties.is_some() && nodes[NodeSlot::MIN].details.as_ref().is_some_and(|source| { - !source.type_urls.is_empty() + !types.is_empty() && source .type_urls .iter()