diff --git a/codegen/src/facts.rs b/codegen/src/facts.rs index 577e675..e3ce10d 100644 --- a/codegen/src/facts.rs +++ b/codegen/src/facts.rs @@ -15,7 +15,7 @@ use std::collections::BTreeMap; // Crate Uses -use crate::connections::{self, PackageConnection}; +use crate::connections::{self, PackageConnection, Provider}; use crate::protocol::{EncodingMapping, GenerateRequest}; // External Uses @@ -400,6 +400,9 @@ pub struct HelloProtocol { /// the encodings that connection uses: the package's /// format (JSON on a framing that writes it), the connection's framing, and /// `comline@2` itself. +/// +/// [`hello_parts`] has the same pieces per protocol, for a Hello of fewer +/// protocols than the connection's. pub fn hello(req: &GenerateRequest, connection: &PackageConnection) -> Result<(Vec, Vec)> { let facts = req.facts.as_ref().ok_or_else(|| { eyre!( @@ -511,6 +514,217 @@ pub fn hello(req: &GenerateRequest, connection: &PackageConnection) -> Result<(V Ok((out, protocols.into_iter().map(|p| p.4).collect())) } +/// One protocol's entry of a Hello: what the peer is told about it, and what +/// the generator keeps beside it. The entry carries no direction on the wire; +/// [`provider`](Self::provider) is for the generated code, which knows +/// `@provider` and so which side provides the protocol and which calls it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HelloEntry { + pub namespace: String, + pub name: String, + /// Its wire number, unique in the package. + pub number: u16, + /// The release it last broke at, as the facts spell it. + pub line: String, + /// [`line`](Self::line) as `(major, minor, patch)`. + pub release: (u32, u32, u32), + /// The first 16 bytes of its shape hash. + pub shape_hash: [u8; 16], + /// Each function's name and wire number, in declaration order. + pub functions: Vec<(String, u16)>, + /// Which side implements it (`@provider`). + pub provider: Provider, +} + +impl HelloEntry { + /// The function numbers the Hello lists: ascending, once each. + pub fn function_numbers(&self) -> Vec { + let mut numbers: Vec = self.functions.iter().map(|(_, number)| *number).collect(); + numbers.sort_unstable(); + numbers.dedup(); + numbers + } + + /// The entry as the Hello's protocol list carries it. + pub fn encode(&self, out: &mut Vec) { + varint(self.number as u32, out); + put_release(self.release, out); + out.extend_from_slice(&self.shape_hash); + let functions = self.function_numbers(); + varint(functions.len() as u32, out); + for f in functions { + varint(f as u32, out); + } + } +} + +/// One encoding the connection uses, as the Hello lists it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct HelloEncoding { + /// `0` format, `1` framing, `2` handshake. + pub kind: u8, + /// The first 16 bytes of its parameter hash. + pub hash: [u8; 16], +} + +/// What a `comline@2` Hello is built from, per protocol: [`hello`]'s pieces +/// before they are put together, so generated code can put together the +/// Hello of the protocols a side provides and calls, instead of the one +/// Hello of the package's whole connection. +/// +/// Everything that doesn't depend on which protocols are listed is here once: +/// the exchange, the package's id and version, and the connection's encodings +/// (the same whichever protocols a Hello lists). The package connection's +/// protocols are [`protocols`](Self::protocols), one [`HelloEntry`] each. +/// +/// **Order.** A Hello is canonical: encodings ascending by kind then hash, +/// protocols ascending by wire number, each protocol's function numbers +/// ascending. [`encodings`](Self::encodings) and [`protocols`](Self::protocols) +/// are in that order already, and [`assemble`](Self::assemble) puts any subset +/// of the entries in it, whichever order they are given. The order the +/// protocols are provided or called in doesn't reach the wire. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HelloParts { + /// `offer` or `symmetric`: a Hello's exchange flag. + pub exchange: String, + /// The package's `id`. + pub package: [u8; 16], + /// The version's release. + pub version: (u32, u32, u32), + /// The version's pre-release (`dev.2`), empty for a release. + pub pre: String, + pub encodings: Vec, + /// Every protocol on the package connection, ascending by wire number. + pub protocols: Vec, +} + +impl HelloParts { + /// The Hello listing exactly `protocols`: those entries, in canonical + /// order, with the package, its version and the connection's encodings. + /// An entry given twice is listed once. With every entry + /// ([`protocols`](Self::protocols)) it is [`hello`]'s output, byte for + /// byte. + pub fn assemble<'a>(&self, protocols: impl IntoIterator) -> Vec { + let mut listed: Vec<&HelloEntry> = protocols.into_iter().collect(); + listed.sort_by_key(|p| p.number); + listed.dedup_by_key(|p| p.number); + let mut out = vec![b'C', b'O', 2, 1, if self.exchange == "symmetric" { 1 } else { 0 }]; + out.extend_from_slice(&self.package); + put_release(self.version, &mut out); + varint(self.pre.len() as u32, &mut out); + out.extend_from_slice(self.pre.as_bytes()); + varint(self.encodings.len() as u32, &mut out); + for encoding in &self.encodings { + out.push(encoding.kind); + out.extend_from_slice(&encoding.hash); + } + varint(listed.len() as u32, &mut out); + for protocol in listed { + protocol.encode(&mut out); + } + out + } + + /// The Hello of the whole connection: every protocol listed. + pub fn assemble_all(&self) -> Vec { + self.assemble(&self.protocols) + } +} + +/// What this end's Hello is built from, per protocol ([`HelloParts`]): the +/// same facts, checks and errors as [`hello`], and the same protocols (those +/// on the package connection), but kept apart instead of put together. +/// +/// A generator emits them, and generated `accept` / `connect` assemble the +/// Hello of the protocols actually provided and called. With every protocol +/// among them, that Hello is [`hello`]'s, byte for byte. +pub fn hello_parts(req: &GenerateRequest, connection: &PackageConnection) -> Result { + let facts = req.facts.as_ref().ok_or_else(|| { + eyre!( + "the package's connection runs `comline@2`, and the request carries no facts: `comline generation` \ + supplies them from the package's build history (run `comline build` first)" + ) + })?; + let id = facts.id.as_deref().ok_or_else(|| { + eyre!( + "the package's connection runs `comline@2`, which identifies the package by its `id`, and version {} \ + was built without one: run `comline build`, which writes one into config.idp", + facts.version + ) + })?; + let package = hash128(id).ok_or_else(|| eyre!("package id `{id}` isn't 32 hex digits"))?; + let (package_version, pre) = version(&facts.version) + .ok_or_else(|| eyre!("package version `{}` isn't `major.minor.patch[-pre]`", facts.version))?; + + let format = if crate::encodings::writes_json(&req.encodings, &connection.framing) { + FORMAT_JSON.to_string() + } else { + crate::encodings::canonical(&req.wire_format) + }; + let formats = facts.encodings.iter().filter(|e| e.kind == "format").count(); + let mut encodings = vec![]; + for e in &facts.encodings { + let (kind, wanted) = match e.kind.as_str() { + // One format in play is the connection's; of several, the one by name. + "format" => (0, formats == 1 || e.name == format), + "framing" => (1, e.name == connection.framing), + "handshake" => (2, e.name == connection.handshake), + _ => continue, + }; + if wanted { + let hash = hash128(&e.param_hash).ok_or_else(|| eyre!("encoding `{}`'s hash isn't hex", e.name))?; + encodings.push(HelloEncoding { kind, hash }); + } + } + encodings.sort(); + encodings.dedup(); + for (kind, what) in [(0, "format"), (1, "framing"), (2, "handshake")] { + if !encodings.iter().any(|e| e.kind == kind) { + bail!("the package's facts record no {what} for its `comline@2` connection: run `comline build`"); + } + } + + let mut protocols = vec![]; + for (namespace, name, parameters, functions) in connections::protocols(&req.schemas) { + // A dependency's protocol isn't on the connection, so not this + // package's to declare. + if !connections::reached(&req.dependencies, namespace, name) + || !connections::on_connection_among(&req.schemas, &req.dependencies, parameters) + { + continue; + } + let p = protocol(Some(facts), namespace, name) + .ok_or_else(|| missing(Some(facts), &req.dependencies, namespace, name, parameters))?; + let addressing = addressing_in(req, namespace, name, parameters, functions)?; + let names = functions.iter().filter_map(|f| match f { + FrozenUnit::Function { name, .. } => Some(name.clone()), + _ => None, + }); + let release = version(&p.line).map(|(r, _)| r).ok_or_else(|| eyre!("line `{}` isn't a release", p.line))?; + let shape_hash = hash128(&p.shape_hash).ok_or_else(|| eyre!("shape hash of `{}` isn't hex", p.path))?; + protocols.push(HelloEntry { + namespace: namespace.to_string(), + name: name.to_string(), + number: p.number, + line: p.line.clone(), + release, + shape_hash, + functions: names.zip(addressing.checks.unwrap_or_default()).collect(), + provider: Provider::of(parameters), + }); + } + protocols.sort_by_key(|p| p.number); + + Ok(HelloParts { + exchange: Comline2Params::of(&connection.handshake_value).exchange, + package, + version: package_version, + pre: pre.to_string(), + encodings, + protocols, + }) +} + /// The first 16 bytes of a hex hash: `comline@2`'s 128-bit hashes. fn hash128(hex: &str) -> Option<[u8; 16]> { if hex.len() < 32 || !hex.is_ascii() { @@ -810,6 +1024,49 @@ mod tests { assert_eq!(hello, [("Chat", 7)]); } + #[test] + fn the_hello_parts_give_the_hello_and_its_errors() { + let mut req = request(&["Chat", "Other"]); + req.facts = Some(facts_of(&["Chat", "Other"])); + let connection = comline2_connection(&req).unwrap().unwrap(); + let (bytes, _) = hello(&req, &connection).unwrap(); + let parts = hello_parts(&req, &connection).unwrap(); + assert_eq!(parts.assemble_all(), bytes); + let numbers: Vec<_> = parts.protocols.iter().map(|p| (p.name.as_str(), p.number)).collect(); + assert_eq!(numbers, [("Chat", 7), ("Other", 8)]); + assert_eq!(parts.encodings.iter().map(|e| e.kind).collect::>(), [0, 1, 2]); + + // The one listing fewer protocols lists exactly those. + let other = parts.protocols.iter().filter(|p| p.name == "Other"); + let subset = parts.assemble(other); + assert!(subset.len() < bytes.len()); + // One protocol: number 8, line 1.0.0, its shape hash, and function 3. + let mut entry = vec![1, 8, 1, 0, 0]; + entry.extend([0x5a; 16]); + entry.extend([1, 3]); + assert!(subset.ends_with(&entry), "{subset:?}"); + + // The same refusals, in the same words. + let mut no_id = req.clone(); + no_id.facts.as_mut().unwrap().id = None; + assert_eq!( + hello_parts(&no_id, &connection).unwrap_err().to_string(), + hello(&no_id, &connection).unwrap_err().to_string() + ); + let mut no_facts = req.clone(); + no_facts.facts = None; + assert_eq!( + hello_parts(&no_facts, &connection).unwrap_err().to_string(), + hello(&no_facts, &connection).unwrap_err().to_string() + ); + let mut no_framing = req.clone(); + no_framing.facts.as_mut().unwrap().encodings.retain(|e| e.kind != "framing"); + assert_eq!( + hello_parts(&no_framing, &connection).unwrap_err().to_string(), + hello(&no_framing, &connection).unwrap_err().to_string() + ); + } + #[test] fn versions_and_hashes_parse() { assert_eq!(version("0.4.0-dev.2"), Some(((0, 4, 0), "dev.2"))); diff --git a/conformance/src/lib.rs b/conformance/src/lib.rs index b3a2e3d..81973ab 100644 --- a/conformance/src/lib.rs +++ b/conformance/src/lib.rs @@ -149,6 +149,9 @@ pub fn corpus() -> Vec { // A protocol in a nested schema, whose facts name its namespace the // way core does (the nested-namespace facts fix). nested_protocol(), + // Two applications in one package, so a side can provide a subset of + // its protocols and the Hello lists only that (generation#153). + connections_subset(), ] } @@ -980,6 +983,85 @@ pub fn nested_protocol() -> Fixture { ) } +/// Two applications in one package, each with a protocol per side, all on the +/// defaults (`datagram@3`, `comline@2`): `Chat` and `Events` in `conformance` +/// are the public application, `Admin` and `Alerts` in `conformance/admin` the +/// private one. A server can provide `Chat` alone, or `Chat` and `Admin`, and a +/// client only `Events`, and the Hello each sends then lists just those and +/// the protocols it calls. With all four provided the Hello is the package's +/// whole one. (`tests/hello_entries.rs` checks what every generator builds +/// from; the goldens hold each generator's own output.) +fn connections_subset() -> Fixture { + let protocol = |name: &str, provider: &str, functions: Vec| FrozenUnit::Protocol { + docstring: String::new(), + parameters: [ + // `Server` is the default and isn't written. + (provider != "Server") + .then(|| FrozenUnit::Property { + name: "provider".to_string(), + value: ConfigValue::Name(provider.to_string()), + }) + .into_iter() + .collect::>(), + vec![ + FrozenUnit::Property { + name: "framing".to_string(), + value: ConfigValue::Str("datagram@3".to_string()), + }, + FrozenUnit::Property { + name: "handshake".to_string(), + value: ConfigValue::Str("comline@2".to_string()), + }, + ], + ] + .concat(), + name: name.to_string(), + functions, + span: (0, 0), + }; + Fixture::package( + "connections_subset", + vec![ + ( + "conformance".to_string(), + vec![ + protocol( + "Chat", + "Server", + vec![ + function("send", vec![arg("n", PrimitiveType::U32)], Some("u32"), vec![]), + function("leave", vec![], None, vec![]), + ], + ), + protocol( + "Events", + "Client", + vec![function("shown", vec![arg("n", PrimitiveType::U32)], Some("bool"), vec![])], + ), + ], + ), + ( + "conformance/admin".to_string(), + vec![ + protocol( + "Admin", + "Server", + vec![ + function("kick", vec![arg("user", PrimitiveType::U32)], None, vec![]), + function("ban", vec![arg("user", PrimitiveType::U32)], None, vec![]), + ], + ), + protocol( + "Alerts", + "Client", + vec![function("raised", vec![arg("code", PrimitiveType::U32)], None, vec![])], + ), + ], + ), + ], + ) +} + /// The dependency `localdeps`'s one schema, `greeting`. fn dependency_greeting(units: Vec) -> (String, Vec) { ("localdeps/greeting".to_string(), units) diff --git a/conformance/tests/hello_entries.rs b/conformance/tests/hello_entries.rs new file mode 100644 index 0000000..8b9cd83 --- /dev/null +++ b/conformance/tests/hello_entries.rs @@ -0,0 +1,183 @@ +//! A Hello is built from per-protocol entries, so a side can list only the +//! protocols it provides and calls (generation#153): what +//! `facts::hello_parts` gives every generator, checked against the one +//! package-wide Hello they all embed today (`facts::hello`). +//! +//! What every generator shares is checked here, on the request each gets. The +//! goldens hold each generator's own output for the fixtures. + +use comline_codegen::connections::Provider; +use comline_codegen::validation::ValidateOnDecode; +use comline_codegen::{connections, facts, GenerateRequest, PackageMeta, TargetLanguage}; +use comline_conformance::{corpus, facts_for, Fixture}; +use comline_core::schema::idl::annotations::DEFAULT_FORMAT; +use comline_core::schema::ir::frozen::unit::FrozenUnit; + +/// The request a generator for `language` gets for `fx`, as the harness +/// builds it: with the facts of the package's own schemas when they're needed. +fn request(fx: &Fixture, language: &str) -> GenerateRequest { + let wire_format = fx.wire_format.unwrap_or(DEFAULT_FORMAT); + let dependencies: Vec = fx.dependencies.iter().map(|d| d.to_string()).collect(); + let mut req = GenerateRequest { + mode: fx.mode, + schemas: fx.schemas.clone(), + package: PackageMeta { name: "conformance".into(), version: "0.0.0".into() }, + target: TargetLanguage { language: language.into(), version: "1.85.0".into() }, + wire_format: wire_format.into(), + external_std: fx.external_std, + validate_on_decode: ValidateOnDecode::Off, + async_api: fx.async_api, + encodings: comline_codegen::encodings::official_for(language), + facts: None, + dependencies, + }; + if facts::needed_in(&req) { + let own: Vec<_> = fx + .schemas + .iter() + .filter(|(namespace, _)| !connections::is_dependency(&req.dependencies, namespace)) + .cloned() + .collect(); + req.facts = Some(facts_for(&own, wire_format)); + } + req +} + +/// Every fixture whose package connection runs `comline@2`, for each +/// language: its request and connection. +fn hellos() -> Vec<(String, GenerateRequest, connections::PackageConnection)> { + let mut out = vec![]; + for fx in corpus() { + for language in ["rust", "typescript"] { + let req = request(&fx, language); + // A package whose protocols disagree on a connection has no Hello + // to build, and no generator emits one. + if let Ok(Some(connection)) = facts::comline2_connection(&req) { + out.push((format!("{} ({language})", fx.name), req, connection)); + } + } + } + out +} + +#[test] +fn every_protocol_assembled_is_the_package_wide_hello_for_every_fixture() { + let hellos = hellos(); + // Not vacuous: the corpus has several packages with a Hello. + let names: Vec<&str> = hellos.iter().map(|(name, _, _)| name.as_str()).collect(); + for expected in [ + "handshake_comline2 (rust)", + "connections_default (typescript)", + "nested_protocol (rust)", + "dependency_and_own_protocols (typescript)", + "connections_subset (rust)", + "connections_subset (typescript)", + ] { + assert!(names.contains(&expected), "{expected} has no Hello: {names:?}"); + } + for (name, req, connection) in &hellos { + let (bytes, protocols) = facts::hello(req, connection).unwrap(); + let parts = facts::hello_parts(req, connection).unwrap(); + assert_eq!(parts.assemble_all(), bytes, "{name}"); + assert_eq!(parts.assemble(parts.protocols.iter().rev()), bytes, "{name}, given in reverse"); + // The same protocols, with what a generator writes beside the Hello. + let entries: Vec<_> = parts + .protocols + .iter() + .map(|p| (p.namespace.as_str(), p.name.as_str(), p.number, p.line.as_str(), p.functions.clone())) + .collect(); + let listed: Vec<_> = protocols + .iter() + .map(|p| (p.namespace.as_str(), p.name.as_str(), p.number, p.line.as_str(), p.functions.clone())) + .collect(); + assert_eq!(entries, listed, "{name}"); + // Canonical: ascending by number, each protocol's function numbers too. + assert!(parts.protocols.windows(2).all(|w| w[0].number < w[1].number), "{name}"); + assert!(parts.encodings.windows(2).all(|w| w[0] < w[1]), "{name}"); + } +} + +/// The corpus's `connections_subset`, with its connection. +fn subset() -> (GenerateRequest, connections::PackageConnection) { + let fx = corpus().into_iter().find(|fx| fx.name == "connections_subset").unwrap(); + let req = request(&fx, "rust"); + let connection = facts::comline2_connection(&req).unwrap().unwrap(); + (req, connection) +} + +/// `req` with only the protocols named `keep` left in, and in its facts: the +/// package a side that provides and calls just those would be, whose +/// package-wide Hello is what a subset's Hello must be. The protocols keep +/// their wire numbers, which the package's history gave them. +fn only(req: &GenerateRequest, keep: &[&str]) -> GenerateRequest { + let mut reduced = req.clone(); + for (_, units) in &mut reduced.schemas { + units.retain(|u| !matches!(u, FrozenUnit::Protocol { name, .. } if !keep.contains(&name.as_str()))); + } + reduced.facts.as_mut().unwrap().protocols.retain(|p| keep.contains(&p.name.as_str())); + reduced +} + +#[test] +fn the_subset_fixture_has_two_applications_with_a_protocol_per_side() { + let (req, connection) = subset(); + let parts = facts::hello_parts(&req, &connection).unwrap(); + let entries: Vec<_> = parts + .protocols + .iter() + .map(|p| (p.namespace.as_str(), p.name.as_str(), p.number, p.provider)) + .collect(); + assert_eq!( + entries, + [ + ("conformance", "Chat", 0, Provider::Server), + ("conformance", "Events", 1, Provider::Client), + ("conformance/admin", "Admin", 2, Provider::Server), + ("conformance/admin", "Alerts", 3, Provider::Client), + ] + ); + // The facts number functions backwards from their positions; the Hello + // lists their numbers ascending, and a generator keeps the names. + let chat = &parts.protocols[0]; + assert_eq!(chat.functions, [("send".to_string(), 1), ("leave".to_string(), 0)]); + assert_eq!(chat.function_numbers(), [0, 1]); +} + +#[test] +fn any_subset_assembled_is_the_hello_of_a_package_of_just_those_protocols() { + let (req, connection) = subset(); + let parts = facts::hello_parts(&req, &connection).unwrap(); + let names: Vec<&str> = parts.protocols.iter().map(|p| p.name.as_str()).collect(); + // The encodings are the connection's, so they're the same bytes in every + // Hello, up to the protocols: a Hello that lists none is those and a zero + // count. + let mut header = parts.assemble(std::iter::empty()); + assert_eq!(header.pop(), Some(0)); + + for mask in 1u32..(1 << names.len()) { + let chosen: Vec<&str> = + names.iter().enumerate().filter(|(i, _)| mask & (1 << i) != 0).map(|(_, n)| *n).collect(); + let reduced = only(&req, &chosen); + let connection = facts::comline2_connection(&reduced).unwrap().unwrap(); + let (expected, _) = facts::hello(&reduced, &connection).unwrap(); + + let entries: Vec<_> = parts.protocols.iter().filter(|p| chosen.contains(&p.name.as_str())).collect(); + let bytes = parts.assemble(entries.iter().copied()); + assert_eq!(bytes, expected, "{chosen:?}"); + assert_eq!(&bytes[..header.len()], &header[..], "{chosen:?}: the encodings are listed once, the same"); + + // However they are given, and however many times. + let mut shuffled: Vec<_> = entries.iter().rev().chain(entries.iter()).copied().collect(); + shuffled.rotate_left(chosen.len() / 2); + assert_eq!(parts.assemble(shuffled), expected, "{chosen:?}, out of order and doubled"); + } +} + +#[test] +fn a_hello_of_no_protocol_lists_none() { + let (req, connection) = subset(); + let parts = facts::hello_parts(&req, &connection).unwrap(); + let bytes = parts.assemble(std::iter::empty()); + assert_eq!(bytes.last(), Some(&0)); + assert!(bytes.len() < parts.assemble_all().len()); +}