diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..35049cb --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --" diff --git a/.github/workflows/format-gate.yml b/.github/workflows/format-gate.yml new file mode 100644 index 0000000..5f986f4 --- /dev/null +++ b/.github/workflows/format-gate.yml @@ -0,0 +1,46 @@ +name: Format gate + +# The IR format gate's checks that look at the repository (core#268): the +# seal of every superseded format (core/formats/MANIFEST), and, on a pull +# request, what the change to the format has to come with: a bump of +# CURRENT_IR_FORMAT_VERSION needs its step, its directory, the previous +# format's seal and fixtures; moved golden hashes +# (core/tests/migrations/expected) need a ledger row. See `cargo xtask`. +# +# The tests that need a build (`live == frozen(current)`, the golden fresh +# build) are core's own tests, in the Cargo Build & Test workflow. +on: + push: + branches: [master] + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + format_gate: + name: Format gate + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The merge base with the pull request's base. + fetch-depth: 0 + persist-credentials: false + - name: The seals, and what a change to the format comes with + env: + BASE: ${{ github.event.pull_request.base.sha }} + run: | + if [ -n "$BASE" ]; then + cargo xtask check-format-bump --base "$BASE" + else + cargo xtask check-format-bump + fi diff --git a/Cargo.toml b/Cargo.toml index 767de6f..d24eeb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "core", "core_stdlib", + "xtask", ] [profile.release] diff --git a/core/Cargo.toml b/core/Cargo.toml index b3cd4dd..83c27e2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -21,6 +21,9 @@ exclude = ["benches/benchmark_results.html", "benches/run_bench.sh", "fuzz/", "o [dependencies] # The embedded standard library (`use std::…`) comline-core-stdlib = { path = "../core_stdlib", version = "0.1.0" } +# The IR formats stored under `.comline/`, frozen: what an older store is +# decoded with, and what the live types are held to. In the IR-only build too. +comline-ir-formats = { path = "formats" } # The specification's catalog of shared names: guarantees, facts and their # values, the official encodings' aliases and frozen spellings. no_std and # dependency-free, so the IR-only build has it too. @@ -100,6 +103,10 @@ iai-callgrind = "=0.16.1" # crate. Exact pin on proptest, like iai-callgrind. proptest = "=1.5.0" comline-core-oracle = { path = "oracle", features = ["proptest"] } +# `live == frozen(current)` over values no real build produces +# (tests/migrations/live_frozen.rs). +comline-ir-formats = { path = "formats", features = ["arbitrary"] } +arbitrary = "1" [[bench]] name = "parser_benchmark" diff --git a/core/formats/Cargo.toml b/core/formats/Cargo.toml new file mode 100644 index 0000000..b4f02af --- /dev/null +++ b/core/formats/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "comline-ir-formats" +version = "0.0.0" +edition = "2021" +license = "GPL-3.0-only" +publish = false +rust-version = "1.82" +description = "The IR formats comline has stored under .comline/, frozen: types, codec, document and ledger of each (core#268)" + +# Internal, never published at first. One module per format (`v3`), each with +# its `SCHEMA.md`, the frozen copies of its types, its codec and its ledger. +# It has no dependency on `comline-core`, the way the oracle has none, so that +# a frozen type can't reach for today's: core depends on it to read old +# stores, and tests that `live == frozen(current)`. Its dependencies are what +# the IR-only (`wasm`) build carries; it never writes a store. See +# SCHEMA.md in each module and `formats/MANIFEST`. + +[features] +# `Arbitrary` for the frozen types, for core's `live == frozen` test over +# values no real build produces. +arbitrary = ["dep:arbitrary"] + +[dependencies] +serde = "1.0.164" +serde_derive = "1.0.164" +ciborium = "0.2" +arbitrary = { version = "1", features = ["derive"], optional = true } + +[dev-dependencies] +toml = "0.8" diff --git a/core/formats/MANIFEST b/core/formats/MANIFEST new file mode 100644 index 0000000..ca4276c --- /dev/null +++ b/core/formats/MANIFEST @@ -0,0 +1,5 @@ +# The seal of each superseded IR format (core#268): the blake3 of its +# directory under src/, over every file's relative path and bytes in path +# order. `cargo xtask check-seal` fails when one differs; a format is sealed +# when the next is introduced, with `cargo xtask seal-format --why `, +# which is also the only way to move a seal. diff --git a/core/formats/src/lib.rs b/core/formats/src/lib.rs new file mode 100644 index 0000000..eb6f71a --- /dev/null +++ b/core/formats/src/lib.rs @@ -0,0 +1,17 @@ +//! The IR formats comline has stored under `.comline/`, frozen (core#268). +//! +//! One module per format, `v`, each with its `SCHEMA.md`, `types`, `codec` +//! and `ledger.toml`. Nothing here depends on `comline-core`: a frozen type +//! can't reach for today's. Reading only; nothing in this crate writes a +//! store. +//! +//! A format is held in place by the gate (core#268, step 1.2): the current +//! one by core's `live == frozen(current)` test, and one that a later format +//! superseded by its seal in `formats/MANIFEST`. + +pub mod v3; + +/// The newest format this crate holds. Core's `CURRENT_IR_FORMAT_VERSION` +/// is tested to be this number: a bump that doesn't add the format here +/// fails. +pub const CURRENT: u32 = v3::FORMAT; diff --git a/core/formats/src/v3/SCHEMA.md b/core/formats/src/v3/SCHEMA.md new file mode 100644 index 0000000..7fba833 --- /dev/null +++ b/core/formats/src/v3/SCHEMA.md @@ -0,0 +1,265 @@ +# IR format 3 + +The format of what comline stores under `.comline/` when `FORMAT_VERSION` +holds `3`. This is the source of truth for the format: the frozen types in +[`types.rs`](types.rs) are its copy in code, [`codec.rs`](codec.rs) is the only +code that reads and writes its bytes, and [`ledger.toml`](ledger.toml) lists +what was added to it without a bump. The design is +[IR migrations](https://github.com/ComlineProject/docs/tree/main/docs/docs/proposals/ir-migrations) +(core#268). The guide on the CAS describes only the current format and links +here for the others. + +Format 3 is **the current format** and is not sealed: core's test +`live == frozen(current)` holds the live types to the copy in this directory. +The directory is sealed in `formats/MANIFEST` when format 4 is introduced. + +## Stamps + +| Format | Stamp | Introduced | What it is | +|---|---|---|---| +| 0 | none | 2026-01-10 | bincode, which tags fields and variants by position. Never marked, never migrated | +| 1 | `1` | 2026-10-11 | Name-tagged CBOR. The stamp is introduced | +| 2 | `2` | 2026-10-12 | `Property.expression: Option` became `value: ConfigValue` | +| 3 | `3` | 2026-10-18 | `KindValue` text became `TypeRef`, `Literal` and `Condition` | + +Formats 0 to 2 have no document and no frozen copy: nothing reads them, and a +store in one is never touched by a command on its own. + +## The store + +| Path | Holds | +|---|---| +| `FORMAT_VERSION` | The stamp: the format number as decimal text | +| `objects//` | One object each, named by the lowercase hex of its hash: the first two digits are the directory. The file is the object's CBOR, compressed with LZ4 (block, the size prepended). The hash is the BLAKE3 of the **uncompressed** CBOR | +| `refs/heads/main` | The hash of the newest commit, in hex. After a Git merge, conflict markers around two | +| `deps-cache/` | Git checkouts of dependencies. A cache, not part of the store | + +The container (the object layout, CBOR, LZ4, BLAKE3, `Blob`, `Tree`, `Commit`, +the ref file) is the same in every CBOR format so far. A change to it is a +format change like any other. + +## Objects + +CBOR here is what `ciborium` writes for the serde derive of each type: structs +as maps keyed by field name, enums externally tagged (a unit variant is its +name; any other, a one-entry map from the name), tuples and `Vec` as arrays, +`Option` as the value or null. A field marked *skipped when empty* is left out +of the map when it holds `None`, an empty `Vec` or an empty map, and a field +marked *defaults* reads as its default when absent. Both exist so that content +that doesn't use a field keeps the bytes it had before the field. + +**Hash** is a BLAKE3 digest, serialized as the array of its 32 bytes. + +**Blob** `{ content: bytes }`: raw content, which the blobs below describe. + +**Tree** `{ entries: [TreeEntry] }`, in the order written. A +`TreeEntry` is `{ mode, name, hash }`, where `mode` is `Blob` or `Tree`. + +**Commit** `{ tree, parents, author, timestamp, message, version, release? }` + +| Field | Meaning | Absent reads as | +|---|---|---| +| `tree` | The root tree's hash | | +| `parents` | Parent commits, first parent first. Walks follow the first; a second is what a reconcile after a Git merge records | | +| `author`, `message` | Text. The author is the user name | | +| `timestamp` | Unix seconds | | +| `version` | Semver: a release (`0.3.0`) or a build between releases (`0.3.0-dev.2`) | | +| `release` | Set on a commit `release` made. *Defaults, skipped when absent* | a release only if `version` has no pre-release part (3.05) | + +**Release** `{ computed, applied, reason, covered, stable, since }`: +`computed` and `applied` are a `VersionBump` (`None`, `Patch`, `Minor`, +`Major`), `reason` is optional text, `covered` and `stable` are booleans, and +`since` is the previous release's commit hash, if there is one. + +## A version's root tree + +The root tree of a commit holds: + +| Entry | Mode | Holds | Absent reads as | +|---|---|---|---| +| `schema_` | Tree | One schema file's frozen units, `` its index among the package's schemas | | +| `dep_` | Tree | A vendored dependency, `` its declared name: [below](#a-dependency-tree) | the dependency was not vendored (built without `deps`) | +| `config` | Blob | The package's frozen config: CBOR of `[ConfigUnit]` | the version was written before configs were kept: **unseen** | +| `docs` | Blob | The `//!` headers: CBOR of `FrozenDocs`. Written only if there are any | no headers (3.10) | +| `facts` | Blob | The protocols' history: CBOR of `History` | computed when read (3.11) | +| `unknown` | Blob | CBOR of `[UnknownRecord]`. **Reserved**: nothing writes it | no unknowns (3.21) | +| `produced_by` | Blob | CBOR of `ProducedBy`. **Reserved**: nothing writes it | unknown (3.22) | + +Entries are in the order a build adds them: the schemas, the dependencies, +`config`, `docs`, then `facts`. A reader finds entries by name and mode, and +skips a name it doesn't know; the ledger is why a name may be new. + +A `schema_` tree has one Blob per frozen unit of the file, in the order the +compiler froze them, each `{ content }` holding the CBOR of one `SchemaUnit`. +The names are `namespace_`, `struct_`, `enum_`, +`protocol_`, `const_`, `import_`, and `unit_` for +the rest; they only make a tree readable, and a reader doesn't depend on them. + +### A dependency tree + +`dep_` holds `schema_` trees like a root tree's, and Blobs: + +| Entry | Holds | Absent reads as | +|---|---|---| +| `manifest` | The content hash verified when the dependency was vendored, as text: its hex digits | | +| `config` | The dependency's own frozen config: CBOR of `[ConfigUnit]`, written even when empty | **unseen**: not empty, not defaulted (3.19) | +| `dependency_configs` | The frozen configs of the dependency's own dependencies: CBOR of `[(name, [ConfigUnit])]`, in the order the compile saw them. Written only where it has any | **unseen** (3.19) | + +## Schema units + +`SchemaUnit` (`FrozenUnit` in `comline_core::schema::ir::frozen::unit`) is +one of the variants below. `span` is `(start, end)`, byte offsets into the +source file. A span is part of a unit's identity on purpose: a change that only +moves text is a different blob. + +| Variant | Fields | +|---|---| +| `Namespace(String)`, `Name(String)` | | +| `Import(path, alias?, span)` | the resolved path; the `X` of `use p as X` | +| `Constant` | `docstring?`, `name`, `ty: TypeRef`, `value?: Literal`, `span` | +| `Property` | `name`, `value: ConfigValue`: an `@key = value` annotation | +| `ExpressionBlock` | `asserts: [SchemaUnit]` | +| `Assert` | `condition`, `message`, `references`, `expr: Condition`, `message_span?`, `uninterpolated` (the last two *defaults, skipped when empty*) | +| `Enum` | `docstring?`, `name`, `variants: [SchemaUnit]`, `span` | +| `EnumVariant(name, span)` | | +| `Settings` | `docstring?`, `name?`, `values: SettingsDict`, `spans: SettingsSpans` | +| `Struct` | `docstring?`, `parameters`, `name`, `fields`, `span` | +| `Protocol` | `docstring`, `parameters`, `name`, `functions`, `span` | +| `Function` | `docstring`, `parameters`, `name`, `arguments: [FrozenArgument]`, `_return?: TypeRef`, `throws: [u16]` (the `ordinal` of each error), `span` | +| `Error` | `docstring?`, `parameters`, `ordinal: u16`, `imported_from?`, `name`, `message`, `fields`, `span?` (*defaults*), `message_span?`, `uninterpolated` (*defaults, skipped when empty*) | +| `Validator` | `docstring?`, `properties`, `name`, `targets: [TypeRef]`, `expression_block: SchemaUnit`, `span` (*defaults*), `validate_span?` (*defaults, skipped when absent*) | +| `ValidatorRef` | `name`, `args`, `span` (*defaults*), `arg_spans` (*defaults*) | +| `Field` | `docstring?`, `parameters`, `optional`, `name`, `ty`, `default?: Literal`, `span` | +| `TypeAlias` | `docstring?`, `parameters` (*defaults*), `name`, `target: TypeRef`, `span` | +| `Deprecated(DeprecatedValue)` | `False`, `True`, `Message(String)`, `Malformed(String)` | +| `Unverified([Guarantee])` | a protocol's frozen set of given-up guarantees | + +`parameters` is a list of `Property`, `Unverified` and `Deprecated` units. +`FrozenArgument` is `{ name, ty, span, parameters }`, `parameters` *defaults, +skipped when empty*. `ConfigValue` is `Marker`, `Bool`, `Integer(i64)`, +`Str`, `Name`, `Path`, `FString` (text) or `Dict([(String, ConfigValue)])`. +`Guarantee` is `SchemaAgreement`, `EncodingAgreement`, `EncodingConformance`, +`VersionHonesty` or `BoundedSize`. + +### Types and values + +`TypeRef` is `Primitive(PrimitiveType)`, `Named(String)`, +`Array { element, length?: u64, max?: u64 }` (`max` *defaults, skipped when +absent*), `Union([TypeRef])` or `Unit`. `PrimitiveType` is `S8`, `S16`, `S32`, +`S64`, `U8`, `U16`, `U32`, `U64`, `F32`, `F64`, `Bool`, `String`, `U128`, +`S128` or `Char`. + +`Literal` is `Integer(i64)`, `Bool`, `String` or `Reference(String)`. +`Condition` is `{ first: Comparison, rest: [(LogicalOp, Comparison)] }`, a +`Comparison` is `{ left, op, right }` with `Operand` sides (`Path([String])`, +`Integer(i64)` or `String`), `LogicalOp` is `And` or `Or`, and `CompareOp` is +`Eq`, `Ne`, `Ge`, `Le`, `Gt` or `Lt`. + +`SettingsDict` is a map from a key to a `SettingsValue` (`Bool`, +`Integer(i64)`, `Str`, `Identifier` or `Dict(SettingsDict)`), ordered by key. +`SettingsSpans` is a map from a dotted path to a span, ordered by path. + +## Config units + +A `config` blob is a list of `ConfigUnit`, in the order the config was frozen: + +| Variant | Fields | +|---|---| +| `Namespace(String)`, `PackageId(String)`, `PackageVersion(String)`, `SchemaPath(String)` | | +| `SpecificationVersion(u8)` | the language edition the package chose, not this format | +| `Dependency` | `author`, `project`, `version` | +| `CodeGeneration` | `name`: a language and its version | +| `PublishRegistry((name, PublishRegistry))` | `kind` (`LocalStorage` or `RegistryServer`), `uri` | +| `Settings(SettingsDict)` | | +| `Handshake(String)`, `HandshakeParams(map)` | | +| `Wire` | `format?`, `framing?` | +| `Encoding` | `name`, `kind` (`format`, `framing` or `handshake`), `profile_hash`, `map: { name: { type, package?, version? } }`, `establishes: [Guarantee]` and `has_vectors` (*defaults*), `grammar?`, `calls?`, `directions?` (*defaults, skipped when absent*) | +| `Unverified(ConfigValue)` | | + +## The `docs` blob + +`FrozenDocs { package?, modules }`: the `//!` header of `config.idp`, and each +own module's by its path. Both *default, skipped when empty*; a module without +a header isn't listed. + +## The `facts` blob + +`History { protocols, retired, numbered }`, `retired` *defaults, skipped when +empty* and `numbered` *defaults* to `false`: + +- `ProtocolHistory { path, line, number, functions, retired, encodings }`: + `number` *defaults* to 0; `retired` and `encodings` *default, skipped when + empty*. +- `FunctionFacts { name, since, number }`: `number` *defaults* to 0. +- `Retired { number: u16, name }`. +- `EncodingFacts { kind, name, params: map, param_hash }`. + +It is a record of what the build computed under the rules in force, not an +input: a core that reads it may recompute it, and does when `numbered` is +false. + +## Reserved entries + +Two root-tree entries exist in the format, and nothing writes them yet. They +are here so that the readers fail closed before any step or build writes one +(`codec::decode_unknown`, `codec::decode_produced_by`, and core's +`require_known`). + +`unknown`: CBOR of `[UnknownRecord]`, each `{ path, step, reason, original? }`. +`path` is a JSON pointer into the version's model, `step` the step that +wrote it as `-`, `reason` why the format it came from could not know +it, and `original` the bytes that format recorded there, if it recorded any. +A version with a non-empty list has parts nobody recorded: a reader that can't +say what they are refuses the version. + +`produced_by`: CBOR of `ProducedBy { core, spec }`: the `comline-core` +version, and the specification version of the catalog it was built against. +A version without it reads as written by an unknown core and spec, never as +the reading ones. + +Both records refuse a field this copy of the format doesn't have. + +## Absence readings and the ledger + +Format 3 gained twenty-two additions without a bump. The rule that allowed it +was that a new entry, a new field with a default, or a new variant needs none, +and it is the ledger that makes it checked: a store holds commits written by +cores at different rows, so "format 3" doesn't say which of the rows below a +version has, and each row says what the version before it reads as. **Without a +bump, the only changes allowed are** + +- a new entry in a root or dependency tree whose absence reads as unknown; +- a new field with `#[serde(default)]` and `skip_serializing_if` for the + default, so that content which doesn't use it keeps its bytes; +- a new enum variant, if nothing existing needs it. + +Each is a row of [`ledger.toml`](ledger.toml), the lines in `types.rs`, and the +text here, in one change, and a change to what a build writes for a package +that doesn't use it moves the golden hashes +(`tests/migrations/expected/`), which the gate then requires a row for. +Any other change is a bump. + +| Row | Commit | Date | Added | A version written before it reads as | +|---|---|---|---|---| +| 3.01 | 0983015 | 2026-10-18 | validator call `span`, `arg_spans` | unknown span, `(0, 0)` | +| 3.02 | 8243d98 | 2026-10-18 | type alias `parameters` | none | +| 3.03 | ad2b3f5 | 2026-10-19 | `Assert.message_span`, `uninterpolated`; validator `validate_span` | none | +| 3.04 | bd9d9f4 | 2026-10-20 | `Unverified(Vec)` in a protocol's parameters; the `Guarantee` enum | no recorded set | +| 3.05 | af0371f | 2026-10-20 | `Commit.release` | a release only if the version has no pre-release part | +| 3.06 | 6571780 | 2026-10-20 | config units `Handshake`, `HandshakeParams` | the default handshake of the reading core | +| 3.07 | 031d74c | 2026-10-25 | config unit `Wire` | the default format and framing of the reading core | +| 3.08 | a5d6f85 | 2026-10-25 | `ConfigValue::Dict` | a variant older readers can't decode | +| 3.09 | f643d51 | 2026-10-31 | config unit `Encoding` | no declaration | +| 3.10 | 038591e | 2026-11-01 | root `docs` blob | no `//!` headers | +| 3.11 | f873d98 | 2026-11-01 | root `facts` blob; config unit `PackageId` | computed when read; no id | +| 3.12 | 4ba7c57 | 2026-11-01 | `Encoding.establishes`, `has_vectors` | the strict reading: nothing established, no vectors | +| 3.13 | 4eb6228 | 2026-11-02 | `PrimitiveType::Char`, `TypeRef::Array.max` | a variant older readers can't decode; no maximum | +| 3.14 | 2b6d636 | 2026-11-02 | `Guarantee::BoundedSize` | a variant older readers can't decode | +| 3.15 | 8ec27df | 2026-11-03 | facts' `number`, `retired`, `numbered` | `numbered = false`: computed when read | +| 3.16 | 4da05ab | 2026-11-07 | `Encoding.grammar`, `calls`, `directions` | none | +| 3.17 | bbbd957, 9a6482e | 2026-11-08 | facts' per-protocol `encodings`; the defaults moved to `datagram@3` and `comline@2` | resolved when read, with what an absent value was (`IMPLICIT_FRAMING`, `IMPLICIT_HANDSHAKE`) | +| 3.18 | 9ea142e | 2026-11-10 | config unit `Unverified` | none | +| 3.19 | 965e781 | 2026-11-12 | `dep_/config`, `dependency_configs` | unseen: not empty, not defaulted | +| 3.20 | 66feecb | 2026-11-12 | `FrozenArgument.parameters` (markers on arguments) | none | +| 3.21 | core#268, step 1.2 | 2026-11-15 | root `unknown` blob: what a step could not know, with the original (reserved: nothing writes it) | none: the version has no unknowns. A reader refuses a version that has one | +| 3.22 | core#268, step 1.2 | 2026-11-15 | root `produced_by` blob: the core and specification versions that wrote the version (reserved: nothing writes it) | unknown: not the reading core's, not defaulted | diff --git a/core/formats/src/v3/codec.rs b/core/formats/src/v3/codec.rs new file mode 100644 index 0000000..72205cd --- /dev/null +++ b/core/formats/src/v3/codec.rs @@ -0,0 +1,164 @@ +//! Format 3's bytes: the one place that encodes and decodes them. +//! +//! Every payload is CBOR, name-tagged by serde. A reader here never guesses: +//! bytes that don't decode, or that carry what this copy of the format +//! doesn't know, are an error and not an empty value. + +use std::fmt; + +use serde::de::DeserializeOwned; +use serde::Serialize; + +use super::types::{ConfigUnit, FrozenDocs, History, ProducedBy, SchemaUnit, UnknownRecord}; + +/// Bytes that are not a value of the format. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CodecError { + what: &'static str, + reason: String, +} + +impl fmt::Display for CodecError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "not a format 3 {}: {}", self.what, self.reason) + } +} + +impl std::error::Error for CodecError {} + +/// `value` as CBOR. +pub fn encode(value: &T) -> Result, CodecError> { + let mut bytes = Vec::new(); + ciborium::into_writer(value, &mut bytes).map_err(|e| CodecError { + what: "value", + reason: e.to_string(), + })?; + Ok(bytes) +} + +/// A `T` from CBOR, naming `what` it was meant to be when it isn't one. +pub fn decode(what: &'static str, bytes: &[u8]) -> Result { + ciborium::de::from_reader(bytes).map_err(|e| CodecError { + what, + reason: e.to_string(), + }) +} + +/// A schema unit's blob content. +pub fn decode_unit(bytes: &[u8]) -> Result { + decode("schema unit", bytes) +} + +/// A `config` blob's content: a version's, or a vendored dependency's. +pub fn decode_config(bytes: &[u8]) -> Result, CodecError> { + decode("config", bytes) +} + +/// A `docs` blob's content. +pub fn decode_docs(bytes: &[u8]) -> Result { + decode("docs", bytes) +} + +/// A `facts` blob's content. +pub fn decode_history(bytes: &[u8]) -> Result { + decode("facts", bytes) +} + +/// An `unknown` blob's content. Fails on a record with a field this copy of +/// the format doesn't have: such a version is refused, not read short. +pub fn decode_unknown(bytes: &[u8]) -> Result, CodecError> { + decode("unknown list", bytes) +} + +/// A `produced_by` blob's content, with the same refusal. +pub fn decode_produced_by(bytes: &[u8]) -> Result { + decode("produced_by record", bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record() -> UnknownRecord { + UnknownRecord { + path: "/schemas/0/1/ty".into(), + step: "3-4".into(), + reason: "format 3 froze `f32` as `float`".into(), + original: Some(b"float".to_vec()), + } + } + + #[test] + fn an_unknown_list_round_trips() { + let list = vec![ + record(), + UnknownRecord { + original: None, + ..record() + }, + ]; + assert_eq!(decode_unknown(&encode(&list).unwrap()).unwrap(), list); + assert_eq!( + decode_unknown(&encode(&Vec::::new()).unwrap()).unwrap(), + vec![] + ); + } + + #[test] + fn an_unknown_record_with_a_field_this_copy_lacks_is_refused() { + #[derive(Serialize)] + struct Newer { + path: String, + step: String, + reason: String, + confidence: u8, + } + let bytes = encode(&vec![Newer { + path: "/x".into(), + step: "3-4".into(), + reason: "r".into(), + confidence: 1, + }]) + .unwrap(); + assert!(decode_unknown(&bytes).is_err()); + } + + #[test] + fn bytes_that_are_not_an_unknown_list_are_an_error_not_an_empty_one() { + for bytes in [&b""[..], &b"\xff"[..], &b"\x01"[..], &b"\xa0"[..]] { + assert!(decode_unknown(bytes).is_err(), "{bytes:?}"); + } + } + + #[test] + fn produced_by_round_trips_and_refuses_what_it_does_not_know() { + let record = ProducedBy { + core: "0.1.0".into(), + spec: "0.11".into(), + }; + assert_eq!( + decode_produced_by(&encode(&record).unwrap()).unwrap(), + record + ); + + #[derive(Serialize)] + struct Newer { + core: String, + spec: String, + toolchain: String, + } + let newer = Newer { + core: "0.2.0".into(), + spec: "0.12".into(), + toolchain: "1.99".into(), + }; + let error = decode_produced_by(&encode(&newer).unwrap()).unwrap_err(); + assert!( + error + .to_string() + .starts_with("not a format 3 produced_by record"), + "{error}" + ); + assert!(decode_produced_by(b"").is_err()); + } +} diff --git a/core/formats/src/v3/ledger.toml b/core/formats/src/v3/ledger.toml new file mode 100644 index 0000000..474d90a --- /dev/null +++ b/core/formats/src/v3/ledger.toml @@ -0,0 +1,172 @@ +# The additions made to format 3 without a bump, one row per addition, in the +# order they were made (core#268, docs/proposals/ir-migrations/formats.md). +# +# id "3.": how SCHEMA.md and a fixture name the row +# commit where it was added; the rows from before this file are the +# commits that added them to `master` +# date when +# added what was added +# absent what a version written before it reads as: the absence reading +# fixture (optional) the store under tests/migrations/stores/format-3/ that +# has the row. None yet: the first fixtures are built for the step +# out of format 3, and `cargo xtask check-format-bump` requires one +# per row before format 4 can be introduced. +# +# A row is added in the same change as the lines in types.rs and SCHEMA.md +# that carry it, and never removed. The first twenty are what format 3 had +# gained when the gate was written; the last two are the entries that +# nothing writes yet. + +[[row]] +id = "3.01" +commit = "0983015" +date = "2026-10-18" +added = "validator call `span`, `arg_spans`" +absent = "unknown span, `(0, 0)`" + +[[row]] +id = "3.02" +commit = "8243d98" +date = "2026-10-18" +added = "type alias `parameters`" +absent = "none" + +[[row]] +id = "3.03" +commit = "ad2b3f5" +date = "2026-10-19" +added = "`Assert.message_span`, `uninterpolated`; validator `validate_span`" +absent = "none" + +[[row]] +id = "3.04" +commit = "bd9d9f4" +date = "2026-10-20" +added = "`Unverified(Vec)` in a protocol's parameters; the `Guarantee` enum" +absent = "no recorded set" + +[[row]] +id = "3.05" +commit = "af0371f" +date = "2026-10-20" +added = "`Commit.release`" +absent = "a release only if the version has no pre-release part" + +[[row]] +id = "3.06" +commit = "6571780" +date = "2026-10-20" +added = "config units `Handshake`, `HandshakeParams`" +absent = "the default handshake of the reading core" + +[[row]] +id = "3.07" +commit = "031d74c" +date = "2026-10-25" +added = "config unit `Wire`" +absent = "the default format and framing of the reading core" + +[[row]] +id = "3.08" +commit = "a5d6f85" +date = "2026-10-25" +added = "`ConfigValue::Dict`" +absent = "a variant older readers can't decode" + +[[row]] +id = "3.09" +commit = "f643d51" +date = "2026-10-31" +added = "config unit `Encoding`" +absent = "no declaration" + +[[row]] +id = "3.10" +commit = "038591e" +date = "2026-11-01" +added = "root `docs` blob" +absent = "no `//!` headers" + +[[row]] +id = "3.11" +commit = "f873d98" +date = "2026-11-01" +added = "root `facts` blob; config unit `PackageId`" +absent = "computed when read; no id" + +[[row]] +id = "3.12" +commit = "4ba7c57" +date = "2026-11-01" +added = "`Encoding.establishes`, `has_vectors`" +absent = "the strict reading: nothing established, no vectors" + +[[row]] +id = "3.13" +commit = "4eb6228" +date = "2026-11-02" +added = "`PrimitiveType::Char`, `TypeRef::Array.max`" +absent = "a variant older readers can't decode; no maximum" + +[[row]] +id = "3.14" +commit = "2b6d636" +date = "2026-11-02" +added = "`Guarantee::BoundedSize`" +absent = "a variant older readers can't decode" + +[[row]] +id = "3.15" +commit = "8ec27df" +date = "2026-11-03" +added = "facts' `number`, `retired`, `numbered`" +absent = "`numbered = false`: computed when read" + +[[row]] +id = "3.16" +commit = "4da05ab" +date = "2026-11-07" +added = "`Encoding.grammar`, `calls`, `directions`" +absent = "none" + +[[row]] +id = "3.17" +commit = "bbbd957, 9a6482e" +date = "2026-11-08" +added = "facts' per-protocol `encodings`; the defaults moved to `datagram@3` and `comline@2`" +absent = "resolved when read, with what an absent value was (`IMPLICIT_FRAMING`, `IMPLICIT_HANDSHAKE`)" + +[[row]] +id = "3.18" +commit = "9ea142e" +date = "2026-11-10" +added = "config unit `Unverified`" +absent = "none" + +[[row]] +id = "3.19" +commit = "965e781" +date = "2026-11-12" +added = "`dep_/config`, `dependency_configs`" +absent = "unseen: not empty, not defaulted" + +[[row]] +id = "3.20" +commit = "66feecb" +date = "2026-11-12" +added = "`FrozenArgument.parameters` (markers on arguments)" +absent = "none" + +[[row]] +id = "3.21" +commit = "core#268, step 1.2" +date = "2026-11-15" +added = "root `unknown` blob: what a step could not know, with the original (reserved: nothing writes it)" +absent = "none: the version has no unknowns. A reader refuses a version that has one" + +[[row]] +id = "3.22" +commit = "core#268, step 1.2" +date = "2026-11-15" +added = "root `produced_by` blob: the core and specification versions that wrote the version (reserved: nothing writes it)" +absent = "unknown: not the reading core's, not defaulted" diff --git a/core/formats/src/v3/mod.rs b/core/formats/src/v3/mod.rs new file mode 100644 index 0000000..69f1b87 --- /dev/null +++ b/core/formats/src/v3/mod.rs @@ -0,0 +1,38 @@ +//! IR format 3: `.comline/FORMAT_VERSION` holding `3`. +//! +//! [`types`] are the frozen copies of what the format stores, [`codec`] is +//! the only code that reads and writes its bytes, `SCHEMA.md` documents it +//! and `ledger.toml` lists what was added to it without a bump. The format +//! is not sealed while it is the current one; core's tests hold the live +//! types to it instead. + +pub mod codec; +pub mod types; + +/// The format's number. +pub const FORMAT: u32 = 3; + +/// The names of the entries a version's trees can hold. +pub mod entry { + /// A root tree's `schema_` subtrees, one per schema file, and a + /// dependency tree's. + pub const SCHEMA_PREFIX: &str = "schema_"; + /// A root tree's subtree for a vendored dependency, `dep_`. + pub const DEPENDENCY_PREFIX: &str = "dep_"; + /// The package's frozen config, a list of config units. + pub const CONFIG: &str = "config"; + /// The `//!` headers ([`FrozenDocs`](super::types::FrozenDocs)). + pub const DOCS: &str = "docs"; + /// The protocols' history ([`History`](super::types::History)). + pub const FACTS: &str = "facts"; + /// What the steps that made the version could not know (reserved). + pub const UNKNOWN: &str = "unknown"; + /// The core and the specification that wrote the version (reserved). + pub const PRODUCED_BY: &str = "produced_by"; + /// In a `dep_` tree: the content hash verified when vendored, as text. + pub const MANIFEST: &str = "manifest"; + /// In a `dep_` tree: the dependency's own frozen config. + pub const DEPENDENCY_CONFIG: &str = "config"; + /// In a `dep_` tree: the configs of the dependency's dependencies. + pub const DEPENDENCY_CONFIGS: &str = "dependency_configs"; +} diff --git a/core/formats/src/v3/types.rs b/core/formats/src/v3/types.rs new file mode 100644 index 0000000..e79cc3d --- /dev/null +++ b/core/formats/src/v3/types.rs @@ -0,0 +1,543 @@ +//! The frozen types of IR format 3, as they are today. +//! +//! Copies of the types core persists under `.comline/`, not re-exports: a +//! change to a live type can't change format 3 by changing this file's view +//! of it. Serde derives and nothing else, with every `#[serde(...)]` +//! attribute that decides the bytes. No behaviour and no dependency on +//! `comline-core` (the `live == frozen(3)` test is in core, which depends on +//! this crate). +//! +//! Names differ from core's where core has two types of one name: core's +//! `schema::ir::frozen::unit::FrozenUnit` is [`SchemaUnit`] here, and +//! `package::config::ir::frozen::FrozenUnit` is [`ConfigUnit`]. A CBOR +//! encoding names fields and variants, never types, so that changes no +//! bytes. [`EncodingKind`] is core's `ProfileKind`. +//! +//! Where each type is stored, and what an absent field or entry means, is in +//! `SCHEMA.md` beside this file. + +use std::collections::BTreeMap; + +use serde_derive::{Deserialize, Serialize}; + +// ---------- objects: the container ---------- + +/// A BLAKE3 digest, serialized as the 32 bytes' sequence. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Hash(pub [u8; 32]); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Blob { + pub content: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Tree { + pub entries: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct TreeEntry { + pub mode: EntryMode, + pub name: String, + pub hash: Hash, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum EntryMode { + Blob, + Tree, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Commit { + pub tree: Hash, + pub parents: Vec, + pub author: String, + pub timestamp: i64, + pub message: String, + pub version: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub release: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Release { + pub computed: VersionBump, + pub applied: VersionBump, + pub reason: Option, + pub covered: bool, + pub stable: bool, + pub since: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum VersionBump { + None, + Patch, + Minor, + Major, +} + +// ---------- schema units: one blob per unit, in a `schema_` tree ---------- + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum SchemaUnit { + Namespace(String), + Name(String), + Import(String, Option, (usize, usize)), + Constant { + docstring: Option, + name: String, + ty: TypeRef, + value: Option, + span: (usize, usize), + }, + Property { + name: String, + value: ConfigValue, + }, + ExpressionBlock { + asserts: Vec, + }, + Assert { + condition: String, + message: String, + references: Vec, + expr: Condition, + #[serde(default, skip_serializing_if = "Option::is_none")] + message_span: Option<(usize, usize)>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + uninterpolated: Vec, + }, + Enum { + docstring: Option, + name: String, + variants: Vec, + span: (usize, usize), + }, + EnumVariant(String, (usize, usize)), + Settings { + docstring: Option, + name: Option, + values: SettingsDict, + spans: SettingsSpans, + }, + Struct { + docstring: Option, + parameters: Vec, + name: String, + fields: Vec, + span: (usize, usize), + }, + Protocol { + docstring: String, + parameters: Vec, + name: String, + functions: Vec, + span: (usize, usize), + }, + Function { + docstring: String, + parameters: Vec, + name: String, + arguments: Vec, + _return: Option, + throws: Vec, + span: (usize, usize), + }, + Error { + docstring: Option, + parameters: Vec, + ordinal: u16, + imported_from: Option, + name: String, + message: String, + fields: Vec, + #[serde(default)] + span: Option<(usize, usize)>, + #[serde(default, skip_serializing_if = "Option::is_none")] + message_span: Option<(usize, usize)>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + uninterpolated: Vec, + }, + Validator { + docstring: Option, + properties: Vec, + name: String, + targets: Vec, + expression_block: Box, + #[serde(default)] + span: (usize, usize), + #[serde(default, skip_serializing_if = "Option::is_none")] + validate_span: Option<(usize, usize)>, + }, + ValidatorRef { + name: String, + args: Vec, + #[serde(default)] + span: (usize, usize), + #[serde(default)] + arg_spans: Vec<(usize, usize)>, + }, + Field { + docstring: Option, + parameters: Vec, + optional: bool, + name: String, + ty: TypeRef, + default: Option, + span: (usize, usize), + }, + TypeAlias { + docstring: Option, + #[serde(default)] + parameters: Vec, + name: String, + target: TypeRef, + span: (usize, usize), + }, + Deprecated(DeprecatedValue), + Unverified(Vec), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ConfigValue { + Marker, + Bool(bool), + Integer(i64), + Str(String), + Name(String), + Path(String), + FString(String), + Dict(Vec<(String, ConfigValue)>), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum DeprecatedValue { + False, + True, + Message(String), + Malformed(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct FrozenArgument { + pub name: String, + pub ty: TypeRef, + pub span: (usize, usize), + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub parameters: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum Guarantee { + SchemaAgreement, + EncodingAgreement, + EncodingConformance, + VersionHonesty, + BoundedSize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum PrimitiveType { + S8, + S16, + S32, + S64, + U8, + U16, + U32, + U64, + F32, + F64, + Bool, + String, + U128, + S128, + Char, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum TypeRef { + Primitive(PrimitiveType), + Named(String), + Array { + element: Box, + length: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + max: Option, + }, + Union(Vec), + Unit, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum Literal { + Integer(i64), + Bool(bool), + String(String), + Reference(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Condition { + pub first: Comparison, + pub rest: Vec<(LogicalOp, Comparison)>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Comparison { + pub left: Operand, + pub op: CompareOp, + pub right: Operand, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum LogicalOp { + And, + Or, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum CompareOp { + Eq, + Ne, + Ge, + Le, + Gt, + Lt, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum Operand { + Path(Vec), + Integer(i64), + String(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum SettingsValue { + Bool(bool), + Integer(i64), + Str(String), + Identifier(String), + Dict(SettingsDict), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct SettingsDict(pub BTreeMap); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct SettingsSpans(pub BTreeMap); + +// ---------- config units: the `config` blob and a dependency's ---------- + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ConfigUnit { + Namespace(String), + PackageId(String), + SpecificationVersion(u8), + PackageVersion(String), + SchemaPath(String), + Dependency(Dependency), + CodeGeneration(LanguageDetails), + PublishRegistry((String, PublishRegistry)), + Settings(SettingsDict), + Handshake(String), + HandshakeParams(BTreeMap), + Wire { + format: Option, + framing: Option, + }, + Encoding { + name: String, + kind: EncodingKind, + profile_hash: String, + map: EncodingMap, + #[serde(default)] + establishes: Vec, + #[serde(default)] + has_vectors: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + grammar: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + calls: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + directions: Option, + }, + Unverified(ConfigValue), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Dependency { + pub author: String, + pub project: String, + pub version: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct LanguageDetails { + pub name: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct PublishRegistry { + pub kind: RegistryKind, + pub uri: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum RegistryKind { + LocalStorage, + RegistryServer, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[serde(rename_all = "snake_case")] +pub enum EncodingKind { + Format, + Framing, + Handshake, +} + +pub type EncodingMap = BTreeMap; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct EncodingMapping { + #[serde(rename = "type")] + pub ty: String, + pub package: Option, + pub version: Option, +} + +// ---------- the `docs` blob ---------- + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct FrozenDocs { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub package: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub modules: BTreeMap, +} + +// ---------- the `facts` blob ---------- + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct History { + pub protocols: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub retired: Vec, + #[serde(default)] + pub numbered: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ProtocolHistory { + pub path: String, + pub line: String, + #[serde(default)] + pub number: u16, + pub functions: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub retired: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub encodings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct FunctionFacts { + pub name: String, + pub since: String, + #[serde(default)] + pub number: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Retired { + pub number: u16, + pub name: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct EncodingFacts { + pub kind: String, + pub name: String, + pub params: BTreeMap, + pub param_hash: String, +} + +// ---------- reserved: nothing writes these yet ---------- + +/// One element of the root tree's `unknown` blob (a CBOR array of these): +/// something a step could not know, and what it recorded. Reserved: no +/// release writes it yet, and a reader that meets one refuses the version +/// (`codec::decode_unknown`, and core's `require_known`). +/// `deny_unknown_fields`, so that a record with a field this build doesn't +/// know is refused rather than read short. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct UnknownRecord { + /// A JSON pointer into the version's model. + pub path: String, + /// The step that wrote it, as `"-"` (`"3-4"`). + pub step: String, + /// Why the format it came from could not know it. + pub reason: String, + /// What that format recorded there, where it recorded something. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub original: Option>, +} + +/// The root tree's `produced_by` blob: the core and the specification that +/// wrote the version. Reserved: no build writes it yet. A version without it +/// was written by one that is not known, and reads as unknown, never as the +/// reading core. `deny_unknown_fields`, as for [`UnknownRecord`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct ProducedBy { + /// The `comline-core` version, as `CARGO_PKG_VERSION` spells it. + pub core: String, + /// The specification the catalog was taken from (`comline_catalog::SPEC_VERSION`). + pub spec: String, +} diff --git a/core/formats/tests/documents.rs b/core/formats/tests/documents.rs new file mode 100644 index 0000000..d465bed --- /dev/null +++ b/core/formats/tests/documents.rs @@ -0,0 +1,111 @@ +//! A format's document and its ledger say the same things. + +use std::fs; +use std::path::Path; + +fn read(relative: &str) -> String { + fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join(relative)) + .unwrap_or_else(|e| panic!("{relative}: {e}")) +} + +/// The ledger's rows: `(id, added, absent)`. +fn rows(format: u32) -> Vec<(String, String, String)> { + let table: toml::Table = read(&format!("src/v{format}/ledger.toml")) + .parse() + .expect("ledger.toml is TOML"); + table["row"] + .as_array() + .expect("`row` is an array of tables") + .iter() + .map(|row| { + let field = |name: &str| { + row[name] + .as_str() + .unwrap_or_else(|| panic!("a row has no `{name}`: {row}")) + .to_string() + }; + for name in ["commit", "date"] { + field(name); + } + (field("id"), field("added"), field("absent")) + }) + .collect() +} + +#[test] +fn the_current_format_is_the_one_this_crate_ends_with() { + assert_eq!(comline_ir_formats::CURRENT, 3); + assert_eq!(comline_ir_formats::v3::FORMAT, 3); +} + +#[test] +fn the_ledger_is_numbered_in_order_and_each_row_says_what_it_added_and_what_absence_reads_as() { + let rows = rows(3); + assert!(rows.len() >= 22, "the ledger is a history and only grows"); + for (n, (id, added, absent)) in rows.iter().enumerate() { + assert_eq!(id, &format!("3.{:02}", n + 1)); + assert!(!added.is_empty() && !absent.is_empty(), "{id}"); + } +} + +#[test] +fn the_document_has_each_ledger_row_and_only_those() { + let schema = read("src/v3/SCHEMA.md"); + assert!(schema.starts_with("# IR format 3\n")); + + let table: Vec> = schema + .lines() + .filter(|l| l.starts_with("| 3.")) + .map(|l| l.trim_matches('|').split('|').map(str::trim).collect()) + .collect(); + let rows = rows(3); + assert_eq!( + table.len(), + rows.len(), + "SCHEMA.md's ledger table and ledger.toml differ in length" + ); + for (cells, (id, added, absent)) in table.iter().zip(&rows) { + assert_eq!(cells[0], id); + assert_eq!(cells[3], added, "{id}: what was added"); + assert_eq!(cells[4], absent, "{id}: the absence reading"); + } +} + +#[test] +fn every_row_a_document_cites_exists() { + let schema = read("src/v3/SCHEMA.md"); + let ids: Vec = rows(3).into_iter().map(|(id, _, _)| id).collect(); + for cited in schema.split("(3.").skip(1) { + let id = format!("3.{}", &cited[..2]); + assert!( + ids.contains(&id), + "SCHEMA.md cites {id}, which the ledger doesn't have" + ); + } +} + +#[test] +fn the_format_directory_has_its_four_files() { + for file in ["SCHEMA.md", "types.rs", "codec.rs", "ledger.toml"] { + assert!( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("src/v3") + .join(file) + .is_file(), + "{file}" + ); + } +} + +#[test] +fn nothing_is_sealed_while_format_3_is_current() { + let manifest = read("MANIFEST"); + let sealed: Vec<&str> = manifest + .lines() + .filter(|l| !l.trim().is_empty() && !l.starts_with('#')) + .collect(); + assert!( + sealed.is_empty(), + "format 3 is current and is sealed when format 4 is introduced: {sealed:?}" + ); +} diff --git a/core/src/compat.rs b/core/src/compat.rs index 46580fc..9ee911b 100644 --- a/core/src/compat.rs +++ b/core/src/compat.rs @@ -184,6 +184,7 @@ pub struct EncodingFacts { /// A version's stored history, if its tree has one. pub fn read_history(store: &ObjectStore, tree: &Tree) -> Result> { + crate::package::build::cas::entries::require_known(store, tree)?; let Some(entry) = tree.entries.iter().find(|e| e.name == FACTS_ENTRY && e.mode == EntryMode::Blob) else { return Ok(None); diff --git a/core/src/doc.rs b/core/src/doc.rs index 32943f1..c95c3e5 100644 --- a/core/src/doc.rs +++ b/core/src/doc.rs @@ -518,6 +518,7 @@ pub fn write_docs(docs: &FrozenDocs, store: &ObjectStore) -> eyre::Result /// The headers a built version's root tree froze; empty for a version built /// before they were stored, or for a package without any. pub fn read_docs(store: &ObjectStore, root: &Tree) -> eyre::Result { + crate::package::build::cas::entries::require_known(store, root)?; let Some(entry) = root .entries .iter() diff --git a/core/src/package/build/cas/build.rs b/core/src/package/build/cas/build.rs index 1eefac5..d2de170 100644 --- a/core/src/package/build/cas/build.rs +++ b/core/src/package/build/cas/build.rs @@ -169,6 +169,11 @@ pub fn compare(store: &ObjectStore, old: &Tree, new: &Tree) -> Result` entry is a vendored // dependency's own copy, kept so the commit is reproducible. let local = |tree: &Tree| -> Result> { diff --git a/core/src/package/build/cas/entries.rs b/core/src/package/build/cas/entries.rs new file mode 100644 index 0000000..3ed1ac4 --- /dev/null +++ b/core/src/package/build/cas/entries.rs @@ -0,0 +1,90 @@ +// The root-tree entries the IR migrations framework reserves (core#268): +// what a step could not know (`unknown`), and the core and specification that +// wrote a version (`produced_by`). Nothing writes either yet. They are read +// here first, and fail closed, so that no step or build can write one into a +// store that a reader would take for complete. + +use std::fmt; + +use comline_ir_formats::v3::codec; +use comline_ir_formats::v3::entry::{PRODUCED_BY, UNKNOWN}; +use comline_ir_formats::v3::types::{ProducedBy, UnknownRecord}; +use eyre::{eyre, Result}; + +use super::objects::{Blob, EntryMode, Tree}; +use super::ObjectStore; + +/// A version records parts that the format it came from could not know, and +/// this comline can't say what they are: whatever it would read from the +/// version it would read as complete. **Nothing was changed**, and the +/// version isn't used. Recovered with `downcast_ref` from the `eyre::Report` +/// it travels in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VersionHasUnknowns { + /// What the version's `unknown` entry lists. + pub records: Vec, +} + +impl fmt::Display for VersionHasUnknowns { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let paths: Vec<&str> = self.records.iter().map(|r| r.path.as_str()).collect(); + write!( + f, + "this version records {} part(s) that its IR format could not know ({}), and this \ + comline can't read them yet: it doesn't use the rest of the version as if it were \ + complete. Nothing was changed. Use a comline that reads them.", + self.records.len(), + paths.join(", ") + ) + } +} + +impl std::error::Error for VersionHasUnknowns {} + +/// The blob a root tree holds under `name`, if it holds one. An entry of that +/// name that isn't a blob is an error: the name is reserved, not skipped. +fn read_entry(store: &ObjectStore, tree: &Tree, name: &str) -> Result>> { + let Some(entry) = tree.find_entry(name) else { + return Ok(None); + }; + if entry.mode != EntryMode::Blob { + return Err(eyre!("the `{name}` entry of a version is not a blob")); + } + Ok(Some(Blob::from_bytes(&store.read(&entry.hash)?)?.content)) +} + +/// What the root tree's `unknown` entry lists: nothing for a version without +/// one. An entry that doesn't decode, or that has a field this comline +/// doesn't know, is an error and not an empty list. +pub fn read_unknown(store: &ObjectStore, tree: &Tree) -> Result> { + match read_entry(store, tree, UNKNOWN)? { + None => Ok(vec![]), + Some(bytes) => { + codec::decode_unknown(&bytes).map_err(|e| eyre!("the `{UNKNOWN}` entry: {e}")) + } + } +} + +/// Fail closed: an error ([`VersionHasUnknowns`]) if the version has unknowns, +/// or if its `unknown` entry can't be read. Called before a version is used +/// as complete: compared, documented, its history read. +pub fn require_known(store: &ObjectStore, tree: &Tree) -> Result<()> { + let records = read_unknown(store, tree)?; + if records.is_empty() { + Ok(()) + } else { + Err(VersionHasUnknowns { records }.into()) + } +} + +/// The core and specification that wrote the version, if it says. `None` is +/// **unknown**, not the reading core's: a version without the entry was +/// written before it existed, or by a step that couldn't know. +pub fn read_produced_by(store: &ObjectStore, tree: &Tree) -> Result> { + match read_entry(store, tree, PRODUCED_BY)? { + None => Ok(None), + Some(bytes) => codec::decode_produced_by(&bytes) + .map(Some) + .map_err(|e| eyre!("the `{PRODUCED_BY}` entry: {e}")), + } +} diff --git a/core/src/package/build/cas/mod.rs b/core/src/package/build/cas/mod.rs index f4d28ac..6a97bb1 100644 --- a/core/src/package/build/cas/mod.rs +++ b/core/src/package/build/cas/mod.rs @@ -28,6 +28,7 @@ pub mod refs; // Git-style references management #[cfg(feature = "compiler")] pub mod build; // Build process implementation pub mod version; // Version bump types +pub mod entries; // Entries reserved for unknowns and who wrote a version #[cfg(feature = "compiler")] pub mod release; // Releases on top of builds #[cfg(feature = "compiler")] diff --git a/core/src/package/build/cas/object_store.rs b/core/src/package/build/cas/object_store.rs index f5d7e27..1aaf9ef 100644 --- a/core/src/package/build/cas/object_store.rs +++ b/core/src/package/build/cas/object_store.rs @@ -10,8 +10,8 @@ use std::path::{Path, PathBuf}; /// `specification_version` (that's the user's own chosen language edition, /// unrelated to core's internal IR representation). /// -/// CBOR tags struct fields and enum variants by name, so growing or -/// reordering `FrozenUnit` needs no bump here - only *renaming* or +/// CBOR tags struct fields and enum variants by name, so growing (with a +/// ledger row, below) or reordering `FrozenUnit` needs no bump here - only *renaming* or /// *removing* an existing field/variant does, since that's the one case a /// self-describing format can't absorb for free. Bump this by hand when you /// do one of those; [`ensure_compatible_store`] then refuses any `.comline/` @@ -20,6 +20,13 @@ use std::path::{Path, PathBuf}; /// changes nothing: no store is deleted, moved or rewritten to change its /// format (core#268, `docs/proposals/ir-migrations`). /// +/// What a format is, and what may change in it without a bump, is written +/// down and checked: `formats/src/v3/` holds its `SCHEMA.md`, the frozen +/// copy of its types and a ledger of every addition made without a bump. +/// A test holds the live types to the copy, golden hashes hold what a fresh +/// build writes, and `cargo xtask check-format-bump` says what a bump has +/// to come with. +/// /// History: /// - 1: CBOR, replacing bincode's position-based encoding. /// - 2: `FrozenUnit::Property`'s `expression: Option` became @@ -29,9 +36,10 @@ use std::path::{Path, PathBuf}; /// `KindValue` text: `Field`/`Constant` `kind_value` became `ty` plus a /// literal, `EnumVariant` holds its name, and `Assert` gained `expr`. /// -/// Additions that needed no bump: the root tree's `docs` blob (the `//!` -/// headers, `crate::doc::FrozenDocs`, core#145) is a new entry that older -/// readers skip and newer ones treat as absent on older versions. So are a +/// Additions that needed no bump (each is a row of the ledger): the root +/// tree's `docs` blob (the `//!` headers, `crate::doc::FrozenDocs`, core#145) +/// is a new entry that older readers skip and newer ones treat as absent on +/// older versions. So are a /// vendored dependency's `config` and `dependency_configs` blobs, in its /// `dep_` tree (core#251): a tree without them reads as having not /// kept the dependency's config diff --git a/core/tests/cas/entries_tests.rs b/core/tests/cas/entries_tests.rs new file mode 100644 index 0000000..67b35f7 --- /dev/null +++ b/core/tests/cas/entries_tests.rs @@ -0,0 +1,188 @@ +// The entries the IR migrations framework reserves (core#268, step 1.2): +// `unknown` and `produced_by`. Nothing writes them yet, so these trees are +// made by hand; what is tested is that every reader fails closed. + +use comline_core::doc; +use comline_core::package::build::cas::entries::{ + read_produced_by, read_unknown, require_known, VersionHasUnknowns, +}; +use comline_core::package::build::cas::objects::{Blob, EntryMode, Tree}; +use comline_core::package::build::cas::storage::to_cbor; +use comline_core::package::build::cas::ObjectStore; +use comline_ir_formats::v3::types::{ProducedBy, UnknownRecord}; +use serde_derive::Serialize; +use tempfile::TempDir; + +fn store() -> (TempDir, ObjectStore) { + let dir = TempDir::new().unwrap(); + let store = ObjectStore::new(dir.path()); + store.init().unwrap(); + (dir, store) +} + +/// A root tree with one entry, `name`, of `mode`, holding the blob `content`. +fn tree_with(store: &ObjectStore, name: &str, mode: EntryMode, content: &[u8]) -> Tree { + let hash = store + .write(&Blob::new(content.to_vec()).to_bytes().unwrap()) + .unwrap(); + let mut tree = Tree::new(); + tree.add_entry(mode, name.to_string(), hash); + tree +} + +fn record() -> UnknownRecord { + UnknownRecord { + path: "/schemas/0/units/4/ty".into(), + step: "3-4".into(), + reason: "format 3 froze `f32` as `float`".into(), + original: Some(b"float".to_vec()), + } +} + +fn unknown_entry(store: &ObjectStore, records: &[UnknownRecord]) -> Tree { + tree_with( + store, + "unknown", + EntryMode::Blob, + &to_cbor(records).unwrap(), + ) +} + +#[test] +fn a_version_without_the_entries_has_no_unknowns_and_an_unknown_writer() { + let (_dir, store) = store(); + let tree = Tree::new(); + assert_eq!(read_unknown(&store, &tree).unwrap(), vec![]); + require_known(&store, &tree).unwrap(); + assert_eq!(read_produced_by(&store, &tree).unwrap(), None); +} + +#[test] +fn an_empty_unknown_list_is_no_unknowns() { + let (_dir, store) = store(); + require_known(&store, &unknown_entry(&store, &[])).unwrap(); +} + +#[test] +fn a_version_with_unknowns_is_refused_and_says_what_they_are() { + let (_dir, store) = store(); + let tree = unknown_entry(&store, &[record()]); + assert_eq!(read_unknown(&store, &tree).unwrap(), vec![record()]); + + let error = require_known(&store, &tree).unwrap_err(); + let refused = error + .downcast_ref::() + .expect("a typed error"); + assert_eq!(refused.records, vec![record()]); + let message = error.to_string(); + assert!( + message.contains("/schemas/0/units/4/ty") && message.contains("Nothing was changed"), + "{message}" + ); +} + +#[test] +fn an_unknown_entry_that_cannot_be_read_is_an_error_and_not_an_empty_list() { + let (_dir, store) = store(); + + #[derive(Serialize)] + struct Newer { + path: String, + step: String, + reason: String, + severity: u8, + } + let newer = [Newer { + path: "/x".into(), + step: "3-4".into(), + reason: "r".into(), + severity: 2, + }]; + let cases = [ + ( + "garbage", + tree_with(&store, "unknown", EntryMode::Blob, b"\xff\xff"), + ), + ("empty", tree_with(&store, "unknown", EntryMode::Blob, b"")), + ( + "a field this comline doesn't know", + tree_with( + &store, + "unknown", + EntryMode::Blob, + &to_cbor(&newer).unwrap(), + ), + ), + ( + "a tree, not a blob", + tree_with( + &store, + "unknown", + EntryMode::Tree, + &to_cbor(&[record()]).unwrap(), + ), + ), + ]; + for (what, tree) in cases { + let error = require_known(&store, &tree).expect_err(what); + assert!( + error.downcast_ref::().is_none(), + "{what}: {error}" + ); + assert!(read_unknown(&store, &tree).is_err(), "{what}"); + } +} + +#[test] +fn produced_by_is_read_when_there_and_refused_when_it_cannot_be() { + let (_dir, store) = store(); + let writer = ProducedBy { + core: "0.1.0".into(), + spec: "0.11".into(), + }; + let tree = tree_with( + &store, + "produced_by", + EntryMode::Blob, + &to_cbor(&writer).unwrap(), + ); + assert_eq!(read_produced_by(&store, &tree).unwrap(), Some(writer)); + + let damaged = tree_with(&store, "produced_by", EntryMode::Blob, b"\xff"); + assert!(read_produced_by(&store, &damaged).is_err()); + let tree = tree_with(&store, "produced_by", EntryMode::Tree, b""); + assert!(read_produced_by(&store, &tree).is_err()); +} + +#[test] +fn every_consumer_of_a_version_refuses_one_with_unknowns() { + use comline_core::package::build::cas::build::compare; + + let (_dir, store) = store(); + let with = unknown_entry(&store, &[record()]); + let without = Tree::new(); + + let is_refused = |error: eyre::Report| error.downcast_ref::().is_some(); + assert!( + is_refused(doc::read_docs(&store, &with).unwrap_err()), + "doc" + ); + assert!( + is_refused(comline_core::compat::read_history(&store, &with).unwrap_err()), + "history" + ); + assert!( + is_refused(compare(&store, &with, &without).map(|_| ()).unwrap_err()), + "compare, old" + ); + assert!( + is_refused(compare(&store, &without, &with).map(|_| ()).unwrap_err()), + "compare, new" + ); + + // And the same readers take a version without any. + doc::read_docs(&store, &without).unwrap(); + assert!(comline_core::compat::read_history(&store, &without) + .unwrap() + .is_none()); +} diff --git a/core/tests/cas/ir_roundtrip_tests.rs b/core/tests/cas/ir_roundtrip_tests.rs index 5252293..95218b4 100644 --- a/core/tests/cas/ir_roundtrip_tests.rs +++ b/core/tests/cas/ir_roundtrip_tests.rs @@ -15,78 +15,9 @@ use comline_core::schema::ir::frozen::cas::blob::{ }; use comline_core::schema::ir::frozen::unit::FrozenUnit; -/// A schema using every declaration kind and type shape the grammar has. -const TOUR: &str = r#" -use std::validators::StringBounds - -/// Limits. -const MAX_NAME: u32 = 64 -const GREETING: string = "hello" -const STRICT: bool = true - -type UserId = u64 -type Names = char[max 64][max 16] - -validator Range for u8, u32 { - min: u32 = 0 - max: u32 = 100 - validate { - assert(value >= params.min and value <= params.max, f"{value.name} is out of range") - } -} - -enum Role { - Admin - Member -} - -@deprecated = "use User" -struct LegacyUser { - id: UserId -} - -settings Strict { - validation.unknown = "error" -} - -@settings = Strict -struct User { - /// The user's id. - id: UserId - @validators = [StringBounds(min_chars = 1, max_chars = 32)] - name: char[max 32] - @validators = [Range(min = 1, max = 99)] - age: u8 = 18 - optional nickname: char[max 32] - role: Role - tags: Names - scores: u32[max 100] - digest: u8[32] - grid: s16[max 8][max 8] - ratio: f64 - weight: f32 = 1 - either: union(char[max 16] u32) - flag: bool = false -} - -error NotFound { - message = "user {id} not found" - id: UserId -} - -error Denied { - message = "denied" -} - -protocol Users { - /// Look a user up. - @timeout_ms = 500 - function get(id: UserId) -> User ! NotFound, Denied; - function ping() -> (); - @idempotent - function forget(UserId); -} -"#; +/// A schema using every declaration kind and type shape the grammar has: the +/// `tour` package of the migrations corpus. +const TOUR: &str = include_str!("../migrations/corpus/tour/src/tour.ids"); fn units_of(context: &ProjectContext) -> Vec<(String, Vec)> { context diff --git a/core/tests/cas/mod.rs b/core/tests/cas/mod.rs index 74536af..5418eeb 100644 --- a/core/tests/cas/mod.rs +++ b/core/tests/cas/mod.rs @@ -19,3 +19,4 @@ pub mod std_reach_tests; pub mod compat_tests; pub mod reconcile_tests; pub mod store_never_wiped_tests; +pub mod entries_tests; diff --git a/core/tests/migrations/corpus.rs b/core/tests/migrations/corpus.rs new file mode 100644 index 0000000..b0ce842 --- /dev/null +++ b/core/tests/migrations/corpus.rs @@ -0,0 +1,123 @@ +// The fixture packages, and a fresh build of each. + +use std::fs; +use std::path::{Path, PathBuf}; + +use comline_core::package::build::build; +use comline_core::package::build::cas::objects::{Commit, Tree}; +use comline_core::package::build::cas::{REF_HEAD_MAIN_FILENAME, read_ref, ObjectStore}; +use tempfile::TempDir; + +/// One package a migration is proved on. +pub struct Fixture { + /// The name its golden file has (`expected/.txt`). + pub name: &'static str, + /// The directory it is copied from, under `core/tests/`. + pub source: &'static str, + /// The package inside the copy that is built: `.` for the directory + /// itself, or one of several that depend on each other. + pub package: &'static str, + /// Whether it needs dependency resolution (the `deps` feature). + pub deps: bool, +} + +/// The corpus, in the order it is documented. Only ever added to. +pub const FIXTURES: &[Fixture] = &[ + // A schema that uses every construct the grammar has. + Fixture { + name: "tour", + source: "migrations/corpus/tour", + package: ".", + deps: false, + }, + // The package the CAS tests build. + Fixture { + name: "test", + source: "fixtures/packages/test", + package: ".", + deps: false, + }, + // `//!` headers: the `docs` blob. + Fixture { + name: "headers", + source: "migrations/corpus/headers", + package: ".", + deps: false, + }, + // `@unverified` at each scope, `handshake = "none"` and a `wire`. + Fixture { + name: "unverified", + source: "migrations/corpus/unverified", + package: ".", + deps: false, + }, + // A dependency that declares an encoding and has one of its own that + // declares another: `dep_` trees, `config` and `dependency_configs`. + Fixture { + name: "dependencies", + source: "migrations/corpus/dependencies", + package: "app", + deps: true, + }, +]; + +/// The fixtures this build of the tests can build. +pub fn available() -> impl Iterator { + FIXTURES + .iter() + .filter(|f| !f.deps || cfg!(feature = "deps")) +} + +pub fn tests_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests") +} + +fn copy_dir(from: &Path, to: &Path) { + fs::create_dir_all(to).unwrap(); + for entry in fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + fs::copy(entry.path(), target).unwrap(); + } + } +} + +/// A fixture, copied to a scratch directory and built once. +pub struct Built { + _dir: TempDir, + /// The package that was built. + pub package: PathBuf, + pub store: ObjectStore, +} + +impl Built { + /// A copy of `fixture`, not built yet. + pub fn copy(fixture: &Fixture) -> Self { + let dir = TempDir::new().unwrap(); + copy_dir(&tests_dir().join(fixture.source), dir.path()); + let package = dir.path().join(fixture.package); + let store = ObjectStore::new(&package); + Self { + _dir: dir, + package, + store, + } + } + + pub fn of(fixture: &Fixture) -> Self { + let built = Self::copy(fixture); + build(&built.package).unwrap_or_else(|e| panic!("{} builds: {e:?}", fixture.name)); + built + } + + /// The commit `refs/heads/main` points to, and its root tree. + pub fn head(&self) -> (Commit, Tree) { + let hash = read_ref(&self.package, REF_HEAD_MAIN_FILENAME).unwrap(); + let commit = Commit::from_bytes(&self.store.read(&hash).unwrap()).unwrap(); + let tree = Tree::from_bytes(&self.store.read(&commit.tree).unwrap()).unwrap(); + (commit, tree) + } +} diff --git a/core/tests/migrations/corpus/dependencies/acme/config.idp b/core/tests/migrations/corpus/dependencies/acme/config.idp new file mode 100644 index 0000000..abe68f5 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/acme/config.idp @@ -0,0 +1,9 @@ +congregation acme +specification_version = 1 +settings = { annotations = { framing = { allowed = true } unverified = { allowed = true } } } +dependencies = { + base = { path = "../base" } +} +encodings = { + "com.acme.lines@1" = { kind = "framing" profile = "profiles/lines.json" } +} diff --git a/core/tests/migrations/corpus/dependencies/acme/profiles/lines.json b/core/tests/migrations/corpus/dependencies/acme/profiles/lines.json new file mode 100644 index 0000000..9cf07cb --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/acme/profiles/lines.json @@ -0,0 +1,11 @@ +{ + "name": "com.acme.lines@1", + "kind": "framing", + "facts": { + "calls": { "value": "by_name", "proven_by": ["lines/request"] }, + "directions": { "value": "both", "proven_by": ["lines/request"] }, + "errors": { "value": "by_name", "proven_by": ["lines/error"] }, + "payload": { "value": "json", "proven_by": ["lines/request"] }, + "protocol": { "value": "named", "proven_by": ["lines/request"] } + } +} diff --git a/core/tests/migrations/corpus/dependencies/acme/profiles/lines.vectors.json b/core/tests/migrations/corpus/dependencies/acme/profiles/lines.vectors.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/acme/profiles/lines.vectors.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/core/tests/migrations/corpus/dependencies/acme/src/api.ids b/core/tests/migrations/corpus/dependencies/acme/src/api.ids new file mode 100644 index 0000000..59e8f4b --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/acme/src/api.ids @@ -0,0 +1,3 @@ +struct Thing { + id: u64 +} diff --git a/core/tests/migrations/corpus/dependencies/app/config.idp b/core/tests/migrations/corpus/dependencies/app/config.idp new file mode 100644 index 0000000..e24b0f6 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/app/config.idp @@ -0,0 +1,7 @@ +congregation app +specification_version = 1 +settings = { annotations = { framing = { allowed = true } unverified = { allowed = true } } } +dependencies = { + acme = { path = "../acme" } +} +wire = { framing = "com.acme.lines@1" } diff --git a/core/tests/migrations/corpus/dependencies/app/src/api.ids b/core/tests/migrations/corpus/dependencies/app/src/api.ids new file mode 100644 index 0000000..e688565 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/app/src/api.ids @@ -0,0 +1,7 @@ +struct S { + a: u32 +} + +protocol P { + function get() -> S; +} diff --git a/core/tests/migrations/corpus/dependencies/base/config.idp b/core/tests/migrations/corpus/dependencies/base/config.idp new file mode 100644 index 0000000..9c66652 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/base/config.idp @@ -0,0 +1,6 @@ +congregation base +specification_version = 1 +settings = { annotations = { framing = { allowed = true } unverified = { allowed = true } } } +encodings = { + "com.acme.cbor@1" = { kind = "format" profile = "profiles/cbor.json" } +} diff --git a/core/tests/migrations/corpus/dependencies/base/profiles/cbor.json b/core/tests/migrations/corpus/dependencies/base/profiles/cbor.json new file mode 100644 index 0000000..a84c8af --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/base/profiles/cbor.json @@ -0,0 +1,13 @@ +{ + "name": "com.acme.cbor@1", + "kind": "format", + "facts": { + "struct_fields": { "value": "by_name", "proven_by": [] }, + "unknown_field": { "value": "ignored", "proven_by": [] }, + "missing_optional": { "value": "absent", "proven_by": [] }, + "enum_variants": { "value": "by_name", "proven_by": [] }, + "unknown_variant": { "value": "rejected", "proven_by": [] }, + "integer_width": { "value": "by_value", "proven_by": [] }, + "output": { "value": "bytes", "proven_by": [] } + } +} diff --git a/core/tests/migrations/corpus/dependencies/base/profiles/cbor.vectors.json b/core/tests/migrations/corpus/dependencies/base/profiles/cbor.vectors.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/base/profiles/cbor.vectors.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/core/tests/migrations/corpus/dependencies/base/src/api.ids b/core/tests/migrations/corpus/dependencies/base/src/api.ids new file mode 100644 index 0000000..2f117a3 --- /dev/null +++ b/core/tests/migrations/corpus/dependencies/base/src/api.ids @@ -0,0 +1,3 @@ +struct Base { + id: u64 +} diff --git a/core/tests/migrations/corpus/headers/config.idp b/core/tests/migrations/corpus/headers/config.idp new file mode 100644 index 0000000..c6ab5bf --- /dev/null +++ b/core/tests/migrations/corpus/headers/config.idp @@ -0,0 +1,6 @@ +//! A package with headers. +//! +//! The second paragraph of the package's header. + +congregation headers +specification_version = 1 diff --git a/core/tests/migrations/corpus/headers/src/api.ids b/core/tests/migrations/corpus/headers/src/api.ids new file mode 100644 index 0000000..d578906 --- /dev/null +++ b/core/tests/migrations/corpus/headers/src/api.ids @@ -0,0 +1,11 @@ +//! The API module. + +/// A ping. +struct Ping { + seq: u32 +} + +/// Answers a ping. +protocol Pinger { + function ping(p: Ping) -> Ping; +} diff --git a/core/tests/migrations/corpus/headers/src/bare.ids b/core/tests/migrations/corpus/headers/src/bare.ids new file mode 100644 index 0000000..bad8f03 --- /dev/null +++ b/core/tests/migrations/corpus/headers/src/bare.ids @@ -0,0 +1,3 @@ +struct Pong { + seq: u32 +} diff --git a/core/tests/migrations/corpus/tour/config.idp b/core/tests/migrations/corpus/tour/config.idp new file mode 100644 index 0000000..dbccdaa --- /dev/null +++ b/core/tests/migrations/corpus/tour/config.idp @@ -0,0 +1,8 @@ +congregation tour +specification_version = 1 + +code_generation = { + languages = { + rust#1.85.0 = {} + } +} diff --git a/core/tests/migrations/corpus/tour/src/tour.ids b/core/tests/migrations/corpus/tour/src/tour.ids new file mode 100644 index 0000000..fc155b9 --- /dev/null +++ b/core/tests/migrations/corpus/tour/src/tour.ids @@ -0,0 +1,70 @@ + +use std::validators::StringBounds + +/// Limits. +const MAX_NAME: u32 = 64 +const GREETING: string = "hello" +const STRICT: bool = true + +type UserId = u64 +type Names = char[max 64][max 16] + +validator Range for u8, u32 { + min: u32 = 0 + max: u32 = 100 + validate { + assert(value >= params.min and value <= params.max, f"{value.name} is out of range") + } +} + +enum Role { + Admin + Member +} + +@deprecated = "use User" +struct LegacyUser { + id: UserId +} + +settings Strict { + validation.unknown = "error" +} + +@settings = Strict +struct User { + /// The user's id. + id: UserId + @validators = [StringBounds(min_chars = 1, max_chars = 32)] + name: char[max 32] + @validators = [Range(min = 1, max = 99)] + age: u8 = 18 + optional nickname: char[max 32] + role: Role + tags: Names + scores: u32[max 100] + digest: u8[32] + grid: s16[max 8][max 8] + ratio: f64 + weight: f32 = 1 + either: union(char[max 16] u32) + flag: bool = false +} + +error NotFound { + message = "user {id} not found" + id: UserId +} + +error Denied { + message = "denied" +} + +protocol Users { + /// Look a user up. + @timeout_ms = 500 + function get(id: UserId) -> User ! NotFound, Denied; + function ping() -> (); + @idempotent + function forget(UserId); +} diff --git a/core/tests/migrations/corpus/unverified/config.idp b/core/tests/migrations/corpus/unverified/config.idp new file mode 100644 index 0000000..97646ff --- /dev/null +++ b/core/tests/migrations/corpus/unverified/config.idp @@ -0,0 +1,13 @@ +congregation unverified +specification_version = 1 + +settings = { annotations = { unverified = { allowed = true } } } + +handshake = "none" +wire = { format = "json@1" framing = "datagram@3" } + +code_generation = { + languages = { + rust#1.85.0 = {} + } +} diff --git a/core/tests/migrations/corpus/unverified/src/mail.ids b/core/tests/migrations/corpus/unverified/src/mail.ids new file mode 100644 index 0000000..ec81711 --- /dev/null +++ b/core/tests/migrations/corpus/unverified/src/mail.ids @@ -0,0 +1,14 @@ +@unverified(bounded_size = "legacy blob") +type Blob = string + +struct Message { + @unverified(bounded_size = "legacy blob") + body: string + id: u64 +} + +@unverified = { schema_agreement = "pinned out of band", encoding_agreement = "one codebase" } +protocol Mail { + function send(@unverified(bounded_size = "legacy blob") note: string, message: Message); + function stash(blob: Blob); +} diff --git a/core/tests/migrations/expected/dependencies.txt b/core/tests/migrations/expected/dependencies.txt new file mode 100644 index 0000000..398f5d1 --- /dev/null +++ b/core/tests/migrations/expected/dependencies.txt @@ -0,0 +1,25 @@ +# The objects a fresh build of this fixture package writes (core#268): each +# entry of the version's root tree, recursively, with its hash, as +# ` ` (a tree's path ends in `/`). The standard library's +# schemas are compiled into every build and aren't recorded. +# These move only with a ledger row or a format bump. Regenerate: +# COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden +f17991caa9359f62cd6f44a221f071de26941e8f2ac08e17e7c89d1d7738b731 schema_0/ +9c72d0cb016fb272dc1b98aa028e57655fd788c6466b853871c70e1060d00cb9 schema_0/namespace_api +c934e2c3708ab4ec23c5bee7a4d575532e2231faa5966db6fd4e6e89a5729872 schema_0/struct_S +4ce161d760ce061cae27b6a6faf27f3c9b97006b7b8980363e982e94ddde62db schema_0/protocol_P +d68ad8e1474206db87c917b5b59f69c4b20b34cc86e4cf5996940242ee363031 schema_1/ +1d6d177bc0aacff1945dc2fee61549a8fe41778f56d1c943e49243f3db4b7f59 schema_1/namespace_acme::api +6ef6da6ca27d5bed33bd14b9f3ce488184d27d9f00a864d480f2fa9d4b013c84 schema_1/struct_Thing +2b7d10eeef3bb2304a4e4240f4786e993453ae533bb1344578d90b33498e96dd dep_acme/ +625abf3e6225eff2e6373de24f1c9e573cfc0c55ff015fe186ebfed20d58da79 dep_acme/schema_0/ +9c72d0cb016fb272dc1b98aa028e57655fd788c6466b853871c70e1060d00cb9 dep_acme/schema_0/namespace_api +6ef6da6ca27d5bed33bd14b9f3ce488184d27d9f00a864d480f2fa9d4b013c84 dep_acme/schema_0/struct_Thing +5697b80a0edc2485bcc452bec97fa5156a8ac24d61b0ca8a4ad52420c78f971b dep_acme/schema_1/ +b1605906dbed3541ecf998af25bb24d07733cbd6b0dabb0946afc95f3b8991c2 dep_acme/schema_1/namespace_base::api +2f29a339943185dda90772614da40755a0d0020c4f4cfed6bf2133efb1da4db0 dep_acme/schema_1/struct_Base +3f52b6636e818df6da938a4216a01006cdaf8f86f665bca590c7e465411c5512 dep_acme/manifest +02d6cf7ca34df430c25e95c7482917d23c1e1296f4bf3d02c5b344ab3cfa0753 dep_acme/config +df319a59f0e1bf9705a7924eeca52681147b3035d8d10bd7d30e7e34a1a047f4 dep_acme/dependency_configs +883dc7289e77d2bc4dedf4c99e7407fa356f3416313e7f57196c9c7499268fc3 config +5f106eb3a858cebcfc246fccc9728e0ddd3cfa82c302231f942a16da1c7cc658 facts diff --git a/core/tests/migrations/expected/headers.txt b/core/tests/migrations/expected/headers.txt new file mode 100644 index 0000000..cfe309e --- /dev/null +++ b/core/tests/migrations/expected/headers.txt @@ -0,0 +1,16 @@ +# The objects a fresh build of this fixture package writes (core#268): each +# entry of the version's root tree, recursively, with its hash, as +# ` ` (a tree's path ends in `/`). The standard library's +# schemas are compiled into every build and aren't recorded. +# These move only with a ledger row or a format bump. Regenerate: +# COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden +22f6ebd3b5ebd7188472133287c318ac69b61258226cd0ee907c155498ee5587 schema_0/ +9c72d0cb016fb272dc1b98aa028e57655fd788c6466b853871c70e1060d00cb9 schema_0/namespace_api +65c42d5d1c676934b9a70b9a5a5f281c652cc19a2006f62cb1e6acd0c955cd21 schema_0/struct_Ping +d76d71aa3af81e38053b39ffb2827e0e8449105a1d9071787fb5b3b5103902b5 schema_0/protocol_Pinger +41db64954221d249f3cdb5cb4fbfcdb0af49e4a953d66ea990e80d6e1804b5d7 schema_1/ +b8006c2a8a63ff122e2938e9155df21c17f456390947867cd0255797e0bff38e schema_1/namespace_bare +307fd8ee67f98aa7337af05f4e89d1088b4e0237e54fb03adc8047f2c5835990 schema_1/struct_Pong +4e7b4eab6b80c67bdb16d309583c602fa3282c2e7260672819361129ceffa7cd config +94d6a7138535940caab2b1cd0f4297b528d6d8c025dcac7c5b3f591ec257e89a docs +9907bd17c7ab8df8275f3fd96fe90c4e8183014c93f2a2ad5d38b2c442f5f71e facts diff --git a/core/tests/migrations/expected/test.txt b/core/tests/migrations/expected/test.txt new file mode 100644 index 0000000..de610d8 --- /dev/null +++ b/core/tests/migrations/expected/test.txt @@ -0,0 +1,19 @@ +# The objects a fresh build of this fixture package writes (core#268): each +# entry of the version's root tree, recursively, with its hash, as +# ` ` (a tree's path ends in `/`). The standard library's +# schemas are compiled into every build and aren't recorded. +# These move only with a ledger row or a format bump. Regenerate: +# COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden +c2292624fe6e84a71c78aa71ef1ad8203d55e854c443e53e56dd0f16a94b17d7 schema_0/ +373fda9bc517fdf47fc92768efea5fff6ba55d9ee0d104595d7ec75f3afcfd9e schema_0/namespace_health +660957249a223491d4c706f1a9108a1e7094629958a029517db97b408d365cdc schema_0/struct_Capabilities +ee39a37757823afc5e217b018a978424bdd7a42a47c4e07daa879b0abb4d7283 schema_0/protocol_HealthCheck +62c20565165085bec641359579b4cb0bc39abfa64081cbb70e03a2eba1c16748 schema_1/ +c113ffcd4b11b8793f404dd1b06218f5d595a14412030758200d2ca422758445 schema_1/namespace_ping +2527227422d11420c678d7ee48e887672cfc61a6bea73dc343771254d7841d09 schema_1/import_1 +6c5c7579cd67e95c5563b773c7453afae42f1f88b50344d8c2b23bd9af39ba60 schema_1/const_LOW_PING_RATE +1f20e6d35544392fa29a88b3a875967647b5baf32d664be539ad167c6a0b6968 schema_1/protocol_Ping +e93f5ea69c6232afeda1921b968114e77ffb33086ece5943bf3341cad4593724 schema_2/ +753a0d189b371384992d76a29f9a75a4e0e80604b8dfdb7e14a3868961865c58 schema_2/namespace_utils::test +fe22eea8959cf1ccb5a9de0119bc746e42650b675edf9c93de4b3aecbda4fd9f config +a5d9e50987ef5e5571af3580a231289625574ceae72d3839b622d5d605854d7c facts diff --git a/core/tests/migrations/expected/tour.txt b/core/tests/migrations/expected/tour.txt new file mode 100644 index 0000000..28bd2f8 --- /dev/null +++ b/core/tests/migrations/expected/tour.txt @@ -0,0 +1,25 @@ +# The objects a fresh build of this fixture package writes (core#268): each +# entry of the version's root tree, recursively, with its hash, as +# ` ` (a tree's path ends in `/`). The standard library's +# schemas are compiled into every build and aren't recorded. +# These move only with a ledger row or a format bump. Regenerate: +# COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden +a4c3687e96c132608795540fb3c4992bb414d1067f443a2a817a4e91ed41087c schema_0/ +0d118251eb6d357fa3240d2b5c3d14d312fe9c833cbd7b6655290e07ba10f654 schema_0/namespace_tour +a8fda6bc0529b4ae86ffa18840374b92f2de97e6769aabda2dfc4753e8ebd817 schema_0/import_1 +302c24934a12492ad3ed249ba29b9bff17ea3d2d77cdf9e202a7dd03be35e3f4 schema_0/const_MAX_NAME +8d44a2f110fdc34aa35798f2764d1cd20e39846bb26ce36ee0bdd34f479b9776 schema_0/const_GREETING +aecdd5636e4ed8dcd4f0c79816971497920e38d29e4a4faece0fa5538f18ce8a schema_0/const_STRICT +6123a69d4ed247b2ae0f1c8e2a840037428a20dd012c3d050e7c84cb9e6c1b06 schema_0/unit_5 +269b09077c79ffdf2e94da5ce2ca5b9202151e8c4e8ec529be6c6ba578f11421 schema_0/unit_6 +a584f7ae55dd9d83a374dcdcb45b06fad2c21add86b0eb22d583df049c14edd9 schema_0/unit_7 +3880bbd88849bdc8e64fd759278b8a26701e0e4f44f73718447ea02bde67fa9f schema_0/enum_Role +62e961898a2dcfc1f06fdbdef91efc5d022b9ea582db5f92f0cfb83a8cba5a7e schema_0/struct_LegacyUser +da8d0f0d3cd8755069b08f00ed0bd566c8a9791cc518034767ac9b4a2356b6e1 schema_0/unit_10 +9b2a40410f0837a32dc734c861d620f788d3a927524adeede0d2d08e3e7785b6 schema_0/struct_User +19c1cdb55f45aa43e93e42c7a9220ce2195e942d048e83f055debd3915df5da7 schema_0/unit_12 +bcfae81d4e9dd4ec3b6f4144c412923555e8796832f4d99d5e6b33de119ca894 schema_0/unit_13 +381ac80dbc49019211c4b2db9d95d35c9cfc91720ba858251cc0c71563a83f12 schema_0/protocol_Users +- schema_1/ (standard library, not recorded) +cd96832b2835fee19bbe37804d7d28d79365ac6a8d8741ba7ea5d72485ffa34f config +c688ba8996c406b04654ebc9c71e3b72372775bb4256ff47320c615512f35b6a facts diff --git a/core/tests/migrations/expected/unverified.txt b/core/tests/migrations/expected/unverified.txt new file mode 100644 index 0000000..2089f7f --- /dev/null +++ b/core/tests/migrations/expected/unverified.txt @@ -0,0 +1,13 @@ +# The objects a fresh build of this fixture package writes (core#268): each +# entry of the version's root tree, recursively, with its hash, as +# ` ` (a tree's path ends in `/`). The standard library's +# schemas are compiled into every build and aren't recorded. +# These move only with a ledger row or a format bump. Regenerate: +# COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden +766b414e8da615c1bdb0e237964bc70c1fda7f46e5b968aaed2647ca54e09fe8 schema_0/ +4732c3cea971ad47c197a979b6aea750a763ac6f24081c23f0fcb566e8d02719 schema_0/namespace_mail +54a32f355dd2140a6c2fd29c5d2a79adcfa8b280f02268e38dc3d60b6f201742 schema_0/unit_1 +0086a1e9901d7788d2f5bdd1dfa6fdeb8266e44b1e55e010b9767c79216605e5 schema_0/struct_Message +6d6ff2a118b6ac6ecc2fc6dc7dde930157b30b37c1257d3398010162eb01a0eb schema_0/protocol_Mail +e8c7700b2f600ac654f23a2ef4b49dfb876d177a44f0d618d909c9d81f25c212 config +80b731323b06d294eb8cd23b3c7e45511abef53ab1b90e8e1b3aa5f3e920f746 facts diff --git a/core/tests/migrations/golden.rs b/core/tests/migrations/golden.rs new file mode 100644 index 0000000..897ac1b --- /dev/null +++ b/core/tests/migrations/golden.rs @@ -0,0 +1,130 @@ +// Golden fresh-build hashes (core#268, step 1.2). +// +// A fresh build of each fixture package writes the same objects as the file +// in `expected/` says, to the byte: one line for every entry of the version's +// root tree, recursively, with its hash. Without it nothing notices a change +// to what a build writes for a package that doesn't use the thing that +// changed, which is what a field added without `skip_serializing_if` is. +// +// A line that moves is a ledger row (and the same lines in the frozen copy) +// or a format bump, and `cargo xtask check-format-bump` says so on the pull +// request. The standard library's schemas are compiled into every build and +// are not recorded: a change to `std` is not a change to the format. To +// regenerate after such a change: +// +// COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden + +use std::fs; + +use comline_core::package::build::cas::objects::{EntryMode, Tree}; +use comline_core::package::build::cas::ObjectStore; + +use super::corpus::{self, Built, Fixture}; + +const HEADER: &str = "\ +# The objects a fresh build of this fixture package writes (core#268): each +# entry of the version's root tree, recursively, with its hash, as +# ` ` (a tree's path ends in `/`). The standard library's +# schemas are compiled into every build and aren't recorded. +# These move only with a ledger row or a format bump. Regenerate: +# COMLINE_UPDATE_GOLDEN=1 cargo test -p comline-core --features deps --test mod migrations::golden +"; + +/// A schema tree of the standard library: its first unit is the namespace. +fn is_std(tree: &Tree) -> bool { + tree.entries + .first() + .is_some_and(|e| e.name.starts_with("namespace_std")) +} + +fn describe(store: &ObjectStore, tree: &Tree, prefix: &str, lines: &mut Vec) { + for entry in &tree.entries { + let path = format!("{prefix}{}", entry.name); + match entry.mode { + EntryMode::Blob => lines.push(format!("{} {path}", entry.hash)), + EntryMode::Tree => { + let tree = Tree::from_bytes(&store.read(&entry.hash).unwrap()).unwrap(); + if is_std(&tree) { + lines.push(format!("- {path}/ (standard library, not recorded)")); + } else { + lines.push(format!("{} {path}/", entry.hash)); + describe(store, &tree, &format!("{path}/"), lines); + } + } + } + } +} + +fn golden(fixture: &Fixture) -> String { + let built = Built::of(fixture); + let (_, tree) = built.head(); + let mut lines = vec![]; + describe(&built.store, &tree, "", &mut lines); + format!("{HEADER}{}\n", lines.join("\n")) +} + +#[test] +fn a_fresh_build_of_each_fixture_writes_the_recorded_objects() { + let update = std::env::var_os("COMLINE_UPDATE_GOLDEN").is_some(); + for fixture in corpus::available() { + let path = corpus::tests_dir().join(format!("migrations/expected/{}.txt", fixture.name)); + let fresh = golden(fixture); + if update { + fs::write(&path, &fresh).unwrap(); + continue; + } + let expected = fs::read_to_string(&path).unwrap_or_else(|e| { + panic!( + "{}: no golden file ({e}); COMLINE_UPDATE_GOLDEN=1 writes it", + fixture.name + ) + }); + if fresh != expected { + let moved: Vec = expected + .lines() + .zip(fresh.lines()) + .filter(|(a, b)| a != b) + .map(|(a, b)| format!(" was {a}\n now {b}")) + .collect(); + panic!( + "{}: a fresh build writes different objects than {} records ({} line(s) differ, or the number of lines):\n{}\n\ + This is a change to the format: a ledger row, with the same lines in the frozen copy \ + (core/formats/src/v{}), or a bump. See core/formats/src/v{}/SCHEMA.md.", + fixture.name, + path.display(), + moved.len(), + moved.join("\n"), + comline_ir_formats::CURRENT, + comline_ir_formats::CURRENT, + ); + } + } +} + +#[test] +fn a_build_is_the_same_twice() { + // The golden hashes are only worth anything if a build is deterministic: + // two builds of one package in two directories write the same objects. + for fixture in corpus::available() { + assert_eq!(golden(fixture), golden(fixture), "{}", fixture.name); + } +} + +#[test] +fn every_golden_file_is_a_fixture_of_the_corpus() { + let dir = corpus::tests_dir().join("migrations/expected"); + let mut files: Vec = fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + files.sort(); + let mut names: Vec = corpus::FIXTURES + .iter() + .map(|f| format!("{}.txt", f.name)) + .collect(); + names.sort(); + assert_eq!( + files, names, + "a fixture without its golden file, or a golden file without its fixture" + ); +} diff --git a/core/tests/migrations/live_frozen.rs b/core/tests/migrations/live_frozen.rs new file mode 100644 index 0000000..af3f929 --- /dev/null +++ b/core/tests/migrations/live_frozen.rs @@ -0,0 +1,266 @@ +// `live == frozen(current)` (core#268, step 1.2). +// +// The types core persists under `.comline/` and the frozen copy of the +// current format (`comline-ir-formats`) have to encode to the same bytes and +// decode each other's. Three ways of looking: every object of the stores the +// fixture packages build (what core really writes, with a release and a +// second build so a commit has a parent and a release record), values of the +// frozen types no build produces (what only a frozen copy that is wrong +// about a field shows), and the names the trees use. +// +// A change to a live type without the same change in the frozen copy fails +// here. So does a root-tree entry the frozen copy doesn't know: it is a +// ledger row. + +use std::fmt::Debug; +use std::fs; + +use arbitrary::{Arbitrary, Unstructured}; +use comline_core::compat::{self, History}; +use comline_core::doc::{self, FrozenDocs}; +use comline_core::package::build::cas::object_store::CURRENT_IR_FORMAT_VERSION; +use comline_core::package::build::cas::objects::{Blob, Commit, EntryMode, Tree}; +use comline_core::package::build::cas::storage::{from_cbor, to_cbor}; +use comline_core::package::build::cas::ObjectStore; +use comline_core::package::build::{build, release, ReleaseRequest}; +use comline_core::package::config::ir::frozen::cas::blob as config_blob; +use comline_core::package::config::ir::frozen::FrozenUnit as LiveConfigUnit; +use comline_core::schema::ir::frozen::unit::FrozenUnit as LiveUnit; +use comline_ir_formats::v3::codec; +use comline_ir_formats::v3::entry; +use comline_ir_formats::v3::types as frozen; +use proptest::prelude::*; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use super::corpus::{self, Built}; + +/// `bytes` decode as a `L` (the live type) and as a `F` (the frozen copy), +/// and each encodes back to `bytes`. +fn same(what: &str, bytes: &[u8]) +where + L: Serialize + DeserializeOwned, + F: Serialize + DeserializeOwned, +{ + let live: L = + from_cbor(bytes).unwrap_or_else(|e| panic!("{what}: the live type can't decode it: {e}")); + let copy: F = codec::decode("value", bytes) + .unwrap_or_else(|e| panic!("{what}: the frozen copy can't decode what core wrote: {e}")); + assert_eq!( + to_cbor(&live).unwrap(), + bytes, + "{what}: the live type encodes it differently" + ); + assert_eq!( + codec::encode(©).unwrap(), + bytes, + "{what}: the frozen copy encodes it differently" + ); +} + +fn blob_content(store: &ObjectStore, hash: &comline_core::package::build::cas::Hash) -> Vec { + let bytes = store.read(hash).unwrap(); + same::("a blob", &bytes); + Blob::from_bytes(&bytes).unwrap().content +} + +fn schema_tree(store: &ObjectStore, tree: &Tree, whose: &str) { + for entry in &tree.entries { + assert_eq!( + entry.mode, + EntryMode::Blob, + "{whose}: a schema tree holds blobs" + ); + let content = blob_content(store, &entry.hash); + same::(&format!("{whose}/{}", entry.name), &content); + } +} + +fn tree_of(store: &ObjectStore, hash: &comline_core::package::build::cas::Hash) -> Tree { + let bytes = store.read(hash).unwrap(); + same::("a tree", &bytes); + Tree::from_bytes(&bytes).unwrap() +} + +fn dependency_tree(store: &ObjectStore, tree: &Tree, whose: &str) { + for entry in &tree.entries { + let whose = format!("{whose}/{}", entry.name); + match (entry.mode, entry.name.as_str()) { + (EntryMode::Tree, name) if name.starts_with(entry::SCHEMA_PREFIX) => { + schema_tree(store, &tree_of(store, &entry.hash), &whose) + } + (EntryMode::Blob, entry::MANIFEST) => { + let text = blob_content(store, &entry.hash); + assert!( + String::from_utf8(text) + .is_ok_and(|t| !t.is_empty() && t.bytes().all(|b| b.is_ascii_hexdigit())), + "{whose}: the manifest is the content hash in hex" + ); + } + (EntryMode::Blob, entry::DEPENDENCY_CONFIG) => { + same::, Vec>(&whose, &blob_content(store, &entry.hash)) + } + (EntryMode::Blob, entry::DEPENDENCY_CONFIGS) => { + same::)>, Vec<(String, Vec)>>( + &whose, + &blob_content(store, &entry.hash), + ) + } + _ => panic!( + "{whose}: a dependency tree entry the frozen copy of format {CURRENT_IR_FORMAT_VERSION} doesn't know. \ + Adding one is a ledger row, with the lines in core/formats/src/v{CURRENT_IR_FORMAT_VERSION}" + ), + } + } +} + +fn root_tree(store: &ObjectStore, tree: &Tree, whose: &str) { + for entry in &tree.entries { + let whose = format!("{whose}/{}", entry.name); + let name = entry.name.as_str(); + match (entry.mode, name) { + (EntryMode::Tree, name) if name.starts_with(entry::SCHEMA_PREFIX) => { + schema_tree(store, &tree_of(store, &entry.hash), &whose) + } + (EntryMode::Tree, name) if name.starts_with(entry::DEPENDENCY_PREFIX) => { + dependency_tree(store, &tree_of(store, &entry.hash), &whose) + } + (EntryMode::Blob, entry::CONFIG) => { + same::, Vec>(&whose, &blob_content(store, &entry.hash)) + } + (EntryMode::Blob, entry::DOCS) => { + same::(&whose, &blob_content(store, &entry.hash)) + } + (EntryMode::Blob, entry::FACTS) => { + same::(&whose, &blob_content(store, &entry.hash)) + } + (EntryMode::Blob, entry::UNKNOWN | entry::PRODUCED_BY) => { + panic!("{whose}: nothing writes this entry yet, and a fresh build has it") + } + _ => panic!( + "{whose}: a root-tree entry the frozen copy of format {CURRENT_IR_FORMAT_VERSION} doesn't know. \ + Adding one is a ledger row, with the lines in core/formats/src/v{CURRENT_IR_FORMAT_VERSION}" + ), + } + } +} + +/// Every commit from the head back, each root tree under it. +fn history(built: &Built, whose: &str) { + let mut next = Some( + comline_core::package::build::cas::read_ref( + &built.package, + comline_core::package::build::cas::REF_HEAD_MAIN_FILENAME, + ) + .unwrap(), + ); + let mut commits = 0; + while let Some(hash) = next { + let bytes = built.store.read(&hash).unwrap(); + same::(&format!("{whose}: a commit"), &bytes); + let commit = Commit::from_bytes(&bytes).unwrap(); + let tree = tree_of(&built.store, &commit.tree); + root_tree(&built.store, &tree, whose); + next = commit.parents.first().copied(); + commits += 1; + } + assert!( + commits >= 2, + "{whose}: the history has a release and a build after it" + ); +} + +#[test] +fn every_object_core_writes_for_the_fixtures_is_the_frozen_formats() { + for fixture in corpus::available() { + let built = Built::of(fixture); + // A release record, then a build with a parent. + release(&built.package, &ReleaseRequest::default()) + .unwrap_or_else(|e| panic!("{} releases: {e:?}", fixture.name)); + fs::write( + built.package.join("src/zz_added.ids"), + "struct Added {\n a: u32\n}\n", + ) + .unwrap(); + build(&built.package).unwrap_or_else(|e| panic!("{} builds again: {e:?}", fixture.name)); + history(&built, fixture.name); + } +} + +#[test] +fn the_frozen_copy_is_the_current_format_and_names_what_core_names() { + assert_eq!(comline_ir_formats::CURRENT, CURRENT_IR_FORMAT_VERSION); + assert_eq!(entry::CONFIG, config_blob::CONFIG_ENTRY); + assert_eq!(entry::DOCS, doc::DOCS_ENTRY); + assert_eq!(entry::FACTS, compat::FACTS_ENTRY); + assert_eq!(entry::DEPENDENCY_PREFIX, config_blob::DEPENDENCY_PREFIX); + assert_eq!( + entry::DEPENDENCY_CONFIG, + config_blob::DEPENDENCY_CONFIG_ENTRY + ); + assert_eq!( + entry::DEPENDENCY_CONFIGS, + config_blob::DEPENDENCY_CONFIGS_ENTRY + ); +} + +/// A value of the frozen type `F` drawn from `bytes`: what the frozen copy +/// encodes, the live type `L` decodes and encodes back to the same bytes, and +/// the frozen copy decodes to the value. Values no build produces, so a +/// frozen type that is wrong about a field (a name, a default, a skipped +/// field) shows up where no fixture has it. +fn random(what: &str, bytes: &[u8]) +where + F: for<'a> Arbitrary<'a> + Serialize + DeserializeOwned + PartialEq + Debug, + L: Serialize + DeserializeOwned, +{ + let Ok(value) = F::arbitrary(&mut Unstructured::new(bytes)) else { + return; + }; + let encoded = codec::encode(&value).unwrap(); + let back: F = match codec::decode("value", &encoded) { + Ok(back) => back, + // Nested deeper than the CBOR reader goes: nothing to compare. + Err(e) if e.to_string().to_lowercase().contains("recursion") => return, + Err(e) => panic!("{what}: the frozen copy can't decode its own encoding of {value:?}: {e}"), + }; + assert_eq!( + back, value, + "{what}: the frozen copy changed a value through its own bytes" + ); + let live: L = from_cbor(&encoded).unwrap_or_else(|e| { + panic!("{what}: the live type can't decode what the frozen copy encodes of {value:?}: {e}") + }); + assert_eq!( + to_cbor(&live).unwrap(), + encoded, + "{what}: the live type encodes {value:?} differently" + ); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn random_schema_units_are_the_same_bytes(bytes in prop::collection::vec(any::(), 0..256)) { + random::("schema unit", &bytes); + } + + #[test] + fn random_config_units_are_the_same_bytes(bytes in prop::collection::vec(any::(), 0..256)) { + random::("config unit", &bytes); + } + + #[test] + fn random_docs_and_facts_are_the_same_bytes(bytes in prop::collection::vec(any::(), 0..256)) { + random::("docs", &bytes); + random::("facts", &bytes); + } + + #[test] + fn random_objects_are_the_same_bytes(bytes in prop::collection::vec(any::(), 0..256)) { + random::("commit", &bytes); + random::("tree", &bytes); + random::("blob", &bytes); + } +} diff --git a/core/tests/migrations/mod.rs b/core/tests/migrations/mod.rs new file mode 100644 index 0000000..07247e3 --- /dev/null +++ b/core/tests/migrations/mod.rs @@ -0,0 +1,9 @@ +// The format gate's tests that need a build (core#268, step 1.2): the fixture +// packages the migrations are proved on, what a fresh build of each hashes +// to, and the live types held to the frozen copy of the current format. The +// checks that look at the repository instead (the seal, the bump check) are +// `cargo xtask`'s. + +mod corpus; +mod golden; +mod live_frozen; diff --git a/core/tests/mod.rs b/core/tests/mod.rs index 55d812c..77e6015 100644 --- a/core/tests/mod.rs +++ b/core/tests/mod.rs @@ -5,3 +5,4 @@ mod autodoc; mod utils; mod cas; mod settings; +mod migrations; diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000..6c97033 --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "xtask" +version = "0.0.0" +edition = "2021" +license = "GPL-3.0-only" +publish = false +description = "Repository tasks: `cargo xtask check-format-bump`, `check-seal`, `seal-format` (core#268)" + +# Run as `cargo xtask ` (the alias is in .cargo/config.toml). It reads +# the repository's files and runs `git`; it depends on no comline crate, so +# the gate can't be moved by the code it guards. + +[dependencies] +blake3 = "1.4.1" +toml = "0.8" + +[dev-dependencies] +tempfile = "3.8" diff --git a/xtask/src/bump.rs b/xtask/src/bump.rs new file mode 100644 index 0000000..99349e4 --- /dev/null +++ b/xtask/src/bump.rs @@ -0,0 +1,447 @@ +//! What a format change has to come with. +//! +//! - Always: the current format has its directory with the four files, and +//! the seals hold. +//! - A bump of `CURRENT_IR_FORMAT_VERSION` (against the merge base) is one +//! format at a time, and comes with the new format's directory including +//! its step (`step.rs`, with `UNKNOWN`, `up` and `down`), the previous +//! format's seal, and a fixture for every ledger row of the previous +//! format. +//! - Without a bump, a move of the golden fresh-build hashes +//! (`core/tests/migrations/expected/`) needs a new ledger row, and a ledger +//! row needs `SCHEMA.md` and the frozen copy changed in the same change. + +use std::path::Path; + +use crate::repo::{current_format_in, Repo, EXPECTED, FORMAT_FILES, STAMP_FILE}; +use crate::seal; + +/// What the step of a bump has to define, as it appears in `step.rs`. +const STEP_PARTS: [(&str, &str); 3] = [ + ("UNKNOWN", "the list of paths it writes as unknown"), + ("fn up", "`up`"), + ("fn down", "`down`, the witness"), +]; + +/// The fixtures of format `format`'s stores. +fn stores_dir(repo: &Repo, format: u32) -> std::path::PathBuf { + repo.path(&format!("core/tests/migrations/stores/format-{format}")) +} + +/// One row of a format's ledger. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row { + pub id: String, + /// The fixture that has the row, a directory under + /// `tests/migrations/stores/format-/`. + pub fixture: Option, +} + +/// The rows of a `ledger.toml`. +pub fn ledger(text: &str) -> Result, String> { + let table: toml::Table = text.parse().map_err(|e| format!("ledger.toml: {e}"))?; + let rows = match table.get("row") { + None => return Ok(vec![]), + Some(toml::Value::Array(rows)) => rows, + Some(_) => return Err("ledger.toml: `row` is not an array of tables".into()), + }; + rows.iter() + .map(|row| { + let field = |name: &str| row.get(name).and_then(|v| v.as_str()).map(str::to_string); + Ok(Row { + id: field("id").ok_or("ledger.toml: a row has no `id`")?, + fixture: field("fixture"), + }) + }) + .collect() +} + +fn ledger_of(repo: &Repo, format: u32) -> Result, String> { + ledger(&repo.read(&format!("core/formats/src/v{format}/ledger.toml"))?) +} + +fn missing_files(repo: &Repo, format: u32, files: &[&str]) -> Vec { + let dir = repo.format_dir(format); + files + .iter() + .filter(|f| !dir.join(f).is_file()) + .map(|f| format!("format {format} has no `{f}` in core/formats/src/v{format}/")) + .collect() +} + +/// Everything the change from `base` (a revision) to `HEAD` is missing; with +/// no `base`, only what is true of the tree itself. +pub fn check(repo: &Repo, base: Option<&str>) -> Result, String> { + let current = repo.current_format()?; + let mut problems = missing_files(repo, current, &FORMAT_FILES); + problems.extend(seal::verify(repo)?); + + let Some(base) = base else { + return Ok(problems); + }; + let merge_base = repo.git(&["merge-base", base, "HEAD"])?.trim().to_string(); + let before = current_format_in( + &repo + .file_at(&merge_base, STAMP_FILE) + .ok_or_else(|| format!("{STAMP_FILE} doesn't exist at {base}"))?, + )?; + if current < before { + problems.push(format!( + "CURRENT_IR_FORMAT_VERSION went down, from {before} to {current}" + )); + } else if current > before { + problems.extend(bump(repo, before, current)?); + } else { + problems.extend(no_bump(repo, &merge_base, current)?); + } + Ok(problems) +} + +/// A bump from `before` to `current`. +fn bump(repo: &Repo, before: u32, current: u32) -> Result, String> { + let mut problems = vec![]; + if current != before + 1 { + problems.push(format!( + "CURRENT_IR_FORMAT_VERSION went from {before} to {current}: a change moves one format, with a step each" + )); + return Ok(problems); + } + + problems.extend(missing_files(repo, current, &["step.rs"])); + if let Ok(step) = std::fs::read_to_string(repo.format_dir(current).join("step.rs")) { + for (part, what) in STEP_PARTS { + if !step.contains(part) { + problems.push(format!( + "the step {before} -> {current} (v{current}/step.rs) doesn't define {what}" + )); + } + } + } + if let Ok(schema) = std::fs::read_to_string(repo.format_dir(current).join("SCHEMA.md")) { + if !schema + .lines() + .any(|l| l.starts_with(&format!("# IR format {current}"))) + { + problems.push(format!( + "v{current}/SCHEMA.md doesn't start a `# IR format {current}` document" + )); + } + } + + // The seal of `before` is checked with every seal, by `seal::verify`. + for row in ledger_of(repo, before)? { + match &row.fixture { + None => problems.push(format!( + "ledger row {} of format {before} has no fixture: the step {before} -> {current} is proved on a store of each level", + row.id + )), + Some(fixture) if !Path::new(&stores_dir(repo, before)).join(fixture).is_dir() => { + problems.push(format!( + "ledger row {} of format {before} names the fixture `{fixture}`, which isn't in core/tests/migrations/stores/format-{before}/", + row.id + )) + } + Some(_) => {} + } + } + Ok(problems) +} + +/// A change that leaves the format number alone. +fn no_bump(repo: &Repo, merge_base: &str, current: u32) -> Result, String> { + let mut problems = vec![]; + let changed: Vec = repo + .git(&["diff", "--name-only", merge_base, "HEAD"])? + .lines() + .map(str::to_string) + .collect(); + let touched = |path: &str| changed.iter().any(|c| c == path); + + let ledger_path = format!("core/formats/src/v{current}/ledger.toml"); + let rows_before = match repo.file_at(merge_base, &ledger_path) { + Some(text) => ledger(&text)?.len(), + None => 0, + }; + let rows_now = ledger_of(repo, current)?.len(); + + if rows_now < rows_before { + problems.push(format!( + "a row was removed from format {current}'s ledger: it is a history" + )); + } + if rows_now > rows_before { + let dir = format!("core/formats/src/v{current}"); + if !touched(&format!("{dir}/SCHEMA.md")) { + problems.push(format!("format {current}'s ledger has a new row, and SCHEMA.md of the format didn't change with it")); + } + if !touched(&format!("{dir}/types.rs")) && !touched(&format!("{dir}/codec.rs")) { + problems.push(format!("format {current}'s ledger has a new row, and the frozen copy (types.rs, codec.rs) didn't change with it")); + } + } + if changed + .iter() + .any(|c| c.starts_with(&format!("{EXPECTED}/"))) + && rows_now <= rows_before + { + problems.push(format!( + "the golden fresh-build hashes in {EXPECTED}/ moved, and format {current}'s ledger has no new row: \ + a change to what a build writes is a ledger row (with the same lines in the frozen copy) or a bump" + )); + } + Ok(problems) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::fs; + use std::ops::Deref; + + /// A repository on disk, removed when dropped. + pub struct Scratch { + _dir: tempfile::TempDir, + repo: Repo, + } + + impl Deref for Scratch { + type Target = Repo; + fn deref(&self) -> &Repo { + &self.repo + } + } + + fn write(repo: &Repo, relative: &str, text: &str) { + let path = repo.path(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + + fn format_files(repo: &Repo, format: u32) { + let dir = format!("core/formats/src/v{format}"); + write( + repo, + &format!("{dir}/SCHEMA.md"), + &format!("# IR format {format}\n"), + ); + write(repo, &format!("{dir}/types.rs"), "// types\n"); + write(repo, &format!("{dir}/codec.rs"), "// codec\n"); + write(repo, &format!("{dir}/ledger.toml"), "[[row]]\nid = \"1\"\n"); + } + + /// A tree whose current format is `current`, with a directory for each + /// format up to it, no seals and no git. + pub fn scratch(current: u32) -> Scratch { + let dir = tempfile::tempdir().unwrap(); + let repo = Repo::new(dir.path()); + write( + &repo, + STAMP_FILE, + &format!("pub const CURRENT_IR_FORMAT_VERSION: u32 = {current};\n"), + ); + for format in 3..=current { + format_files(&repo, format); + } + write(&repo, crate::repo::MANIFEST, "# seals\n"); + write(&repo, &format!("{EXPECTED}/tour.txt"), "tree 00\n"); + Scratch { _dir: dir, repo } + } + + fn commit(repo: &Repo, message: &str) { + repo.git(&["add", "-A"]).unwrap(); + repo.git(&[ + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "-m", + message, + ]) + .unwrap(); + } + + /// A committed tree at format 3, tagged `base`. + fn based() -> Scratch { + let repo = scratch(3); + repo.git(&["init", "-q", "-b", "main"]).unwrap(); + commit(&repo, "base"); + repo.git(&["tag", "base"]).unwrap(); + repo + } + + fn contains(problems: &[String], text: &str) -> bool { + problems.iter().any(|p| p.contains(text)) + } + + #[test] + fn a_change_that_leaves_the_formats_alone_passes() { + let repo = based(); + write(&repo, "README.md", "unrelated\n"); + commit(&repo, "unrelated"); + assert_eq!(check(&repo, Some("base")).unwrap(), Vec::::new()); + } + + #[test] + fn the_current_format_has_its_four_files() { + let repo = based(); + fs::remove_file(repo.format_dir(3).join("codec.rs")).unwrap(); + let problems = check(&repo, None).unwrap(); + assert!( + contains(&problems, "format 3 has no `codec.rs`"), + "{problems:?}" + ); + } + + #[test] + fn a_bump_without_its_step_its_seal_or_its_fixtures_fails_and_with_them_passes() { + let repo = based(); + write( + &repo, + STAMP_FILE, + "pub const CURRENT_IR_FORMAT_VERSION: u32 = 4;\n", + ); + format_files(&repo, 4); + commit(&repo, "bump"); + + let problems = check(&repo, Some("base")).unwrap(); + assert!( + contains(&problems, "format 4 has no `step.rs`"), + "{problems:?}" + ); + assert!( + contains(&problems, "format 3 is superseded by 4 and not sealed"), + "{problems:?}" + ); + assert!( + contains(&problems, "ledger row 1 of format 3 has no fixture"), + "{problems:?}" + ); + + // A step without the witness is not a step. + write( + &repo, + "core/formats/src/v4/step.rs", + "pub const UNKNOWN: &[&str] = &[];\npub fn up() {}\n", + ); + let problems = check(&repo, Some("base")).unwrap(); + assert!( + contains(&problems, "doesn't define `down`, the witness"), + "{problems:?}" + ); + + write( + &repo, + "core/formats/src/v4/step.rs", + "pub const UNKNOWN: &[&str] = &[];\npub fn up() {}\npub fn down() {}\n", + ); + seal::seal(&repo, 3, "format 4 is introduced").unwrap(); + write( + &repo, + "core/formats/src/v3/ledger.toml", + "[[row]]\nid = \"1\"\nfixture = \"tour\"\n", + ); + // Editing the ledger changed format 3 after it was sealed: seal last. + seal::seal(&repo, 3, "format 4 is introduced").unwrap(); + let problems = check(&repo, Some("base")).unwrap(); + assert!( + contains(&problems, "names the fixture `tour`, which isn't in"), + "{problems:?}" + ); + + write( + &repo, + "core/tests/migrations/stores/format-3/tour/PRODUCED_BY", + "rev\n", + ); + commit(&repo, "bump, complete"); + assert_eq!(check(&repo, Some("base")).unwrap(), Vec::::new()); + } + + #[test] + fn a_bump_skipping_a_format_fails() { + let repo = based(); + write( + &repo, + STAMP_FILE, + "pub const CURRENT_IR_FORMAT_VERSION: u32 = 5;\n", + ); + format_files(&repo, 5); + commit(&repo, "bump"); + let problems = check(&repo, Some("base")).unwrap(); + assert!(contains(&problems, "went from 3 to 5"), "{problems:?}"); + } + + #[test] + fn golden_hashes_that_moved_need_a_ledger_row_with_the_frozen_copy() { + let repo = based(); + write(&repo, &format!("{EXPECTED}/tour.txt"), "tree 01\n"); + commit(&repo, "moved"); + let problems = check(&repo, Some("base")).unwrap(); + assert!(contains(&problems, "ledger has no new row"), "{problems:?}"); + + // A row alone is not enough: the document and the frozen copy change with it. + write( + &repo, + "core/formats/src/v3/ledger.toml", + "[[row]]\nid = \"1\"\n[[row]]\nid = \"2\"\n", + ); + commit(&repo, "row"); + let problems = check(&repo, Some("base")).unwrap(); + assert!( + contains(&problems, "SCHEMA.md of the format didn't change"), + "{problems:?}" + ); + assert!( + contains( + &problems, + "the frozen copy (types.rs, codec.rs) didn't change" + ), + "{problems:?}" + ); + + write( + &repo, + "core/formats/src/v3/SCHEMA.md", + "# IR format 3\nrow 2\n", + ); + write( + &repo, + "core/formats/src/v3/types.rs", + "// types\n// row 2\n", + ); + commit(&repo, "row, document and copy"); + assert_eq!(check(&repo, Some("base")).unwrap(), Vec::::new()); + } + + #[test] + fn a_ledger_row_cannot_be_removed() { + let repo = based(); + write(&repo, "core/formats/src/v3/ledger.toml", ""); + commit(&repo, "removed"); + let problems = check(&repo, Some("base")).unwrap(); + assert!(contains(&problems, "a row was removed"), "{problems:?}"); + } + + #[test] + fn a_ledger_is_read() { + let rows = + ledger("[[row]]\nid = \"3.01\"\nfixture = \"tour\"\n[[row]]\nid = \"3.02\"\n").unwrap(); + assert_eq!( + rows[0], + Row { + id: "3.01".into(), + fixture: Some("tour".into()) + } + ); + assert_eq!( + rows[1], + Row { + id: "3.02".into(), + fixture: None + } + ); + assert_eq!(ledger("# none\n").unwrap(), vec![]); + assert!(ledger("[[row]]\nfixture = \"x\"\n").is_err()); + } +} diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs new file mode 100644 index 0000000..9852511 --- /dev/null +++ b/xtask/src/lib.rs @@ -0,0 +1,11 @@ +//! The format gate's checks that look at the repository and not at a build +//! (core#268, step 1.2): the seal of a superseded format, and what a format +//! bump (or a moved golden hash) has to come with. The tests that need a +//! build are in `core/tests/migrations`. +//! +//! Every check returns the problems it found, as sentences, and an empty +//! list when there is none. + +pub mod bump; +pub mod repo; +pub mod seal; diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000..4791c54 --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,71 @@ +//! `cargo xtask `: the repository's own checks (core#268, step 1.2). + +use std::process::ExitCode; + +use xtask::repo::Repo; +use xtask::{bump, seal}; + +const USAGE: &str = "\ +usage: cargo xtask + + check-format-bump [--base ] what a format change has to come with: with --base, + a bump of CURRENT_IR_FORMAT_VERSION needs its step, + directory, seal and fixtures, and moved golden hashes + need a ledger row; always, the seals hold + check-seal the seals in core/formats/MANIFEST hold + seal-format --why record the seal of the superseded format N, or move it"; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let repo = Repo::here(); + let result = match args.first().map(String::as_str) { + Some("check-format-bump") => flag(&args, "--base") + .and_then(|base| bump::check(&repo, base.as_deref())) + .map(|problems| report("the format gate", problems)), + Some("check-seal") => seal::verify(&repo).map(|problems| report("the seals", problems)), + Some("seal-format") => seal_format(&repo, &args), + _ => Err(USAGE.to_string()), + }; + match result { + Ok(code) => code, + Err(message) => { + eprintln!("{message}"); + ExitCode::FAILURE + } + } +} + +/// The value after `name` in `args`, if the flag is there. +fn flag(args: &[String], name: &str) -> Result, String> { + match args.iter().position(|a| a == name) { + None => Ok(None), + Some(at) => args + .get(at + 1) + .cloned() + .map(Some) + .ok_or(format!("{name} needs a value")), + } +} + +fn report(what: &str, problems: Vec) -> ExitCode { + if problems.is_empty() { + println!("{what}: ok"); + return ExitCode::SUCCESS; + } + for problem in &problems { + eprintln!("error: {problem}"); + } + eprintln!("{what}: {} problem(s)", problems.len()); + ExitCode::FAILURE +} + +fn seal_format(repo: &Repo, args: &[String]) -> Result { + let format: u32 = args + .get(1) + .and_then(|n| n.parse().ok()) + .ok_or("seal-format needs a format number")?; + let why = flag(args, "--why")?.ok_or("a seal is moved with a reason: `--why `")?; + seal::seal(repo, format, &why)?; + println!("sealed format {format}"); + Ok(ExitCode::SUCCESS) +} diff --git a/xtask/src/repo.rs b/xtask/src/repo.rs new file mode 100644 index 0000000..768c461 --- /dev/null +++ b/xtask/src/repo.rs @@ -0,0 +1,114 @@ +//! Where things are in the repository, and reading them. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// The file that holds `CURRENT_IR_FORMAT_VERSION`. +pub const STAMP_FILE: &str = "core/src/package/build/cas/object_store.rs"; +/// The frozen formats' crate. +pub const FORMATS: &str = "core/formats"; +/// The seals. +pub const MANIFEST: &str = "core/formats/MANIFEST"; +/// The golden fresh-build hashes of the fixture packages. +pub const EXPECTED: &str = "core/tests/migrations/expected"; + +/// The files every format's directory has. +pub const FORMAT_FILES: [&str; 4] = ["SCHEMA.md", "types.rs", "codec.rs", "ledger.toml"]; + +/// A checkout of the repository. +pub struct Repo { + pub root: PathBuf, +} + +impl Repo { + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } + } + + /// The checkout the running `xtask` belongs to. + pub fn here() -> Self { + Self::new(Path::new(env!("CARGO_MANIFEST_DIR")).join("..")) + } + + pub fn path(&self, relative: &str) -> PathBuf { + self.root.join(relative) + } + + pub fn format_dir(&self, format: u32) -> PathBuf { + self.path(&format!("{FORMATS}/src/v{format}")) + } + + pub fn read(&self, relative: &str) -> Result { + fs::read_to_string(self.path(relative)).map_err(|e| format!("can't read {relative}: {e}")) + } + + /// The formats that have a directory in the crate, ascending. + pub fn formats(&self) -> Result, String> { + let src = self.path(&format!("{FORMATS}/src")); + let mut formats: Vec = fs::read_dir(&src) + .map_err(|e| format!("can't read {}: {e}", src.display()))? + .filter_map(Result::ok) + .filter(|e| e.path().is_dir()) + .filter_map(|e| e.file_name().to_str()?.strip_prefix('v')?.parse().ok()) + .collect(); + formats.sort_unstable(); + Ok(formats) + } + + /// `CURRENT_IR_FORMAT_VERSION` in the working tree. + pub fn current_format(&self) -> Result { + current_format_in(&self.read(STAMP_FILE)?) + } + + /// Run `git` in the checkout. + pub fn git(&self, args: &[&str]) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(&self.root) + .args(args) + .output() + .map_err(|e| format!("can't run git: {e}"))?; + if !output.status.success() { + return Err(format!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } + + /// A file as `revision` had it, `None` if it had none. + pub fn file_at(&self, revision: &str, relative: &str) -> Option { + self.git(&["show", &format!("{revision}:{relative}")]).ok() + } +} + +/// The number in `pub const CURRENT_IR_FORMAT_VERSION: u32 = ;`. +pub fn current_format_in(source: &str) -> Result { + let line = source + .lines() + .find(|l| { + l.trim_start() + .starts_with("pub const CURRENT_IR_FORMAT_VERSION") + }) + .ok_or("no `CURRENT_IR_FORMAT_VERSION` in the stamp file")?; + line.split('=') + .nth(1) + .and_then(|v| v.trim().trim_end_matches(';').trim().parse().ok()) + .ok_or_else(|| format!("can't read a number from `{}`", line.trim())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_stamp_is_read_from_its_constant() { + let source = "/// docs\npub const CURRENT_IR_FORMAT_VERSION: u32 = 3;\nfn x() {}\n"; + assert_eq!(current_format_in(source), Ok(3)); + assert!(current_format_in("const X: u32 = 3;").is_err()); + assert!(current_format_in("pub const CURRENT_IR_FORMAT_VERSION: u32 = three;").is_err()); + } +} diff --git a/xtask/src/seal.rs b/xtask/src/seal.rs new file mode 100644 index 0000000..e58a1e6 --- /dev/null +++ b/xtask/src/seal.rs @@ -0,0 +1,272 @@ +//! The seal: `formats/MANIFEST` records the blake3 of each superseded +//! format's directory, so that a frozen type, a codec or a document can't +//! change after its format was replaced. +//! +//! One line per sealed format: `v blake3: # `. +//! The current format is not sealed, and every older one is. + +use std::fs; +use std::path::Path; + +use crate::repo::{Repo, MANIFEST}; + +/// The header a manifest starts with. +const HEADER: &str = "\ +# The seal of each superseded IR format (core#268): the blake3 of its +# directory under src/, over every file's relative path and bytes in path +# order. `cargo xtask check-seal` fails when one differs; a format is sealed +# when the next is introduced, with `cargo xtask seal-format --why `, +# which is also the only way to move a seal. +"; + +/// A line of the manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Seal { + pub format: u32, + pub hash: String, + pub why: String, +} + +/// The hash of the directory `dir`: for each file under it in path order, +/// its path relative to `dir` (with `/`), a zero byte, its length as a +/// little-endian `u64`, and its bytes. +pub fn dir_hash(dir: &Path) -> Result { + fn walk(base: &Path, dir: &Path, files: &mut Vec) -> Result<(), String> { + for entry in fs::read_dir(dir).map_err(|e| format!("can't read {}: {e}", dir.display()))? { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.is_dir() { + walk(base, &path, files)?; + } else { + let relative = path.strip_prefix(base).map_err(|e| e.to_string())?; + let parts: Vec<_> = relative.iter().map(|p| p.to_string_lossy()).collect(); + files.push(parts.join("/")); + } + } + Ok(()) + } + + let mut files = vec![]; + walk(dir, dir, &mut files)?; + files.sort(); + let mut hasher = blake3::Hasher::new(); + for file in &files { + let bytes = fs::read(dir.join(file)).map_err(|e| format!("can't read {file}: {e}"))?; + hasher.update(file.as_bytes()); + hasher.update(&[0]); + hasher.update(&(bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + } + Ok(format!("blake3:{}", hasher.finalize().to_hex())) +} + +/// The seals in `text`. +pub fn parse(text: &str) -> Result, String> { + let mut seals = vec![]; + for (number, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (entry, why) = line.split_once('#').unwrap_or((line, "")); + let mut words = entry.split_whitespace(); + let (Some(format), Some(hash), None) = (words.next(), words.next(), words.next()) else { + return Err(format!( + "MANIFEST line {}: expected `v blake3: # `", + number + 1 + )); + }; + let format = format + .strip_prefix('v') + .and_then(|n| n.parse().ok()) + .ok_or_else(|| format!("MANIFEST line {}: `{format}` is not a format", number + 1))?; + let valid = hash.strip_prefix("blake3:").is_some_and(|h| { + h.len() == 64 + && h.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }); + if !valid { + return Err(format!( + "MANIFEST line {}: `{hash}` is not `blake3:` and 64 hex digits", + number + 1 + )); + } + if seals.iter().any(|s: &Seal| s.format == format) { + return Err(format!( + "MANIFEST line {}: format {format} is sealed twice", + number + 1 + )); + } + seals.push(Seal { + format, + hash: hash.to_string(), + why: why.trim().to_string(), + }); + } + Ok(seals) +} + +fn render(seals: &[Seal]) -> String { + let mut text = HEADER.to_string(); + for seal in seals { + text.push_str(&format!("v{} {} # {}\n", seal.format, seal.hash, seal.why)); + } + text +} + +/// Everything wrong with the seals: a superseded format that isn't sealed, +/// one that changed since it was, and a seal on the current format or on one +/// that is gone. +pub fn verify(repo: &Repo) -> Result, String> { + let current = repo.current_format()?; + let seals = parse(&repo.read(MANIFEST)?)?; + let mut problems = vec![]; + for seal in &seals { + let dir = repo.format_dir(seal.format); + if seal.format >= current { + problems.push(format!( + "format {} is sealed, but the current format is {current}: a format is sealed when the next one is introduced", + seal.format + )); + } else if !dir.is_dir() { + problems.push(format!( + "format {} is sealed, but its directory is gone", + seal.format + )); + } else if dir_hash(&dir)? != seal.hash { + problems.push(format!( + "sealed format {} changed: its frozen types, codec or document differ from the seal in MANIFEST. \ + A superseded format is never edited; if this is deliberate, move the seal with \ + `cargo xtask seal-format {} --why `", + seal.format, seal.format + )); + } + } + for format in repo.formats()? { + if format < current && !seals.iter().any(|s| s.format == format) { + problems.push(format!( + "format {format} is superseded by {current} and not sealed: `cargo xtask seal-format {format} --why `" + )); + } + } + Ok(problems) +} + +/// Record the seal of `format`, replacing the one it has. `why` is written +/// beside it. +pub fn seal(repo: &Repo, format: u32, why: &str) -> Result<(), String> { + if why.trim().is_empty() { + return Err("a seal is moved with a reason: `--why `".into()); + } + let current = repo.current_format()?; + if format >= current { + return Err(format!( + "format {format} is the current format (or newer) and can't be sealed yet" + )); + } + let dir = repo.format_dir(format); + if !dir.is_dir() { + return Err(format!("format {format} has no directory")); + } + let mut seals = parse(&repo.read(MANIFEST)?)?; + seals.retain(|s| s.format != format); + seals.push(Seal { + format, + hash: dir_hash(&dir)?, + why: why.trim().to_string(), + }); + seals.sort_by_key(|s| s.format); + fs::write(repo.path(MANIFEST), render(&seals)).map_err(|e| format!("can't write MANIFEST: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bump::tests::scratch; + + #[test] + fn a_manifest_parses_and_renders() { + let text = format!( + "{HEADER}v3 blake3:{} # format 4 is introduced\n", + "ab".repeat(32) + ); + let seals = parse(&text).unwrap(); + assert_eq!(seals.len(), 1); + assert_eq!( + (seals[0].format, seals[0].why.as_str()), + (3, "format 4 is introduced") + ); + assert_eq!(render(&seals), text); + assert!(parse("v3 sha:abc").is_err()); + assert!(parse("three blake3:00").is_err()); + let twice = format!("v3 blake3:{0}\nv3 blake3:{0}\n", "ab".repeat(32)); + assert!(parse(&twice).is_err()); + } + + #[test] + fn the_hash_follows_every_file_and_its_path() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("a"), "one").unwrap(); + let first = dir_hash(dir.path()).unwrap(); + assert_eq!(dir_hash(dir.path()).unwrap(), first); + fs::write(dir.path().join("a"), "two").unwrap(); + let changed = dir_hash(dir.path()).unwrap(); + assert_ne!(changed, first); + fs::rename(dir.path().join("a"), dir.path().join("b")).unwrap(); + assert_ne!(dir_hash(dir.path()).unwrap(), changed); + } + + #[test] + fn the_current_format_is_not_sealed_and_an_older_one_has_to_be() { + let repo = scratch(4); + let problems = verify(&repo).unwrap(); + assert!( + problems + .iter() + .any(|p| p.contains("format 3 is superseded by 4 and not sealed")), + "{problems:?}" + ); + assert!( + seal(&repo, 4, "why").is_err(), + "the current format can't be sealed" + ); + assert!(seal(&repo, 3, " ").is_err(), "a seal needs a reason"); + + seal(&repo, 3, "format 4 is introduced").unwrap(); + assert_eq!(verify(&repo).unwrap(), Vec::::new()); + } + + #[test] + fn a_sealed_format_that_changes_fails_until_the_seal_is_moved_with_a_reason() { + let repo = scratch(4); + seal(&repo, 3, "format 4 is introduced").unwrap(); + + fs::write(repo.format_dir(3).join("types.rs"), "// edited\n").unwrap(); + let problems = verify(&repo).unwrap(); + assert!( + problems + .iter() + .any(|p| p.contains("sealed format 3 changed")), + "{problems:?}" + ); + + seal(&repo, 3, "a typo in a document").unwrap(); + assert_eq!(verify(&repo).unwrap(), Vec::::new()); + assert!(fs::read_to_string(repo.path(MANIFEST)) + .unwrap() + .contains("# a typo in a document")); + } + + #[test] + fn a_seal_on_the_current_format_fails() { + let repo = scratch(3); + let manifest = format!("{HEADER}v3 blake3:{} # early\n", "00".repeat(32)); + fs::write(repo.path(MANIFEST), manifest).unwrap(); + let problems = verify(&repo).unwrap(); + assert!( + problems + .iter() + .any(|p| p.contains("is sealed, but the current format is 3")), + "{problems:?}" + ); + } +}