Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,17 @@ jobs:
- run: cargo test --verbose
- name: std-extra/ matches a fresh regen
run: cargo run -p xtask -- --check

std_pin:
# `STD_REV` can only point at a merge commit once that commit exists, so
# this runs after the merge rather than on the PR that regenerates
# `std-extra/`. Needs full history: the check diffs HEAD against STD_REV.
name: comline_std pin is current
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- run: rustup update stable && rustup default stable
- run: cargo run -p xtask -- --check-pin
4 changes: 2 additions & 2 deletions codegen/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ license = "GPL-3.0-only"
# repository structure — no crates.io publishing at this stage). Pinned to the
# same generation rev `comline-typescript` uses, so a CLI build has one
# `comline-codegen`. Local iteration: add a `[patch]` onto a sibling checkout.
comline-codegen = { git = "https://github.com/ComlineProject/generation", rev = "52803df6436f49aec5f2266a94c89d4ced4b229c" }
comline-core = { git = "https://github.com/ComlineProject/core", rev = "5c1f3474b462ed912f4f2f5a518b87a5f6576aaa" }
comline-codegen = { git = "https://github.com/ComlineProject/generation", rev = "69a3e37ae788a91c5f72dd047d6196aa4c95fdef" }
comline-core = { git = "https://github.com/ComlineProject/core", rev = "016e38603152444f9b51fddfd46b1e6b1d73f1d8" }

eyre = "0.6.8"

Expand Down
33 changes: 29 additions & 4 deletions codegen/src/generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const RUNTIME_REV: &str = "0dda42de5105e75e339bf66fa38085bd53ab9ab9";
/// `comline-runtime` above, pointing at `comline-rust`'s own `std-extra/`
/// (see `xtask/`, which regenerates it and is what this pin must match).
const STD_GIT: &str = "https://github.com/ComlineProject/comline-rust";
const STD_REV: &str = "3fde4a968319cbf6baac1e5c812aaa042cc8a0da";
pub const STD_REV: &str = "3fde4a968319cbf6baac1e5c812aaa042cc8a0da";

/// Whether a `/`-joined namespace is `std` or nested under it - the only
/// provenance signal std's schemas carry by the time they reach this
Expand Down Expand Up @@ -90,7 +90,9 @@ pub fn generate_rust(req: &GenRequest) -> Result<Vec<GeneratedFile>> {
let mut files = vec![
GeneratedFile {
path: PathBuf::from("Cargo.toml"),
contents: cargo_toml(&req.package, has_protocol, needs_std_dep),
contents: cargo_toml(
&req.package, &req.lang_version, has_protocol, needs_std_dep,
),
},
GeneratedFile {
path: PathBuf::from("src/lib.rs"),
Expand Down Expand Up @@ -125,6 +127,18 @@ impl<'a> ModuleTree<'a> {
fn build(schemas: &'a [(String, Vec<FrozenUnit>)]) -> Result<Self> {
let mut root = Self::empty();
for (namespace, units) in schemas {
// `src/lib.rs` is this crate's root, written before any schema
// file - a schema of the same name would be emitted over it and
// take the whole `pub mod` tree with it. Core refuses the name
// when a package is compiled; this catches a `GenRequest` built
// directly, since that is a public contract.
if namespace == "lib" {
bail!(
"rust lib mode can't generate `lib`: `src/lib.rs` is the \
crate root, so a schema of that name would overwrite the \
module tree - rename the schema"
);
}
let mut node = &mut root;
for segment in namespace.split('/') {
node = node.children.entry(segment.to_string()).or_insert_with(Self::empty);
Expand Down Expand Up @@ -190,7 +204,15 @@ impl<'a> ModuleTree<'a> {
}
}

fn cargo_toml(pkg: &PackageMeta, has_protocol: bool, needs_std_dep: bool) -> String {
/// `lang_version` is the toolchain the target declared (`rust#1.70.0`), so
/// the crate states the version it was generated for rather than leaving a
/// consumer to discover it by failing to build.
fn cargo_toml(
pkg: &PackageMeta,
lang_version: &str,
has_protocol: bool,
needs_std_dep: bool,
) -> String {
let mut deps = String::from("serde = { version = \"1\", features = [\"derive\"] }\n");
if has_protocol {
deps.push_str(&format!(
Expand All @@ -206,8 +228,11 @@ fn cargo_toml(pkg: &PackageMeta, has_protocol: bool, needs_std_dep: bool) -> Str
name = \"{}\"\n\
version = \"{}\"\n\
edition = \"2021\"\n\
rust-version = \"{lang_version}\"\n\
# A schema namespace named `main` would otherwise be picked up as a\n\
# binary target (`src/main.rs`) with no `fn main`.\n\
# binary target (`src/main.rs`) with no `fn main`. Core refuses that\n\
# name outright now; this stays as a belt-and-braces guard for a\n\
# `GenRequest` built by hand.\n\
autobins = false\n\
\n\
[dependencies]\n\
Expand Down
10 changes: 8 additions & 2 deletions codegen/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,15 @@

mod generator;

pub use generator::generate_rust;
pub use generator::{generate_rust, STD_REV};

/// The toolchain version this generator registers under - what a
/// congregation names as `rust#1.70.0`. Public so `xtask`, which generates
/// `comline_std` through the very same entry point every consumer uses, can
/// state it rather than repeat the literal.
pub const LANG_VERSION: &str = "1.70.0";

/// Contribute the Rust generator to a CLI's [`Registry`](comline_codegen::Registry).
pub fn register(registry: &mut comline_codegen::Registry) {
registry.register("rust", "rs", "1.70.0", generate_rust);
registry.register("rust", "rs", LANG_VERSION, generate_rust);
}
56 changes: 50 additions & 6 deletions codegen/tests/compiles.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,22 +179,66 @@ fn a_generated_protocol_crate_builds() {
("rpc".to_string(), rpc_schema()),
("clock".to_string(), rpc_default_schema()),
];
// `chat` opts back to datagram with `@framing = "datagram"`; `clock`
// (unannotated) takes this default; `rpc` names `jsonrpc` itself.
build_generated_crate("protocol", &schemas, Some("jsonrpc".into()), false);
}

/// `lib` mode links `comline_std` rather than inlining std, which only
/// works if the crate it emits actually resolves against the real pinned
/// `comline_std` - the import rewriting, the dependency line, and the
/// absence of std's own files all have to agree. Emitting the right text is
/// not the same claim as building, so this builds it.
#[test]
fn a_generated_crate_linking_comline_std_builds() {
let schemas = vec![
("chat".to_string(), uses_std_http()),
// Std's own schema, present the way a real package sees it. With
// `external_std` on it is filtered out of the output and turned
// into the dependency instead.
("std/http".to_string(), vec![]),
];
build_generated_crate("externalstd", &schemas, None, true);
}

/// A struct whose field is typed by `std`'s `Request`, with the leaf import
/// core expands `use std::http::Request;` into before the generator sees it.
fn uses_std_http() -> Vec<FrozenUnit> {
vec![
FrozenUnit::Import("std::http::Request".to_string(), None, (0, 0)),
FrozenUnit::Struct {
docstring: None,
parameters: vec![],
name: "Call".to_string(),
fields: vec![field("request", "Request")],
span: (0, 0),
},
]
}

/// Generate into a temp dir and `cargo build` it. Left in place on failure,
/// so the generated source can be read rather than guessed at.
fn build_generated_crate(
label: &str,
schemas: &[(String, Vec<FrozenUnit>)],
default_framing: Option<String>,
external_std: bool,
) {
let req = GenRequest {
mode: Mode::Lib,
schemas: &schemas,
schemas,
package: PackageMeta {
name: "comline-codegen-rust-compiletest".into(),
version: "0.0.0".into(),
},
// `chat` opts back to datagram with `@framing = "datagram"`; `clock`
// (unannotated) takes this default; `rpc` names `jsonrpc` itself.
default_framing: Some("jsonrpc".into()),
external_std: false,
lang_version: comline_codegen_rust::LANG_VERSION.into(),
default_framing,
external_std,
};
let files = generate_rust(&req).expect("generation");

let dir = std::env::temp_dir().join(format!(
"comline-rust-compiletest-{}",
"comline-rust-compiletest-{label}-{}",
std::process::id()
));
let _ = fs::remove_dir_all(&dir);
Expand Down
29 changes: 29 additions & 0 deletions codegen/tests/generate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ fn code_req(schemas: &[(String, Vec<FrozenUnit>)]) -> GenRequest<'_> {
mode: Mode::Code,
schemas,
package: PackageMeta { name: "test".into(), version: "0.1.0".into() },
lang_version: comline_codegen_rust::LANG_VERSION.into(),
default_framing: None,
external_std: false,
}
Expand All @@ -18,6 +19,7 @@ fn lib_req(schemas: &[(String, Vec<FrozenUnit>)]) -> GenRequest<'_> {
mode: Mode::Lib,
schemas,
package: PackageMeta { name: "chat".into(), version: "0.3.0".into() },
lang_version: comline_codegen_rust::LANG_VERSION.into(),
default_framing: None,
external_std: false,
}
Expand Down Expand Up @@ -543,6 +545,33 @@ fn lib_mode_rejects_a_namespace_thats_both_a_schema_and_a_parent() {
assert!(err.contains("account"), "{err}");
}

/// `src/lib.rs` is the crate root this mode writes first; a schema of the
/// same name was emitted straight over it, taking the whole `pub mod` tree
/// with it and leaving a crate whose modules had all vanished. Core refuses
/// the name when a package is compiled, but `GenRequest` is public, so a
/// caller building one by hand gets told rather than silently wrecked.
#[test]
fn lib_mode_rejects_a_schema_named_lib() {
let schemas = vec![("lib".to_string(), vec![user_struct()])];
let err = generate_rust(&lib_req(&schemas)).unwrap_err().to_string();
assert!(err.contains("crate root"), "{err}");
}

/// The target declares `rust#1.70.0`; the manifest should say so rather
/// than leaving a consumer to discover the toolchain by failing to build.
#[test]
fn lib_mode_records_the_declared_toolchain() {
let schemas = vec![("chat".to_string(), vec![user_struct()])];
let req = GenRequest { lang_version: "1.74.0".into(), ..lib_req(&schemas) };
let files = generate_rust(&req).unwrap();
let manifest = &files
.iter()
.find(|f| f.path.to_str() == Some("Cargo.toml"))
.expect("a lib build has a manifest")
.contents;
assert!(manifest.contains("rust-version = \"1.74.0\""), "{manifest}");
}

/// `main` imports and uses `Shared`, declared over in the `shared` schema -
/// core already expands `use shared::Shared;` into this exact leaf
/// `FrozenUnit::Import` before it ever reaches the generator.
Expand Down
5 changes: 4 additions & 1 deletion std-extra/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
name = "comline_std"
version = "0.1.0"
edition = "2021"
rust-version = "1.70.0"
# A schema namespace named `main` would otherwise be picked up as a
# binary target (`src/main.rs`) with no `fn main`.
# binary target (`src/main.rs`) with no `fn main`. Core refuses that
# name outright now; this stays as a belt-and-braces guard for a
# `GenRequest` built by hand.
autobins = false

[dependencies]
Expand Down
4 changes: 2 additions & 2 deletions xtask/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ publish = false
[dependencies]
# Same rev `codegen` pins, so this tool and the generator it drives agree on
# one `comline-core` - see codegen/Cargo.toml's own comment.
comline-core = { git = "https://github.com/ComlineProject/core", rev = "5c1f3474b462ed912f4f2f5a518b87a5f6576aaa" }
comline-codegen = { git = "https://github.com/ComlineProject/generation", rev = "52803df6436f49aec5f2266a94c89d4ced4b229c" }
comline-core = { git = "https://github.com/ComlineProject/core", rev = "016e38603152444f9b51fddfd46b1e6b1d73f1d8" }
comline-codegen = { git = "https://github.com/ComlineProject/generation", rev = "69a3e37ae788a91c5f72dd047d6196aa4c95fdef" }
comline-codegen-rust = { path = "../codegen" }

eyre = "0.6.8"
60 changes: 59 additions & 1 deletion xtask/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,25 @@

use std::fs;
use std::path::PathBuf;
use std::process::Command;

use comline_codegen::{GenRequest, Mode, PackageMeta};
use comline_codegen_rust::generate_rust;
use comline_codegen_rust::{generate_rust, STD_REV};
use comline_core::package::build::PackageSources;
use comline_core::package::stdlib;
use comline_core::schema::ir::frozen::unit::FrozenUnit;

fn main() -> eyre::Result<()> {
let check = std::env::args().any(|a| a == "--check");
// Deliberately its own flag, not part of `--check`. Regenerating
// `std-extra/` and repointing `STD_REV` at it cannot happen in one
// commit - the rev to point at is the merge commit that does not exist
// yet - so a PR that legitimately changes std's output would always
// fail a combined check. CI runs `--check` on every push and this one
// only on the default branch, where the merge commit exists.
if std::env::args().any(|a| a == "--check-pin") {
std::process::exit(if stale_pin()? { 1 } else { 0 });
}

// std's own real manifest (`congregation std`) - compiling it standalone
// means every schema's own namespace comes out prefixed with that
Expand Down Expand Up @@ -48,6 +58,9 @@ fn main() -> eyre::Result<()> {
mode: Mode::Lib,
schemas: &schemas,
package: PackageMeta { name: "comline_std".to_string(), version: "0.1.0".to_string() },
// The one generator this crate registers, so the version it
// registers under is the one `comline_std`'s own manifest states.
lang_version: comline_codegen_rust::LANG_VERSION.to_string(),
default_framing: None,
// comline_std is generated inline, as an ordinary standalone
// crate - external_std only matters to a *consumer* referencing
Expand Down Expand Up @@ -82,3 +95,48 @@ fn main() -> eyre::Result<()> {

Ok(())
}

/// Whether `generator.rs`'s `STD_REV` points at a commit older than the
/// `std-extra/` sitting beside it.
///
/// Checking the *contents* is only half the guarantee. Every consumer
/// generating in `lib` mode links `comline_std` by default, and reaches it
/// through that pinned rev - so a correctly regenerated `std-extra/` with a
/// pin still on the previous commit ships the old std to everyone, and
/// nothing fails anywhere. This is what stands between a regeneration and
/// that.
///
/// Skipped (not failed) when git can't answer - a published tarball, a
/// shallow clone that doesn't contain `STD_REV`. A missing guard beats a
/// false alarm in a tree that cannot run the guard at all.
fn stale_pin() -> eyre::Result<bool> {
let repo = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");

let reachable = Command::new("git")
.args(["cat-file", "-e", &format!("{STD_REV}^{{commit}}")])
.current_dir(&repo)
.status();
match reachable {
Ok(status) if status.success() => {}
_ => {
eprintln!("note: STD_REV ({STD_REV}) isn't in this checkout - skipping the pin check");
return Ok(false);
}
}

let diff = Command::new("git")
.args(["diff", "--quiet", STD_REV, "HEAD", "--", "std-extra"])
.current_dir(&repo)
.status()?;
if diff.success() {
return Ok(false);
}

eprintln!(
"STD_REV ({STD_REV}) predates changes to std-extra/ - every `lib`-mode \
consumer would link the old std.\n\
Merge this, then repoint STD_REV in codegen/src/generator.rs at the \
merge commit."
);
Ok(true)
}
Loading