From b12ad53bcf72df032bdf2b11df4617cdee5a8e6c Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 13 Sep 2026 05:39:58 -0700 Subject: [PATCH] feat(source): add bounded C++ function analysis (refs #209) --- .github/workflows/ci.yml | 1 + Cargo.lock | 39 ++++ Cargo.toml | 4 + ci/check_compilation_boundary_dependencies.py | 2 + docs/source-cpp.md | 32 +++ src/lib.rs | 4 + src/source.rs | 195 ++++++++++++++++++ tests/source_cpp.rs | 108 ++++++++++ 8 files changed, 385 insertions(+) create mode 100644 docs/source-cpp.md create mode 100644 src/source.rs create mode 100644 tests/source_cpp.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4639acc..a6f86cd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,7 @@ jobs: - text-similarity - command-arguments - config-toml + - source-cpp - terminal-style - terminal-input - event-stream diff --git a/Cargo.lock b/Cargo.lock index 875ef582..9d803b9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2593,6 +2593,8 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", + "tree-sitter", + "tree-sitter-cpp", "ureq", "url", "wasmparser", @@ -4506,6 +4508,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -4789,6 +4792,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "string_cache" version = "0.9.0" @@ -5680,6 +5689,36 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tree-sitter" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1c71c1c4cc0920b20d6b0f6572e7682cd07a6a2faec71067a31fa394c586df" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index fe8931fe..df488f78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ secure-random = ["dep:getrandom"] text-similarity = ["dep:strsim"] command-arguments = ["dep:shell-words"] config-toml = ["dep:toml"] +source-cpp = ["dep:tree-sitter", "dep:tree-sitter-cpp"] terminal-style = [] # Native terminal capture and key decoding without PTY process spawning. terminal-input = [] @@ -226,6 +227,8 @@ getrandom = { version = "=0.4.3", optional = true } strsim = { version = "=0.11.1", optional = true } toml = { version = "=0.8.23", optional = true } shell-words = { version = "=1.1.1", optional = true } +tree-sitter = { version = "=0.26.11", optional = true } +tree-sitter-cpp = { version = "=0.23.4", optional = true } globset = { version = "=0.4.18", optional = true } interprocess = { version = "=2.4.3", optional = true } jwalk = { version = "=0.8.1", optional = true } @@ -387,6 +390,7 @@ toml = "=0.8.23" features = [ "full", "config-toml", + "source-cpp", "daemon-identity", "daemon-frame-v1", "daemon-registration", diff --git a/ci/check_compilation_boundary_dependencies.py b/ci/check_compilation_boundary_dependencies.py index 2e833c17..545eb381 100644 --- a/ci/check_compilation_boundary_dependencies.py +++ b/ci/check_compilation_boundary_dependencies.py @@ -15,6 +15,8 @@ ("pty", "portable-pty"), ("text-similarity", "strsim"), ("command-arguments", "shell-words"), + ("source-cpp", "tree-sitter"), + ("source-cpp", "tree-sitter-cpp"), ("wasm-sketch-host", "wasmtime"), ("ipc", "interprocess"), ("tokio-console", "console-subscriber"), diff --git a/docs/source-cpp.md b/docs/source-cpp.md new file mode 100644 index 00000000..4a159358 --- /dev/null +++ b/docs/source-cpp.md @@ -0,0 +1,32 @@ +# C++ function analysis + +The optional `source-cpp` feature privately owns an exact-pinned C++ grammar and +parser. `source::analyze_cpp` returns owned `FunctionDefinition` records in source +order, with byte ranges and namespace/aggregate/explicit-linkage context flags. +It does not expose parser nodes, trees, grammar symbols, or backend traits. + +The implementation retains provenance from FastLED/fbuild's source scanner at +`1e75ccf5a4ca922b4d922a6da286b965fac8832d`, via the FastLED WASM adapter. It uses +syntax nodes to remove function parameter defaults, rather than splitting on +commas or guessing expression nesting. Headers retain their source formatting, +including newlines terminating line comments. Do not flatten or trim those +newlines before appending a declaration terminator. +They are not guaranteed to be standalone declarations: this is syntax analysis, +not a C++ compiler, preprocessor, or name-resolution service. + +Applications own record selection, deduplication, Arduino `setup`/`loop` policy, +tab order, generated includes, source maps, and editor publication. Definitions +inside another function body are not inventoried. Invalid or incomplete syntax +returns an error without partial records, allowing an editor to keep its last +good product output. + +The source limit is 8 MiB, checked before parsing. The parser observes a +two-second cooperative deadline; this is not process containment or a hard CPU +deadline. Subsequent iterative traversal is limited to 131,072 node visits, +depth 256, and 16,384 output records. Signature traversal shares the visit budget. +These are traversal/output bounds, not independent parser allocation quotas. +Diagnostics do not contain source contents. No filesystem, runtime, or ambient +process state is created by analysis. + +Issue #209 tracks contract expansion, platform verification, and FastLED +adoption. The initial implementation is not yet release-accepted. diff --git a/src/lib.rs b/src/lib.rs index a4d1dd96..8af26f06 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,10 @@ pub mod config; /// an incremental hasher and key-derivation domain separation. pub mod hash; +/// Bounded source analysis; callers own language integration and product policy. +#[cfg(feature = "source-cpp")] +pub mod source; + /// Bounded, fallible operating-system entropy without token-format policy. #[cfg(feature = "secure-random")] pub mod random; diff --git a/src/source.rs b/src/source.rs new file mode 100644 index 00000000..95a87af1 --- /dev/null +++ b/src/source.rs @@ -0,0 +1,195 @@ +//! C++ syntax mechanics adapted from FastLED/fbuild's source scanner at +//! 1e75ccf5a4ca922b4d922a6da286b965fac8832d via fastled-wasm. No Arduino policy. + +use std::ops::{ControlFlow, Range}; +use std::time::{Duration, Instant}; +use tree_sitter::{Node, ParseOptions, Parser}; + +pub const MAX_SOURCE_BYTES: usize = 8 * 1024 * 1024; +pub const MAX_FUNCTIONS: usize = 16_384; +pub const MAX_VISITED_NODES: usize = 131_072; +pub const MAX_DEPTH: usize = 256; + +/// Enclosing syntax contexts, not compiler-resolved semantic scopes. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct FunctionContext { + pub namespace: bool, + pub aggregate: bool, + pub explicit_linkage: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FunctionDefinition { + /// Definition header without its body or function parameter defaults. + /// Formatting and comments are retained. This is not a semantic C++ compiler + /// and does not promise that every header can become a standalone declaration. + pub signature: String, + pub source_range: Range, + pub context: FunctionContext, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum AnalysisError { + #[error("C++ source exceeds the byte limit")] + InputTooLarge, + #[error("C++ syntax is incomplete or invalid")] + InvalidSyntax, + #[error("C++ parsing exceeded its cooperative deadline")] + TimedOut, + #[error("C++ analysis exceeded its traversal or output limit")] + LimitExceeded, + #[error("C++ definition form is unsupported")] + UnsupportedDefinition, + #[error("C++ parser initialization failed")] + ParserUnavailable, +} + +/// Analyze definitions in source order, without deduplication or name filtering. +/// Definitions nested inside a function body are not inventoried. +/// +/// The byte limit is checked before parsing. A two-second cooperative parser +/// deadline is not a hard CPU deadline. Traversal depth/node and record limits +/// apply afterward; they are not independent parser-allocation quotas. Failure +/// returns no partial records. Source is never included in errors. +pub fn analyze_cpp(source: &str) -> Result, AnalysisError> { + if source.len() > MAX_SOURCE_BYTES { + return Err(AnalysisError::InputTooLarge); + } + let mut parser = Parser::new(); + parser + .set_language(&tree_sitter_cpp::LANGUAGE.into()) + .map_err(|_| AnalysisError::ParserUnavailable)?; + let started = Instant::now(); + let mut progress = |_: &tree_sitter::ParseState| { + if started.elapsed() >= Duration::from_secs(2) { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }; + let bytes = source.as_bytes(); + let tree = parser + .parse_with_options( + &mut |offset, _| bytes.get(offset..).unwrap_or_default(), + None, + Some(ParseOptions::new().progress_callback(&mut progress)), + ) + .ok_or(AnalysisError::TimedOut)?; + if tree.root_node().has_error() { + return Err(AnalysisError::InvalidSyntax); + } + let mut remaining = MAX_VISITED_NODES; + let mut pending = vec![(tree.root_node(), 0, FunctionContext::default())]; + let mut output = Vec::new(); + while let Some((node, depth, mut context)) = pending.pop() { + charge(&mut remaining, depth)?; + match node.kind() { + "namespace_definition" => context.namespace = true, + "class_specifier" | "struct_specifier" | "union_specifier" => context.aggregate = true, + "linkage_specification" => context.explicit_linkage = true, + "function_definition" => { + if output.len() == MAX_FUNCTIONS { + return Err(AnalysisError::LimitExceeded); + } + output.push(definition(node, source, context, depth, &mut remaining)?); + continue; + } + _ => {} + } + if pending.len().saturating_add(node.child_count()) > remaining { + return Err(AnalysisError::LimitExceeded); + } + for index in (0..node.child_count()).rev() { + let index = u32::try_from(index).map_err(|_| AnalysisError::LimitExceeded)?; + if let Some(child) = node.child(index) { + pending.push((child, depth + 1, context)); + } + } + } + Ok(output) +} + +fn charge(remaining: &mut usize, depth: usize) -> Result<(), AnalysisError> { + if depth > MAX_DEPTH { + return Err(AnalysisError::LimitExceeded); + } + *remaining = remaining + .checked_sub(1) + .ok_or(AnalysisError::LimitExceeded)?; + Ok(()) +} + +fn definition( + node: Node<'_>, + source: &str, + context: FunctionContext, + root_depth: usize, + remaining: &mut usize, +) -> Result { + let header = node + .parent() + .filter(|parent| parent.kind() == "template_declaration") + .unwrap_or(node); + let body = node + .child_by_field_name("body") + .ok_or(AnalysisError::UnsupportedDefinition)?; + let range = header.start_byte()..body.start_byte(); + let mut removals = Vec::new(); + let mut pending = vec![(node, root_depth)]; + while let Some((part, depth)) = pending.pop() { + charge(remaining, depth)?; + if part.start_byte() >= range.end { + continue; + } + if part.kind() == "optional_parameter_declaration" { + let mut cursor = part.walk(); + let equals = part + .children(&mut cursor) + .find(|child| child.kind() == "=") + .ok_or(AnalysisError::UnsupportedDefinition)?; + let mut start = equals.start_byte(); + // A newline can terminate a preceding // comment. Removing it + // would comment out the following comma or closing parenthesis. + while start > part.start_byte() && matches!(source.as_bytes()[start - 1], b' ' | b'\t') + { + start -= 1; + } + removals.push(start..part.end_byte()); + continue; + } + if pending.len().saturating_add(part.child_count()) > *remaining { + return Err(AnalysisError::LimitExceeded); + } + for index in (0..part.child_count()).rev() { + let index = u32::try_from(index).map_err(|_| AnalysisError::LimitExceeded)?; + if let Some(child) = part.child(index) { + pending.push((child, depth + 1)); + } + } + } + let original = source + .get(range.clone()) + .ok_or(AnalysisError::UnsupportedDefinition)?; + // Copy retained spans once. Repeated in-place deletion would make a wide + // parameter list quadratic even though its source and node count are bounded. + let mut signature = String::with_capacity(original.len()); + let mut retained_start = range.start; + for remove in removals { + if remove.start < retained_start || remove.end > range.end { + return Err(AnalysisError::UnsupportedDefinition); + } + signature.push_str(&source[retained_start..remove.start]); + retained_start = remove.end; + } + signature.push_str(&source[retained_start..range.end]); + Ok(FunctionDefinition { + // Likewise keep a trailing newline before the removed function body: + // the caller may append a semicolon after a line-commented header. + signature: signature + .trim_start() + .trim_end_matches([' ', '\t']) + .to_owned(), + source_range: range, + context, + }) +} diff --git a/tests/source_cpp.rs b/tests/source_cpp.rs new file mode 100644 index 00000000..29fd4c76 --- /dev/null +++ b/tests/source_cpp.rs @@ -0,0 +1,108 @@ +#![cfg(feature = "source-cpp")] + +use kernal_api::source::{analyze_cpp, AnalysisError}; + +#[test] +fn extracts_definitions_without_arduino_policy() { + let functions = + analyze_cpp("void setup() {}\nint helper(int value = 1) { return value; }").unwrap(); + assert_eq!(functions.len(), 2); + assert_eq!(functions[0].signature, "void setup()"); + assert_eq!(functions[1].signature, "int helper(int value)"); + assert!(!functions[1].context.namespace); +} + +#[test] +fn preserves_scope_and_linkage_for_consumer_selection() { + let functions = analyze_cpp( + "namespace n { void f() {} }\nstruct S { void m() {} };\nextern \"C\" { void hook() {} }", + ) + .unwrap(); + assert_eq!(functions.len(), 3); + assert!(functions[0].context.namespace); + assert!(functions[1].context.aggregate); + assert!(functions[2].context.explicit_linkage); +} + +#[test] +fn rejects_incomplete_syntax_without_partial_results() { + assert_eq!( + analyze_cpp("void good() {}\nvoid bad("), + Err(AnalysisError::InvalidSyntax) + ); +} + +#[test] +fn strips_syntax_defaults_not_commas_inside_expressions() { + let functions = analyze_cpp( + r#"int helper(int x = (1 < 2 ? 3 : 4), const char* text = "a,b") { return x; }"#, + ) + .unwrap(); + assert_eq!( + functions[0].signature, + "int helper(int x, const char* text)" + ); + let functions = + analyze_cpp("template \nT f(T value = T{}) { return value; }").unwrap(); + assert_eq!( + functions[0].signature, + "template \nT f(T value)" + ); +} + +#[test] +fn enforces_source_and_traversal_bounds() { + use kernal_api::source::{MAX_DEPTH, MAX_SOURCE_BYTES}; + assert_eq!( + analyze_cpp(&" ".repeat(MAX_SOURCE_BYTES + 1)), + Err(AnalysisError::InputTooLarge) + ); + let source = format!( + "{}void f() {{}}{}", + "namespace n {".repeat(MAX_DEPTH + 1), + "}".repeat(MAX_DEPTH + 1) + ); + assert_eq!(analyze_cpp(&source), Err(AnalysisError::LimitExceeded)); +} + +#[test] +fn preserves_line_comment_terminators_when_removing_defaults() { + let functions = analyze_cpp( + "int f(int x // keep parameter comment\n = 1) // keep header comment\n { return x; }", + ) + .unwrap(); + assert_eq!( + functions[0].signature, + "int f(int x // keep parameter comment\n) // keep header comment\n" + ); + // Appending a declaration terminator must not put it inside a comment. + let declaration = format!("{};", functions[0].signature); + assert_eq!(analyze_cpp(&declaration).unwrap(), vec![]); +} + +#[test] +fn retains_attributes_function_pointer_parameters_and_source_ranges() { + let source = + "// prefix\n[[nodiscard]] int f(int (*callback)(int) = nullptr) noexcept { return 1; }"; + let functions = analyze_cpp(source).unwrap(); + assert_eq!( + functions[0].signature, + "[[nodiscard]] int f(int (*callback)(int)) noexcept" + ); + assert_eq!( + &source[functions[0].source_range.clone()], + "[[nodiscard]] int f(int (*callback)(int) = nullptr) noexcept " + ); +} + +#[test] +fn multiple_defaults_with_lambdas_and_braced_values_keep_parameter_order() { + let functions = analyze_cpp( + "void f(int x = [] { return 1; }(), Pair y = Pair{1, 2}, const char* z = \"é,終\") {}", + ) + .unwrap(); + assert_eq!( + functions[0].signature, + "void f(int x, Pair y, const char* z)" + ); +}