diff --git a/Cargo.lock b/Cargo.lock index 9b45f76..0ddd055 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1809,7 +1809,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stackql-deploy" -version = "2.0.9" +version = "2.1.0" dependencies = [ "base64", "chrono", diff --git a/Cargo.toml b/Cargo.toml index f522b1c..564bda4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stackql-deploy" -version = "2.0.9" +version = "2.1.0" edition = "2021" rust-version = "1.75" description = "Infrastructure-as-code framework for declarative cloud resource management using StackQL" diff --git a/ci-scripts/get-contributors.iql b/ci-scripts/get-contributors.iql index 3e44756..ffa4a0d 100644 --- a/ci-scripts/get-contributors.iql +++ b/ci-scripts/get-contributors.iql @@ -1,20 +1,25 @@ -SELECT login FROM -( -SELECT login, SUM(contributions) total_contributions FROM -(SELECT login, contributions -FROM github.repos.contributors -WHERE owner = 'stackql' -AND repo = 'stackql' -UNION -SELECT login, contributions -FROM github.repos.contributors -WHERE owner = 'stackql' -AND repo = 'stackql-deploy' -UNION -SELECT login, contributions -FROM github.repos.contributors -WHERE owner = 'stackql' -AND repo = 'stackql-deploy-rs') t -GROUP BY login -ORDER BY total_contributions DESC -) t1 +-- Contributors across the stackql, stackql-deploy and stackql-deploy-rs repos, +-- ordered by total contributions. Non-human contributors are excluded: +-- GitHub-flagged bot accounts (type = 'Bot') and AI agent user accounts. +SELECT login FROM +( +SELECT login, SUM(contributions) total_contributions FROM +(SELECT login, type, contributions +FROM github.repos.contributors +WHERE owner = 'stackql' +AND repo = 'stackql' +UNION +SELECT login, type, contributions +FROM github.repos.contributors +WHERE owner = 'stackql' +AND repo = 'stackql-deploy' +UNION +SELECT login, type, contributions +FROM github.repos.contributors +WHERE owner = 'stackql' +AND repo = 'stackql-deploy-rs') t +WHERE type <> 'Bot' +AND login NOT IN ('claude') +GROUP BY login +ORDER BY total_contributions DESC +) t1 diff --git a/docs/exports.md b/docs/exports.md index 4cadff3..471c854 100644 --- a/docs/exports.md +++ b/docs/exports.md @@ -127,7 +127,30 @@ Sensitive values can be masked in log output by listing them under ``` The actual value is still stored in the context and usable by templates; -only the log messages are masked. +only the log messages are masked. Protected export values are masked +everywhere they appear in log output, including rendered queries shown +using `--dry-run` or `--show-queries` and `DEBUG` level logging. + +## Protected inputs (globals and props) + +To mask sensitive input values (rather than exported values), set +`protected: true` on a global variable or resource property: + +```yaml +globals: + - name: postgres_master_password + value: "{{ POSTGRES_MASTER_PASSWORD }}" + protected: true +resources: + - name: operational_db + props: + - name: master_user_password + value: "{{ postgres_master_password }}" + protected: true +``` + +The rendered value is masked (shown as `********`) in all log output; the +real value is still sent to the provider in queries. ## Stack-level exports diff --git a/src/commands/base.rs b/src/commands/base.rs index 781ee34..6e20195 100644 --- a/src/commands/base.rs +++ b/src/commands/base.rs @@ -1271,6 +1271,8 @@ impl CommandRunner { ); println!("{}", sep); for (name, val) in &rows { + // Mask protected values in the displayed table (files keep real values) + let val = crate::core::secrets::redact(val); let display_val = if val.len() > max_val_len { format!("{}...", &val[..max_val_len - 3]) } else { diff --git a/src/core/config.rs b/src/core/config.rs index b59e77f..6e05d37 100644 --- a/src/core/config.rs +++ b/src/core/config.rs @@ -182,6 +182,9 @@ pub fn render_globals( } let sql_compat = to_sql_compatible_json(&rendered); + if global_var.protected { + crate::core::secrets::register_secret(&sql_compat); + } debug!( "Setting global variable [{}] to {}", global_var.name, sql_compat @@ -210,6 +213,9 @@ pub fn render_properties( if let Some(ref value) = prop.value { let rendered = render_value(engine, value, &resource_context); let sql_compat = to_sql_compatible_json(&rendered); + if prop.protected { + crate::core::secrets::register_secret(&sql_compat); + } debug!("Setting property [{}] to {}", prop.name, sql_compat); prop_context.insert(prop.name.clone(), sql_compat.clone()); resource_context.insert(prop.name.clone(), sql_compat); @@ -219,6 +225,9 @@ pub fn render_properties( if let Some(env_val) = values.get(stack_env) { let rendered = render_value(engine, &env_val.value, &resource_context); let sql_compat = to_sql_compatible_json(&rendered); + if prop.protected { + crate::core::secrets::register_secret(&sql_compat); + } debug!( "Setting property [{}] using env-specific value to {}", prop.name, sql_compat @@ -293,6 +302,9 @@ pub fn render_properties( if let Some(merged_val) = base_value { let processed = serde_json::to_string(&merged_val).unwrap_or_default(); + if prop.protected { + crate::core::secrets::register_secret(&processed); + } prop_context.insert(prop.name.clone(), processed.clone()); resource_context.insert(prop.name.clone(), processed); } @@ -437,7 +449,7 @@ pub fn is_json(s: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::resource::manifest::{Property, Resource}; + use crate::resource::manifest::{Property, PropertyValue, Resource}; /// Helper to create a minimal Resource for testing. fn make_resource(name: &str, props: Vec) -> Resource { @@ -466,6 +478,7 @@ mod tests { values: None, description: String::new(), merge: None, + protected: false, } } @@ -654,4 +667,115 @@ mod tests { assert_eq!(ctx.get("client_token").unwrap(), token); } + + // ------------------------------------------------------------------ + // protected (secret) value tests + // ------------------------------------------------------------------ + + #[test] + fn test_protected_prop_registered_for_redaction() { + let engine = TemplateEngine::new(); + let global_context = HashMap::new(); + + let mut prop = make_prop("master_user_password", "Cfg-Prop-S3cret-Value-1"); + prop.protected = true; + + let ctx = render_properties(&engine, &[prop], &global_context, "dev"); + + // Value is stored unmasked in the context (real queries need it) + assert_eq!( + ctx.get("master_user_password").unwrap(), + "Cfg-Prop-S3cret-Value-1" + ); + // But the log scrubber masks it wherever it appears + let redacted = + crate::core::secrets::redact("INSERT ... SELECT 'Cfg-Prop-S3cret-Value-1', ..."); + assert!( + !redacted.contains("Cfg-Prop-S3cret-Value-1"), + "protected prop value leaked: {}", + redacted + ); + } + + #[test] + fn test_protected_prop_env_specific_value_registered_for_redaction() { + let engine = TemplateEngine::new(); + let global_context = HashMap::new(); + + let mut values = HashMap::new(); + values.insert( + "dev".to_string(), + PropertyValue { + value: serde_yaml::Value::String("Cfg-EnvProp-S3cret-Value-2".to_string()), + }, + ); + let prop = Property { + name: "api_key".to_string(), + value: None, + values: Some(values), + description: String::new(), + merge: None, + protected: true, + }; + + let ctx = render_properties(&engine, &[prop], &global_context, "dev"); + + assert_eq!(ctx.get("api_key").unwrap(), "Cfg-EnvProp-S3cret-Value-2"); + let redacted = crate::core::secrets::redact("key = 'Cfg-EnvProp-S3cret-Value-2'"); + assert!(!redacted.contains("Cfg-EnvProp-S3cret-Value-2")); + } + + #[test] + fn test_protected_global_registered_for_redaction() { + let engine = TemplateEngine::new(); + let mut vars = HashMap::new(); + vars.insert( + "DB_PASSWORD".to_string(), + "Cfg-Global-S3cret-Value-3".to_string(), + ); + + let manifest: Manifest = serde_yaml::from_str( + r#" +version: 1 +name: test-stack +providers: + - aws +globals: + - name: db_password + value: "{{ DB_PASSWORD }}" + protected: true + - name: region + value: us-east-1 +"#, + ) + .unwrap(); + + let ctx = render_globals(&engine, &vars, &manifest, "dev", "test-stack"); + + // Stored unmasked + assert_eq!(ctx.get("db_password").unwrap(), "Cfg-Global-S3cret-Value-3"); + // Masked in log output + let redacted = crate::core::secrets::redact("password = 'Cfg-Global-S3cret-Value-3'"); + assert!(!redacted.contains("Cfg-Global-S3cret-Value-3")); + // Non-protected global is not masked + let not_redacted = crate::core::secrets::redact("region = 'us-east-1'"); + assert!(not_redacted.contains("us-east-1")); + } + + #[test] + fn test_unprotected_prop_not_registered() { + let engine = TemplateEngine::new(); + let global_context = HashMap::new(); + + let prop = make_prop("instance_class", "Cfg-Plain-Value-Not-Secret-4"); + + let ctx = render_properties(&engine, &[prop], &global_context, "dev"); + + assert_eq!( + ctx.get("instance_class").unwrap(), + "Cfg-Plain-Value-Not-Secret-4" + ); + let out = crate::core::secrets::redact("class = 'Cfg-Plain-Value-Not-Secret-4'"); + assert!(out.contains("Cfg-Plain-Value-Not-Secret-4")); + } } diff --git a/src/core/mod.rs b/src/core/mod.rs index a80ccbc..c589556 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -8,5 +8,6 @@ pub mod config; pub mod env; pub mod errors; +pub mod secrets; pub mod templating; pub mod utils; diff --git a/src/core/secrets.rs b/src/core/secrets.rs new file mode 100644 index 0000000..fa664b0 --- /dev/null +++ b/src/core/secrets.rs @@ -0,0 +1,161 @@ +// core/secrets.rs + +//! # Secrets Module +//! +//! Process-wide registry of sensitive values that must never appear in log +//! output. Values are registered when a manifest global or resource property +//! marked `protected: true` is rendered (and when resource-level `protected` +//! exports are captured), then every log line is scrubbed of registered +//! values at the logger sink (see `utils::logging`). +//! +//! Redaction is display-only: the real values are still stored in the +//! template context and sent to the server in queries. + +use std::sync::RwLock; + +use log::warn; + +/// Fixed-width mask used in place of secret values. A fixed width is used so +/// that redacted output does not leak the length of the secret. +pub const MASK: &str = "********"; + +/// Values shorter than this are not registered: masking very short strings +/// (e.g. "1", "gp3") would garble unrelated log output. +const MIN_SECRET_LEN: usize = 4; + +/// Registered secret values, kept sorted longest-first so that overlapping +/// secrets are replaced correctly. +static SECRETS: RwLock> = RwLock::new(Vec::new()); + +/// Register a sensitive value for log redaction. +/// +/// Also registers the JSON-string-escaped form of the value when it differs +/// (secrets can appear inside serialized JSON structures, e.g. tags). +pub fn register_secret(value: &str) { + if value.is_empty() { + return; + } + if value.len() < MIN_SECRET_LEN { + warn!( + "protected value is too short to mask reliably ({} chars), it will not be redacted from logs", + value.len() + ); + return; + } + + let mut candidates = vec![value.to_string()]; + + // JSON-escaped form (without the surrounding quotes), e.g. a secret + // containing a double quote or backslash appears escaped inside JSON. + if let Ok(escaped) = serde_json::to_string(value) { + let inner = escaped.trim_matches('"'); + if inner != value { + candidates.push(inner.to_string()); + } + } + + let mut secrets = SECRETS.write().unwrap(); + for candidate in candidates { + if candidate.len() < MIN_SECRET_LEN || secrets.contains(&candidate) { + continue; + } + // Insert maintaining longest-first order + let pos = secrets + .iter() + .position(|s| s.len() < candidate.len()) + .unwrap_or(secrets.len()); + secrets.insert(pos, candidate); + } +} + +/// Replace every occurrence of a registered secret value in `text` with the +/// fixed mask. Returns the input unchanged when no secrets are registered. +pub fn redact(text: &str) -> String { + let secrets = SECRETS.read().unwrap(); + if secrets.is_empty() { + return text.to_string(); + } + let mut out = text.to_string(); + for secret in secrets.iter() { + if out.contains(secret.as_str()) { + out = out.replace(secret.as_str(), MASK); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + // NOTE: the registry is process-global and tests run in parallel, so each + // test uses unique secret values and never clears the registry. + + #[test] + fn test_redact_masks_registered_value() { + register_secret("Sup3r-S3cret-Passw0rd"); + let out = redact("MasterUserPassword = 'Sup3r-S3cret-Passw0rd',"); + assert_eq!(out, format!("MasterUserPassword = '{}',", MASK)); + } + + #[test] + fn test_redact_masks_multiple_occurrences() { + register_secret("mult1-Occurrence-secret"); + let out = redact("a mult1-Occurrence-secret b mult1-Occurrence-secret c"); + assert_eq!(out, format!("a {} b {} c", MASK, MASK)); + } + + #[test] + fn test_redact_leaves_other_text_alone() { + register_secret("unrelated-secret-xyz123"); + let out = redact("no secrets here"); + assert_eq!(out, "no secrets here"); + } + + #[test] + fn test_short_values_not_registered() { + register_secret("ab1"); + let out = redact("value is ab1"); + assert_eq!(out, "value is ab1"); + } + + #[test] + fn test_empty_value_not_registered() { + register_secret(""); + let out = redact("some text"); + assert_eq!(out, "some text"); + } + + #[test] + fn test_overlapping_secrets_longest_first() { + register_secret("overlap-secret"); + register_secret("overlap-secret-longer-form"); + let out = redact("x overlap-secret-longer-form y"); + // The longer secret must be replaced whole, not partially by the shorter one + assert_eq!(out, format!("x {} y", MASK)); + } + + #[test] + fn test_json_escaped_form_registered() { + register_secret(r#"pa"ss\word-with-specials"#); + // As it would appear inside a serialized JSON string + let json = serde_json::to_string(&serde_json::json!({ + "password": r#"pa"ss\word-with-specials"# + })) + .unwrap(); + let out = redact(&json); + assert!( + !out.contains("word-with-specials"), + "escaped secret leaked: {}", + out + ); + } + + #[test] + fn test_duplicate_registration_is_idempotent() { + register_secret("dup-registration-secret"); + register_secret("dup-registration-secret"); + let out = redact("dup-registration-secret"); + assert_eq!(out, MASK); + } +} diff --git a/src/core/utils.rs b/src/core/utils.rs index d18211b..2eff7d6 100644 --- a/src/core/utils.rs +++ b/src/core/utils.rs @@ -608,6 +608,12 @@ pub fn export_vars( ) { for (key, value) in export_data { let is_protected = protected_exports.contains(key); + if is_protected { + // Register for global log redaction so the value is also masked + // anywhere else it surfaces (e.g. interpolated into a downstream + // resource's query shown via --dry-run or --show-queries). + crate::core::secrets::register_secret(value); + } let display_value = if is_protected { "*".repeat(value.len()) } else { @@ -1143,6 +1149,32 @@ mod tests { ); } + #[test] + fn test_export_vars_protected_values_registered_for_redaction() { + // Protected export values must be masked anywhere they later surface + // in log output (e.g. interpolated into a downstream query) + let mut ctx: HashMap = HashMap::new(); + let mut data = HashMap::new(); + data.insert( + "generated_password".to_string(), + "Utils-Exported-S3cret-1".to_string(), + ); + + export_vars( + &mut ctx, + "vault", + &data, + &["generated_password".to_string()], + ); + + let redacted = crate::core::secrets::redact("SELECT 'Utils-Exported-S3cret-1' AS password"); + assert!( + !redacted.contains("Utils-Exported-S3cret-1"), + "protected export value leaked: {}", + redacted + ); + } + // ------------------------------------------------------------------ // has_returning_clause // ------------------------------------------------------------------ diff --git a/src/resource/manifest.rs b/src/resource/manifest.rs index 7c44829..5228c9b 100644 --- a/src/resource/manifest.rs +++ b/src/resource/manifest.rs @@ -89,6 +89,11 @@ pub struct GlobalVar { /// Optional description #[serde(default)] pub description: String, + + /// When true, the rendered value is masked in all log output + /// (dry-run queries, --show-queries, debug logs) + #[serde(default)] + pub protected: bool, } /// Represents a resource in the manifest. @@ -209,6 +214,11 @@ pub struct Property { /// Items to merge with the value #[serde(default)] pub merge: Option>, + + /// When true, the rendered value is masked in all log output + /// (dry-run queries, --show-queries, debug logs) + #[serde(default)] + pub protected: bool, } /// Represents a value for a property in a specific environment. diff --git a/src/utils/logging.rs b/src/utils/logging.rs index e609411..56154cf 100644 --- a/src/utils/logging.rs +++ b/src/utils/logging.rs @@ -55,16 +55,17 @@ pub fn initialize_logger(log_level: &str) { let color = LevelColors::get_color(level_str); let reset = LevelColors::RESET; + // Scrub protected (secret) values from every log line, regardless of + // level or origin. This is the single chokepoint that keeps protected + // manifest values out of dry-run output, --show-queries, and debug logs. + let message = crate::core::secrets::redact(&record.args().to_string()); + if record.level() <= log::Level::Info { // For info, warn, error: [timestamp LEVEL stackql_deploy] message writeln!( buf, "[{} {}{}{} stackql_deploy] {}", - timestamp, - color, - level_str, - reset, - record.args() + timestamp, color, level_str, reset, message ) } else { // For debug, trace: [timestamp LEVEL file_name (line_num)] message @@ -83,7 +84,7 @@ pub fn initialize_logger(log_level: &str) { reset, file_name, record.line().unwrap_or(0), - record.args() + message ) } }); diff --git a/website/docs/manifest-file.md b/website/docs/manifest-file.md index e446778..2af4b9f 100644 --- a/website/docs/manifest-file.md +++ b/website/docs/manifest-file.md @@ -69,6 +69,12 @@ the fields within the __`stackql_manifest.yml`__ file are described in further d *** +### `global.protected` + + + +*** + ### `resources` @@ -215,6 +221,12 @@ When `return_vals` successfully captures an identifier from `RETURNING *`, the f *** +### `resource.prop.protected` + + + +*** + ### `exports` diff --git a/website/docs/manifest_fields/globals/protected.mdx b/website/docs/manifest_fields/globals/protected.mdx new file mode 100644 index 0000000..a523620 --- /dev/null +++ b/website/docs/manifest_fields/globals/protected.mdx @@ -0,0 +1,17 @@ +import File from '/src/components/File'; +import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; + + + +When set to `true` (defaults to `false`), the rendered value of the global variable is masked (shown as `********`) in all log output, including rendered queries shown using `--dry-run` or `--show-queries` and `DEBUG` level logging. Use this for sensitive values sourced from external environment variables or secrets. The actual value is still available to templates and sent to the provider in queries, masking applies to displayed output only. + + + +```yaml {4} +globals: + - name: postgres_master_password + value: "{{ POSTGRES_MASTER_PASSWORD }}" + protected: true +``` + + diff --git a/website/docs/manifest_fields/index.js b/website/docs/manifest_fields/index.js index 9592759..45a3d99 100644 --- a/website/docs/manifest_fields/index.js +++ b/website/docs/manifest_fields/index.js @@ -5,6 +5,7 @@ export { default as Globals } from "./globals.mdx"; export { default as GlobalName } from "./globals/name.mdx"; export { default as GlobalDescription } from "./globals/description.mdx"; export { default as GlobalValue } from "./globals/value.mdx"; +export { default as GlobalProtected } from "./globals/protected.mdx"; export { default as Resources } from "./resources.mdx"; export { default as ResourceName } from "./resources/name.mdx"; export { default as ResourceType } from "./resources/type.mdx"; @@ -22,5 +23,6 @@ export { default as ResourcePropDescription } from "./resources/props/descriptio export { default as ResourcePropValue } from "./resources/props/value.mdx"; export { default as ResourcePropValues } from "./resources/props/values.mdx"; export { default as ResourcePropMerge } from "./resources/props/merge.mdx"; +export { default as ResourcePropProtected } from "./resources/props/protected.mdx"; export { default as Exports } from "./exports.mdx"; export { default as Version } from "./version.mdx"; diff --git a/website/docs/manifest_fields/resources/props/protected.mdx b/website/docs/manifest_fields/resources/props/protected.mdx new file mode 100644 index 0000000..10d45de --- /dev/null +++ b/website/docs/manifest_fields/resources/props/protected.mdx @@ -0,0 +1,23 @@ +import File from '@site/src/components/File'; +import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; + + + +When set to `true` (defaults to `false`), the rendered value of the property is masked (shown as `********`) in all log output, including rendered queries shown using `--dry-run` or `--show-queries` and `DEBUG` level logging. Use this for sensitive property values such as passwords or API keys. The actual value is still sent to the provider in queries, masking applies to displayed output only. + + + +```yaml {7,10} +resources: + - name: operational_db + file: aws/rds_postgres.iql + props: + - name: master_username + value: "{{ postgres_master_username }}" + protected: true + - name: master_user_password + value: "{{ postgres_master_password }}" + protected: true +``` + + diff --git a/website/docs/manifest_fields/resources/protected.mdx b/website/docs/manifest_fields/resources/protected.mdx index 079c879..0b1ad70 100644 --- a/website/docs/manifest_fields/resources/protected.mdx +++ b/website/docs/manifest_fields/resources/protected.mdx @@ -3,7 +3,7 @@ import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; -Protected variables from the `resource`, these variables are masked in the output logs. Protected variables are a subset of `exports` +Protected variables from the `resource`, these variables are masked in the output logs. Protected variables are a subset of `exports`. Protected export values are masked wherever they appear in log output, including rendered queries shown using `--dry-run` or `--show-queries` and `DEBUG` level logging. To mask sensitive input values (rather than exported values), see `global.protected` and `resource.prop.protected`.