From 1caa54306a9fe8b35235ea786d03963009d20f87 Mon Sep 17 00:00:00 2001 From: Greg Anders Date: Thu, 10 Sep 2026 16:15:04 -0500 Subject: [PATCH] Add support for Prometheus service version labels Add a config option that automatically adds a version label to all registered metrics. --- examples/http_server/example_conf.yaml | 5 + foundations/src/telemetry/metrics/init.rs | 56 +++++++++-- foundations/src/telemetry/metrics/mod.rs | 96 ++++++++++++++++++- foundations/src/telemetry/settings/metrics.rs | 7 ++ foundations/tests/custom_metric_via_facade.rs | 2 +- foundations/tests/info_metrics.rs | 2 +- foundations/tests/metrics.rs | 4 +- foundations/tests/metrics_negotiation.rs | 2 +- .../tests/metrics_service_version_label.rs | 55 +++++++++++ 9 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 foundations/tests/metrics_service_version_label.rs diff --git a/examples/http_server/example_conf.yaml b/examples/http_server/example_conf.yaml index 11632aad..3ca475ec 100644 --- a/examples/http_server/example_conf.yaml +++ b/examples/http_server/example_conf.yaml @@ -52,6 +52,11 @@ telemetry: service_name_format: metric_prefix # Whether to report optional metrics in the telemetry server. report_optional: false + # Label name used to add `ServiceInfo::version` to every registered metric. + # + # A metric row that already uses this name with a different value is not + # collected. + service_version_label_name: null # Server settings. server: # Enables telemetry server diff --git a/foundations/src/telemetry/metrics/init.rs b/foundations/src/telemetry/metrics/init.rs index 07041f97..3053d3d2 100644 --- a/foundations/src/telemetry/metrics/init.rs +++ b/foundations/src/telemetry/metrics/init.rs @@ -25,7 +25,13 @@ struct RuntimeInfo { static UNINITIALISED_SERVICE_NAME: &str = "undefined"; #[cfg(feature = "foundations-metrics-backend")] -static SERVICE_NAME: OnceLock = OnceLock::new(); +struct ServiceIdentity { + name: String, + version: &'static str, +} + +#[cfg(feature = "foundations-metrics-backend")] +static SERVICE_IDENTITY: OnceLock = OnceLock::new(); /// Returns the service name to apply when collecting metrics. /// @@ -34,12 +40,17 @@ static SERVICE_NAME: OnceLock = OnceLock::new(); /// taking effect. #[cfg(feature = "foundations-metrics-backend")] pub(super) fn service_name() -> &'static str { - SERVICE_NAME + SERVICE_IDENTITY .get() - .map(String::as_str) + .map(|identity| identity.name.as_str()) .unwrap_or(UNINITIALISED_SERVICE_NAME) } +#[cfg(feature = "foundations-metrics-backend")] +pub(super) fn service_version() -> Option<&'static str> { + SERVICE_IDENTITY.get().map(|identity| identity.version) +} + /// Initializes the metric system with a system-wide metric prefix. /// /// Must be called before any use of metrics defined @@ -53,7 +64,10 @@ pub(crate) fn init( settings: &MetricsSettings, ) -> crate::BootstrapResult<()> { #[cfg(feature = "foundations-metrics-backend")] - validate_service_name_format(settings)?; + { + validate_service_name_format(settings)?; + validate_service_version_label_name(settings)?; + } #[cfg(not(feature = "foundations-metrics-backend"))] let first_install = Registries::init(service_info, settings); @@ -69,8 +83,11 @@ pub(crate) fn init( super::report_nonfatal_collect_error(&args); }); - SERVICE_NAME - .set(service_info.name_in_metrics.clone()) + SERVICE_IDENTITY + .set(ServiceIdentity { + name: service_info.name_in_metrics.clone(), + version: service_info.version, + }) .is_ok() }; @@ -108,6 +125,22 @@ fn validate_service_name_format(settings: &MetricsSettings) -> crate::BootstrapR Ok(()) } +#[cfg(feature = "foundations-metrics-backend")] +pub(super) fn validate_service_version_label_name( + settings: &MetricsSettings, +) -> crate::BootstrapResult<()> { + if let Some(label_name) = &settings.service_version_label_name + && !foundations_metrics::is_valid_name(label_name) + { + anyhow::bail!( + "metrics.service_version_label_name {label_name:?} cannot be encoded; expected {}", + foundations_metrics::NAME_REQUIREMENT, + ); + } + + Ok(()) +} + /// Tested here rather than through `telemetry::init`, which refuses to run twice /// per process and so cannot assert the accepting and rejecting cases together. #[cfg(all(test, feature = "foundations-metrics-backend", feature = "settings"))] @@ -147,4 +180,15 @@ mod service_name_format_tests { fn metric_prefix_format_is_not_subject_to_the_check() { assert!(validate(ServiceNameFormat::MetricPrefix).is_ok()); } + + #[test] + fn unencodable_service_version_label_names_are_rejected() { + let error = validate_service_version_label_name(&MetricsSettings { + service_version_label_name: Some("ver\0sion".to_owned()), + ..Default::default() + }) + .expect_err("an unencodable service version label name must be rejected"); + + assert!(error.to_string().contains("service_version_label_name")); + } } diff --git a/foundations/src/telemetry/metrics/mod.rs b/foundations/src/telemetry/metrics/mod.rs index 86041ec8..af917b3d 100644 --- a/foundations/src/telemetry/metrics/mod.rs +++ b/foundations/src/telemetry/metrics/mod.rs @@ -112,16 +112,100 @@ fn collection_options(settings: &MetricsSettings) -> foundations_metrics::Collec #[cfg(feature = "foundations-metrics-backend")] fn collect_registered_metrics( settings: &MetricsSettings, -) -> Vec { +) -> Result> { + init::validate_service_version_label_name(settings)?; + #[cfg(target_os = "linux")] process::register(); - foundations_metrics::collect(collection_options(settings)) + let mut families = foundations_metrics::collect(collection_options(settings)); + if let Some(name) = settings.service_version_label_name.as_deref() { + let version = init::service_version() + .ok_or("metrics.service_version_label_name requires telemetry to be initialized")?; + apply_service_version_label(&mut families, name, version); + } + Ok(families) +} + +#[cfg(feature = "foundations-metrics-backend")] +fn apply_service_version_label( + families: &mut [foundations_metrics::MetricFamily], + name: &str, + value: &str, +) { + let version_label = foundations_metrics::proto::LabelPair { + name: Some(name.to_owned()), + value: Some(value.to_owned()), + }; + + for family in families { + let family_name = family.name.as_deref().unwrap_or_default(); + family.metric.retain_mut(|metric| { + match metric + .label + .iter() + .find(|label| label.name.as_deref() == Some(name)) + { + Some(label) if label.value.as_deref() != Some(value) => { + report_nonfatal_collect_error(&format_args!( + "skipped row in metric family {family_name:?}; service version label {name:?} already has a different value" + )); + false + } + Some(_) => true, + None => { + metric.label.insert(0, version_label.clone()); + true + } + } + }); + } +} + +#[cfg(all(test, feature = "foundations-metrics-backend"))] +mod service_version_label_tests { + use foundations_metrics::proto::{LabelPair, Metric, MetricFamily}; + + use super::apply_service_version_label; + + fn label(name: &str, value: &str) -> LabelPair { + LabelPair { + name: Some(name.to_owned()), + value: Some(value.to_owned()), + } + } + + #[test] + fn version_label_is_idempotent_and_drops_conflicting_rows() { + let wanted = label("version", "wanted"); + let mut families = [MetricFamily { + metric: vec![ + Metric { + label: vec![wanted.clone()], + ..Default::default() + }, + Metric { + label: vec![label("version", "other")], + ..Default::default() + }, + ], + ..Default::default() + }]; + + apply_service_version_label(&mut families, "version", "wanted"); + + assert_eq!(families[0].metric.len(), 1); + assert_eq!(families[0].metric[0].label, vec![wanted]); + } } /// Collects all metrics in [Prometheus text format]. /// /// [Prometheus text format]: https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format +/// +/// # Errors +/// +/// Fails when the service version label is invalid or telemetry is not initialized. #[cfg_attr( feature = "foundations-metrics-backend", deprecated = "Only ever produces text. Use `collect_format` instead, serving the body it returns with the matching `ScrapeFormat::content_type`." @@ -142,7 +226,9 @@ pub fn collect(settings: &MetricsSettings) -> Result { /// Fails when `format` cannot represent everything this process exposes. /// [`ScrapeFormat::Protobuf`] cannot carry the output of a registered extra /// producer, which is opaque text; [`allow_protobuf`] reports whether it may be -/// asked for. [`ScrapeFormat::fallback`] never fails for this reason. +/// asked for. The configured service version label must be valid, and telemetry +/// must be initialized before it is used. [`ScrapeFormat::fallback`] never fails +/// for format incompatibility. #[cfg(feature = "foundations-metrics-backend")] pub fn collect_format(format: ScrapeFormat, settings: &MetricsSettings) -> Result> { collect_encoded(format, settings) @@ -180,7 +266,7 @@ fn collect_protobuf(settings: &MetricsSettings) -> Result> { ); } - let families = collect_registered_metrics(settings); + let families = collect_registered_metrics(settings)?; Ok(foundations_metrics::encode_to_protobuf(&families)) } @@ -197,7 +283,7 @@ fn collect_text(settings: &MetricsSettings) -> Result { #[cfg(feature = "foundations-metrics-backend")] { - let families = collect_registered_metrics(settings); + let families = collect_registered_metrics(settings)?; buffer.extend_from_slice(foundations_metrics::encode_to_text(&families).as_bytes()); diff --git a/foundations/src/telemetry/settings/metrics.rs b/foundations/src/telemetry/settings/metrics.rs index 2fca35f6..5b308cff 100644 --- a/foundations/src/telemetry/settings/metrics.rs +++ b/foundations/src/telemetry/settings/metrics.rs @@ -11,6 +11,13 @@ pub struct MetricsSettings { /// Whether to report optional metrics in the telemetry server. pub report_optional: bool, + + /// Label name used to add [`crate::ServiceInfo::version`] to every registered metric. + /// + /// A metric row that already uses this name with a different value is not + /// collected. + #[cfg(feature = "foundations-metrics-backend")] + pub service_version_label_name: Option, } /// Service name format. diff --git a/foundations/tests/custom_metric_via_facade.rs b/foundations/tests/custom_metric_via_facade.rs index daecfd1e..2eeb92fb 100644 --- a/foundations/tests/custom_metric_via_facade.rs +++ b/foundations/tests/custom_metric_via_facade.rs @@ -59,7 +59,7 @@ fn a_custom_metric_is_exposed_through_the_facade() { let settings = MetricsSettings { service_name_format: ServiceNameFormat::MetricPrefix, - report_optional: false, + ..Default::default() }; let text = collect_text(&settings); diff --git a/foundations/tests/info_metrics.rs b/foundations/tests/info_metrics.rs index 343e5b2d..4f2e5bc3 100644 --- a/foundations/tests/info_metrics.rs +++ b/foundations/tests/info_metrics.rs @@ -41,7 +41,7 @@ async fn init_reports_build_and_runtime_info_unprefixed() { let settings = MetricsSettings { service_name_format: ServiceNameFormat::MetricPrefix, - report_optional: false, + ..Default::default() }; let text = collect_text(&settings); diff --git a/foundations/tests/metrics.rs b/foundations/tests/metrics.rs index e9b96b03..6c9c289e 100644 --- a/foundations/tests/metrics.rs +++ b/foundations/tests/metrics.rs @@ -52,7 +52,7 @@ fn metrics_unprefixed() { let settings = MetricsSettings { service_name_format: ServiceNameFormat::MetricPrefix, - report_optional: false, + ..Default::default() }; let metrics = collect_text(&settings); @@ -91,7 +91,7 @@ undefined_encode_error_valid 1 let settings = MetricsSettings { service_name_format: ServiceNameFormat::MetricPrefix, - report_optional: false, + ..Default::default() }; let metrics = collect_text(&settings); dbg!(&metrics); diff --git a/foundations/tests/metrics_negotiation.rs b/foundations/tests/metrics_negotiation.rs index 98acaca7..9fa917e3 100644 --- a/foundations/tests/metrics_negotiation.rs +++ b/foundations/tests/metrics_negotiation.rs @@ -122,7 +122,7 @@ async fn metrics_endpoint_serves_the_negotiated_format() { let label_settings = MetricsSettings { service_name_format: ServiceNameFormat::LabelWithName("service".to_owned()), - report_optional: false, + ..Default::default() }; let labelled = collect_text(&label_settings); diff --git a/foundations/tests/metrics_service_version_label.rs b/foundations/tests/metrics_service_version_label.rs new file mode 100644 index 00000000..d44354e0 --- /dev/null +++ b/foundations/tests/metrics_service_version_label.rs @@ -0,0 +1,55 @@ +//! A configured service version label must apply to every registered metric. +#![cfg(all(feature = "foundations-metrics-backend", feature = "settings"))] + +use foundations::ServiceInfo; +use foundations::telemetry::metrics::{Counter, ScrapeFormat, collect_format, metrics}; +use foundations::telemetry::settings::TelemetrySettings; +use foundations::telemetry::{TelemetryConfig, TelemetryContext}; + +const VERSION: &str = "2026.09.10-1-abcdef"; + +#[metrics] +mod requests { + pub fn total() -> Counter; +} + +#[tokio::test] +async fn every_metric_includes_the_service_version() { + let _context = TelemetryContext::test(); + let service_info = ServiceInfo { + name: "test-service", + name_in_metrics: "test_service".to_owned(), + version: VERSION, + author: "Cloudflare", + description: "Test service", + }; + let mut settings = TelemetrySettings::default(); + settings.server.enabled = false; + settings.metrics.service_version_label_name = Some("version".to_owned()); + + foundations::telemetry::init(TelemetryConfig { + service_info: &service_info, + settings: &settings, + custom_server_routes: vec![], + }) + .expect("initialize telemetry"); + requests::total().inc(); + + let output = String::from_utf8( + collect_format(ScrapeFormat::Text { utf8_names: false }, &settings.metrics) + .expect("collect text metrics"), + ) + .expect("metrics are UTF-8"); + let samples: Vec<_> = output + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .collect(); + + assert!(!samples.is_empty()); + assert!( + samples + .iter() + .all(|sample| { sample.contains(&format!(r#"version="{VERSION}""#)) }), + "all samples must contain the service version: {output}" + ); +}