diff --git a/lib/bencher_comment/src/lib.rs b/lib/bencher_comment/src/lib.rs index 3def941364..6d0ceea3f4 100644 --- a/lib/bencher_comment/src/lib.rs +++ b/lib/bencher_comment/src/lib.rs @@ -1,7 +1,7 @@ #![expect(clippy::format_push_string, reason = "todo")] use std::{ - collections::{BTreeMap, HashSet, btree_map::Entry}, + collections::{BTreeMap, BTreeSet, HashSet}, ops::{BitOr, BitOrAssign}, time::Duration, }; @@ -9,14 +9,15 @@ use std::{ #[cfg(feature = "plus")] use bencher_json::SpecUuid; use bencher_json::{ - AlertUuid, BenchmarkSlug, BranchSlug, HeadUuid, JsonAlert, JsonBenchmark, JsonBoundary, - JsonMeasure, JsonPerfQuery, JsonReport, MeasureSlug, ModelUuid, ProjectSlug, ReportUuid, - ResourceName, TestbedSlug, ThresholdUuid, Units, + AlertUuid, BenchmarkSlug, BenchmarkUuid, BranchSlug, HeadUuid, JsonAlert, JsonBenchmark, + JsonBoundary, JsonMeasure, JsonPerfQuery, JsonReport, MeasureSlug, MetricName, ModelUuid, + ParameterSet, ProjectSlug, ReportUuid, ResourceName, TestbedSlug, ThresholdUuid, Units, project::{ alert::AlertStatus, boundary::BoundaryLimit, plot::{LOWER_BOUNDARY, UPPER_BOUNDARY}, report::{JsonReportIteration, JsonReportMeasure, JsonReportResult}, + threshold::JsonThresholdModel, }, }; use ordered_float::OrderedFloat; @@ -39,6 +40,7 @@ pub struct ReportComment { multiple_iterations: bool, benchmark_count: usize, missing_threshold: HashSet, + variant_benchmarks: HashSet, json_report: JsonReport, sub_adapter: SubAdapter, source: String, @@ -64,12 +66,46 @@ impl ReportComment { multiple_iterations: results.len() > 1, benchmark_count: results.iter().map(Vec::len).sum(), missing_threshold: Measure::missing_threshold(&json_report), + variant_benchmarks: variant_benchmarks(&json_report), json_report, sub_adapter, source, } } + /// The name a row gives its benchmark, and the variant that row carries. + /// + /// A benchmark whose every row in this comment carries the empty variant keeps + /// the bare benchmark name it has always had. One non-empty variant among its + /// rows names them all, the empty variant among them included, which reads `{}`. + /// The parameters are spelled in their canonical form and follow the benchmark + /// name, the way the perf image spells them. + fn benchmark_label(&self, benchmark: &JsonBenchmark, parameters: &ParameterSet) -> String { + if self.variant_benchmarks.contains(&benchmark.uuid) { + format!( + "{name} {parameters}", + name = benchmark.name, + parameters = parameters.canonical() + ) + } else { + benchmark.name.to_string() + } + } + + /// The name a row gives a checked measure: the measure, and the metric the + /// threshold checks when that is not the conventional `value`. + /// + /// A threshold that names no metric checks `value`, so it reads as the measure + /// alone, which is every threshold an older client could create. + fn measure_label(measure: &JsonMeasure, metric: Option<&MetricName>) -> String { + match metric { + Some(metric) if *metric != MetricName::value() => { + format!("{name} ({metric})", name = measure.name) + }, + _ => measure.name.to_string(), + } + } + fn results(&self) -> &[JsonReportIteration] { self.json_report.results.as_deref().unwrap_or_default() } @@ -116,7 +152,8 @@ impl ReportComment { for report_measure in &result.measures { text.push_str(&format!( "\n- {benchmark} ({measure}): {console_url}", - benchmark = result.benchmark.name, + benchmark = + self.benchmark_label(&result.benchmark, &result.variant.parameters), measure = report_measure.measure.name, console_url = self.perf_url( &result.benchmark, @@ -138,8 +175,9 @@ impl ReportComment { for alert in self.alerts() { text.push_str(&format!( "\n- {benchmark_name} ({measure_name}){iter}: {console_url}", - benchmark_name = alert.benchmark.name, - measure_name = alert.threshold.measure.name, + benchmark_name = self.benchmark_label(&alert.benchmark, &alert.variant.parameters), + measure_name = + Self::measure_label(&alert.threshold.measure, alert.threshold.metric.as_ref()), iter = if self.multiple_iterations { format!(" (Iteration {iteration})", iteration = alert.iteration) } else { @@ -346,12 +384,13 @@ impl ReportComment { html.push_str(&format!( "{benchmark}", url = self.resource_url(Resource::Benchmark(alert.benchmark.slug.clone())), - benchmark = alert.benchmark.name, + benchmark = self.benchmark_label(&alert.benchmark, &alert.variant.parameters), )); html.push_str(&format!( "{measure}
{units}
", url = self.resource_url(Resource::Measure(alert.threshold.measure.slug.clone())), - measure = alert.threshold.measure.name, + measure = + Self::measure_label(&alert.threshold.measure, alert.threshold.metric.as_ref()), )); self.html_alerts_table_view_cell(html, alert); value_cell( @@ -443,24 +482,24 @@ impl ReportComment { iteration: &JsonReportIteration, require_threshold: bool, ) { - let mbl = boundary_limits_map(iteration, require_threshold); + let columns = measure_columns(iteration, require_threshold); html.push_str(""); - self.html_iteration_table_header(html, &mbl); - self.html_iteration_table_body(html, iteration, &mbl); + self.html_iteration_table_header(html, &columns); + self.html_iteration_table_body(html, iteration, &columns); html.push_str("
"); } fn html_iteration_table_header( &self, html: &mut String, - mbl: &BTreeMap, + columns: &BTreeMap, ) { html.push_str(""); html.push_str(""); html.push_str("Benchmark"); - for (measure, boundary_limits) in mbl { - let units = Units::new(boundary_limits.min.into(), measure.units.clone()).scale_units(); + for (measure, measure_columns) in columns { + let units = Units::new(measure_columns.min.into(), measure.units.clone()).scale_units(); html.push_str(&format!( "{measure}", @@ -468,25 +507,34 @@ impl ReportComment { measure = measure.name, )); - html.push_str(""); - if boundary_limits.has_limit() { - html.push_str("Benchmark Result
"); - } - html.push_str(units.as_ref()); - if boundary_limits.has_limit() { - html.push_str("
(Result Δ%)"); - } - html.push_str(""); + if let Some(boundary_limits) = measure_columns.point_estimate { + html.push_str(""); + if boundary_limits.has_limit() { + html.push_str("Benchmark Result
"); + } + html.push_str(units.as_ref()); + if boundary_limits.has_limit() { + html.push_str("
(Result Δ%)"); + } + html.push_str(""); - if boundary_limits.lower { - html.push_str(&format!( - "Lower Boundary
{units}
(Limit %)" - )); + if boundary_limits.lower { + html.push_str(&format!( + "Lower Boundary
{units}
(Limit %)" + )); + } + + if boundary_limits.upper { + html.push_str(&format!( + "Upper Boundary
{units}
(Limit %)" + )); + } } - if boundary_limits.upper { + for name in &measure_columns.names { html.push_str(&format!( - "Upper Boundary
{units}
(Limit %)" + "{measure} ({name})
{units}", + measure = measure.name, )); } } @@ -498,7 +546,7 @@ impl ReportComment { &self, html: &mut String, iteration: &JsonReportIteration, - mbl: &BTreeMap, + columns: &BTreeMap, ) { html.push_str(""); for result in iteration { @@ -506,78 +554,122 @@ impl ReportComment { html.push_str(&format!( "{name}", url = self.resource_url(Resource::Benchmark(result.benchmark.slug.clone())), - name = result.benchmark.name, + name = self.benchmark_label(&result.benchmark, &result.variant.parameters), )); - for (measure, boundary_limits) in mbl { - let (factor, units_symbol) = { - let units = Units::new(boundary_limits.min.into(), measure.units.clone()); - (units.scale_factor(), units.scale_units_symbol()) - }; - - let report_measure = result - .measures - .iter() - .find(|m| m.measure.slug == measure.slug); - // The point estimate. A measure that named no `value` has nothing for - // this table to draw, so its cells stay empty. - let point_estimate = report_measure.and_then(|m| m.metric.as_ref()); - let alert = self.find_alert(result, measure); - - if let Some(report_measure) = report_measure { - self.html_iteration_table_view_cell( + for (measure, measure_columns) in columns { + self.html_iteration_table_measure_cells(html, result, measure, measure_columns); + } + html.push_str(""); + } + html.push_str(""); + } + + /// Every cell one measure contributes to one row: the view cell, the point + /// estimate's cells when it has columns, and one cell per name of its own. + fn html_iteration_table_measure_cells( + &self, + html: &mut String, + result: &JsonReportResult, + measure: &Measure, + measure_columns: &MeasureColumns, + ) { + let (factor, units_symbol) = { + let units = Units::new(measure_columns.min.into(), measure.units.clone()); + (units.scale_factor(), units.scale_units_symbol()) + }; + + let report_measure = result + .measures + .iter() + .find(|m| m.measure.slug == measure.slug); + // The point estimate. A measure that named no `value` has nothing for + // this table to draw, so its cells stay empty. + let point_estimate = report_measure.and_then(|m| m.metric.as_ref()); + let alert = self.find_alert(result, measure, &MetricName::value()); + + if let Some(report_measure) = report_measure { + self.html_iteration_table_view_cell( + html, + result, + report_measure, + measure_columns.point_estimate.unwrap_or_default(), + alert, + ); + } else { + html.push_str(EMPTY_CELL); + } + + if let Some(boundary_limits) = measure_columns.point_estimate { + if let (Some(report_measure), Some(metric)) = (report_measure, point_estimate) { + value_cell( + html, + metric.value, + report_measure.boundary.and_then(|b| b.baseline), + factor, + &units_symbol, + alert.is_some(), + ); + } else { + html.push_str(EMPTY_CELL); + } + if boundary_limits.lower { + if let (Some(report_measure), Some(metric)) = (report_measure, point_estimate) { + lower_limit_cell( html, - result, - report_measure, - *boundary_limits, - alert, + metric.value, + report_measure.boundary.and_then(|b| b.lower_limit), + factor, + &units_symbol, + alert.is_some_and(|a| a.limit == BoundaryLimit::Lower), ); } else { html.push_str(EMPTY_CELL); } + } + if boundary_limits.upper { if let (Some(report_measure), Some(metric)) = (report_measure, point_estimate) { - value_cell( + upper_limit_cell( html, metric.value, - report_measure.boundary.and_then(|b| b.baseline), + report_measure.boundary.and_then(|b| b.upper_limit), factor, &units_symbol, - alert.is_some(), + alert.is_some_and(|a| a.limit == BoundaryLimit::Upper), ); } else { html.push_str(EMPTY_CELL); } - if boundary_limits.lower { - if let (Some(report_measure), Some(metric)) = (report_measure, point_estimate) { - lower_limit_cell( - html, - metric.value, - report_measure.boundary.and_then(|b| b.lower_limit), - factor, - &units_symbol, - alert.is_some_and(|a| a.limit == BoundaryLimit::Lower), - ); - } else { - html.push_str(EMPTY_CELL); - } - } - if boundary_limits.upper { - if let (Some(report_measure), Some(metric)) = (report_measure, point_estimate) { - upper_limit_cell( - html, - metric.value, - report_measure.boundary.and_then(|b| b.upper_limit), - factor, - &units_symbol, - alert.is_some_and(|a| a.limit == BoundaryLimit::Upper), - ); - } else { - html.push_str(EMPTY_CELL); - } - } } - html.push_str(""); } - html.push_str(""); + + for name in &measure_columns.names { + let named = + report_measure.and_then(|m| m.metrics.iter().find(|metric| metric.name == *name)); + if let Some(named) = named { + // A metric has one column, so what a threshold computed + // for it rides inside the cell the way a baseline does. + // + // The boundaries are in threshold creation order, oldest first, and + // that order is not a ranking: the first is not the winner. It is + // taken because a cell draws one baseline and the choice has to be + // deterministic. This is display only and decides nothing. + let baseline = named + .boundaries + .first() + .and_then(|boundary| boundary.boundary.baseline); + let named_alert = self.find_alert(result, measure, name); + value_cell( + html, + named.value, + baseline, + factor, + &units_symbol, + named_alert.is_some(), + ); + } else { + html.push_str(EMPTY_CELL); + } + } } fn html_iteration_table_view_cell( @@ -597,7 +689,7 @@ impl ReportComment { Some(boundary_limits) ) )); - if let Some(threshold) = &report_measure.threshold { + if let Some(threshold) = view_threshold(report_measure) { html.push_str("
"); html.push_str(&format!( "🚷 view threshold", @@ -672,7 +764,7 @@ impl ReportComment { for iteration in self.results() { for result in iteration { for report_measure in &result.measures { - if report_measure.threshold.is_some() { + if has_threshold(report_measure) { return true; } } @@ -685,10 +777,32 @@ impl ReportComment { !self.alerts().is_empty() } - pub fn find_alert(&self, result: &JsonReportResult, measure: &Measure) -> Option<&JsonAlert> { + /// The alert this row's cell shows, if any. + /// + /// Four dimensions are matched: the benchmark, the variant, the measure, and + /// the name. Two variants of one benchmark are two rows and two names of one + /// measure are two columns, so a cell that matched fewer would show an alert + /// that fired somewhere else. + /// + /// The iteration is a fifth dimension and is deliberately not matched, which is + /// how this has always behaved. Matching it would change what an existing + /// multi-iteration BMF v0 comment renders, so it stays as it is. + pub fn find_alert( + &self, + result: &JsonReportResult, + measure: &Measure, + metric: &MetricName, + ) -> Option<&JsonAlert> { self.alerts().iter().find(|alert| { alert.benchmark.slug == result.benchmark.slug + && alert.variant.uuid == result.variant.uuid && alert.threshold.measure.slug == measure.slug + && alert + .threshold + .metric + .clone() + .unwrap_or_else(MetricName::value) + == *metric }) } @@ -897,7 +1011,7 @@ impl Measure { result .measures .iter() - .filter(|&report_measure| report_measure.threshold.is_none()) + .filter(|&report_measure| !has_threshold(report_measure)) .map(|report_measure| Measure::from(report_measure.measure.clone())) }) }) @@ -1117,19 +1231,95 @@ impl BoundaryLimits { } } -fn boundary_limits_map( +/// The columns one measure contributes to an iteration table. +#[derive(Clone)] +struct MeasureColumns { + /// The smallest number any column of this measure shows, which is what scales + /// the units the whole measure is spelled in. + min: OrderedFloat, + /// The point estimate's columns: the value column, and whichever boundary + /// columns the results produced. + /// + /// Absent when no result of this measure named a point estimate, which BMF v1 + /// permits. Such a measure renders its named columns instead of nothing. + point_estimate: Option, + /// Every name this measure carries beyond the conventional trio, in name order. + /// + /// One column each, after the point estimate's columns. The trio is what the + /// existing columns already draw, so it never earns a column of its own. + names: BTreeSet, +} + +/// The threshold this measure's cell links to, if anything checked it. +/// +/// The deprecated singular threshold is the bare one, which checks the conventional +/// `value` name of every variant. Every threshold a BMF v0 payload can create is +/// bare, so a v0 measure that anything checked has it and this never looks any +/// further: the v0 cell is the cell it always was. A measure checked only by a +/// threshold that names a metric or filters variants has to be read off the rows +/// themselves, which is the only way the cell, `--ci-only-thresholds`, and the no +/// threshold warning can agree about the same measure. +/// +/// The rows are already in threshold creation order, oldest first, and nothing about +/// that order is a ranking. The first is taken because a cell links to one threshold +/// and the choice has to be deterministic, not because it won anything. +fn view_threshold(report_measure: &JsonReportMeasure) -> Option<&JsonThresholdModel> { + report_measure.threshold.as_ref().or_else(|| { + report_measure + .metrics + .iter() + .flat_map(|metric| metric.boundaries.iter()) + .map(|boundary| &boundary.threshold) + .next() + }) +} + +/// Whether anything checked this measure of this variant. +/// +/// One predicate for the cell, `--ci-only-thresholds`, and the no threshold warning, +/// so no comment can shout `NO THRESHOLD` in a cell while staying silent about that +/// measure in the warning above it. +fn has_threshold(report_measure: &JsonReportMeasure) -> bool { + view_threshold(report_measure).is_some() +} + +/// Whether a name is one of the three the metric triple maps onto. +fn is_conventional(name: &MetricName) -> bool { + *name == MetricName::value() + || *name == MetricName::lower_value() + || *name == MetricName::upper_value() +} + +/// Every benchmark whose rows in this comment name the variant they carry. +/// +/// A benchmark is in here when any row of it, result or alert, carries non-empty +/// parameters. Results and alerts are counted together so the two tables of one +/// comment never disagree about how a benchmark is named. +fn variant_benchmarks(json_report: &JsonReport) -> HashSet { + let mut benchmarks = HashSet::new(); + for iteration in json_report.results.as_deref().unwrap_or_default() { + for result in iteration { + if !result.variant.parameters.is_empty() { + benchmarks.insert(result.benchmark.uuid); + } + } + } + for alert in json_report.alerts.as_deref().unwrap_or_default() { + if !alert.variant.parameters.is_empty() { + benchmarks.insert(alert.benchmark.uuid); + } + } + benchmarks +} + +fn measure_columns( iteration: &JsonReportIteration, require_threshold: bool, -) -> BTreeMap { - let mut map = BTreeMap::new(); +) -> BTreeMap { + let mut map: BTreeMap = BTreeMap::new(); for result in iteration { for report_measure in &result.measures { - // A measure that named no point estimate has no row in this table. - let Some(metric) = report_measure.metric.as_ref() else { - continue; - }; - let measure = Measure::from(report_measure.measure.clone()); - let min = { + let point_estimate = report_measure.metric.as_ref().and_then(|metric| { let mut min = metric.value; if let Some(lower_limit) = report_measure.boundary.and_then(|b| b.lower_limit) { min = min.min(lower_limit); @@ -1137,28 +1327,50 @@ fn boundary_limits_map( if let Some(upper_limit) = report_measure.boundary.and_then(|b| b.upper_limit) { min = min.min(upper_limit); } - min - }; - let lower = report_measure - .boundary - .and_then(|b| b.lower_limit) - .is_some(); - let upper = report_measure - .boundary - .and_then(|b| b.upper_limit) - .is_some(); - let boundary_limits = BoundaryLimits { min, lower, upper }; - if require_threshold && !boundary_limits.has_limit() { + let lower = report_measure + .boundary + .and_then(|b| b.lower_limit) + .is_some(); + let upper = report_measure + .boundary + .and_then(|b| b.upper_limit) + .is_some(); + let boundary_limits = BoundaryLimits { min, lower, upper }; + (!require_threshold || boundary_limits.has_limit()).then_some(boundary_limits) + }); + + // A name beyond the trio earns a column of its own. Under + // `--ci-only-thresholds` only a checked name does, the same way only a + // bounded point estimate does. + let named = report_measure + .metrics + .iter() + .filter(|metric| !is_conventional(&metric.name)) + .filter(|metric| !require_threshold || !metric.boundaries.is_empty()) + .collect::>(); + + // A measure with neither a point estimate nor a name of its own has + // nothing for this table to draw. + if point_estimate.is_none() && named.is_empty() { continue; } - match map.entry(measure) { - Entry::Occupied(mut entry) => { - let entry = entry.get_mut(); - *entry |= boundary_limits; - }, - Entry::Vacant(entry) => { - entry.insert(boundary_limits); - }, + + let measure = Measure::from(report_measure.measure.clone()); + let columns = map.entry(measure).or_insert_with(|| MeasureColumns { + min: f64::INFINITY.into(), + point_estimate: None, + names: BTreeSet::new(), + }); + if let Some(boundary_limits) = point_estimate { + columns.min = columns.min.min(boundary_limits.min); + match &mut columns.point_estimate { + Some(point_estimate) => *point_estimate |= boundary_limits, + None => columns.point_estimate = Some(boundary_limits), + } + } + for metric in named { + columns.min = columns.min.min(metric.value); + columns.names.insert(metric.name.clone()); } } } @@ -1171,7 +1383,7 @@ mod tests { DateTime, JsonBranch, JsonHead, JsonProject, JsonReport, JsonReportCounts, JsonTestbed, project::{ Visibility, - report::{Adapter, JsonReportResults}, + report::{Adapter, JsonReportAlerts, JsonReportResults}, }, }; use ordered_float::OrderedFloat; @@ -1253,11 +1465,46 @@ mod tests { } /// One iteration with two measures of one benchmark: one that named a point - /// estimate and one that named only a percentile. + /// estimate and one that named only a percentile, with nothing checking either. + fn value_less_results() -> JsonReportResults { + value_less_results_checked(&serde_json::json!([])) + } + + /// The same iteration, with `boundaries` on the percentile row. /// + /// A threshold that names a metric is never the bare one, so the deprecated + /// singular `threshold` stays absent however many of these there are. That is + /// the wire shape a BMF v1 report produces and the shape this pins. + fn named_check() -> serde_json::Value { + let date = serde_json::to_value(DateTime::TEST).unwrap(); + serde_json::json!([{ + "threshold": { + "uuid": "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", + "project": "11111111-1111-1111-1111-111111111111", + "model": { + "uuid": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "test": "t_test", + "min_sample_size": null, + "max_sample_size": null, + "window": null, + "lower_boundary": null, + "upper_boundary": 0.95, + "created": date, + "replaced": null, + }, + "created": date, + }, + "boundary": { + "baseline": 1.0, + "lower_limit": null, + "upper_limit": 3.0, + }, + }]) + } + /// Built from JSON rather than constructors because the absent deprecated /// `metric` is the wire shape under test. - fn value_less_results() -> JsonReportResults { + fn value_less_results_checked(boundaries: &serde_json::Value) -> JsonReportResults { let date = serde_json::to_value(DateTime::TEST).unwrap(); let measure = |uuid: &str, name: &str, slug: &str| { serde_json::json!({ @@ -1310,7 +1557,7 @@ mod tests { "uuid": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "p99", "value": 2.0, - "boundaries": [], + "boundaries": boundaries, }], "threshold": null, "boundary": null, @@ -1320,9 +1567,148 @@ mod tests { .unwrap() } + /// One iteration of one benchmark on `sets`, one variant per set, each with + /// one measure that named a point estimate. + /// + /// Built from JSON rather than constructors because the parameters on the wire + /// are what is under test. + fn variant_results(sets: &[serde_json::Value]) -> JsonReportResults { + let date = serde_json::to_value(DateTime::TEST).unwrap(); + let results = sets + .iter() + .enumerate() + .map(|(index, set)| { + serde_json::json!({ + "iteration": 0, + "benchmark": { + "uuid": "66666666-6666-6666-6666-666666666666", + "project": "11111111-1111-1111-1111-111111111111", + "name": "bench", + "slug": "bench", + "created": date, + "modified": date, + "archived": null, + }, + "variant": { + "uuid": format!("77777777-7777-7777-7777-77777777777{index}"), + "parameters": set, + }, + "measures": [{ + "measure": { + "uuid": "88888888-8888-8888-8888-888888888888", + "project": "11111111-1111-1111-1111-111111111111", + "name": "Latency", + "slug": "latency", + "units": "nanoseconds (ns)", + "created": date, + "modified": date, + "archived": null, + }, + "metrics": [{ + "uuid": "99999999-9999-9999-9999-999999999999", + "name": "value", + "value": 1.0, + "boundaries": [], + }], + "metric": { + "uuid": "99999999-9999-9999-9999-999999999999", + "value": 1.0, + "lower_value": null, + "upper_value": null, + }, + "threshold": null, + "boundary": null, + }], + }) + }) + .collect::>(); + serde_json::from_value(serde_json::json!([results])).unwrap() + } + + /// One alert on the `p99` row of one variant of `bench`. + fn named_alert(set: &serde_json::Value) -> JsonReportAlerts { + let date = serde_json::to_value(DateTime::TEST).unwrap(); + serde_json::from_value(serde_json::json!([{ + "uuid": "dddddddd-dddd-dddd-dddd-dddddddddddd", + "report": "00000000-0000-0000-0000-000000000000", + "iteration": 0, + "benchmark": { + "uuid": "66666666-6666-6666-6666-666666666666", + "project": "11111111-1111-1111-1111-111111111111", + "name": "bench", + "slug": "bench", + "created": date, + "modified": date, + "archived": null, + }, + "variant": { + "uuid": "77777777-7777-7777-7777-777777777770", + "benchmark": "66666666-6666-6666-6666-666666666666", + "parameters": set, + "created": date, + "modified": date, + "archived": null, + }, + "value": 2.0, + "threshold": { + "uuid": "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", + "project": "11111111-1111-1111-1111-111111111111", + "branch": { + "uuid": "33333333-3333-3333-3333-333333333333", + "project": "11111111-1111-1111-1111-111111111111", + "name": "main", + "slug": "main", + "head": { + "uuid": "44444444-4444-4444-4444-444444444444", + "start_point": null, + "version": null, + "created": date, + "replaced": null, + }, + "created": date, + "modified": date, + "archived": null, + }, + "testbed": { + "uuid": "55555555-5555-5555-5555-555555555555", + "project": "11111111-1111-1111-1111-111111111111", + "name": "localhost", + "slug": "localhost", + "created": date, + "modified": date, + "archived": null, + }, + "measure": { + "uuid": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "project": "11111111-1111-1111-1111-111111111111", + "name": "Throughput", + "slug": "throughput", + "units": "nanoseconds (ns)", + "created": date, + "modified": date, + "archived": null, + }, + "metric": "p99", + "model": null, + "created": date, + "modified": date, + }, + "boundary": { + "baseline": 1.0, + "lower_limit": null, + "upper_limit": 1.5, + }, + "limit": "upper", + "status": "active", + "created": date, + "modified": date, + }])) + .unwrap() + } + // A measure that named no point estimate reaches the comment with the deprecated - // `metric` absent. There is nothing for the table to draw, so it has no column, - // and rendering neither panics nor invents a zero. + // `metric` absent. It has no point estimate column, and every name it did carry + // beyond the conventional trio gets a column of its own. #[test] fn report_table_value_less_measure() { let mut report = json_report(Visibility::Public); @@ -1338,8 +1724,16 @@ mod tests { "unexpected table: {html}" ); assert!( - !html.contains(">Throughput"), - "the measure that named no point estimate has no column: {html}" + html.contains(">Throughput"), + "the measure that named no point estimate still has its named column: {html}" + ); + assert!( + html.contains("Throughput (p99)
nanoseconds (ns)"), + "the metric has a column of its own: {html}" + ); + assert!( + html.contains("2.00 ns"), + "the metric is drawn: {html}" ); // It is still a measure of this report, so the threshold warning names it. assert!( @@ -1348,6 +1742,177 @@ mod tests { ); } + // A measure checked only by a threshold that names a metric has no bare threshold, + // so the deprecated singular field is absent. The cell links to the threshold + // that did check it and never shouts NO THRESHOLD, which is what keeps the cell + // and the report level warning telling the same story about one measure. + #[test] + fn report_table_named_threshold_checks_the_cell() { + let mut report = json_report(Visibility::Public); + report.results = Some(value_less_results_checked(&named_check())); + + let comment = report_comment_for(report); + let html = comment.html(false, None); + + // The measure the named threshold checked: linked, not warned about. + assert!( + html.contains( + "/thresholds/eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee?model=ffffffff-ffff-ffff-ffff-ffffffffffff\">view threshold" + ), + "the cell links the threshold that checked the metric: {html}" + ); + // The warning lists a measure as ` ()`, which the table header + // never does, so this is the warning and not the column. + assert!( + !html.contains("/measures/throughput\">Throughput (nanoseconds (ns))"), + "the warning does not name a measure something checked: {html}" + ); + // The measure nothing checked is still warned about, in the cell and above it. + assert_eq!( + html.matches("⚠️ NO THRESHOLD").count(), + 1, + "only the unchecked measure's cell says so: {html}" + ); + assert!( + html.contains("/measures/latency\">Latency (nanoseconds (ns))"), + "the warning still names the unchecked measure: {html}" + ); + // One checked measure is enough for `--ci-only-thresholds` to post. + assert!(comment.has_threshold(), "unexpected report: {html}"); + } + + // Nothing checked either measure, so every reader agrees the other way too. + #[test] + fn report_table_unchecked_measures_have_no_threshold() { + let mut report = json_report(Visibility::Public); + report.results = Some(value_less_results()); + + let comment = report_comment_for(report); + assert!(!comment.has_threshold()); + let html = comment.html(false, None); + assert!( + !html.contains("view threshold"), + "unexpected table: {html}" + ); + assert!(html.contains("⚠️ NO THRESHOLD"), "unexpected table: {html}"); + } + + // A BMF v0 report has nothing but the conventional trio, so no measure earns a + // named column and the point estimate keeps every column it always had. + #[test] + fn report_table_v0_has_no_named_columns() { + let mut report = json_report(Visibility::Public); + report.results = Some(variant_results(&[serde_json::json!({})])); + + let html = report_comment_for(report).html(false, None); + assert!( + html.contains(">Latencynanoseconds (ns)"), + "unexpected table: {html}" + ); + assert!( + !html.contains("Latency ("), + "the conventional trio never earns a column of its own: {html}" + ); + } + + // A benchmark whose every row in the comment carries the empty variant keeps + // the bare benchmark name it has always had, in the table and in the human text + // alike. + #[test] + fn report_labels_stay_bare_without_variants() { + let mut report = json_report(Visibility::Public); + report.results = Some(variant_results(&[serde_json::json!({})])); + let comment = report_comment_for(report); + + let html = comment.html(false, None); + assert!( + html.contains("/benchmarks/bench\">bench"), + "unexpected table: {html}" + ); + assert!( + !html.contains("bench {}"), + "a benchmark with no variants is named the way it always was: {html}" + ); + assert!( + comment.human().contains("\n- bench (Latency): "), + "unexpected text: {}", + comment.human() + ); + } + + // Two variants of one benchmark are two rows, so each row names the set it + // carries, and the empty set among them reads `{}`. + #[test] + fn report_labels_name_each_variant() { + let mut report = json_report(Visibility::Public); + report.results = Some(variant_results(&[ + serde_json::json!({}), + serde_json::json!({ "size_mb": 16 }), + ])); + let comment = report_comment_for(report); + + let html = comment.html(false, None); + assert!( + html.contains("/benchmarks/bench\">bench {}"), + "unexpected table: {html}" + ); + assert!( + html.contains(r#"/benchmarks/bench">bench {"size_mb":16}"#), + "unexpected table: {html}" + ); + let human = comment.human(); + assert!(human.contains("\n- bench {} (Latency): "), "{human}"); + assert!( + human.contains("\n- bench {\"size_mb\":16} (Latency): "), + "{human}" + ); + } + + // The alert table reads the flat value, names the variant the alert fired on, + // and names the metric its threshold checks when that is not `value`. + #[test] + fn report_alert_names_the_variant_and_the_metric() { + let mut report = json_report(Visibility::Public); + report.results = Some(variant_results(&[serde_json::json!({ "size_mb": 16 })])); + report.alerts = Some(named_alert(&serde_json::json!({ "size_mb": 16 }))); + + let html = report_comment_for(report).html(false, None); + assert!( + html.contains(r#"/benchmarks/bench">bench {"size_mb":16}"#), + "the alert names the variant it fired on: {html}" + ); + assert!( + html.contains(">Throughput (p99)
nanoseconds (ns)"), + "the alert names the metric its threshold checks: {html}" + ); + // The flat `value`, drawn against the boundary's baseline. + assert!( + html.contains("2.00 ns"), + "the alert draws its scalar: {html}" + ); + } + + // A threshold that names no metric checks the conventional `value`, so the alert + // reads as the measure alone, which is every alert an older client could raise. + #[test] + fn report_alert_without_a_metric_names_the_measure_alone() { + let mut alerts = named_alert(&serde_json::json!({})); + alerts[0].threshold.metric = None; + let mut report = json_report(Visibility::Public); + report.results = Some(variant_results(&[serde_json::json!({})])); + report.alerts = Some(alerts); + + let html = report_comment_for(report).html(false, None); + assert!( + html.contains(">Throughput
nanoseconds (ns)"), + "unexpected table: {html}" + ); + assert!( + html.contains("/benchmarks/bench\">bench"), + "unexpected table: {html}" + ); + } + #[test] fn report_table_public_project() { let html = report_comment(Visibility::Public).html(false, None); diff --git a/lib/bencher_json/src/project/perf.rs b/lib/bencher_json/src/project/perf.rs index 5cfa6b1fa6..750a67460a 100644 --- a/lib/bencher_json/src/project/perf.rs +++ b/lib/bencher_json/src/project/perf.rs @@ -533,7 +533,10 @@ pub mod table { use bencher_valid::GitHash; use ordered_float::OrderedFloat; - use tabled::{Table, Tabled}; + use tabled::{ + Table, Tabled, + settings::{Remove, location::ByColumnName}, + }; use crate::{ DateTime, JsonBenchmark, JsonBranch, JsonMeasure, JsonMetricTriple, JsonPerf, JsonProject, @@ -541,10 +544,26 @@ pub mod table { project::{head::VersionNumber, report::Iteration}, }; + /// The header of the column that names each line's variant. + /// + /// A project that never reported any parameters has nothing to tell its lines + /// apart by, so the column is removed rather than filled with `{}`, and this is + /// what names it for removal. The `tabled` rename attribute takes a literal, so + /// the field below spells the same string out. + const PARAMETERS: &str = "Parameters"; + impl From for Table { fn from(json_perf: JsonPerf) -> Self { + // One non-empty variant anywhere in the query is what makes the column worth + // a column: without one, every line is the benchmark's only variant. + let has_variants = json_perf + .results + .iter() + .any(|result| !result.variant.parameters.is_empty()); + let mut perf_table = Vec::new(); for result in json_perf.results { + let parameters = result.variant.parameters.canonical(); for metric in result.metrics { let (baseline, lower_limit, upper_limit) = if let Some(boundary) = metric.boundary { @@ -565,6 +584,7 @@ pub mod table { branch: result.branch.clone(), testbed: result.testbed.clone(), benchmark: result.benchmark.clone(), + parameters: parameters.clone(), measure: result.measure.clone(), iteration: metric.iteration, start_time: metric.start_time, @@ -578,7 +598,11 @@ pub mod table { }); } } - Self::new(perf_table) + let mut table = Self::new(perf_table); + if !has_variants { + table.with(Remove::column(ByColumnName::new(PARAMETERS))); + } + table } } @@ -592,6 +616,13 @@ pub mod table { pub testbed: JsonTestbed, #[tabled(rename = "Benchmark")] pub benchmark: JsonBenchmark, + /// The canonical spelling of the variant this line plots. + /// + /// The column sits between the benchmark and the measure, the way the + /// variant sits between them in a perf result, and it is removed entirely + /// when no line of the query plots a non-empty variant. + #[tabled(rename = "Parameters")] + pub parameters: String, #[tabled(rename = "Measure")] pub measure: JsonMeasure, #[tabled(rename = "Iteration")] @@ -629,4 +660,148 @@ pub mod table { } } } + + #[cfg(test)] + mod tests { + use tabled::Table; + + use crate::JsonPerf; + + /// A one line perf query whose benchmark plots `set`. + fn line(benchmark: &str, set: &str, value: f64) -> String { + format!( + r#"{{ + "branch": {{ + "uuid": "7d7e73de-78c2-43f7-bc2a-da31a5b9a819", + "project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf", + "name": "master", + "slug": "master", + "head": {{ + "uuid": "7d7e73de-78c2-43f7-bc2a-da31a5b9a819", + "start_point": null, + "version": null, + "created": "2023-07-02T12:53:33Z", + "replaced": null + }}, + "created": "2023-07-02T12:53:33Z", + "modified": "2023-07-02T12:53:33Z" + }}, + "testbed": {{ + "uuid": "e095df48-52a6-474b-aaa7-1a8546c235b6", + "project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf", + "name": "base", + "slug": "base", + "created": "2023-07-02T12:53:33Z", + "modified": "2023-07-02T12:53:33Z" + }}, + "benchmark": {{ + "uuid": "dbb90f5c-e7e2-438c-9533-ce86792174ee", + "project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf", + "name": "{benchmark}", + "slug": "dbb90f5c-e7e2-438c-9533-ce86792174ee", + "created": "2023-07-02T12:53:33Z", + "modified": "2023-07-02T12:53:33Z" + }}, + "variant": {{ + "uuid": "b23b1a5e-0f4f-4b8a-9a35-2b7a2f5f0a2f", + "benchmark": "dbb90f5c-e7e2-438c-9533-ce86792174ee", + "parameters": {set}, + "created": "2023-07-02T12:53:33Z", + "modified": "2023-07-02T12:53:33Z", + "archived": null + }}, + "measure": {{ + "uuid": "61a385d0-f19d-4f20-895a-e3c684ec6cbc", + "project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf", + "name": "Latency", + "slug": "latency", + "units": "nanoseconds (ns)", + "created": "2023-07-02T12:53:33Z", + "modified": "2023-07-02T12:53:33Z" + }}, + "metrics": [ + {{ + "report": "ef582192-c7f4-47a0-8668-55cf7d99d8cc", + "iteration": 0, + "start_time": "2023-07-02T12:53:33Z", + "end_time": "2023-07-02T12:53:33Z", + "version": {{ "number": 0, "hash": null }}, + "threshold": null, + "boundary": null, + "alert": null, + "metrics": {{ "value": {{ "value": {value} }} }}, + "metric": {{ + "uuid": "00000000-0000-0000-0000-000000000000", + "value": {value}, + "lower_value": null, + "upper_value": null + }} + }} + ] + }}"# + ) + } + + fn json_perf(lines: &[String]) -> JsonPerf { + let results = lines.join(","); + serde_json::from_str(&format!( + r#"{{ + "project": {{ + "uuid": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf", + "organization": "4142ce9a-f0a0-44d5-94cd-fc76c77d9098", + "name": "The Computer", + "slug": "the-computer", + "url": null, + "visibility": "public", + "bmf_version": 0, + "created": "2023-07-02T12:53:33Z", + "modified": "2023-07-02T12:53:33Z" + }}, + "start_time": null, + "end_time": null, + "results": [{results}] + }}"# + )) + .expect("Failed to parse perf JSON") + } + + fn table(lines: &[String]) -> String { + Table::from(json_perf(lines)).to_string() + } + + // A query whose every line plots the empty variant is the query every + // project made before a benchmark could have more than one variant. It + // prints exactly the table it always printed, column for column. + #[test] + fn table_without_variants() { + let table = table(&[line("bencher::mock_0", "{}", 7.0)]); + assert_eq!( + table, + concat!( + "+--------------+--------+---------+-----------------+---------------------------+-----------+-------------------------+-------------------------+----------------+--------------+--------------+-------------------+----------------------+----------------------+\n", + "| Project | Branch | Testbed | Benchmark | Measure | Iteration | Start Time | End Time | Version Number | Version Hash | Metric Value | Boundary Baseline | Lower Boundary Limit | Upper Boundary Limit |\n", + "+--------------+--------+---------+-----------------+---------------------------+-----------+-------------------------+-------------------------+----------------+--------------+--------------+-------------------+----------------------+----------------------+\n", + "| The Computer | master | base | bencher::mock_0 | Latency: nanoseconds (ns) | 0 | 2023-07-02 12:53:33 UTC | 2023-07-02 12:53:33 UTC | 0 | | 7 | | | |\n", + "+--------------+--------+---------+-----------------+---------------------------+-----------+-------------------------+-------------------------+----------------+--------------+--------------+-------------------+----------------------+----------------------+", + ), + "the table a project without variants prints" + ); + } + + // One non-empty set anywhere in the query earns the column, and every line + // spells the set it plots, the empty set among them as `{}`. + #[test] + fn table_with_variants() { + let table = table(&[ + line("bencher::mock_0", "{}", 7.0), + line("bencher::mock_0", r#"{"size_mb": 16}"#, 8.0), + ]); + assert!(table.contains("Parameters"), "unexpected table: {table}"); + assert!(table.contains("| {} "), "unexpected table: {table}"); + assert!( + table.contains(r#"| {"size_mb":16} "#), + "unexpected table: {table}" + ); + } + } } diff --git a/services/console/public/v1/bmf.json b/services/console/public/v1/bmf.json new file mode 100644 index 0000000000..12157a857d --- /dev/null +++ b/services/console/public/v1/bmf.json @@ -0,0 +1,38 @@ +{ + "$id": "https://bencher.dev/v1/bmf.json", + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "patternProperties": { + ".+": { + "type": "array", + "items": { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "maxProperties": 8, + "patternProperties": { + ".+": { + "type": ["string", "number", "boolean"] + } + } + }, + "measures": { + "type": "object", + "patternProperties": { + ".+": { + "type": "object", + "patternProperties": { + ".+": { + "type": "number" + } + } + } + } + } + }, + "required": ["measures"] + } + } + } +} diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-example.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-example.mdx new file mode 100644 index 0000000000..3e6bf66216 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-example.mdx @@ -0,0 +1,51 @@ +```json +{ + "benchmark_name": [ + { + "measures": { + "latency": { + "value": 88.0, + "lower_value": 87.42, + "upper_value": 88.88 + } + } + }, + { + "parameters": { + "op": "read", + "size_mb": 16 + }, + "measures": { + "latency": { + "value": 132.0, + "lower_value": 130.1, + "upper_value": 133.9 + }, + "throughput": { + "value": 5.55, + "p95": 4.1, + "p99": 3.14 + } + } + }, + { + "parameters": { + "op": "read", + "size_mb": 32 + }, + "measures": { + "latency": { + "value": 261.0, + "lower_value": 258.3, + "upper_value": 263.7 + }, + "throughput": { + "value": 5.42, + "p95": 4.02, + "p99": 3.01 + } + } + } + ] +} +``` diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-schema.mdx new file mode 100644 index 0000000000..0d70e4e6af --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-schema.mdx @@ -0,0 +1,40 @@ +```json +{ + "$id": "https://bencher.dev/v1/bmf.json", + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "patternProperties": { + ".+": { + "type": "array", + "items": { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "maxProperties": 8, + "patternProperties": { + ".+": { + "type": ["string", "number", "boolean"] + } + } + }, + "measures": { + "type": "object", + "patternProperties": { + ".+": { + "type": "object", + "patternProperties": { + ".+": { + "type": "number" + } + } + } + } + } + }, + "required": ["measures"] + } + } + } +} +``` diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/de/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/de/schema.mdx index 9e42b87464..83616e7ad9 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/de/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/de/schema.mdx @@ -8,7 +8,9 @@ Dies ist das [JSON-Schema][json schema] für Bencher Metric Format (BMF) JSON: ### Schema-Versionen: -- Neueste: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (aktuell): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (Standard): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +Die Version ohne Versionsangabe, [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json), ist die Standardversion BMF `v0`. [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/de/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/de/v1.mdx new file mode 100644 index 0000000000..64bf6f0380 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/de/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +Mit BMF `v1` kann ein [Benchmark][benchmark] mehr als ein Ergebnis melden und eine [Measure][measure] mehr als einen Wert. +Ein Benchmark-Name wird einem Array von Einträgen zugeordnet. +Jeder Eintrag benennt die `parameters`, mit denen der Benchmark ausgeführt wurde, und die `measures`, die er erzeugt hat. + + + +In diesem Beispiel meldet `benchmark_name` drei Ergebnisse: +eines ohne Parameter, eines mit `{"op":"read","size_mb":16}` und eines mit `{"op":"read","size_mb":32}`. +Jedes der drei ist eine eigene Variante, mit eigener Historie und eigenen [Alarmen][alert]. + +Die `bmf_version` des Projekts ist die Version, als die ein Bericht gelesen wird, der selbst keine angibt, +und nur ein Server-Administrator kann sie ändern. +BMF `v1` erfordert eine aktuelle Version [der `bencher` CLI][bencher run]. + +### Parameter + +Das `parameters`-Objekt ist die Permutation der Eingaben, mit denen ein Benchmark ausgeführt wurde. +Jeder Schlüssel wird einem JSON-Skalar zugeordnet: einer Zeichenkette, einer Zahl oder einem Boolean. +`null`, Arrays und Objekte werden abgelehnt. +Ein Schlüssel und ein Zeichenketten-Wert sind jeweils höchstens `64` Zeichen lang, und die Parameter eines Benchmarks tragen höchstens `8` Schlüssel. + +Parameter werden in ihrer kanonischen Form verglichen, [RFC 8785 (JSON Canonicalization Scheme)][jcs]. +Die Reihenfolge der Schlüssel spielt keine Rolle, und Zahlen sind ECMAScript-Doubles, +sodass `16`, `16.0` und `1.6e1` ein Wert sind und `{"size_mb": 16}` und `{"size_mb": 16.0}` dieselben Parameter. + +Ein Eintrag ohne `parameters` und ein Eintrag mit `"parameters": {}` sind dieselbe Variante: +die leere Variante, mit der jeder Benchmark geboren wird. +Zwei Einträge, die zu denselben Parametern aufgelöst werden, sind eine Variante, +sodass ihre Measures zusammengeführt werden, statt eine Reihe abzuspalten. + +### Metriken + +Jede Measure wird einem Objekt aus Metriken zugeordnet, und jede Metrik ist eine JSON-Zahl. +Ein Name ist höchstens `64` Zeichen lang, und eine Measure trägt höchstens `8` Namen. +Namen jenseits der Obergrenze werden verworfen: die konventionellen Namen bleiben immer erhalten, +und der Rest wird in lexikographischer Reihenfolge bis zur Obergrenze behalten. + +Drei Namen sind konventionell, nicht privilegiert: + +- `value`: die Punktschätzung, also der Wert, den der Perf-Plot zeichnet und den ein [Threshold][threshold] standardmäßig prüft +- `lower_value`: die untere Grenze der Punktschätzung +- `upper_value`: die obere Grenze der Punktschätzung + +Jeder andere Name, etwa `p95` oder `p99`, wird genau auf dieselbe Weise gespeichert und abgefragt. +Eine Measure darf nur `value` benennen, nur andere Namen oder eine beliebige Mischung aus beidem. + +### Der Array-Wrapper + +⚠️ Ein BMF-`v1`-Payload, das ohne seinen Array-Wrapper geschrieben wird, ist ein gültiges BMF-`v0`-Payload. + +Der generische [`json`-Adapter][json adapter] akzeptiert beide Versionen. +Ein BMF-`v0`-Payload wird als BMF `v0` geparst, und nichts geht verloren. +Aber ein Eintragsobjekt, das dort steht, wo das Array hingehört, sieht genau aus wie eine BMF-`v0`-Measure-Map, +sodass `parameters` und jede Metrik als Measures des Benchmarks gelesen werden +und die beabsichtigten Ergebnisse stillschweigend verloren gehen. + +Wählen Sie für ein BMF-`v1`-Payload den `json_v1`-Adapter. +Der `json_v1`-Adapter akzeptiert nichts außer BMF `v1`, +sodass ein Payload, dem sein Array-Wrapper fehlt, nicht geparst werden kann +und der Bericht abgelehnt statt stillschweigend herabgestuft wird. + +Setzen Sie `bmf_version` im Bericht ebenfalls auf `1`. +Das Feld `bmf_version` gibt die Version an, in der das Payload geschrieben ist, +und es ist das, was dem `json`-Adapter sagt, zuerst BMF `v1` zu versuchen. +Ein Bericht, der keine `bmf_version` setzt, übernimmt die `bmf_version` des Projekts, die standardmäßig `0` ist. + +### Thresholds + +Ein Bericht deklariert seine [Thresholds][threshold] im Feld `thresholds.models`. +Bei BMF `v0` ist dieses Feld eine Map von Measure auf Modell. +Bei BMF `v1` ist es eine Liste von Einträgen, und jeder Eintrag benennt alles, was der Threshold prüft: + +- `measure`: die UUID, der Slug oder der Name der Measure +- `metric`: der Name der Metrik, die der Threshold prüft. Wenn nicht gesetzt, prüft der Threshold den konventionellen Namen `value`. Ein Threshold prüft immer genau einen Namen. +- `parameters`: die Varianten, die der Threshold prüft, als Parameterfilter. Eine Variante passt, wenn irgendein Eintrag des Filters eine Teilmenge ihrer Parameter ist, sodass ein Eintrag nur die Schlüssel benennt, die für ihn wichtig sind, und eine Variante, die mehr Schlüssel festlegt, dennoch passt. Wenn nicht gesetzt oder auf eine leere Liste gesetzt, prüft der Threshold jede Variante. +- `model`: das zu verwendende Threshold-Modell + +Jeder Threshold, der passt, löst aus. +Ein Threshold, der jede Variante prüft, und ein Threshold, der nur `{"size_mb":16}` prüft, +prüfen beide die Variante `{"size_mb":16}`, +sodass eine Regression dort zwei Alarme auslöst, einen für jeden Threshold. + +Das Feld `thresholds.reset` löscht die Thresholds, welche die Form des Payloads adressieren kann. +Eine BMF-`v0`-Map benennt eine Measure und sonst nichts, sodass `reset` nur die Thresholds erreicht, +die den konventionellen Namen `value` jeder Variante prüfen. +Eine BMF-`v1`-Liste kann jeden Threshold adressieren, sodass `reset` sie alle erreicht. + +### Fold + +Fold wird für BMF `v1` nicht unterstützt. +Der Mittelwert der `p99`-Werte je Iteration ist nicht der `p99` der zusammengefassten Stichprobe, +sodass ein für ein BMF-`v1`-Payload angefordertes Fold eine Warnung auslöst +und die Ergebnisse ungefaltet aufgenommen werden, eine Iteration pro Payload. + +### Bencher Metric Format (BMF) v1 JSON-Schema + +Dies ist das [JSON-Schema][json schema] für BMF-`v1`-JSON: + + + +[benchmark]: /de/docs/explanation/benchmarking/#benchmark +[measure]: /de/docs/explanation/benchmarking/#measure +[alert]: /de/docs/explanation/thresholds/#alerts +[threshold]: /de/docs/explanation/thresholds/ +[bencher run]: /de/docs/explanation/bencher-run/ +[json adapter]: /de/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/en/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/en/schema.mdx index 71f7d7b128..56097c8c54 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/en/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/en/schema.mdx @@ -8,7 +8,9 @@ This is the [JSON schema][json schema] for Bencher Metric Format (BMF) JSON: ### Schema Versions: -- Latest: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (latest): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (default): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +The unversioned [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) is the default version, BMF `v0`. [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/en/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/en/v1.mdx new file mode 100644 index 0000000000..dcd08afc3a --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/en/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +BMF `v1` lets one [Benchmark][benchmark] report more than one result and one [Measure][measure] report more than one value. +A Benchmark name maps to an array of entries. +Each entry names the `parameters` the Benchmark ran with and the `measures` it produced. + + + +In this example, `benchmark_name` reports three results: +one with no parameters, one with `{"op":"read","size_mb":16}`, and one with `{"op":"read","size_mb":32}`. +Each of the three is its own variant, with its own history and its own [Alerts][alert]. + +A Project's `bmf_version` is the version a Report that declares none is read as, +and only a server admin can change it. +BMF `v1` requires a current version of [the `bencher` CLI][bencher run]. + +### Parameters + +The `parameters` object is the permutation of inputs a Benchmark ran with. +Each key maps to a JSON scalar: a string, a number, or a boolean. +`null`, arrays, and objects are rejected. +A key and a string value are each at most `64` characters, and a Benchmark's parameters carry at most `8` keys. + +Parameters are compared in their canonical form, [RFC 8785 (JSON Canonicalization Scheme)][jcs]. +Key order does not matter, and numbers are ECMAScript doubles, +so `16`, `16.0`, and `1.6e1` are one value and `{"size_mb": 16}` and `{"size_mb": 16.0}` are the same parameters. + +An entry without `parameters` and an entry with `"parameters": {}` are the same variant: +the empty variant that every Benchmark is born with. +Two entries that resolve to the same parameters are one variant, +so their measures merge rather than fork a series. + +### Metrics + +Each Measure maps to an object of Metrics, and every Metric is a JSON number. +A name is at most `64` characters, and a Measure carries at most `8` names. +Names beyond the cap are dropped: the conventional names are always kept, +and the rest are kept in lexicographic order up to the cap. + +Three names are conventional, not privileged: + +- `value`: the point estimate, which is the value the perf plot draws and the value a [Threshold][threshold] checks by default +- `lower_value`: the lower bound of the point estimate +- `upper_value`: the upper bound of the point estimate + +Every other name, such as `p95` or `p99`, is stored and queried exactly the same way. +A Measure may name only `value`, only other names, or any mix of the two. + +### The Array Wrapper + +⚠️ A BMF `v1` payload written without its array wrapper is a valid BMF `v0` payload. + +The generic [`json` adapter][json adapter] accepts both versions. +A BMF `v0` payload parses as BMF `v0` and nothing is dropped. +But an entry object written where the array belongs looks exactly like a BMF `v0` Measure map, +so `parameters` and every Metric are read as Measures of the Benchmark +and the intended results are silently lost. + +Select the `json_v1` adapter for a BMF `v1` payload. +The `json_v1` adapter accepts nothing but BMF `v1`, +so a payload that is missing its array wrapper fails to parse +and the Report is rejected instead of quietly downgraded. + +Set `bmf_version` to `1` on the Report as well. +The `bmf_version` field states the version the payload is written in, +and it is what tells the `json` adapter to try BMF `v1` first. +A Report that sets no `bmf_version` takes the Project's `bmf_version`, which defaults to `0`. + +### Thresholds + +A Report declares its [Thresholds][threshold] in the `thresholds.models` field. +At BMF `v0` that field is a map of Measure to model. +At BMF `v1` it is a list of entries, and each entry names everything the Threshold checks: + +- `measure`: the Measure UUID, slug, or name +- `metric`: the name of the metric the Threshold checks. If not set, the Threshold checks the conventional `value` name. A Threshold always checks exactly one name. +- `parameters`: the variants the Threshold checks, as a parameters filter. A variant matches when any entry in the filter is a subset of its parameters, so an entry names only the keys it cares about and a variant that pins more keys still matches. If not set, or set to an empty list, the Threshold checks every variant. +- `model`: the Threshold model to use + +Every Threshold that matches fires. +A Threshold that checks every variant and a Threshold that checks only `{"size_mb":16}` +both check the `{"size_mb":16}` variant, +so one regression there raises two Alerts, one for each Threshold. + +The `thresholds.reset` field clears the Thresholds the payload's shape can address. +A BMF `v0` map names a Measure and nothing else, so `reset` reaches only the Thresholds +that check the conventional `value` name of every variant. +A BMF `v1` list can address every Threshold, so `reset` reaches all of them. + +### Fold + +Fold is not supported for BMF `v1`. +The mean of per iteration `p99` values is not the `p99` of the pooled sample, +so a fold requested for a BMF `v1` payload is warned about +and the results are ingested unfolded, one iteration per payload. + +### Bencher Metric Format (BMF) v1 JSON Schema + +This is the [JSON schema][json schema] for BMF `v1` JSON: + + + +[benchmark]: /docs/explanation/benchmarking/#benchmark +[measure]: /docs/explanation/benchmarking/#measure +[alert]: /docs/explanation/thresholds/#alerts +[threshold]: /docs/explanation/thresholds/ +[bencher run]: /docs/explanation/bencher-run/ +[json adapter]: /docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/es/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/es/schema.mdx index 2dc91fe465..4378d0b0c9 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/es/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/es/schema.mdx @@ -8,7 +8,9 @@ Este es el [esquema JSON][json schema] para el JSON de Bencher Metric Format (BM ### Versiones del esquema: -- Más reciente: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (más reciente): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (por defecto): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +La dirección sin versión, [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json), es la versión por defecto, BMF `v0`. [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/es/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/es/v1.mdx new file mode 100644 index 0000000000..a9c93fb9a7 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/es/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +BMF `v1` permite que un [Benchmark][benchmark] reporte más de un resultado y que una [Medida][measure] reporte más de un valor. +Un nombre de Benchmark se asigna a un array de entradas. +Cada entrada nombra los `parameters` con los que se ejecutó el Benchmark y las `measures` que produjo. + + + +En este ejemplo, `benchmark_name` reporta tres resultados: +uno sin parámetros, uno con `{"op":"read","size_mb":16}` y uno con `{"op":"read","size_mb":32}`. +Cada uno de los tres es su propia variante, con su propio historial y sus propias [Alertas][alert]. + +El `bmf_version` del Proyecto es la versión con la que se lee un Reporte que no declara ninguna, +y solo un administrador del servidor puede cambiarlo. +BMF `v1` requiere una versión actual de [la CLI `bencher`][bencher run]. + +### Parámetros + +El objeto `parameters` es la permutación de entradas con las que se ejecutó un Benchmark. +Cada clave se asigna a un escalar JSON: una cadena, un número o un booleano. +`null`, los arrays y los objetos son rechazados. +Una clave y un valor de cadena tienen como máximo `64` caracteres cada uno, y los parámetros de un Benchmark llevan como máximo `8` claves. + +Los parámetros se comparan en su forma canónica, [RFC 8785 (JSON Canonicalization Scheme)][jcs]. +El orden de las claves no importa, y los números son dobles de ECMAScript, +así que `16`, `16.0` y `1.6e1` son un mismo valor, y `{"size_mb": 16}` y `{"size_mb": 16.0}` son los mismos parámetros. + +Una entrada sin `parameters` y una entrada con `"parameters": {}` son la misma variante: +la variante vacía con la que nace todo Benchmark. +Dos entradas que se resuelven a los mismos parámetros son una sola variante, +así que sus Medidas se fusionan en lugar de bifurcar una serie. + +### Métricas + +Cada Medida se asigna a un objeto de Métricas, y cada Métrica es un número JSON. +Un nombre tiene como máximo `64` caracteres, y una Medida lleva como máximo `8` nombres. +Los nombres que exceden el límite se descartan: los nombres convencionales siempre se conservan, +y el resto se conserva en orden lexicográfico hasta el límite. + +Tres nombres son convencionales, no privilegiados: + +- `value`: la estimación puntual, que es el valor que dibuja el gráfico de rendimiento y el valor que un [Umbral][threshold] verifica por defecto +- `lower_value`: el límite inferior de la estimación puntual +- `upper_value`: el límite superior de la estimación puntual + +Cualquier otro nombre, como `p95` o `p99`, se almacena y se consulta exactamente de la misma forma. +Una Medida puede nombrar solo `value`, solo otros nombres, o cualquier mezcla de ambos. + +### El envoltorio del array + +⚠️ Un payload de BMF `v1` escrito sin su envoltorio de array es un payload válido de BMF `v0`. + +El [adaptador `json`][json adapter] genérico acepta ambas versiones. +Un payload de BMF `v0` se analiza como BMF `v0` y no se descarta nada. +Pero un objeto de entrada escrito donde corresponde el array se ve exactamente como un mapa de Medidas de BMF `v0`, +así que `parameters` y cada Métrica se leen como Medidas del Benchmark +y los resultados previstos se pierden en silencio. + +Selecciona el adaptador `json_v1` para un payload de BMF `v1`. +El adaptador `json_v1` no acepta nada más que BMF `v1`, +así que un payload al que le falta su envoltorio de array no se puede analizar +y el Reporte se rechaza en lugar de degradarse en silencio. + +Establece también `bmf_version` en `1` en el Reporte. +El campo `bmf_version` indica la versión en la que está escrito el payload, +y es lo que le indica al adaptador `json` que intente primero BMF `v1`. +Un Reporte que no establece `bmf_version` toma el `bmf_version` del Proyecto, que por defecto es `0`. + +### Umbrales + +Un Reporte declara sus [Umbrales][threshold] en el campo `thresholds.models`. +En BMF `v0` ese campo es un mapa de Medida a modelo. +En BMF `v1` es una lista de entradas, y cada entrada nombra todo lo que el Umbral verifica: + +- `measure`: el UUID, el slug o el nombre de la Medida +- `metric`: el nombre de la métrica que el Umbral verifica. Si no se establece, el Umbral verifica el nombre convencional `value`. Un Umbral siempre verifica exactamente un nombre. +- `parameters`: las variantes que el Umbral verifica, como un filtro de parámetros. Una variante coincide cuando cualquier entrada del filtro es un subconjunto de sus parámetros, así que una entrada nombra solo las claves que le importan y una variante que fija más claves aún coincide. Si no se establece, o se establece como una lista vacía, el Umbral verifica todas las variantes. +- `model`: el modelo de Umbral a usar + +Todos los Umbrales que coinciden se disparan. +Un Umbral que verifica todas las variantes y un Umbral que verifica solo `{"size_mb":16}` +verifican ambos la variante `{"size_mb":16}`, +así que una sola regresión ahí genera dos Alertas, una por cada Umbral. + +El campo `thresholds.reset` borra los Umbrales que la forma del payload puede direccionar. +Un mapa de BMF `v0` nombra una Medida y nada más, así que `reset` alcanza solo los Umbrales +que verifican el nombre convencional `value` de todas las variantes. +Una lista de BMF `v1` puede direccionar todos los Umbrales, así que `reset` los alcanza a todos. + +### Fold + +Fold no es compatible con BMF `v1`. +La media de los valores `p99` por iteración no es el `p99` de la muestra agrupada, +así que un fold solicitado para un payload de BMF `v1` genera una advertencia +y los resultados se ingieren sin plegar, una iteración por payload. + +### Esquema JSON de Bencher Metric Format (BMF) v1 + +Este es el [esquema JSON][json schema] para el JSON de BMF `v1`: + + + +[benchmark]: /es/docs/explanation/benchmarking/#benchmark +[measure]: /es/docs/explanation/benchmarking/#measure +[alert]: /es/docs/explanation/thresholds/#alerts +[threshold]: /es/docs/explanation/thresholds/ +[bencher run]: /es/docs/explanation/bencher-run/ +[json adapter]: /es/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/fr/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/fr/schema.mdx index ae285516a8..dae5012429 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/fr/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/fr/schema.mdx @@ -8,7 +8,9 @@ Il s'agit du [schéma JSON][json schema] pour le Bencher Metric Format (BMF) au ### Versions du schéma : -- Dernière version : [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (dernière) : [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (par défaut) : [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1` : [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +L'adresse sans version, [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json), correspond à la version par défaut, BMF `v0`. [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/fr/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/fr/v1.mdx new file mode 100644 index 0000000000..d29616fff8 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/fr/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +Le BMF `v1` permet à un [Benchmark][benchmark] de rapporter plusieurs résultats et à une [Mesure][measure] de rapporter plusieurs valeurs. +Un nom de Benchmark correspond à un tableau d'entrées. +Chaque entrée nomme les `parameters` avec lesquels le Benchmark s'est exécuté et les `measures` qu'il a produites. + + + +Dans cet exemple, `benchmark_name` rapporte trois résultats : +un sans paramètres, un avec `{"op":"read","size_mb":16}` et un avec `{"op":"read","size_mb":32}`. +Chacun des trois est sa propre variante, avec son propre historique et ses propres [Alertes][alert]. + +Le `bmf_version` du Projet est la version dans laquelle est lu un Rapport qui n'en déclare aucune, +et seul un administrateur du serveur peut le modifier. +Le BMF `v1` exige une version à jour de [la CLI `bencher`][bencher run]. + +### Paramètres + +L'objet `parameters` est la permutation des entrées avec lesquelles un Benchmark s'est exécuté. +Chaque clé correspond à un scalaire JSON : une chaîne de caractères, un nombre ou un booléen. +`null`, les tableaux et les objets sont rejetés. +Une clé et une valeur de type chaîne font chacune au plus `64` caractères, et les paramètres d'un Benchmark portent au plus `8` clés. + +Les paramètres sont comparés sous leur forme canonique, [RFC 8785 (JSON Canonicalization Scheme)][jcs]. +L'ordre des clés n'a pas d'importance, et les nombres sont des doubles ECMAScript, +donc `16`, `16.0` et `1.6e1` ne font qu'une seule valeur et `{"size_mb": 16}` et `{"size_mb": 16.0}` sont les mêmes paramètres. + +Une entrée sans `parameters` et une entrée avec `"parameters": {}` sont la même variante : +la variante vide avec laquelle chaque Benchmark naît. +Deux entrées qui se résolvent vers les mêmes paramètres ne forment qu'une seule variante, +donc leurs mesures fusionnent au lieu de créer une série distincte. + +### Metrics + +Chaque Mesure correspond à un objet de Metrics, et chaque Metric est un nombre JSON. +Un nom fait au plus `64` caractères, et une Mesure porte au plus `8` noms. +Les noms au-delà de cette limite sont supprimés : les noms conventionnels sont toujours conservés, +et les autres sont conservés dans l'ordre lexicographique jusqu'à la limite. + +Trois noms sont conventionnels, sans pour autant être privilégiés : + +- `value` : l'estimation ponctuelle, c'est-à-dire la valeur que le graphique de performance trace et la valeur qu'un [Seuil][threshold] vérifie par défaut +- `lower_value` : la borne inférieure de l'estimation ponctuelle +- `upper_value` : la borne supérieure de l'estimation ponctuelle + +Tout autre nom, tel que `p95` ou `p99`, est stocké et interrogé exactement de la même façon. +Une Mesure peut ne nommer que `value`, ne nommer que d'autres noms, ou n'importe quel mélange des deux. + +### L'enveloppe de tableau + +⚠️ Une charge utile BMF `v1` écrite sans son enveloppe de tableau est une charge utile BMF `v0` valide. + +L'[adaptateur `json`][json adapter] générique accepte les deux versions. +Une charge utile BMF `v0` est analysée comme du BMF `v0` et rien n'est perdu. +Mais un objet d'entrée écrit là où le tableau devrait se trouver ressemble exactement à une carte de Mesures BMF `v0`, +donc `parameters` et chaque Metric sont lus comme des Mesures du Benchmark +et les résultats attendus sont silencieusement perdus. + +Sélectionnez l'adaptateur `json_v1` pour une charge utile BMF `v1`. +L'adaptateur `json_v1` n'accepte rien d'autre que du BMF `v1`, +donc une charge utile à laquelle il manque son enveloppe de tableau échoue à l'analyse +et le Rapport est rejeté au lieu d'être discrètement rétrogradé. + +Définissez également `bmf_version` à `1` sur le Rapport. +Le champ `bmf_version` indique la version dans laquelle la charge utile est écrite, +et c'est lui qui indique à l'adaptateur `json` d'essayer d'abord le BMF `v1`. +Un Rapport qui ne définit pas `bmf_version` reprend le `bmf_version` du Projet, qui vaut `0` par défaut. + +### Seuils + +Un Rapport déclare ses [Seuils][threshold] dans le champ `thresholds.models`. +En BMF `v0`, ce champ est une carte de Mesure vers Modèle. +En BMF `v1`, c'est une liste d'entrées, et chaque entrée nomme tout ce que le Seuil vérifie : + +- `measure` : l'UUID, le slug ou le nom de la Mesure +- `metric` : le nom de la métrique que le Seuil vérifie. S'il n'est pas défini, le Seuil vérifie le nom conventionnel `value`. Un Seuil vérifie toujours exactement un nom. +- `parameters` : les variantes que le Seuil vérifie, sous forme de filtre de paramètres. Une variante correspond dès qu'une entrée du filtre est un sous-ensemble de ses paramètres, donc une entrée ne nomme que les clés qui l'intéressent et une variante qui fixe davantage de clés correspond quand même. S'il n'est pas défini, ou défini comme une liste vide, le Seuil vérifie toutes les variantes. +- `model` : le Modèle de Seuil à utiliser + +Chaque Seuil qui correspond se déclenche. +Un Seuil qui vérifie toutes les variantes et un Seuil qui ne vérifie que `{"size_mb":16}` +vérifient tous deux la variante `{"size_mb":16}`, +donc une seule régression à cet endroit génère deux Alertes, une pour chaque Seuil. + +Le champ `thresholds.reset` efface les Seuils que la forme de la charge utile peut désigner. +Une carte BMF `v0` ne nomme qu'une Mesure et rien d'autre, donc `reset` n'atteint que les Seuils +qui vérifient le nom conventionnel `value` de chaque variante. +Une liste BMF `v1` peut désigner n'importe quel Seuil, donc `reset` les atteint tous. + +### Repli + +Le repli n'est pas pris en charge pour le BMF `v1`. +La moyenne des valeurs `p99` de chaque itération n'est pas le `p99` de l'échantillon regroupé, +donc un repli demandé pour une charge utile BMF `v1` fait l'objet d'un avertissement +et les résultats sont ingérés sans repli, une itération par charge utile. + +### Schéma JSON de Bencher Metric Format (BMF) v1 + +Il s'agit du [schéma JSON][json schema] pour le BMF `v1` au format JSON : + + + +[benchmark]: /fr/docs/explanation/benchmarking/#benchmark +[measure]: /fr/docs/explanation/benchmarking/#measure +[alert]: /fr/docs/explanation/thresholds/#alerts +[threshold]: /fr/docs/explanation/thresholds/ +[bencher run]: /fr/docs/explanation/bencher-run/ +[json adapter]: /fr/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/ja/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/ja/schema.mdx index 221baeb92d..ec2641b941 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/ja/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/ja/schema.mdx @@ -8,7 +8,9 @@ import BmfSchema from "../bmf-schema.mdx"; ### スキーマのバージョン: -- 最新: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0`(最新): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0`(デフォルト): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +バージョンを指定しない [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) は、デフォルトのバージョンである BMF `v0` です。 [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/ja/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/ja/v1.mdx new file mode 100644 index 0000000000..1ff25c8371 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/ja/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +BMF `v1` では、1 つの [Benchmark][benchmark] が複数の結果を報告でき、1 つの [Measure][measure] が複数の値を報告できます。 +Benchmark 名は、エントリの配列に対応します。 +各エントリは、その Benchmark が実行された `parameters` と、生成した `measures` を指定します。 + + + +この例では、`benchmark_name` が 3 つの結果を報告しています。 +1 つはパラメータなし、1 つは `{"op":"read","size_mb":16}`、もう 1 つは `{"op":"read","size_mb":32}` です。 +この 3 つはそれぞれが独立したバリアントであり、それぞれ独自の履歴と独自の [Alert][alert] を持ちます。 + +Project の `bmf_version` は、自身でバージョンを宣言しない Report がどのバージョンとして読まれるかを決めるものであり、 +これを変更できるのはサーバー管理者だけです。 +BMF `v1` には、最新バージョンの [`bencher` CLI][bencher run] が必要です。 + +### パラメータ + +`parameters` オブジェクトは、その Benchmark が実行された入力の組み合わせです。 +各キーは JSON のスカラー値、つまり文字列、数値、真偽値のいずれかに対応します。 +`null`、配列、オブジェクトは拒否されます。 +キーと文字列値はそれぞれ最大 `64` 文字で、1 つの Benchmark のパラメータが持てるキーは最大 `8` 個です。 + +パラメータは、正規形である [RFC 8785 (JSON Canonicalization Scheme)][jcs] で比較されます。 +キーの順序は影響せず、数値は ECMAScript の倍精度浮動小数点数として扱われます。 +そのため `16`、`16.0`、`1.6e1` は 1 つの値であり、`{"size_mb": 16}` と `{"size_mb": 16.0}` は同じパラメータです。 + +`parameters` を持たないエントリと `"parameters": {}` を持つエントリは、同じバリアントです。 +これは、すべての Benchmark が最初から持っている空のバリアントです。 +同じパラメータに解決される 2 つのエントリは 1 つのバリアントであり、 +その measures は系列を分岐させるのではなく統合されます。 + +### メトリック + +各 Measure はメトリックのオブジェクトに対応し、メトリックはすべて JSON の数値です。 +名前は最大 `64` 文字で、1 つの Measure が持てる名前は最大 `8` 個です。 +上限を超えた名前は破棄されます。慣例的な名前は常に保持され、 +残りは辞書順で上限まで保持されます。 + +慣例的な名前は 3 つありますが、特別扱いされるわけではありません: + +- `value`: 点推定値。パフォーマンスプロットが描画する値であり、[Threshold][threshold] がデフォルトでチェックする値です +- `lower_value`: 点推定値の下限 +- `upper_value`: 点推定値の上限 + +`p95` や `p99` など、それ以外の名前もまったく同じように保存され、照会されます。 +Measure は `value` だけを指定することも、それ以外の名前だけを指定することも、両者を混在させることもできます。 + +### 配列ラッパー + +⚠️ 配列ラッパーなしで書かれた BMF `v1` のペイロードは、有効な BMF `v0` のペイロードです。 + +汎用の [`json` アダプター][json adapter] は両方のバージョンを受け付けます。 +BMF `v0` のペイロードは BMF `v0` として解析され、何も失われません。 +しかし、配列があるべき場所に書かれたエントリオブジェクトは、BMF `v0` の Measure のマップとまったく同じに見えます。 +そのため `parameters` とメトリックはすべてその Benchmark の Measure として読み取られ、 +意図した結果は何も告げられないまま失われます。 + +BMF `v1` のペイロードには `json_v1` アダプターを選択してください。 +`json_v1` アダプターは BMF `v1` しか受け付けません。 +そのため、配列ラッパーのないペイロードは解析に失敗し、 +Report は黙って格下げされるのではなく拒否されます。 + +Report の `bmf_version` にも `1` を設定してください。 +`bmf_version` フィールドは、そのペイロードがどのバージョンで書かれているかを示すものであり、 +`json` アダプターに BMF `v1` を先に試させるものです。 +`bmf_version` を設定しない Report は Project の `bmf_version` を引き継ぎ、その既定値は `0` です。 + +### Thresholds + +Report は、その [Threshold][threshold] を `thresholds.models` フィールドで宣言します。 +BMF `v0` では、このフィールドは Measure から model へのマップです。 +BMF `v1` では、これはエントリのリストであり、各エントリはその Threshold がチェックするものをすべて指定します: + +- `measure`: Measure の UUID、スラッグ、または名前 +- `metric`: その Threshold がチェックするメトリックの名前。設定されていない場合、Threshold は慣例的な `value` という名前をチェックします。Threshold がチェックする名前は常にちょうど 1 つです。 +- `parameters`: その Threshold がチェックするバリアントを、パラメータフィルタとして指定します。フィルタ内のいずれかのエントリがバリアントのパラメータの部分集合であれば、そのバリアントは一致します。そのため、エントリは関心のあるキーだけを指定すればよく、より多くのキーを固定しているバリアントも一致します。設定されていない場合、または空のリストが設定されている場合、Threshold はすべてのバリアントをチェックします。 +- `model`: 使用する Threshold の model + +一致した Threshold はすべて発火します。 +すべてのバリアントをチェックする Threshold と、`{"size_mb":16}` だけをチェックする Threshold は、 +どちらも `{"size_mb":16}` のバリアントをチェックします。 +そのため、そこで 1 つの性能低下が起きると、Threshold ごとに 1 つずつ、計 2 つの Alert が生成されます。 + +`thresholds.reset` フィールドは、そのペイロードの形式で指定できる Threshold を解除します。 +BMF `v0` のマップは Measure しか指定できないため、`reset` が届くのは、 +すべてのバリアントの慣例的な `value` という名前をチェックする Threshold だけです。 +BMF `v1` のリストはあらゆる Threshold を指定できるため、`reset` はそのすべてに届きます。 + +### Fold + +BMF `v1` では Fold はサポートされません。 +反復ごとの `p99` の平均は、まとめた標本の `p99` ではありません。 +そのため、BMF `v1` のペイロードに対して要求された fold は警告され、 +結果は fold されずに、ペイロードごとに 1 反復として取り込まれます。 + +### Bencher Metric Format (BMF) v1 JSON スキーマ + +これは BMF `v1` JSON の [JSON スキーマ][json schema]です: + + + +[benchmark]: /ja/docs/explanation/benchmarking/#benchmark +[measure]: /ja/docs/explanation/benchmarking/#measure +[alert]: /ja/docs/explanation/thresholds/#alerts +[threshold]: /ja/docs/explanation/thresholds/ +[bencher run]: /ja/docs/explanation/bencher-run/ +[json adapter]: /ja/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/ko/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/ko/schema.mdx index 171d7e82ec..a68237a6c4 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/ko/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/ko/schema.mdx @@ -8,7 +8,9 @@ import BmfSchema from "../bmf-schema.mdx"; ### 스키마 버전: -- 최신: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (최신): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (기본): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +버전을 지정하지 않은 [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json)은 기본 버전인 BMF `v0`입니다. [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/ko/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/ko/v1.mdx new file mode 100644 index 0000000000..383c5c5e44 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/ko/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +BMF `v1`에서는 하나의 [Benchmark][benchmark]가 여러 결과를 보고할 수 있고, 하나의 [Measure][measure]가 여러 값을 보고할 수 있습니다. +Benchmark 이름은 항목의 배열에 대응됩니다. +각 항목은 해당 Benchmark가 실행된 `parameters`와 생성한 `measures`를 지정합니다. + + + +이 예제에서 `benchmark_name`은 세 개의 결과를 보고합니다. +하나는 파라미터가 없고, 하나는 `{"op":"read","size_mb":16}`, 나머지 하나는 `{"op":"read","size_mb":32}`입니다. +이 셋은 각각 독립된 변형이며, 각자의 이력과 각자의 [Alert][alert]를 가집니다. + +Project의 `bmf_version`은 스스로 버전을 선언하지 않은 Report가 어떤 버전으로 읽히는지를 정하며, +이를 바꿀 수 있는 것은 서버 관리자뿐입니다. +BMF `v1`은 최신 버전의 [`bencher` CLI][bencher run]가 필요합니다. + +### 파라미터 + +`parameters` 객체는 해당 Benchmark가 실행된 입력의 조합입니다. +각 키는 JSON 스칼라 값, 즉 문자열, 숫자, 불리언 중 하나에 대응됩니다. +`null`, 배열, 객체는 거부됩니다. +키와 문자열 값은 각각 최대 `64`자이며, 하나의 Benchmark의 파라미터는 최대 `8`개의 키를 가집니다. + +파라미터는 정규 형식인 [RFC 8785 (JSON Canonicalization Scheme)][jcs]로 비교됩니다. +키 순서는 영향을 주지 않으며, 숫자는 ECMAScript 배정밀도 부동소수점으로 취급됩니다. +따라서 `16`, `16.0`, `1.6e1`은 하나의 값이고, `{"size_mb": 16}`과 `{"size_mb": 16.0}`은 같은 파라미터입니다. + +`parameters`가 없는 항목과 `"parameters": {}`를 가진 항목은 같은 변형입니다. +이는 모든 Benchmark가 처음부터 가지고 있는 빈 변형입니다. +같은 파라미터로 해석되는 두 항목은 하나의 변형이므로, +그 measures는 계열을 나누지 않고 병합됩니다. + +### 메트릭 + +각 Measure는 메트릭 객체에 대응되며, 메트릭은 모두 JSON 숫자입니다. +이름은 최대 `64`자이며, 하나의 Measure는 최대 `8`개의 이름을 가집니다. +상한을 넘는 이름은 버려집니다. 관례적인 이름은 항상 유지되고, +나머지는 사전순으로 상한까지 유지됩니다. + +관례적인 이름은 세 가지이며, 특별한 권한을 가지지는 않습니다: + +- `value`: 점 추정치로, 성능 플롯이 그리는 값이자 [Threshold][threshold]가 기본적으로 검사하는 값입니다 +- `lower_value`: 점 추정치의 하한 +- `upper_value`: 점 추정치의 상한 + +`p95`나 `p99` 같은 다른 모든 이름도 정확히 같은 방식으로 저장되고 조회됩니다. +Measure는 `value`만 지정할 수도, 다른 이름만 지정할 수도, 둘을 섞어서 지정할 수도 있습니다. + +### 배열 래퍼 + +⚠️ 배열 래퍼 없이 작성된 BMF `v1` 페이로드는 유효한 BMF `v0` 페이로드입니다. + +범용 [`json` 어댑터][json adapter]는 두 버전을 모두 허용합니다. +BMF `v0` 페이로드는 BMF `v0`으로 파싱되며 아무것도 손실되지 않습니다. +하지만 배열이 있어야 할 자리에 작성된 항목 객체는 BMF `v0`의 Measure 맵과 정확히 똑같아 보입니다. +그래서 `parameters`와 메트릭이 모두 해당 Benchmark의 Measure로 읽히고, +의도한 결과는 아무런 알림 없이 손실됩니다. + +BMF `v1` 페이로드에는 `json_v1` 어댑터를 선택하십시오. +`json_v1` 어댑터는 BMF `v1` 외에는 아무것도 허용하지 않습니다. +따라서 배열 래퍼가 빠진 페이로드는 파싱에 실패하고, +Report는 조용히 하위 버전으로 강등되는 대신 거부됩니다. + +Report에도 `bmf_version`을 `1`로 설정하십시오. +`bmf_version` 필드는 해당 페이로드가 어떤 버전으로 작성되었는지를 나타내고, +`json` 어댑터가 BMF `v1`을 먼저 시도하도록 알려 주는 값입니다. +`bmf_version`을 설정하지 않은 Report는 Project의 `bmf_version`을 따르며, 그 기본값은 `0`입니다. + +### Thresholds + +Report는 자신의 [Threshold][threshold]를 `thresholds.models` 필드에 선언합니다. +BMF `v0`에서 이 필드는 Measure를 model에 대응시키는 맵입니다. +BMF `v1`에서는 항목의 목록이며, 각 항목은 해당 Threshold가 검사하는 모든 것을 지정합니다: + +- `measure`: Measure의 UUID, 슬러그 또는 이름 +- `metric`: 해당 Threshold가 검사하는 메트릭의 이름. 설정하지 않으면 Threshold는 관례적인 `value` 이름을 검사합니다. Threshold는 항상 정확히 하나의 이름을 검사합니다. +- `parameters`: 해당 Threshold가 검사하는 변형을 파라미터 필터로 지정합니다. 필터의 어느 한 항목이 변형의 파라미터의 부분집합이면 그 변형은 일치합니다. 따라서 항목은 관심 있는 키만 지정하면 되고, 더 많은 키를 고정한 변형도 일치합니다. 설정하지 않거나 빈 목록으로 설정하면 Threshold는 모든 변형을 검사합니다. +- `model`: 사용할 Threshold model + +일치하는 모든 Threshold가 발동합니다. +모든 변형을 검사하는 Threshold와 `{"size_mb":16}`만 검사하는 Threshold는 +둘 다 `{"size_mb":16}` 변형을 검사합니다. +따라서 그곳에서 성능 저하가 한 번 발생하면 Threshold마다 하나씩, 두 개의 Alert가 생성됩니다. + +`thresholds.reset` 필드는 해당 페이로드의 형태로 지정할 수 있는 Threshold를 해제합니다. +BMF `v0` 맵은 Measure만 지정할 뿐이므로, `reset`은 모든 변형의 관례적인 `value` 이름을 +검사하는 Threshold에만 도달합니다. +BMF `v1` 목록은 모든 Threshold를 지정할 수 있으므로, `reset`은 그 전부에 도달합니다. + +### Fold + +BMF `v1`에서는 Fold가 지원되지 않습니다. +반복별 `p99` 값의 평균은 합쳐진 표본의 `p99`가 아닙니다. +따라서 BMF `v1` 페이로드에 대해 요청된 fold는 경고가 발생하고, +결과는 fold되지 않은 채 페이로드당 한 번의 반복으로 수집됩니다. + +### Bencher Metric Format (BMF) v1 JSON 스키마 + +다음은 BMF `v1` JSON의 [JSON 스키마][json schema]입니다: + + + +[benchmark]: /ko/docs/explanation/benchmarking/#benchmark +[measure]: /ko/docs/explanation/benchmarking/#measure +[alert]: /ko/docs/explanation/thresholds/#alerts +[threshold]: /ko/docs/explanation/thresholds/ +[bencher run]: /ko/docs/explanation/bencher-run/ +[json adapter]: /ko/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/pt/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/pt/schema.mdx index 268795580b..eed1bf8f64 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/pt/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/pt/schema.mdx @@ -8,7 +8,9 @@ Este é o [esquema JSON][json schema] para o Bencher Metric Format (BMF) em JSON ### Versões do esquema: -- Mais recente: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (mais recente): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (padrão): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) -[json schema]: https://json-schema.org/draft-07/json-schema-release-notes \ No newline at end of file +O endereço sem versão, [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json), é a versão padrão, BMF `v0`. + +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/pt/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/pt/v1.mdx new file mode 100644 index 0000000000..4c31858729 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/pt/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +O BMF `v1` permite que um [Benchmark][benchmark] relate mais de um resultado e que uma [Measure][measure] relate mais de um valor. +Um nome de Benchmark corresponde a um array de entradas. +Cada entrada nomeia os `parameters` com os quais o Benchmark foi executado e as `measures` que ele produziu. + + + +Neste exemplo, `benchmark_name` relata três resultados: +um sem parâmetros, um com `{"op":"read","size_mb":16}` e um com `{"op":"read","size_mb":32}`. +Cada um dos três é sua própria variante, com seu próprio histórico e seus próprios [Alertas][alert]. + +O `bmf_version` do Projeto é a versão com a qual um Relatório que não declara nenhuma é lido, +e apenas um administrador do servidor pode alterá-lo. +O BMF `v1` exige uma versão atual da [CLI `bencher`][bencher run]. + +### Parâmetros + +O objeto `parameters` é a permutação de entradas com as quais um Benchmark foi executado. +Cada chave corresponde a um escalar JSON: uma string, um número ou um booleano. +`null`, arrays e objetos são rejeitados. +Uma chave e um valor de string têm, cada um, no máximo `64` caracteres, e os parâmetros de um Benchmark carregam no máximo `8` chaves. + +Os parâmetros são comparados em sua forma canônica, [RFC 8785 (JSON Canonicalization Scheme)][jcs]. +A ordem das chaves não importa, e os números são doubles do ECMAScript, +portanto `16`, `16.0` e `1.6e1` são um único valor e `{"size_mb": 16}` e `{"size_mb": 16.0}` são os mesmos parâmetros. + +Uma entrada sem `parameters` e uma entrada com `"parameters": {}` são a mesma variante: +a variante vazia com a qual todo Benchmark nasce. +Duas entradas que resolvem para os mesmos parâmetros são uma única variante, +portanto suas measures se fundem em vez de bifurcar uma série. + +### Métricas + +Cada Measure corresponde a um objeto de Métricas, e toda Métrica é um número JSON. +Um nome tem no máximo `64` caracteres, e uma Measure carrega no máximo `8` nomes. +Os nomes que ultrapassam esse limite são descartados: os nomes convencionais são sempre mantidos, +e os demais são mantidos em ordem lexicográfica até o limite. + +Três nomes são convencionais, não privilegiados: + +- `value`: a estimativa pontual, que é o valor que o gráfico de desempenho desenha e o valor que um [Limiar][threshold] verifica por padrão +- `lower_value`: o limite inferior da estimativa pontual +- `upper_value`: o limite superior da estimativa pontual + +Todo outro nome, como `p95` ou `p99`, é armazenado e consultado exatamente da mesma maneira. +Uma Measure pode nomear apenas `value`, apenas outros nomes, ou qualquer mistura dos dois. + +### O Invólucro de Array + +⚠️ Um payload BMF `v1` escrito sem seu invólucro de array é um payload BMF `v0` válido. + +O [adaptador `json`][json adapter] genérico aceita ambas as versões. +Um payload BMF `v0` é analisado como BMF `v0` e nada é descartado. +Mas um objeto de entrada escrito onde o array deveria estar se parece exatamente com um mapa de Measures do BMF `v0`, +portanto `parameters` e cada Métrica são lidos como Measures do Benchmark +e os resultados pretendidos são perdidos silenciosamente. + +Selecione o adaptador `json_v1` para um payload BMF `v1`. +O adaptador `json_v1` não aceita nada além de BMF `v1`, +portanto um payload ao qual falta o invólucro de array não consegue ser analisado +e o Relatório é rejeitado em vez de rebaixado sem aviso. + +Defina também `bmf_version` como `1` no Relatório. +O campo `bmf_version` declara a versão na qual o payload está escrito, +e é ele que diz ao adaptador `json` para tentar o BMF `v1` primeiro. +Um Relatório que não define `bmf_version` assume o `bmf_version` do Projeto, cujo padrão é `0`. + +### Limiares + +Um Relatório declara seus [Limiares][threshold] no campo `thresholds.models`. +No BMF `v0`, esse campo é um mapa de Measure para Modelo. +No BMF `v1`, ele é uma lista de entradas, e cada entrada nomeia tudo o que o Limiar verifica: + +- `measure`: o UUID, o slug ou o nome da Measure +- `metric`: o nome da métrica que o Limiar verifica. Se não for definido, o Limiar verifica o nome convencional `value`. Um Limiar sempre verifica exatamente um nome. +- `parameters`: as variantes que o Limiar verifica, como um filtro de parâmetros. Uma variante corresponde quando qualquer entrada do filtro é um subconjunto de seus parâmetros, portanto uma entrada nomeia apenas as chaves que lhe interessam e uma variante que fixa mais chaves ainda corresponde. Se não for definido, ou se for definido como uma lista vazia, o Limiar verifica todas as variantes. +- `model`: o Modelo de Limiar a ser usado + +Todo Limiar que corresponde é disparado. +Um Limiar que verifica todas as variantes e um Limiar que verifica apenas `{"size_mb":16}` +verificam ambos a variante `{"size_mb":16}`, +portanto uma única regressão ali gera dois Alertas, um para cada Limiar. + +O campo `thresholds.reset` limpa os Limiares que o formato do payload consegue endereçar. +Um mapa BMF `v0` nomeia uma Measure e nada mais, portanto `reset` alcança apenas os Limiares +que verificam o nome convencional `value` de cada variante. +Uma lista BMF `v1` consegue endereçar qualquer Limiar, portanto `reset` alcança todos eles. + +### Combinação + +A combinação não é suportada para o BMF `v1`. +A média dos valores `p99` de cada iteração não é o `p99` da amostra agrupada, +portanto uma combinação solicitada para um payload BMF `v1` gera um aviso +e os resultados são ingeridos sem combinação, uma iteração por payload. + +### Esquema JSON do Bencher Metric Format (BMF) v1 + +Este é o [esquema JSON][json schema] para o BMF `v1` em JSON: + + + +[benchmark]: /pt/docs/explanation/benchmarking/#benchmark +[measure]: /pt/docs/explanation/benchmarking/#measure +[alert]: /pt/docs/explanation/thresholds/#alerts +[threshold]: /pt/docs/explanation/thresholds/ +[bencher run]: /pt/docs/explanation/bencher-run/ +[json adapter]: /pt/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/ru/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/ru/schema.mdx index 999c7b19d7..73539fec6f 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/ru/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/ru/schema.mdx @@ -8,7 +8,9 @@ import BmfSchema from "../bmf-schema.mdx"; ### Версии схемы: -- Последняя: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0` (последняя): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0` (по умолчанию): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +Адрес без указания версии, [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json), соответствует версии по умолчанию, BMF `v0`. [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/ru/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/ru/v1.mdx new file mode 100644 index 0000000000..734bb59f21 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/ru/v1.mdx @@ -0,0 +1,113 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +BMF `v1` позволяет одному [Бенчмарку][benchmark] сообщать более одного результата, а одной [Мере][measure] сообщать более одного значения. +Имя Бенчмарка сопоставляется с массивом записей. +Каждая запись называет `parameters`, с которыми был запущен Бенчмарк, и `measures`, которые он выдал. + + + +В этом примере `benchmark_name` сообщает три результата: +один без параметров, один с `{"op":"read","size_mb":16}` и один с `{"op":"read","size_mb":32}`. +Каждый из этих трех является отдельным вариантом, со своей историей и своими [Оповещениями][alert]. + +`bmf_version` Проекта задает версию, как которую читается Отчет, не объявивший свою, +и изменить ее может только администратор сервера. +BMF `v1` требует актуальной версии [CLI `bencher`][bencher run]. + +### Параметры + +Объект `parameters` задает комбинацию входных данных, с которыми был запущен Бенчмарк. +Каждый ключ сопоставляется со скалярным значением JSON: строкой, числом или логическим значением. +`null`, массивы и объекты отклоняются. +Ключ и строковое значение содержат не более `64` символов каждый, а параметры Бенчмарка несут не более `8` ключей. + +Параметры сравниваются в своей канонической форме, [RFC 8785 (JSON Canonicalization Scheme)][jcs]. +Порядок ключей не имеет значения, а числа являются числами двойной точности ECMAScript, +поэтому `16`, `16.0` и `1.6e1` представляют собой одно значение, а `{"size_mb": 16}` и `{"size_mb": 16.0}` представляют собой одни и те же параметры. + +Запись без `parameters` и запись с `"parameters": {}` являются одним и тем же вариантом: +пустым вариантом, с которым рождается каждый Бенчмарк. +Две записи, которые сводятся к одним и тем же параметрам, являются одним вариантом, +поэтому их меры объединяются, а не порождают отдельный ряд. + +### Метрики + +Каждая Мера сопоставляется с объектом Метрик, и каждая Метрика является числом JSON. +Имя содержит не более `64` символов, а Мера несет не более `8` имен. +Имена сверх лимита отбрасываются: общепринятые имена сохраняются всегда, +а остальные сохраняются в лексикографическом порядке до достижения лимита. + +Три имени являются общепринятыми, но не привилегированными: + +- `value`: точечная оценка, то есть значение, которое рисует график производительности и которое [Порог][threshold] проверяет по умолчанию +- `lower_value`: нижняя граница точечной оценки +- `upper_value`: верхняя граница точечной оценки + +Любое другое имя, например `p95` или `p99`, хранится и запрашивается точно так же. +Мера может называть только `value`, только другие имена или любое их сочетание. + +### Обертка-массив + +⚠️ Полезная нагрузка BMF `v1`, записанная без обертки-массива, является допустимой полезной нагрузкой BMF `v0`. + +Универсальный [адаптер `json`][json adapter] принимает обе версии. +Полезная нагрузка BMF `v0` разбирается как BMF `v0`, и ничего не теряется. +Но объект записи, помещенный туда, где должен быть массив, выглядит в точности как карта Мер BMF `v0`, +поэтому `parameters` и каждая Метрика читаются как Меры Бенчмарка, +а предполагаемые результаты незаметно теряются. + +Для полезной нагрузки BMF `v1` выберите адаптер `json_v1`. +Адаптер `json_v1` не принимает ничего, кроме BMF `v1`, +поэтому полезная нагрузка без обертки-массива не разбирается, +и Отчет отклоняется, а не тихо понижается в версии. + +Задайте `bmf_version` равным `1` и в самом Отчете. +Поле `bmf_version` объявляет версию, в которой записана полезная нагрузка, +и именно оно указывает адаптеру `json` сначала попробовать BMF `v1`. +Отчет, не задающий `bmf_version`, берет `bmf_version` Проекта, который по умолчанию равен `0`. + +### Пороги + +Отчет объявляет свои [Пороги][threshold] в поле `thresholds.models`. +В BMF `v0` это поле является картой соответствия Меры и модели. +В BMF `v1` это список записей, и каждая запись называет все, что проверяет Порог: + +- `measure`: UUID, слаг или имя Меры +- `metric`: имя метрики, которую проверяет Порог. Если не задано, Порог проверяет общепринятое имя `value`. Порог всегда проверяет ровно одно имя. +- `parameters`: варианты, которые проверяет Порог, в виде фильтра параметров. Вариант совпадает, когда любая запись фильтра является подмножеством его параметров, поэтому запись называет только те ключи, которые ей важны, а вариант, закрепляющий больше ключей, все равно совпадает. Если не задано или задано пустым списком, Порог проверяет каждый вариант. +- `model`: модель Порога, которую следует использовать + +Срабатывает каждый совпавший Порог. +Порог, который проверяет каждый вариант, и Порог, который проверяет только `{"size_mb":16}`, +оба проверяют вариант `{"size_mb":16}`, +поэтому одно ухудшение производительности там порождает два Оповещения, по одному на каждый Порог. + +Поле `thresholds.reset` очищает те Пороги, которые способна адресовать форма полезной нагрузки. +Карта BMF `v0` называет Меру и ничего больше, поэтому `reset` достает только до тех Порогов, +которые проверяют общепринятое имя `value` каждого варианта. +Список BMF `v1` может адресовать любой Порог, поэтому `reset` достает до них всех. + +### Свертка + +Свертка не поддерживается для BMF `v1`. +Среднее значений `p99` по итерациям не является `p99` объединенной выборки, +поэтому свертка, запрошенная для полезной нагрузки BMF `v1`, сопровождается предупреждением, +а результаты принимаются без свертки, по одной итерации на полезную нагрузку. + +### JSON-схема Bencher Metric Format (BMF) v1 + +Это [JSON-схема][json schema] для JSON BMF `v1`: + + + +[benchmark]: /ru/docs/explanation/benchmarking/#benchmark +[measure]: /ru/docs/explanation/benchmarking/#measure +[alert]: /ru/docs/explanation/thresholds/#alerts +[threshold]: /ru/docs/explanation/thresholds/ +[bencher run]: /ru/docs/explanation/bencher-run/ +[json adapter]: /ru/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/zh/schema.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/zh/schema.mdx index faa815483e..73a0b8b4e7 100644 --- a/services/console/src/chunks/docs-reference/bencher-metric-format/zh/schema.mdx +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/zh/schema.mdx @@ -8,7 +8,9 @@ import BmfSchema from "../bmf-schema.mdx"; ### 模式版本: -- 最新:[`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) -- `v0`(最新):[`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v0`(默认):[`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json) +- `v1`:[`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json) + +未指定版本的 [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json) 即为默认版本 BMF `v0`。 [json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/chunks/docs-reference/bencher-metric-format/zh/v1.mdx b/services/console/src/chunks/docs-reference/bencher-metric-format/zh/v1.mdx new file mode 100644 index 0000000000..1ccef3ef94 --- /dev/null +++ b/services/console/src/chunks/docs-reference/bencher-metric-format/zh/v1.mdx @@ -0,0 +1,112 @@ +import BmfV1Example from "../bmf-v1-example.mdx"; +import BmfV1Schema from "../bmf-v1-schema.mdx"; + +## Bencher Metric Format (BMF) v1 + +BMF `v1` 让一个[基准测试][benchmark]可以报告多个结果,也让一个[度量][measure]可以报告多个值。 +一个基准测试名称映射到一个条目数组。 +每个条目指明该基准测试运行时所用的 `parameters`,以及它产出的 `measures`。 + + + +在这个示例中,`benchmark_name` 报告了三个结果: +一个不带参数,一个带 `{"op":"read","size_mb":16}`,一个带 `{"op":"read","size_mb":32}`。 +这三个结果各自是一个变体,各有自己的历史和自己的[警报][alert]。 + +项目的 `bmf_version` 决定一份自身未声明版本的报告按哪个版本读取, +只有服务器管理员才能更改它。 +BMF `v1` 要求使用最新版本的 [`bencher` CLI][bencher run]。 + +### 参数 + +`parameters` 对象是该基准测试运行时所用输入的组合。 +每个键映射到一个 JSON 标量:字符串、数字或布尔值。 +`null`、数组和对象会被拒绝。 +键和字符串值各自最多 `64` 个字符,一个基准测试的参数最多携带 `8` 个键。 + +参数按其规范形式进行比较,即 [RFC 8785 (JSON Canonicalization Scheme)][jcs]。 +键的顺序无关紧要,数字是 ECMAScript 双精度浮点数, +所以 `16`、`16.0` 和 `1.6e1` 是同一个值,`{"size_mb": 16}` 和 `{"size_mb": 16.0}` 是相同的参数。 + +没有 `parameters` 的条目与带 `"parameters": {}` 的条目是同一个变体: +每个基准测试与生俱来的空变体。 +两个解析为相同参数的条目就是一个变体, +因此它们的度量会汇合在一起,而不是分出一条新的序列。 + +### 指标 + +每个度量映射到一个指标对象,每个指标都是一个 JSON 数字。 +一个名称最多 `64` 个字符,一个度量最多携带 `8` 个名称。 +超出上限的名称会被丢弃:约定俗成的名称始终保留, +其余的按字典序保留,直到达到上限。 + +有三个名称是约定俗成的,而非享有特权的: + +- `value`:点估计值,也就是性能图所绘制的值,以及[阈值][threshold]默认检查的值 +- `lower_value`:点估计值的下界 +- `upper_value`:点估计值的上界 + +其他任何名称,例如 `p95` 或 `p99`,都以完全相同的方式存储和查询。 +一个度量可以只指定 `value`,只指定其他名称,或者两者任意混用。 + +### 数组包装层 + +⚠️ 缺少数组包装层的 BMF `v1` 载荷是一个有效的 BMF `v0` 载荷。 + +通用的 [`json` 适配器][json adapter]接受这两个版本。 +BMF `v0` 载荷会按 BMF `v0` 解析,不会丢失任何内容。 +但写在数组位置上的条目对象,看起来与 BMF `v0` 的度量映射一模一样, +于是 `parameters` 和每个指标都会被读作该基准测试的度量, +而本来想要的结果就悄无声息地丢失了。 + +对 BMF `v1` 载荷请选择 `json_v1` 适配器。 +`json_v1` 适配器只接受 BMF `v1`, +因此缺少数组包装层的载荷会解析失败, +报告会被拒绝,而不是被悄悄降级。 + +同时也要把报告上的 `bmf_version` 设为 `1`。 +`bmf_version` 字段声明载荷所采用的版本, +也正是它告诉 `json` 适配器先尝试 BMF `v1`。 +未设置 `bmf_version` 的报告采用项目的 `bmf_version`,其默认值为 `0`。 + +### 阈值 + +报告在 `thresholds.models` 字段中声明它的[阈值][threshold]。 +在 BMF `v0` 中,该字段是度量到模型的映射。 +在 BMF `v1` 中,它是一个条目列表,每个条目指明该阈值检查的一切: + +- `measure`:度量的 UUID、标识符或名称 +- `metric`:该阈值检查的指标名称。若未设置,该阈值检查约定俗成的 `value` 名称。一个阈值始终只检查一个名称。 +- `parameters`:该阈值检查的变体,以参数过滤器给出。当过滤器中任一条目是某个变体的参数的子集时,该变体即匹配,因此一个条目只需指明它关心的键,而固定了更多键的变体仍然匹配。若未设置或设为空列表,该阈值检查每一个变体。 +- `model`:要使用的阈值模型 + +每个匹配的阈值都会触发。 +一个检查每个变体的阈值,和一个只检查 `{"size_mb":16}` 的阈值, +都会检查 `{"size_mb":16}` 这个变体, +所以那里的一次性能回归会引发两个警报,每个阈值一个。 + +`thresholds.reset` 字段会清除载荷的形态所能寻址到的阈值。 +BMF `v0` 的映射只能指明一个度量,别无其他,所以 `reset` 只能触及那些检查每个变体约定俗成的 `value` 名称的阈值。 +BMF `v1` 的列表可以寻址任何阈值,所以 `reset` 能触及全部阈值。 + +### 合并 + +BMF `v1` 不支持合并。 +各次迭代 `p99` 值的平均数并不是合并样本的 `p99`, +所以针对 BMF `v1` 载荷请求的合并会收到警告, +结果会不经合并地被摄取,每个载荷一次迭代。 + +### Bencher Metric Format (BMF) v1 的 JSON 模式 + +以下是 BMF `v1` JSON 的 [JSON 模式][json schema]: + + + +[benchmark]: /zh/docs/explanation/benchmarking/#benchmark +[measure]: /zh/docs/explanation/benchmarking/#measure +[alert]: /zh/docs/explanation/thresholds/#alerts +[threshold]: /zh/docs/explanation/thresholds/ +[bencher run]: /zh/docs/explanation/bencher-run/ +[json adapter]: /zh/docs/explanation/adapters/#-json +[jcs]: https://www.rfc-editor.org/rfc/rfc8785 +[json schema]: https://json-schema.org/draft-07/json-schema-release-notes diff --git a/services/console/src/content/docs-reference/de/bencher-metric-format.mdx b/services/console/src/content/docs-reference/de/bencher-metric-format.mdx index c14f3d8c2e..09db7c3ffd 100644 --- a/services/console/src/content/docs-reference/de/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/de/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Bencher-Metrikformat" description: "Beispiel und JSON-Schema für das Bencher-Metrikformat (BMF)" heading: "Bencher-Metrikformat (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/de/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/de/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/de/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/de/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/de/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/de/bencher-mock.mdx"; +
-
\ No newline at end of file + diff --git a/services/console/src/content/docs-reference/en/bencher-metric-format.mdx b/services/console/src/content/docs-reference/en/bencher-metric-format.mdx index a637350113..137463a1c8 100644 --- a/services/console/src/content/docs-reference/en/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/en/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Bencher Metric Format" description: "The Bencher Metric Format (BMF) example and JSON schema" heading: "Bencher Metric Format (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,12 +11,14 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/en/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/en/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/en/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/en/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/en/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/en/bencher-mock.mdx"; +
diff --git a/services/console/src/content/docs-reference/es/bencher-metric-format.mdx b/services/console/src/content/docs-reference/es/bencher-metric-format.mdx index 5b984629a7..59e90c487b 100644 --- a/services/console/src/content/docs-reference/es/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/es/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Formato de Métricas Bencher" description: "Ejemplo del Formato de Métricas Bencher (BMF) y esquema JSON" heading: "Formato de Métricas Bencher (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/es/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/es/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/es/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/es/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/es/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/es/bencher-mock.mdx"; +
-
\ No newline at end of file +
diff --git a/services/console/src/content/docs-reference/fr/bencher-metric-format.mdx b/services/console/src/content/docs-reference/fr/bencher-metric-format.mdx index 63607b6f3d..9e632fabd4 100644 --- a/services/console/src/content/docs-reference/fr/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/fr/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Format de métrique Bencher" description: "Exemple de Format de Métrique Bencher (BMF) et schéma JSON" heading: "Format de métrique Bencher (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/fr/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/fr/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/fr/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/fr/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/fr/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/fr/bencher-mock.mdx"; +
-
\ No newline at end of file + diff --git a/services/console/src/content/docs-reference/ja/bencher-metric-format.mdx b/services/console/src/content/docs-reference/ja/bencher-metric-format.mdx index df8a60b5e8..47f2d94649 100644 --- a/services/console/src/content/docs-reference/ja/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/ja/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "ベンチャーメトリックフォーマット" description: "ベンチャーメトリックフォーマット(BMF)の例とJSONスキーマ" heading: "ベンチャーメトリックフォーマット (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/ja/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/ja/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/ja/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/ja/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/ja/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/ja/bencher-mock.mdx"; +
-
\ No newline at end of file + diff --git a/services/console/src/content/docs-reference/ko/bencher-metric-format.mdx b/services/console/src/content/docs-reference/ko/bencher-metric-format.mdx index f134b2a21e..337fc3c432 100644 --- a/services/console/src/content/docs-reference/ko/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/ko/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Bencher 메트릭 포맷" description: "Bencher 메트릭 포맷(BMF) 예시와 JSON 스키마" heading: "Bencher 메트릭 포맷(BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/ko/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/ko/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/ko/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/ko/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/ko/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/ko/bencher-mock.mdx"; +
-
\ No newline at end of file + diff --git a/services/console/src/content/docs-reference/pt/bencher-metric-format.mdx b/services/console/src/content/docs-reference/pt/bencher-metric-format.mdx index 3602098227..1d8dcbf3c7 100644 --- a/services/console/src/content/docs-reference/pt/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/pt/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Formato de Métrica Bencher" description: "Exemplo do Formato de Métrica Bencher (BMF) e esquema JSON" heading: "Formato de Métrica Bencher (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/pt/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/pt/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/en/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/pt/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/pt/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/pt/bencher-mock.mdx"; +
-
\ No newline at end of file + diff --git a/services/console/src/content/docs-reference/ru/bencher-metric-format.mdx b/services/console/src/content/docs-reference/ru/bencher-metric-format.mdx index 7f89708233..6904bc82c7 100644 --- a/services/console/src/content/docs-reference/ru/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/ru/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "Формат метрик Bencher" description: "Пример Формата Метрик Bencher (BMF) и схема JSON" heading: "Формат метрик Bencher (BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/ru/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/ru/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/ru/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/ru/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/ru/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/ru/bencher-mock.mdx"; +
-
\ No newline at end of file + diff --git a/services/console/src/content/docs-reference/zh/bencher-metric-format.mdx b/services/console/src/content/docs-reference/zh/bencher-metric-format.mdx index d13806f2e6..1ffeaf6d63 100644 --- a/services/console/src/content/docs-reference/zh/bencher-metric-format.mdx +++ b/services/console/src/content/docs-reference/zh/bencher-metric-format.mdx @@ -3,7 +3,7 @@ title: "基准测试指标格式" description: "基准测试指标格式(BMF)示例与JSON Schema" heading: "基准测试指标格式(BMF)" published: "2024-05-12T15:12:00Z" -modified: "2026-01-30T15:12:00Z" +modified: "2026-08-26T15:12:00Z" sortOrder: 3 --- @@ -11,13 +11,15 @@ import Intro from "../../../chunks/docs-reference/bencher-metric-format/zh/intro import Example from "../../../chunks/docs-reference/bencher-metric-format/zh/example.mdx"; import OrderOfPrecedence from "../../../chunks/docs-reference/bencher-metric-format/zh/order-of-precedence.mdx"; import Schema from "../../../chunks/docs-reference/bencher-metric-format/zh/schema.mdx"; +import V1 from "../../../chunks/docs-reference/bencher-metric-format/zh/v1.mdx"; import BencherMock from "../../../chunks/docs-reference/bencher-metric-format/zh/bencher-mock.mdx"; +
-
\ No newline at end of file +