Skip to content

Label the variant in comments, the CLI table, and the BMF v1 reference #1022

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
epompeii wants to merge 1 commit into from
Closed

Label the variant in comments, the CLI table, and the BMF v1 reference #1022

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
819 changes: 692 additions & 127 deletions lib/bencher_comment/src/lib.rs
View file
Open in desktop

Large diffs are not rendered by default.

179 changes: 177 additions & 2 deletions lib/bencher_json/src/project/perf.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -533,18 +533,37 @@ pub mod table {

use bencher_valid::GitHash;
use ordered_float::OrderedFloat;
use tabled::{Table, Tabled};
use tabled::{
Table, Tabled,
settings::{Remove, location::ByColumnName},
};

use crate::{
DateTime, JsonBenchmark, JsonBranch, JsonMeasure, JsonMetricTriple, JsonPerf, JsonProject,
JsonTestbed,
project::{head::VersionNumber, report::Iteration},
};

/// The header of the column that names each line's variant.
///
/// A project that never reported any parameters has nothing to tell its lines
/// apart by, so the column is removed rather than filled with `{}`, and this is
/// what names it for removal. The `tabled` rename attribute takes a literal, so
/// the field below spells the same string out.
const PARAMETERS: &str = "Parameters";

impl From<JsonPerf> for Table {
fn from(json_perf: JsonPerf) -> Self {
// One non-empty variant anywhere in the query is what makes the column worth
// a column: without one, every line is the benchmark's only variant.
let has_variants = json_perf
.results
.iter()
.any(|result| !result.variant.parameters.is_empty());

let mut perf_table = Vec::new();
for result in json_perf.results {
let parameters = result.variant.parameters.canonical();
for metric in result.metrics {
let (baseline, lower_limit, upper_limit) =
if let Some(boundary) = metric.boundary {
Expand All @@ -565,6 +584,7 @@ pub mod table {
branch: result.branch.clone(),
testbed: result.testbed.clone(),
benchmark: result.benchmark.clone(),
parameters: parameters.clone(),
measure: result.measure.clone(),
iteration: metric.iteration,
start_time: metric.start_time,
Expand All @@ -578,7 +598,11 @@ pub mod table {
});
}
}
Self::new(perf_table)
let mut table = Self::new(perf_table);
if !has_variants {
table.with(Remove::column(ByColumnName::new(PARAMETERS)));
}
table
}
}

Expand All @@ -592,6 +616,13 @@ pub mod table {
pub testbed: JsonTestbed,
#[tabled(rename = "Benchmark")]
pub benchmark: JsonBenchmark,
/// The canonical spelling of the variant this line plots.
///
/// The column sits between the benchmark and the measure, the way the
/// variant sits between them in a perf result, and it is removed entirely
/// when no line of the query plots a non-empty variant.
#[tabled(rename = "Parameters")]
pub parameters: String,
#[tabled(rename = "Measure")]
pub measure: JsonMeasure,
#[tabled(rename = "Iteration")]
Expand Down Expand Up @@ -629,4 +660,148 @@ pub mod table {
}
}
}

#[cfg(test)]
mod tests {
use tabled::Table;

use crate::JsonPerf;

/// A one line perf query whose benchmark plots `set`.
fn line(benchmark: &str, set: &str, value: f64) -> String {
format!(
r#"{{
"branch": {{
"uuid": "7d7e73de-78c2-43f7-bc2a-da31a5b9a819",
"project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf",
"name": "master",
"slug": "master",
"head": {{
"uuid": "7d7e73de-78c2-43f7-bc2a-da31a5b9a819",
"start_point": null,
"version": null,
"created": "2023-07-02T12:53:33Z",
"replaced": null
}},
"created": "2023-07-02T12:53:33Z",
"modified": "2023-07-02T12:53:33Z"
}},
"testbed": {{
"uuid": "e095df48-52a6-474b-aaa7-1a8546c235b6",
"project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf",
"name": "base",
"slug": "base",
"created": "2023-07-02T12:53:33Z",
"modified": "2023-07-02T12:53:33Z"
}},
"benchmark": {{
"uuid": "dbb90f5c-e7e2-438c-9533-ce86792174ee",
"project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf",
"name": "{benchmark}",
"slug": "dbb90f5c-e7e2-438c-9533-ce86792174ee",
"created": "2023-07-02T12:53:33Z",
"modified": "2023-07-02T12:53:33Z"
}},
"variant": {{
"uuid": "b23b1a5e-0f4f-4b8a-9a35-2b7a2f5f0a2f",
"benchmark": "dbb90f5c-e7e2-438c-9533-ce86792174ee",
"parameters": {set},
"created": "2023-07-02T12:53:33Z",
"modified": "2023-07-02T12:53:33Z",
"archived": null
}},
"measure": {{
"uuid": "61a385d0-f19d-4f20-895a-e3c684ec6cbc",
"project": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf",
"name": "Latency",
"slug": "latency",
"units": "nanoseconds (ns)",
"created": "2023-07-02T12:53:33Z",
"modified": "2023-07-02T12:53:33Z"
}},
"metrics": [
{{
"report": "ef582192-c7f4-47a0-8668-55cf7d99d8cc",
"iteration": 0,
"start_time": "2023-07-02T12:53:33Z",
"end_time": "2023-07-02T12:53:33Z",
"version": {{ "number": 0, "hash": null }},
"threshold": null,
"boundary": null,
"alert": null,
"metrics": {{ "value": {{ "value": {value} }} }},
"metric": {{
"uuid": "00000000-0000-0000-0000-000000000000",
"value": {value},
"lower_value": null,
"upper_value": null
}}
}}
]
}}"#
)
}

fn json_perf(lines: &[String]) -> JsonPerf {
let results = lines.join(",");
serde_json::from_str(&format!(
r#"{{
"project": {{
"uuid": "c7fd3581-73d1-443c-b30f-6aa5c1c516cf",
"organization": "4142ce9a-f0a0-44d5-94cd-fc76c77d9098",
"name": "The Computer",
"slug": "the-computer",
"url": null,
"visibility": "public",
"bmf_version": 0,
"created": "2023-07-02T12:53:33Z",
"modified": "2023-07-02T12:53:33Z"
}},
"start_time": null,
"end_time": null,
"results": [{results}]
}}"#
))
.expect("Failed to parse perf JSON")
}

fn table(lines: &[String]) -> String {
Table::from(json_perf(lines)).to_string()
}

// A query whose every line plots the empty variant is the query every
// project made before a benchmark could have more than one variant. It
// prints exactly the table it always printed, column for column.
#[test]
fn table_without_variants() {
let table = table(&[line("bencher::mock_0", "{}", 7.0)]);
assert_eq!(
table,
concat!(
"+--------------+--------+---------+-----------------+---------------------------+-----------+-------------------------+-------------------------+----------------+--------------+--------------+-------------------+----------------------+----------------------+\n",
"| Project | Branch | Testbed | Benchmark | Measure | Iteration | Start Time | End Time | Version Number | Version Hash | Metric Value | Boundary Baseline | Lower Boundary Limit | Upper Boundary Limit |\n",
"+--------------+--------+---------+-----------------+---------------------------+-----------+-------------------------+-------------------------+----------------+--------------+--------------+-------------------+----------------------+----------------------+\n",
"| The Computer | master | base | bencher::mock_0 | Latency: nanoseconds (ns) | 0 | 2023-07-02 12:53:33 UTC | 2023-07-02 12:53:33 UTC | 0 | | 7 | | | |\n",
"+--------------+--------+---------+-----------------+---------------------------+-----------+-------------------------+-------------------------+----------------+--------------+--------------+-------------------+----------------------+----------------------+",
),
"the table a project without variants prints"
);
}

// One non-empty set anywhere in the query earns the column, and every line
// spells the set it plots, the empty set among them as `{}`.
#[test]
fn table_with_variants() {
let table = table(&[
line("bencher::mock_0", "{}", 7.0),
line("bencher::mock_0", r#"{"size_mb": 16}"#, 8.0),
]);
assert!(table.contains("Parameters"), "unexpected table: {table}");
assert!(table.contains("| {} "), "unexpected table: {table}");
assert!(
table.contains(r#"| {"size_mb":16} "#),
"unexpected table: {table}"
);
}
}
}
38 changes: 38 additions & 0 deletions services/console/public/v1/bmf.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"$id": "https://bencher.dev/v1/bmf.json",
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"patternProperties": {
".+": {
"type": "array",
"items": {
"type": "object",
"properties": {
"parameters": {
"type": "object",
"maxProperties": 8,
"patternProperties": {
".+": {
"type": ["string", "number", "boolean"]
}
}
},
"measures": {
"type": "object",
"patternProperties": {
".+": {
"type": "object",
"patternProperties": {
".+": {
"type": "number"
}
}
}
}
}
},
"required": ["measures"]
}
}
}
}
51 changes: 51 additions & 0 deletions ...ices/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-example.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
```json
{
"benchmark_name": [
{
"measures": {
"latency": {
"value": 88.0,
"lower_value": 87.42,
"upper_value": 88.88
}
}
},
{
"parameters": {
"op": "read",
"size_mb": 16
},
"measures": {
"latency": {
"value": 132.0,
"lower_value": 130.1,
"upper_value": 133.9
},
"throughput": {
"value": 5.55,
"p95": 4.1,
"p99": 3.14
}
}
},
{
"parameters": {
"op": "read",
"size_mb": 32
},
"measures": {
"latency": {
"value": 261.0,
"lower_value": 258.3,
"upper_value": 263.7
},
"throughput": {
"value": 5.42,
"p95": 4.02,
"p99": 3.01
}
}
}
]
}
```
40 changes: 40 additions & 0 deletions services/console/src/chunks/docs-reference/bencher-metric-format/bmf-v1-schema.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
```json
{
"$id": "https://bencher.dev/v1/bmf.json",
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"patternProperties": {
".+": {
"type": "array",
"items": {
"type": "object",
"properties": {
"parameters": {
"type": "object",
"maxProperties": 8,
"patternProperties": {
".+": {
"type": ["string", "number", "boolean"]
}
}
},
"measures": {
"type": "object",
"patternProperties": {
".+": {
"type": "object",
"patternProperties": {
".+": {
"type": "number"
}
}
}
}
}
},
"required": ["measures"]
}
}
}
}
```
6 changes: 4 additions & 2 deletions services/console/src/chunks/docs-reference/bencher-metric-format/de/schema.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ Dies ist das [JSON-Schema][json schema] für Bencher Metric Format (BMF) JSON:

### Schema-Versionen:

- Neueste: [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json)
- `v0` (aktuell): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json)
- `v0` (Standard): [`https://bencher.dev/v0/bmf.json`](https://bencher.dev/v0/bmf.json)
- `v1`: [`https://bencher.dev/v1/bmf.json`](https://bencher.dev/v1/bmf.json)

Die Version ohne Versionsangabe, [`https://bencher.dev/bmf.json`](https://bencher.dev/bmf.json), ist die Standardversion BMF `v0`.

[json schema]: https://json-schema.org/draft-07/json-schema-release-notes
Loading
Loading