From 58de57d88c619bb6806e66b101deb06b5c32540d Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Mon, 17 Aug 2026 13:20:59 +0200 Subject: [PATCH 1/9] refactor: make the bucketing more explicit Signed-off-by: Jaime Silvela --- DEVELOPERS_DEVELOPERS_DEVELOPERS.md | 11 +- summarize_test_results.py | 300 +++++++++++++++------------- 2 files changed, 167 insertions(+), 144 deletions(-) diff --git a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md index 29aa979..acb6304 100644 --- a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md +++ b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md @@ -23,7 +23,16 @@ The procedure for cutting a release: ## Developing and testing You can test directly with the Python code on the `example-artifacts` directory, -where you can see some JSON artifacts in the expected format. For example: +where you can see some JSON artifacts in the expected format. + +Before running the python scripts, you may need to install the requirements +(just `prettytable` currently). + +``` shell +pip install --no-cache-dir -r requirements.txt +``` + +A basic execution looks like this: ``` shell python summarize_test_results.py --dir example-artifacts diff --git a/summarize_test_results.py b/summarize_test_results.py index c386c64..05cd3f8 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -200,69 +200,82 @@ def track_time_taken(test_results, test_times, suite_times): end_time = datetime.fromisoformat(end_frags[0]) duration = end_time - start_time matrix_id = test_results["matrix_id"] - if name not in test_times["max"]: - test_times["max"][name] = duration - if name not in test_times["min"]: - test_times["min"][name] = duration - if name not in test_times["slowest_branch"]: - test_times["slowest_branch"][name] = matrix_id - - if duration > test_times["max"][name]: - test_times["max"][name] = duration - test_times["slowest_branch"][name] = matrix_id - if duration < test_times["min"][name]: - test_times["min"][name] = duration + if name not in test_times: + test_times[name] = { + "max": duration, + "min": duration, + "slowest_branch": matrix_id, + } + bucket = test_times[name] + + if duration > bucket["max"]: + bucket["max"] = duration + bucket["slowest_branch"] = matrix_id + if duration < bucket["min"]: + bucket["min"] = duration # Track test suite timings. # For each platform-matrix branch, track the earliest start and the latest end platform = test_results["platform"] - if platform not in suite_times["start_time"]: - suite_times["start_time"][platform] = {} - if matrix_id not in suite_times["start_time"][platform]: - suite_times["start_time"][platform][matrix_id] = start_time - if platform not in suite_times["end_time"]: - suite_times["end_time"][platform] = {} - if matrix_id not in suite_times["end_time"][platform]: - suite_times["end_time"][platform][matrix_id] = end_time + if platform not in suite_times: + suite_times[platform] = { + "start_time": start_time, + "end_time": end_time, + matrix_id: { + "start_time": start_time, + "end_time": end_time + } + } + suite_bucket = suite_times[platform] - if start_time < suite_times["start_time"][platform][matrix_id]: - suite_times["start_time"][platform][matrix_id] = start_time - if suite_times["end_time"][platform][matrix_id] < end_time: - suite_times["end_time"][platform][matrix_id] = end_time + if start_time < suite_bucket["start_time"]: + suite_bucket["start_time"] = start_time + if suite_bucket["end_time"] < end_time: + suite_bucket["end_time"] = end_time + if matrix_id not in suite_bucket: + suite_bucket[matrix_id] = { + "start_time": start_time, + "end_time": end_time + } + else: + if start_time < suite_bucket[matrix_id]["start_time"]: + suite_bucket[matrix_id]["start_time"] = start_time + if suite_bucket[matrix_id]["end_time"] < end_time: + suite_bucket[matrix_id]["end_time"] = end_time def count_bucketed_by_test(test_results, by_test): """counts the successes, failures, failing versions of kubernetes, failing versions of postgres, bucketed by test name. """ + newEntry = { + "total": 0, + "failed": 0, + "k8s_versions_failed": {}, + "pg_versions_failed": {}, + "platforms_failed": {} + } name = test_results["name"] - if name not in by_test["total"]: - by_test["total"][name] = 0 - by_test["total"][name] = 1 + by_test["total"][name] + if name not in by_test: + by_test[name] = newEntry + bucket = by_test[name] + + bucket["total"] = 1 + bucket["total"] if is_failed(test_results) and not is_ginkgo_report_failure(test_results): - if name not in by_test["failed"]: - by_test["failed"][name] = 0 - if name not in by_test["k8s_versions_failed"]: - by_test["k8s_versions_failed"][name] = {} - if name not in by_test["pg_versions_failed"]: - by_test["pg_versions_failed"][name] = {} - if name not in by_test["platforms_failed"]: - by_test["platforms_failed"][name] = {} - by_test["failed"][name] = 1 + by_test["failed"][name] + bucket["failed"] = 1 + bucket["failed"] k8s_version = test_results["k8s_version"] pg_version = test_results["pg_version"] platform = test_results["platform"] - by_test["k8s_versions_failed"][name][k8s_version] = True - by_test["pg_versions_failed"][name][pg_version] = True - by_test["platforms_failed"][name][platform] = True + bucket["k8s_versions_failed"][k8s_version] = True + bucket["pg_versions_failed"][pg_version] = True + bucket["platforms_failed"][platform] = True def count_bucketed_by_code(test_results, by_failing_code): """buckets by failed code, with a list of tests where the assertion fails, and a view of the stack trace. """ - name = test_results["name"] if test_results["error"] == "" or test_results["state"] == "ignoreFailed": return # it does not make sense to show failing code that is outside the test, @@ -270,21 +283,24 @@ def count_bucketed_by_code(test_results, by_failing_code): if not is_normal_failure(test_results): return + newEntry = { + "total": 0, + "tests": {}, + "errors": {}, + } + name = test_results["name"] + errfile = test_results["error_file"] errline = test_results["error_line"] err_desc = f"{errfile}:{errline}" - if err_desc not in by_failing_code["total"]: - by_failing_code["total"][err_desc] = 0 - by_failing_code["total"][err_desc] = 1 + by_failing_code["total"][err_desc] - - if err_desc not in by_failing_code["tests"]: - by_failing_code["tests"][err_desc] = {} - by_failing_code["tests"][err_desc][name] = True - - if err_desc not in by_failing_code["errors"]: - by_failing_code["errors"][err_desc] = test_results["error"] + if err_desc not in by_failing_code: + by_failing_code[err_desc] = newEntry + bucket = by_failing_code[err_desc] + bucket["total"] = 1 + bucket["total"] + bucket["tests"][name] = True + bucket["errors"] = test_results["error"] def count_bucketed_by_special_failures(test_results, by_special_failures): """counts the successes, failures, failing versions of kubernetes, @@ -294,6 +310,14 @@ def count_bucketed_by_special_failures(test_results, by_special_failures): if not is_failed(test_results) or is_normal_failure(test_results): return + newEntry = { + "total": 0, + "tests_failed": {}, + "k8s_versions_failed": {}, + "pg_versions_failed": {}, + "platforms_failed": {}, + } + failure = "" if is_external_failure(test_results): failure = test_results["state"] @@ -305,38 +329,34 @@ def count_bucketed_by_special_failures(test_results, by_special_failures): pg_version = test_results["pg_version"] platform = test_results["platform"] - if failure not in by_special_failures["total"]: - by_special_failures["total"][failure] = 0 - - for key in [ - "tests_failed", - "k8s_versions_failed", - "pg_versions_failed", - "platforms_failed", - ]: - if failure not in by_special_failures[key]: - by_special_failures[key][failure] = {} + if failure not in by_special_failures: + by_special_failures[failure] = newEntry + bucket = by_special_failures[failure] - by_special_failures["total"][failure] += 1 - by_special_failures["tests_failed"][failure][test_name] = True - by_special_failures["k8s_versions_failed"][failure][k8s_version] = True - by_special_failures["pg_versions_failed"][failure][pg_version] = True - by_special_failures["platforms_failed"][failure][platform] = True + bucket["total"] += 1 + bucket["tests_failed"][test_name] = True + bucket["k8s_versions_failed"][k8s_version] = True + bucket["pg_versions_failed"][pg_version] = True + bucket["platforms_failed"][platform] = True def count_bucketized_stats(test_results, buckets, field_id): - """counts the success/failures onto a bucket. This means there are two - dictionaries: one for `total` tests, one for `failed` tests. + """bucketizes test results according to the field_id. + For each bucket, it counts the total tests run and the failed tests. """ + newEntry = { + "total": 0, + "failed": 0 + } bucket_id = test_results[field_id] - if bucket_id not in buckets["total"]: - buckets["total"][bucket_id] = 0 - buckets["total"][bucket_id] = 1 + buckets["total"][bucket_id] + if bucket_id not in buckets: + buckets[bucket_id] = newEntry + + bucket = buckets[bucket_id] + bucket["total"] = 1 + bucket["total"] if is_failed(test_results): - if bucket_id not in buckets["failed"]: - buckets["failed"][bucket_id] = 0 - buckets["failed"][bucket_id] = 1 + buckets["failed"][bucket_id] + bucket["failed"] += 1 def compute_bucketized_summary(parameter_buckets): @@ -346,16 +366,16 @@ def compute_bucketized_summary(parameter_buckets): """ failed_buckets_count = 0 total_buckets_count = 0 - for _ in parameter_buckets["total"]: - total_buckets_count = 1 + total_buckets_count - for _ in parameter_buckets["failed"]: - failed_buckets_count = 1 + failed_buckets_count + for k, v in parameter_buckets.items(): + total_buckets_count += v["total"] + failed_buckets_count += v["failed"] return failed_buckets_count, total_buckets_count def compute_test_summary(test_dir): """iterate over the JSON artifact files in `test_dir`, and - bucket them for comprehension. + bucket them in various ways to provide insights as to failure + patterns. Returns a dictionary of dictionaries: @@ -380,36 +400,23 @@ def compute_test_summary(test_dir): total_runs = 0 total_fails = 0 total_special_fails = 0 - by_test = { - "total": {}, - "failed": {}, - "k8s_versions_failed": {}, - "pg_versions_failed": {}, - "platforms_failed": {}, - } - by_failing_code = { - "total": {}, - "tests": {}, - "errors": {}, - } - by_matrix = {"total": {}, "failed": {}} - by_k8s = {"total": {}, "failed": {}} - by_postgres = {"total": {}, "failed": {}} - by_platform = {"total": {}, "failed": {}} + by_test = {} + by_failing_code = {} + by_matrix = {} + by_k8s = {} + by_postgres = {} + by_platform = {} # special failures are not due to the test having failed, but # to something at a higher level, like the E2E suite having been # cancelled, timed out, or having executed improperly - by_special_failures = { - "total": {}, - "tests_failed": {}, - "k8s_versions_failed": {}, - "pg_versions_failed": {}, - "platforms_failed": {}, - } + by_special_failures = {} - test_durations = {"max": {}, "min": {}, "slowest_branch": {}} - suite_durations = {"start_time": {}, "end_time": {}} + # test_durations track the quickest and slowest tests over the + # total execution. + # suite_durations compute the duration of the test suite overall + test_durations = {} + suite_durations = {} # start computation of summary ############################## @@ -426,6 +433,10 @@ def compute_test_summary(test_dir): test_results = combine_postgres_data(parsed) test_results = compress_kubernetes_version(test_results) + # From this point, test_results is the representation of + # one text execution, i.e. one E2E test run on one version + # of CNPG x Postgres x Kubernetes + total_runs = 1 + total_runs if is_failed(test_results): total_fails = 1 + total_fails @@ -440,6 +451,7 @@ def compute_test_summary(test_dir): count_bucketed_by_code(test_results, by_failing_code) # special failures are treated separately + # bucketing by the failure count_bucketed_by_special_failures(test_results, by_special_failures) # bucketing by matrix ID @@ -454,6 +466,8 @@ def compute_test_summary(test_dir): # bucketing by platform count_bucketized_stats(test_results, by_platform, "platform") + # bucketing test_durations by test name + # bucketing suite_durations by platform and also by matrix_id track_time_taken(test_results, test_durations, suite_durations) return { @@ -519,10 +533,9 @@ def compute_systematic_failures_on_metric(summary, metric, embed=True): output = "" has_systematic_failure_in_metric = False counter = 0 - for bucket_hits in summary[metric]["failed"].items(): - bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs - failures = summary[metric]["failed"][bucket] - runs = summary[metric]["total"][bucket] + for bucket, stats in summary[metric].items(): + failures = stats["failed"] + runs = stats["total"] if failures == runs and failures > 1: if not has_systematic_failure_in_metric: output += f"{metric_name(metric)} with systematic failures:\n\n" @@ -624,12 +637,9 @@ def compute_thermometer_on_metric(summary, metric, embed=True): """ output = f"{metric_name(metric)} thermometer:\n\n" - for bucket_hits in summary[metric]["total"].items(): - bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs - failures = 0 - if bucket in summary[metric]["failed"]: - failures = summary[metric]["failed"][bucket] - runs = summary[metric]["total"][bucket] + for bucket, stats in summary[metric].items(): + failures = stats["failed"] + runs = stats["total"] success_percent = (1 - failures / runs) * 100 color = compute_semaphore(success_percent, embed) output += ( @@ -693,11 +703,11 @@ def format_bucket_table(buckets, structure, file_out=None): table.set_style(MARKDOWN) sorted_by_fail = dict( - sorted(buckets["failed"].items(), key=lambda item: item[1], reverse=True) + sorted(buckets.items(), key=lambda item: item[1]["failed"], reverse=True) ) for bucket in sorted_by_fail: - table.add_row([buckets["failed"][bucket], buckets["total"][bucket], bucket]) + table.add_row([buckets[bucket]["failed"], buckets[bucket]["total"], bucket]) print(table, file=file_out) @@ -714,22 +724,22 @@ def format_by_test(summary, structure, file_out=None): sorted_by_fail = dict( sorted( - summary["by_test"]["failed"].items(), - key=lambda item: item[1], + summary["by_test"].items(), + key=lambda item: item[1]["failed"], reverse=True, ) ) for bucket in sorted_by_fail: - failed_k8s = ", ".join(summary["by_test"]["k8s_versions_failed"][bucket].keys()) - failed_pg = ", ".join(summary["by_test"]["pg_versions_failed"][bucket].keys()) + failed_k8s = ", ".join(summary["by_test"][bucket]["k8s_versions_failed"].keys()) + failed_pg = ", ".join(summary["by_test"][bucket]["pg_versions_failed"].keys()) failed_platforms = ", ".join( - summary["by_test"]["platforms_failed"][bucket].keys() + summary["by_test"][bucket]["platforms_failed"].keys() ) table.add_row( [ - summary["by_test"]["failed"][bucket], - summary["by_test"]["total"][bucket], + summary["by_test"][bucket]["failed"], + summary["by_test"][bucket]["total"], failed_k8s, failed_pg, failed_platforms, @@ -752,28 +762,28 @@ def format_by_special_failure(summary, structure, file_out=None): sorted_by_count = dict( sorted( - summary["by_special_failures"]["total"].items(), - key=lambda item: item[1], + summary["by_special_failures"].items(), + key=lambda item: item[1]["total"], reverse=True, ) ) for bucket in sorted_by_count: failed_tests = ", ".join( - summary["by_special_failures"]["tests_failed"][bucket].keys() + summary["by_special_failures"][bucket]["tests_failed"].keys() ) failed_k8s = ", ".join( - summary["by_special_failures"]["k8s_versions_failed"][bucket].keys() + summary["by_special_failures"][bucket]["k8s_versions_failed"].keys() ) failed_pg = ", ".join( - summary["by_special_failures"]["pg_versions_failed"][bucket].keys() + summary["by_special_failures"][bucket]["pg_versions_failed"].keys() ) failed_platforms = ", ".join( - summary["by_special_failures"]["platforms_failed"][bucket].keys() + summary["by_special_failures"][bucket]["platforms_failed"].keys() ) table.add_row( [ - summary["by_special_failures"]["total"][bucket], + summary["by_special_failures"][bucket]["total"], bucket, failed_tests, failed_k8s, @@ -797,24 +807,24 @@ def format_by_code(summary, structure, file_out=None): sorted_by_code = dict( sorted( - summary["by_code"]["total"].items(), - key=lambda item: item[1], + summary["by_code"].items(), + key=lambda item: item[1]["total"], reverse=True, ) ) for bucket in sorted_by_code: - tests = ", ".join(summary["by_code"]["tests"][bucket].keys()) + tests = ", ".join(summary["by_code"][bucket]["tests"].keys()) # replace newlines and pipes to avoid interference with Markdown tables errors = ( - summary["by_code"]["errors"][bucket] + summary["by_code"][bucket]["errors"] .replace("\n", "
") .replace("|", "—") ) err_cell = f"
Click to expand{errors}
" table.add_row( [ - summary["by_code"]["total"][bucket], + summary["by_code"][bucket]["total"], bucket, tests, err_cell, @@ -841,15 +851,17 @@ def format_durations_table(test_times, structure, file_out=None): table.set_style(MARKDOWN) table.field_names = structure["header"] + print(test_times) + sorted_by_longest = dict( - sorted(test_times["max"].items(), key=lambda item: item[1], reverse=True) + sorted(test_times.items(), key=lambda item: item[1]["max"], reverse=True) ) for bucket in sorted_by_longest: name = bucket - longest = format_duration(test_times["max"][bucket]) - shortest = format_duration(test_times["min"][bucket]) - branch = test_times["slowest_branch"][bucket] + longest = format_duration(test_times[bucket]["max"]) + shortest = format_duration(test_times[bucket]["min"]) + branch = test_times[bucket]["slowest_branch"] table.add_row([longest, shortest, branch, name]) print(table, file=file_out) @@ -873,11 +885,13 @@ def format_suite_durations_table(suite_times, structure, file_out=None): "max": {}, "slowest_branch": {}, } - for platform in suite_times["start_time"]: - for matrix_id in suite_times["start_time"][platform]: + for platform in suite_times: + for matrix_id in suite_times[platform]: + if matrix_id == "start_time" or matrix_id == "end_time": + continue duration = ( - suite_times["end_time"][platform][matrix_id] - - suite_times["start_time"][platform][matrix_id] + suite_times[platform][matrix_id]["end_time"] + - suite_times[platform][matrix_id]["start_time"] ) if platform not in suite_durations["max"]: suite_durations["max"][platform] = duration From f7c9ec68db99f557a2ffaac2499d391bb4b8e9cd Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Mon, 17 Aug 2026 15:55:10 +0200 Subject: [PATCH 2/9] fix: unit tests Signed-off-by: Jaime Silvela --- DEVELOPERS_DEVELOPERS_DEVELOPERS.md | 8 +++++++- test_summary.py | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md index acb6304..ab01c6a 100644 --- a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md +++ b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md @@ -38,7 +38,13 @@ A basic execution looks like this: python summarize_test_results.py --dir example-artifacts ``` -or +or, if you're using Python virtual environments, say `pythonVenv` + +``` shell +pythonVenv/bin/python summarize_test_results.py --dir example-artifacts +``` + +you can get the report into a file by setting the `GITHUB_STEP_SUMMARY`: ``` shell GITHUB_STEP_SUMMARY=out.md python summarize_test_results.py --dir example-artifacts diff --git a/test_summary.py b/test_summary.py index 4abc4f6..d7a173a 100644 --- a/test_summary.py +++ b/test_summary.py @@ -31,41 +31,41 @@ def test_compute_summary(self): self.assertEqual(self.summary["total_failed"], 1) self.assertEqual( - self.summary["by_code"]["total"], - {"/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80": 1}, + self.summary["by_code"]["/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80"]["total"], + 1, "unexpected summary", ) self.assertEqual( - self.summary["by_code"]["tests"], + self.summary["by_code"]["/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80"]["tests"], { - "/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80": { "InitDB settings - initdb custom post-init SQL scripts -- can find the" " tables created by the post-init SQL queries": True - } }, "unexpected summary", ) self.assertEqual( - self.summary["by_matrix"], {"total": {"id1": 3}, "failed": {"id1": 1}} + self.summary["by_matrix"], {"id1": {"total": 3, "failed": 1}} ) self.assertEqual( - self.summary["by_k8s"], {"total": {"1.22": 3}, "failed": {"1.22": 1}} + self.summary["by_k8s"], {"1.22": {"total": 3, "failed": 1}} ) self.assertEqual( - self.summary["by_platform"], {"total": {"local": 3}, "failed": {"local": 1}} + self.summary["by_platform"], {"local": {"total": 3, "failed": 1}} ) self.assertEqual( self.summary["by_postgres"], - {"total": {"PostgreSQL-11.1": 3}, "failed": {"PostgreSQL-11.1": 1}}, + {"PostgreSQL-11.1": {"total": 3, "failed": 1}}, ) self.assertEqual( self.summary["suite_durations"], { - "end_time": { - "local": {"id1": datetime.datetime(2021, 11, 29, 18, 31, 7)} - }, - "start_time": { - "local": {"id1": datetime.datetime(2021, 11, 29, 18, 28, 37)} + "local": { + "end_time": datetime.datetime(2021, 11, 29, 18, 31, 7), + "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37), + "id1": { + "end_time": datetime.datetime(2021, 11, 29, 18, 31, 7), + "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37) + } }, }, ) From 70b8668890b2e2683655fb186b3f2248059519b3 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Tue, 18 Aug 2026 10:05:52 +0200 Subject: [PATCH 3/9] chore: heed deprecation warning Signed-off-by: Jaime Silvela --- DEVELOPERS_DEVELOPERS_DEVELOPERS.md | 6 ++++++ summarize_test_results.py | 17 ++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md index ab01c6a..866b621 100644 --- a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md +++ b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md @@ -104,6 +104,12 @@ CIclops has the beginning of a unit test suite. You can run it with: python3 -m unittest ``` +or + +``` sh +python test_summary.py +``` + ## Testing within a calling GitHub workflow Even with unit tests and local tests, it's good to try Ciclops code out from a diff --git a/summarize_test_results.py b/summarize_test_results.py index 05cd3f8..fd4dac6 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -59,9 +59,8 @@ import math import os import pathlib -from prettytable import MARKDOWN from prettytable import PrettyTable - +from prettytable import TableStyle def is_failed(e2e_test): """checks if the test failed. In ginkgo, the passing states are @@ -678,7 +677,7 @@ def format_overview(summary, structure, file_out=None): print("## " + structure["title"] + "\n", file=file_out) table = PrettyTable(align="l") table.field_names = structure["header"] - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) for row in structure["rows"]: table.add_row([summary[row[1]], summary[row[2]], row[0]]) @@ -700,7 +699,7 @@ def format_bucket_table(buckets, structure, file_out=None): print(f"\n

{title}

\n", file=file_out) table = PrettyTable(align="l") table.field_names = structure["header"] - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) sorted_by_fail = dict( sorted(buckets.items(), key=lambda item: item[1]["failed"], reverse=True) @@ -720,7 +719,7 @@ def format_by_test(summary, structure, file_out=None): table = PrettyTable(align="l") table.field_names = structure["header"] - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) sorted_by_fail = dict( sorted( @@ -758,7 +757,7 @@ def format_by_special_failure(summary, structure, file_out=None): table = PrettyTable(align="l") table.field_names = structure["header"] - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) sorted_by_count = dict( sorted( @@ -803,7 +802,7 @@ def format_by_code(summary, structure, file_out=None): table = PrettyTable(align="l") table.field_names = structure["header"] - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) sorted_by_code = dict( sorted( @@ -848,7 +847,7 @@ def format_durations_table(test_times, structure, file_out=None): print(f"\n

{title}

\n", file=file_out) table = PrettyTable(align="l", max_width=80) - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) table.field_names = structure["header"] print(test_times) @@ -874,7 +873,7 @@ def format_suite_durations_table(suite_times, structure, file_out=None): print(f"\n

{title}

\n", file=file_out) table = PrettyTable(align="l", max_width=80) - table.set_style(MARKDOWN) + table.set_style(TableStyle.MARKDOWN) table.field_names = structure["header"] # we want to display a table with one row per platform, giving us the From a9f17fb0605d717fffaa0d35c440c388bd6af201 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Tue, 25 Aug 2026 14:48:41 +0200 Subject: [PATCH 4/9] fix: proper formation of overview table Signed-off-by: Jaime Silvela --- summarize_test_results.py | 87 ++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index fd4dac6..fca0639 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -167,7 +167,9 @@ def combine_postgres_data(test_entry): def track_time_taken(test_results, test_times, suite_times): """computes the running shortest and longest duration of - running each kind of test + running each kind of test, and the longest branch. + Also computes start and end of the test suite per platform + and matrix_id within the platform. """ name = test_results["name"] # tag abnormal failures, e.g.: "[operator was restarted] Imports with …" @@ -205,13 +207,13 @@ def track_time_taken(test_results, test_times, suite_times): "min": duration, "slowest_branch": matrix_id, } - bucket = test_times[name] + test_bucket = test_times[name] - if duration > bucket["max"]: - bucket["max"] = duration - bucket["slowest_branch"] = matrix_id - if duration < bucket["min"]: - bucket["min"] = duration + if duration > test_bucket["max"]: + test_bucket["max"] = duration + test_bucket["slowest_branch"] = matrix_id + if duration < test_bucket["min"]: + test_bucket["min"] = duration # Track test suite timings. # For each platform-matrix branch, track the earliest start and the latest end @@ -225,23 +227,23 @@ def track_time_taken(test_results, test_times, suite_times): "end_time": end_time } } - suite_bucket = suite_times[platform] + platform_bucket = suite_times[platform] - if start_time < suite_bucket["start_time"]: - suite_bucket["start_time"] = start_time - if suite_bucket["end_time"] < end_time: - suite_bucket["end_time"] = end_time + if start_time < platform_bucket["start_time"]: + platform_bucket["start_time"] = start_time + if platform_bucket["end_time"] < end_time: + platform_bucket["end_time"] = end_time - if matrix_id not in suite_bucket: - suite_bucket[matrix_id] = { + if matrix_id not in platform_bucket: + platform_bucket[matrix_id] = { "start_time": start_time, "end_time": end_time } else: - if start_time < suite_bucket[matrix_id]["start_time"]: - suite_bucket[matrix_id]["start_time"] = start_time - if suite_bucket[matrix_id]["end_time"] < end_time: - suite_bucket[matrix_id]["end_time"] = end_time + if start_time < platform_bucket[matrix_id]["start_time"]: + platform_bucket[matrix_id]["start_time"] = start_time + if platform_bucket[matrix_id]["end_time"] < end_time: + platform_bucket[matrix_id]["end_time"] = end_time def count_bucketed_by_test(test_results, by_test): """counts the successes, failures, failing versions of kubernetes, @@ -258,17 +260,17 @@ def count_bucketed_by_test(test_results, by_test): if name not in by_test: by_test[name] = newEntry - bucket = by_test[name] + test_bucket = by_test[name] - bucket["total"] = 1 + bucket["total"] + test_bucket["total"] = 1 + test_bucket["total"] if is_failed(test_results) and not is_ginkgo_report_failure(test_results): - bucket["failed"] = 1 + bucket["failed"] + test_bucket["failed"] = 1 + test_bucket["failed"] k8s_version = test_results["k8s_version"] pg_version = test_results["pg_version"] platform = test_results["platform"] - bucket["k8s_versions_failed"][k8s_version] = True - bucket["pg_versions_failed"][pg_version] = True - bucket["platforms_failed"][platform] = True + test_bucket["k8s_versions_failed"][k8s_version] = True + test_bucket["pg_versions_failed"][pg_version] = True + test_bucket["platforms_failed"][platform] = True def count_bucketed_by_code(test_results, by_failing_code): @@ -296,10 +298,10 @@ def count_bucketed_by_code(test_results, by_failing_code): if err_desc not in by_failing_code: by_failing_code[err_desc] = newEntry - bucket = by_failing_code[err_desc] - bucket["total"] = 1 + bucket["total"] - bucket["tests"][name] = True - bucket["errors"] = test_results["error"] + error_bucket = by_failing_code[err_desc] + error_bucket["total"] = 1 + error_bucket["total"] + error_bucket["tests"][name] = True + error_bucket["errors"] = test_results["error"] def count_bucketed_by_special_failures(test_results, by_special_failures): """counts the successes, failures, failing versions of kubernetes, @@ -330,13 +332,13 @@ def count_bucketed_by_special_failures(test_results, by_special_failures): if failure not in by_special_failures: by_special_failures[failure] = newEntry - bucket = by_special_failures[failure] + failure_bucket = by_special_failures[failure] - bucket["total"] += 1 - bucket["tests_failed"][test_name] = True - bucket["k8s_versions_failed"][k8s_version] = True - bucket["pg_versions_failed"][pg_version] = True - bucket["platforms_failed"][platform] = True + failure_bucket["total"] += 1 + failure_bucket["tests_failed"][test_name] = True + failure_bucket["k8s_versions_failed"][k8s_version] = True + failure_bucket["pg_versions_failed"][pg_version] = True + failure_bucket["platforms_failed"][platform] = True def count_bucketized_stats(test_results, buckets, field_id): @@ -362,12 +364,19 @@ def compute_bucketized_summary(parameter_buckets): """counts the number of buckets with failures and the total number of buckets returns (num-failed-buckets, num-total-buckets) + The input buckets come like so: + bucket_1: {total: X1, failed: Y1}, + bucket_2: {total: X1, failed: Y1} + ... + We should count the number of distinct buckets, and + which of them have `failed` greater than zero. """ failed_buckets_count = 0 total_buckets_count = 0 for k, v in parameter_buckets.items(): - total_buckets_count += v["total"] - failed_buckets_count += v["failed"] + total_buckets_count += 1 + if v["failed"] > 0: + failed_buckets_count += 1 return failed_buckets_count, total_buckets_count @@ -706,6 +715,8 @@ def format_bucket_table(buckets, structure, file_out=None): ) for bucket in sorted_by_fail: + if buckets[bucket]["failed"] == 0: + continue table.add_row([buckets[bucket]["failed"], buckets[bucket]["total"], bucket]) print(table, file=file_out) @@ -730,6 +741,8 @@ def format_by_test(summary, structure, file_out=None): ) for bucket in sorted_by_fail: + if summary["by_test"][bucket]["failed"] == 0: + continue failed_k8s = ", ".join(summary["by_test"][bucket]["k8s_versions_failed"].keys()) failed_pg = ", ".join(summary["by_test"][bucket]["pg_versions_failed"].keys()) failed_platforms = ", ".join( @@ -850,8 +863,6 @@ def format_durations_table(test_times, structure, file_out=None): table.set_style(TableStyle.MARKDOWN) table.field_names = structure["header"] - print(test_times) - sorted_by_longest = dict( sorted(test_times.items(), key=lambda item: item[1]["max"], reverse=True) ) From f5848d25c09e2108d7dc17c5aa4bb357ac0b1902 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Tue, 25 Aug 2026 14:59:06 +0200 Subject: [PATCH 5/9] chore: apply black formatting Signed-off-by: Jaime Silvela --- summarize_test_results.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index fca0639..959da15 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -62,6 +62,7 @@ from prettytable import PrettyTable from prettytable import TableStyle + def is_failed(e2e_test): """checks if the test failed. In ginkgo, the passing states are well-defined but ginkgo 1 -> 2 added new failure kinds. So, check @@ -222,10 +223,7 @@ def track_time_taken(test_results, test_times, suite_times): suite_times[platform] = { "start_time": start_time, "end_time": end_time, - matrix_id: { - "start_time": start_time, - "end_time": end_time - } + matrix_id: {"start_time": start_time, "end_time": end_time}, } platform_bucket = suite_times[platform] @@ -235,16 +233,14 @@ def track_time_taken(test_results, test_times, suite_times): platform_bucket["end_time"] = end_time if matrix_id not in platform_bucket: - platform_bucket[matrix_id] = { - "start_time": start_time, - "end_time": end_time - } + platform_bucket[matrix_id] = {"start_time": start_time, "end_time": end_time} else: if start_time < platform_bucket[matrix_id]["start_time"]: platform_bucket[matrix_id]["start_time"] = start_time if platform_bucket[matrix_id]["end_time"] < end_time: platform_bucket[matrix_id]["end_time"] = end_time + def count_bucketed_by_test(test_results, by_test): """counts the successes, failures, failing versions of kubernetes, failing versions of postgres, bucketed by test name. @@ -254,7 +250,7 @@ def count_bucketed_by_test(test_results, by_test): "failed": 0, "k8s_versions_failed": {}, "pg_versions_failed": {}, - "platforms_failed": {} + "platforms_failed": {}, } name = test_results["name"] @@ -303,6 +299,7 @@ def count_bucketed_by_code(test_results, by_failing_code): error_bucket["tests"][name] = True error_bucket["errors"] = test_results["error"] + def count_bucketed_by_special_failures(test_results, by_special_failures): """counts the successes, failures, failing versions of kubernetes, failing versions of postgres, bucketed by test name. @@ -345,10 +342,7 @@ def count_bucketized_stats(test_results, buckets, field_id): """bucketizes test results according to the field_id. For each bucket, it counts the total tests run and the failed tests. """ - newEntry = { - "total": 0, - "failed": 0 - } + newEntry = {"total": 0, "failed": 0} bucket_id = test_results[field_id] if bucket_id not in buckets: buckets[bucket_id] = newEntry From 2c8f8615f7e26bcb9e900b1d319035a8bd864ddb Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Wed, 26 Aug 2026 09:54:28 +0200 Subject: [PATCH 6/9] chore: unit test overview computation Signed-off-by: Jaime Silvela --- test_summary.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test_summary.py b/test_summary.py index d7a173a..3b2023c 100644 --- a/test_summary.py +++ b/test_summary.py @@ -104,6 +104,28 @@ def test_compute_systematic_failures(self): self.assertEqual(has_alerts, False) self.assertEqual(out, "") + def test_compute_overview(self): + """In this test we look at the stats of the big "example-artifacts" folder, + so that we can get realistic results + """ + self.maxDiff = None + big_summary = summarize_test_results.compute_test_summary("example-artifacts") + overview = summarize_test_results.compile_overview(big_summary) + self.assertEqual(overview, { + 'total_failed': 15, + 'total_run': 18, + 'k8s_failed': 7, + 'k8s_run': 7, + 'matrix_failed': 14, + 'matrix_run': 15, + 'platform_failed': 2, + 'platform_run': 2, + 'postgres_failed': 8, + 'postgres_run': 9, + 'total_special_fails': 7, + 'unique_failed': 4, + 'unique_run': 7 + }) if __name__ == "__main__": unittest.main() From 8877be73366b044483ca683d64503225d616320a Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Wed, 26 Aug 2026 10:14:38 +0200 Subject: [PATCH 7/9] chore: apply PR comments and format Signed-off-by: Jaime Silvela --- summarize_test_results.py | 49 ++++++++++++++++----------------- test_summary.py | 58 +++++++++++++++++++++------------------ 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index 959da15..0b9cec7 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -245,17 +245,16 @@ def count_bucketed_by_test(test_results, by_test): """counts the successes, failures, failing versions of kubernetes, failing versions of postgres, bucketed by test name. """ - newEntry = { - "total": 0, - "failed": 0, - "k8s_versions_failed": {}, - "pg_versions_failed": {}, - "platforms_failed": {}, - } name = test_results["name"] if name not in by_test: - by_test[name] = newEntry + by_test[name] = { + "total": 0, + "failed": 0, + "k8s_versions_failed": {}, + "pg_versions_failed": {}, + "platforms_failed": {}, + } test_bucket = by_test[name] test_bucket["total"] = 1 + test_bucket["total"] @@ -280,11 +279,6 @@ def count_bucketed_by_code(test_results, by_failing_code): if not is_normal_failure(test_results): return - newEntry = { - "total": 0, - "tests": {}, - "errors": {}, - } name = test_results["name"] errfile = test_results["error_file"] @@ -292,12 +286,18 @@ def count_bucketed_by_code(test_results, by_failing_code): err_desc = f"{errfile}:{errline}" if err_desc not in by_failing_code: - by_failing_code[err_desc] = newEntry + by_failing_code[err_desc] = { + "total": 0, + "tests": {}, + # we keep only one "errors" because the stack trace will + # be essentially unchanged for other iterations of this + # error + "errors": test_results["error"], + } error_bucket = by_failing_code[err_desc] error_bucket["total"] = 1 + error_bucket["total"] error_bucket["tests"][name] = True - error_bucket["errors"] = test_results["error"] def count_bucketed_by_special_failures(test_results, by_special_failures): @@ -308,14 +308,6 @@ def count_bucketed_by_special_failures(test_results, by_special_failures): if not is_failed(test_results) or is_normal_failure(test_results): return - newEntry = { - "total": 0, - "tests_failed": {}, - "k8s_versions_failed": {}, - "pg_versions_failed": {}, - "platforms_failed": {}, - } - failure = "" if is_external_failure(test_results): failure = test_results["state"] @@ -328,7 +320,13 @@ def count_bucketed_by_special_failures(test_results, by_special_failures): platform = test_results["platform"] if failure not in by_special_failures: - by_special_failures[failure] = newEntry + by_special_failures[failure] = { + "total": 0, + "tests_failed": {}, + "k8s_versions_failed": {}, + "pg_versions_failed": {}, + "platforms_failed": {}, + } failure_bucket = by_special_failures[failure] failure_bucket["total"] += 1 @@ -342,10 +340,9 @@ def count_bucketized_stats(test_results, buckets, field_id): """bucketizes test results according to the field_id. For each bucket, it counts the total tests run and the failed tests. """ - newEntry = {"total": 0, "failed": 0} bucket_id = test_results[field_id] if bucket_id not in buckets: - buckets[bucket_id] = newEntry + buckets[bucket_id] = {"total": 0, "failed": 0} bucket = buckets[bucket_id] bucket["total"] = 1 + bucket["total"] diff --git a/test_summary.py b/test_summary.py index 3b2023c..9254cf2 100644 --- a/test_summary.py +++ b/test_summary.py @@ -31,24 +31,24 @@ def test_compute_summary(self): self.assertEqual(self.summary["total_failed"], 1) self.assertEqual( - self.summary["by_code"]["/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80"]["total"], + self.summary["by_code"][ + "/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80" + ]["total"], 1, "unexpected summary", ) self.assertEqual( - self.summary["by_code"]["/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80"]["tests"], + self.summary["by_code"][ + "/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80" + ]["tests"], { - "InitDB settings - initdb custom post-init SQL scripts -- can find the" - " tables created by the post-init SQL queries": True + "InitDB settings - initdb custom post-init SQL scripts -- can find the" + " tables created by the post-init SQL queries": True }, "unexpected summary", ) - self.assertEqual( - self.summary["by_matrix"], {"id1": {"total": 3, "failed": 1}} - ) - self.assertEqual( - self.summary["by_k8s"], {"1.22": {"total": 3, "failed": 1}} - ) + self.assertEqual(self.summary["by_matrix"], {"id1": {"total": 3, "failed": 1}}) + self.assertEqual(self.summary["by_k8s"], {"1.22": {"total": 3, "failed": 1}}) self.assertEqual( self.summary["by_platform"], {"local": {"total": 3, "failed": 1}} ) @@ -64,8 +64,8 @@ def test_compute_summary(self): "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37), "id1": { "end_time": datetime.datetime(2021, 11, 29, 18, 31, 7), - "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37) - } + "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37), + }, }, }, ) @@ -111,21 +111,25 @@ def test_compute_overview(self): self.maxDiff = None big_summary = summarize_test_results.compute_test_summary("example-artifacts") overview = summarize_test_results.compile_overview(big_summary) - self.assertEqual(overview, { - 'total_failed': 15, - 'total_run': 18, - 'k8s_failed': 7, - 'k8s_run': 7, - 'matrix_failed': 14, - 'matrix_run': 15, - 'platform_failed': 2, - 'platform_run': 2, - 'postgres_failed': 8, - 'postgres_run': 9, - 'total_special_fails': 7, - 'unique_failed': 4, - 'unique_run': 7 - }) + self.assertEqual( + overview, + { + "total_failed": 15, + "total_run": 18, + "k8s_failed": 7, + "k8s_run": 7, + "matrix_failed": 14, + "matrix_run": 15, + "platform_failed": 2, + "platform_run": 2, + "postgres_failed": 8, + "postgres_run": 9, + "total_special_fails": 7, + "unique_failed": 4, + "unique_run": 7, + }, + ) + if __name__ == "__main__": unittest.main() From d917e5e2b784f5a029d8ed3603e4bd2df26d7164 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Wed, 26 Aug 2026 10:53:37 +0200 Subject: [PATCH 8/9] chore: as per PR comment, clarify platform matrices Signed-off-by: Jaime Silvela --- summarize_test_results.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index 0b9cec7..b5dae23 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -223,7 +223,9 @@ def track_time_taken(test_results, test_times, suite_times): suite_times[platform] = { "start_time": start_time, "end_time": end_time, - matrix_id: {"start_time": start_time, "end_time": end_time}, + "matrices": { + matrix_id: {"start_time": start_time, "end_time": end_time}, + }, } platform_bucket = suite_times[platform] @@ -232,13 +234,16 @@ def track_time_taken(test_results, test_times, suite_times): if platform_bucket["end_time"] < end_time: platform_bucket["end_time"] = end_time - if matrix_id not in platform_bucket: - platform_bucket[matrix_id] = {"start_time": start_time, "end_time": end_time} + if matrix_id not in platform_bucket["matrices"]: + platform_bucket["matrices"][matrix_id] = { + "start_time": start_time, + "end_time": end_time, + } else: - if start_time < platform_bucket[matrix_id]["start_time"]: - platform_bucket[matrix_id]["start_time"] = start_time - if platform_bucket[matrix_id]["end_time"] < end_time: - platform_bucket[matrix_id]["end_time"] = end_time + if start_time < platform_bucket["matrices"][matrix_id]["start_time"]: + platform_bucket["matrices"][matrix_id]["start_time"] = start_time + if platform_bucket["matrices"][matrix_id]["end_time"] < end_time: + platform_bucket["matrices"][matrix_id]["end_time"] = end_time def count_bucketed_by_test(test_results, by_test): @@ -887,12 +892,10 @@ def format_suite_durations_table(suite_times, structure, file_out=None): "slowest_branch": {}, } for platform in suite_times: - for matrix_id in suite_times[platform]: - if matrix_id == "start_time" or matrix_id == "end_time": - continue + for matrix_id in suite_times[platform]["matrices"]: duration = ( - suite_times[platform][matrix_id]["end_time"] - - suite_times[platform][matrix_id]["start_time"] + suite_times[platform]["matrices"][matrix_id]["end_time"] + - suite_times[platform]["matrices"][matrix_id]["start_time"] ) if platform not in suite_durations["max"]: suite_durations["max"][platform] = duration From 358c8c534876fd23fc2d5b927ccda5b7029981de Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Wed, 26 Aug 2026 10:59:27 +0200 Subject: [PATCH 9/9] fix: unit test Signed-off-by: Jaime Silvela --- test_summary.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test_summary.py b/test_summary.py index 9254cf2..8bd8661 100644 --- a/test_summary.py +++ b/test_summary.py @@ -62,9 +62,11 @@ def test_compute_summary(self): "local": { "end_time": datetime.datetime(2021, 11, 29, 18, 31, 7), "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37), - "id1": { - "end_time": datetime.datetime(2021, 11, 29, 18, 31, 7), - "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37), + "matrices": { + "id1": { + "end_time": datetime.datetime(2021, 11, 29, 18, 31, 7), + "start_time": datetime.datetime(2021, 11, 29, 18, 28, 37), + }, }, }, },