diff --git a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md
index 29aa979..866b621 100644
--- a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md
+++ b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md
@@ -23,13 +23,28 @@ The procedure for cutting a release:
## Developing and testing
You can test directly with the Python code on the `example-artifacts` directory,
-where you can see some JSON artifacts in the expected format. For example:
+where you can see some JSON artifacts in the expected format.
+
+Before running the python scripts, you may need to install the requirements
+(just `prettytable` currently).
+
+``` shell
+pip install --no-cache-dir -r requirements.txt
+```
+
+A basic execution looks like this:
``` shell
python summarize_test_results.py --dir example-artifacts
```
-or
+or, if you're using Python virtual environments, say `pythonVenv`
+
+``` shell
+pythonVenv/bin/python summarize_test_results.py --dir example-artifacts
+```
+
+you can get the report into a file by setting the `GITHUB_STEP_SUMMARY`:
``` shell
GITHUB_STEP_SUMMARY=out.md python summarize_test_results.py --dir example-artifacts
@@ -89,6 +104,12 @@ CIclops has the beginning of a unit test suite. You can run it with:
python3 -m unittest
```
+or
+
+``` sh
+python test_summary.py
+```
+
## Testing within a calling GitHub workflow
Even with unit tests and local tests, it's good to try Ciclops code out from a
diff --git a/summarize_test_results.py b/summarize_test_results.py
index c386c64..b5dae23 100644
--- a/summarize_test_results.py
+++ b/summarize_test_results.py
@@ -59,8 +59,8 @@
import math
import os
import pathlib
-from prettytable import MARKDOWN
from prettytable import PrettyTable
+from prettytable import TableStyle
def is_failed(e2e_test):
@@ -168,7 +168,9 @@ def combine_postgres_data(test_entry):
def track_time_taken(test_results, test_times, suite_times):
"""computes the running shortest and longest duration of
- running each kind of test
+ running each kind of test, and the longest branch.
+ Also computes start and end of the test suite per platform
+ and matrix_id within the platform.
"""
name = test_results["name"]
# tag abnormal failures, e.g.: "[operator was restarted] Imports with …"
@@ -200,35 +202,48 @@ def track_time_taken(test_results, test_times, suite_times):
end_time = datetime.fromisoformat(end_frags[0])
duration = end_time - start_time
matrix_id = test_results["matrix_id"]
- if name not in test_times["max"]:
- test_times["max"][name] = duration
- if name not in test_times["min"]:
- test_times["min"][name] = duration
- if name not in test_times["slowest_branch"]:
- test_times["slowest_branch"][name] = matrix_id
-
- if duration > test_times["max"][name]:
- test_times["max"][name] = duration
- test_times["slowest_branch"][name] = matrix_id
- if duration < test_times["min"][name]:
- test_times["min"][name] = duration
+ if name not in test_times:
+ test_times[name] = {
+ "max": duration,
+ "min": duration,
+ "slowest_branch": matrix_id,
+ }
+ test_bucket = test_times[name]
+
+ if duration > test_bucket["max"]:
+ test_bucket["max"] = duration
+ test_bucket["slowest_branch"] = matrix_id
+ if duration < test_bucket["min"]:
+ test_bucket["min"] = duration
# Track test suite timings.
# For each platform-matrix branch, track the earliest start and the latest end
platform = test_results["platform"]
- if platform not in suite_times["start_time"]:
- suite_times["start_time"][platform] = {}
- if matrix_id not in suite_times["start_time"][platform]:
- suite_times["start_time"][platform][matrix_id] = start_time
- if platform not in suite_times["end_time"]:
- suite_times["end_time"][platform] = {}
- if matrix_id not in suite_times["end_time"][platform]:
- suite_times["end_time"][platform][matrix_id] = end_time
+ if platform not in suite_times:
+ suite_times[platform] = {
+ "start_time": start_time,
+ "end_time": end_time,
+ "matrices": {
+ matrix_id: {"start_time": start_time, "end_time": end_time},
+ },
+ }
+ platform_bucket = suite_times[platform]
- if start_time < suite_times["start_time"][platform][matrix_id]:
- suite_times["start_time"][platform][matrix_id] = start_time
- if suite_times["end_time"][platform][matrix_id] < end_time:
- suite_times["end_time"][platform][matrix_id] = end_time
+ if start_time < platform_bucket["start_time"]:
+ platform_bucket["start_time"] = start_time
+ if platform_bucket["end_time"] < end_time:
+ platform_bucket["end_time"] = end_time
+
+ if matrix_id not in platform_bucket["matrices"]:
+ platform_bucket["matrices"][matrix_id] = {
+ "start_time": start_time,
+ "end_time": end_time,
+ }
+ else:
+ if start_time < platform_bucket["matrices"][matrix_id]["start_time"]:
+ platform_bucket["matrices"][matrix_id]["start_time"] = start_time
+ if platform_bucket["matrices"][matrix_id]["end_time"] < end_time:
+ platform_bucket["matrices"][matrix_id]["end_time"] = end_time
def count_bucketed_by_test(test_results, by_test):
@@ -237,32 +252,31 @@ def count_bucketed_by_test(test_results, by_test):
"""
name = test_results["name"]
- if name not in by_test["total"]:
- by_test["total"][name] = 0
- by_test["total"][name] = 1 + by_test["total"][name]
+ if name not in by_test:
+ by_test[name] = {
+ "total": 0,
+ "failed": 0,
+ "k8s_versions_failed": {},
+ "pg_versions_failed": {},
+ "platforms_failed": {},
+ }
+ test_bucket = by_test[name]
+
+ test_bucket["total"] = 1 + test_bucket["total"]
if is_failed(test_results) and not is_ginkgo_report_failure(test_results):
- if name not in by_test["failed"]:
- by_test["failed"][name] = 0
- if name not in by_test["k8s_versions_failed"]:
- by_test["k8s_versions_failed"][name] = {}
- if name not in by_test["pg_versions_failed"]:
- by_test["pg_versions_failed"][name] = {}
- if name not in by_test["platforms_failed"]:
- by_test["platforms_failed"][name] = {}
- by_test["failed"][name] = 1 + by_test["failed"][name]
+ test_bucket["failed"] = 1 + test_bucket["failed"]
k8s_version = test_results["k8s_version"]
pg_version = test_results["pg_version"]
platform = test_results["platform"]
- by_test["k8s_versions_failed"][name][k8s_version] = True
- by_test["pg_versions_failed"][name][pg_version] = True
- by_test["platforms_failed"][name][platform] = True
+ test_bucket["k8s_versions_failed"][k8s_version] = True
+ test_bucket["pg_versions_failed"][pg_version] = True
+ test_bucket["platforms_failed"][platform] = True
def count_bucketed_by_code(test_results, by_failing_code):
"""buckets by failed code, with a list of tests where the assertion fails,
and a view of the stack trace.
"""
- name = test_results["name"]
if test_results["error"] == "" or test_results["state"] == "ignoreFailed":
return
# it does not make sense to show failing code that is outside the test,
@@ -270,20 +284,25 @@ def count_bucketed_by_code(test_results, by_failing_code):
if not is_normal_failure(test_results):
return
+ name = test_results["name"]
+
errfile = test_results["error_file"]
errline = test_results["error_line"]
err_desc = f"{errfile}:{errline}"
- if err_desc not in by_failing_code["total"]:
- by_failing_code["total"][err_desc] = 0
- by_failing_code["total"][err_desc] = 1 + by_failing_code["total"][err_desc]
-
- if err_desc not in by_failing_code["tests"]:
- by_failing_code["tests"][err_desc] = {}
- by_failing_code["tests"][err_desc][name] = True
+ if err_desc not in by_failing_code:
+ by_failing_code[err_desc] = {
+ "total": 0,
+ "tests": {},
+ # we keep only one "errors" because the stack trace will
+ # be essentially unchanged for other iterations of this
+ # error
+ "errors": test_results["error"],
+ }
- if err_desc not in by_failing_code["errors"]:
- by_failing_code["errors"][err_desc] = test_results["error"]
+ error_bucket = by_failing_code[err_desc]
+ error_bucket["total"] = 1 + error_bucket["total"]
+ error_bucket["tests"][name] = True
def count_bucketed_by_special_failures(test_results, by_special_failures):
@@ -305,57 +324,62 @@ def count_bucketed_by_special_failures(test_results, by_special_failures):
pg_version = test_results["pg_version"]
platform = test_results["platform"]
- if failure not in by_special_failures["total"]:
- by_special_failures["total"][failure] = 0
-
- for key in [
- "tests_failed",
- "k8s_versions_failed",
- "pg_versions_failed",
- "platforms_failed",
- ]:
- if failure not in by_special_failures[key]:
- by_special_failures[key][failure] = {}
+ if failure not in by_special_failures:
+ by_special_failures[failure] = {
+ "total": 0,
+ "tests_failed": {},
+ "k8s_versions_failed": {},
+ "pg_versions_failed": {},
+ "platforms_failed": {},
+ }
+ failure_bucket = by_special_failures[failure]
- by_special_failures["total"][failure] += 1
- by_special_failures["tests_failed"][failure][test_name] = True
- by_special_failures["k8s_versions_failed"][failure][k8s_version] = True
- by_special_failures["pg_versions_failed"][failure][pg_version] = True
- by_special_failures["platforms_failed"][failure][platform] = True
+ failure_bucket["total"] += 1
+ failure_bucket["tests_failed"][test_name] = True
+ failure_bucket["k8s_versions_failed"][k8s_version] = True
+ failure_bucket["pg_versions_failed"][pg_version] = True
+ failure_bucket["platforms_failed"][platform] = True
def count_bucketized_stats(test_results, buckets, field_id):
- """counts the success/failures onto a bucket. This means there are two
- dictionaries: one for `total` tests, one for `failed` tests.
+ """bucketizes test results according to the field_id.
+ For each bucket, it counts the total tests run and the failed tests.
"""
bucket_id = test_results[field_id]
- if bucket_id not in buckets["total"]:
- buckets["total"][bucket_id] = 0
- buckets["total"][bucket_id] = 1 + buckets["total"][bucket_id]
+ if bucket_id not in buckets:
+ buckets[bucket_id] = {"total": 0, "failed": 0}
+
+ bucket = buckets[bucket_id]
+ bucket["total"] = 1 + bucket["total"]
if is_failed(test_results):
- if bucket_id not in buckets["failed"]:
- buckets["failed"][bucket_id] = 0
- buckets["failed"][bucket_id] = 1 + buckets["failed"][bucket_id]
+ bucket["failed"] += 1
def compute_bucketized_summary(parameter_buckets):
"""counts the number of buckets with failures and the
total number of buckets
returns (num-failed-buckets, num-total-buckets)
+ The input buckets come like so:
+ bucket_1: {total: X1, failed: Y1},
+ bucket_2: {total: X1, failed: Y1}
+ ...
+ We should count the number of distinct buckets, and
+ which of them have `failed` greater than zero.
"""
failed_buckets_count = 0
total_buckets_count = 0
- for _ in parameter_buckets["total"]:
- total_buckets_count = 1 + total_buckets_count
- for _ in parameter_buckets["failed"]:
- failed_buckets_count = 1 + failed_buckets_count
+ for k, v in parameter_buckets.items():
+ total_buckets_count += 1
+ if v["failed"] > 0:
+ failed_buckets_count += 1
return failed_buckets_count, total_buckets_count
def compute_test_summary(test_dir):
"""iterate over the JSON artifact files in `test_dir`, and
- bucket them for comprehension.
+ bucket them in various ways to provide insights as to failure
+ patterns.
Returns a dictionary of dictionaries:
@@ -380,36 +404,23 @@ def compute_test_summary(test_dir):
total_runs = 0
total_fails = 0
total_special_fails = 0
- by_test = {
- "total": {},
- "failed": {},
- "k8s_versions_failed": {},
- "pg_versions_failed": {},
- "platforms_failed": {},
- }
- by_failing_code = {
- "total": {},
- "tests": {},
- "errors": {},
- }
- by_matrix = {"total": {}, "failed": {}}
- by_k8s = {"total": {}, "failed": {}}
- by_postgres = {"total": {}, "failed": {}}
- by_platform = {"total": {}, "failed": {}}
+ by_test = {}
+ by_failing_code = {}
+ by_matrix = {}
+ by_k8s = {}
+ by_postgres = {}
+ by_platform = {}
# special failures are not due to the test having failed, but
# to something at a higher level, like the E2E suite having been
# cancelled, timed out, or having executed improperly
- by_special_failures = {
- "total": {},
- "tests_failed": {},
- "k8s_versions_failed": {},
- "pg_versions_failed": {},
- "platforms_failed": {},
- }
+ by_special_failures = {}
- test_durations = {"max": {}, "min": {}, "slowest_branch": {}}
- suite_durations = {"start_time": {}, "end_time": {}}
+ # test_durations track the quickest and slowest tests over the
+ # total execution.
+ # suite_durations compute the duration of the test suite overall
+ test_durations = {}
+ suite_durations = {}
# start computation of summary
##############################
@@ -426,6 +437,10 @@ def compute_test_summary(test_dir):
test_results = combine_postgres_data(parsed)
test_results = compress_kubernetes_version(test_results)
+ # From this point, test_results is the representation of
+ # one text execution, i.e. one E2E test run on one version
+ # of CNPG x Postgres x Kubernetes
+
total_runs = 1 + total_runs
if is_failed(test_results):
total_fails = 1 + total_fails
@@ -440,6 +455,7 @@ def compute_test_summary(test_dir):
count_bucketed_by_code(test_results, by_failing_code)
# special failures are treated separately
+ # bucketing by the failure
count_bucketed_by_special_failures(test_results, by_special_failures)
# bucketing by matrix ID
@@ -454,6 +470,8 @@ def compute_test_summary(test_dir):
# bucketing by platform
count_bucketized_stats(test_results, by_platform, "platform")
+ # bucketing test_durations by test name
+ # bucketing suite_durations by platform and also by matrix_id
track_time_taken(test_results, test_durations, suite_durations)
return {
@@ -519,10 +537,9 @@ def compute_systematic_failures_on_metric(summary, metric, embed=True):
output = ""
has_systematic_failure_in_metric = False
counter = 0
- for bucket_hits in summary[metric]["failed"].items():
- bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs
- failures = summary[metric]["failed"][bucket]
- runs = summary[metric]["total"][bucket]
+ for bucket, stats in summary[metric].items():
+ failures = stats["failed"]
+ runs = stats["total"]
if failures == runs and failures > 1:
if not has_systematic_failure_in_metric:
output += f"{metric_name(metric)} with systematic failures:\n\n"
@@ -624,12 +641,9 @@ def compute_thermometer_on_metric(summary, metric, embed=True):
"""
output = f"{metric_name(metric)} thermometer:\n\n"
- for bucket_hits in summary[metric]["total"].items():
- bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs
- failures = 0
- if bucket in summary[metric]["failed"]:
- failures = summary[metric]["failed"][bucket]
- runs = summary[metric]["total"][bucket]
+ for bucket, stats in summary[metric].items():
+ failures = stats["failed"]
+ runs = stats["total"]
success_percent = (1 - failures / runs) * 100
color = compute_semaphore(success_percent, embed)
output += (
@@ -668,7 +682,7 @@ def format_overview(summary, structure, file_out=None):
print("## " + structure["title"] + "\n", file=file_out)
table = PrettyTable(align="l")
table.field_names = structure["header"]
- table.set_style(MARKDOWN)
+ table.set_style(TableStyle.MARKDOWN)
for row in structure["rows"]:
table.add_row([summary[row[1]], summary[row[2]], row[0]])
@@ -690,14 +704,16 @@ def format_bucket_table(buckets, structure, file_out=None):
print(f"\n\n", file=file_out)
table = PrettyTable(align="l")
table.field_names = structure["header"]
- table.set_style(MARKDOWN)
+ table.set_style(TableStyle.MARKDOWN)
sorted_by_fail = dict(
- sorted(buckets["failed"].items(), key=lambda item: item[1], reverse=True)
+ sorted(buckets.items(), key=lambda item: item[1]["failed"], reverse=True)
)
for bucket in sorted_by_fail:
- table.add_row([buckets["failed"][bucket], buckets["total"][bucket], bucket])
+ if buckets[bucket]["failed"] == 0:
+ continue
+ table.add_row([buckets[bucket]["failed"], buckets[bucket]["total"], bucket])
print(table, file=file_out)
@@ -710,26 +726,28 @@ def format_by_test(summary, structure, file_out=None):
table = PrettyTable(align="l")
table.field_names = structure["header"]
- table.set_style(MARKDOWN)
+ table.set_style(TableStyle.MARKDOWN)
sorted_by_fail = dict(
sorted(
- summary["by_test"]["failed"].items(),
- key=lambda item: item[1],
+ summary["by_test"].items(),
+ key=lambda item: item[1]["failed"],
reverse=True,
)
)
for bucket in sorted_by_fail:
- failed_k8s = ", ".join(summary["by_test"]["k8s_versions_failed"][bucket].keys())
- failed_pg = ", ".join(summary["by_test"]["pg_versions_failed"][bucket].keys())
+ if summary["by_test"][bucket]["failed"] == 0:
+ continue
+ failed_k8s = ", ".join(summary["by_test"][bucket]["k8s_versions_failed"].keys())
+ failed_pg = ", ".join(summary["by_test"][bucket]["pg_versions_failed"].keys())
failed_platforms = ", ".join(
- summary["by_test"]["platforms_failed"][bucket].keys()
+ summary["by_test"][bucket]["platforms_failed"].keys()
)
table.add_row(
[
- summary["by_test"]["failed"][bucket],
- summary["by_test"]["total"][bucket],
+ summary["by_test"][bucket]["failed"],
+ summary["by_test"][bucket]["total"],
failed_k8s,
failed_pg,
failed_platforms,
@@ -748,32 +766,32 @@ def format_by_special_failure(summary, structure, file_out=None):
table = PrettyTable(align="l")
table.field_names = structure["header"]
- table.set_style(MARKDOWN)
+ table.set_style(TableStyle.MARKDOWN)
sorted_by_count = dict(
sorted(
- summary["by_special_failures"]["total"].items(),
- key=lambda item: item[1],
+ summary["by_special_failures"].items(),
+ key=lambda item: item[1]["total"],
reverse=True,
)
)
for bucket in sorted_by_count:
failed_tests = ", ".join(
- summary["by_special_failures"]["tests_failed"][bucket].keys()
+ summary["by_special_failures"][bucket]["tests_failed"].keys()
)
failed_k8s = ", ".join(
- summary["by_special_failures"]["k8s_versions_failed"][bucket].keys()
+ summary["by_special_failures"][bucket]["k8s_versions_failed"].keys()
)
failed_pg = ", ".join(
- summary["by_special_failures"]["pg_versions_failed"][bucket].keys()
+ summary["by_special_failures"][bucket]["pg_versions_failed"].keys()
)
failed_platforms = ", ".join(
- summary["by_special_failures"]["platforms_failed"][bucket].keys()
+ summary["by_special_failures"][bucket]["platforms_failed"].keys()
)
table.add_row(
[
- summary["by_special_failures"]["total"][bucket],
+ summary["by_special_failures"][bucket]["total"],
bucket,
failed_tests,
failed_k8s,
@@ -793,28 +811,28 @@ def format_by_code(summary, structure, file_out=None):
table = PrettyTable(align="l")
table.field_names = structure["header"]
- table.set_style(MARKDOWN)
+ table.set_style(TableStyle.MARKDOWN)
sorted_by_code = dict(
sorted(
- summary["by_code"]["total"].items(),
- key=lambda item: item[1],
+ summary["by_code"].items(),
+ key=lambda item: item[1]["total"],
reverse=True,
)
)
for bucket in sorted_by_code:
- tests = ", ".join(summary["by_code"]["tests"][bucket].keys())
+ tests = ", ".join(summary["by_code"][bucket]["tests"].keys())
# replace newlines and pipes to avoid interference with Markdown tables
errors = (
- summary["by_code"]["errors"][bucket]
+ summary["by_code"][bucket]["errors"]
.replace("\n", "
")
.replace("|", "—")
)
err_cell = f"Click to expand
{errors}