Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 34 additions & 13 deletions modules/shell/scripts/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@
# the window, three times.
set -euo pipefail
cd "$(dirname "$0")/.."
mkdir -p .evidence
: > .evidence/e2e.txt
say() { echo "$*" | tee -a .evidence/e2e.txt; }
# Where its text goes: the module's `.evidence`, or a shard's own folder when the run is cut in
# shards side by side (scripts/e2e_sharded.sh, #862); `tray`, a shard that plays the tray's part alone.
out=${KITOKEY_E2E_OUT:-.evidence}
part=${KITOKEY_E2E_PART:-}
mkdir -p "$out"
: > "$out/e2e.txt"
say() { echo "$*" | tee -a "$out/e2e.txt"; }
# Each of the tray's scenarios timed, said when the next one begins or the window's part does: where
# a run's minutes go (#862). `SECONDS` is bash's own clock, whole seconds.
scenario="" scenario_at=0
Expand All @@ -22,7 +26,7 @@ scenario() { scenario_done; scenario=$1; scenario_at=$SECONDS; say "$1. $2"; }
if ! only=$(python3 -I scripts/e2e_only.py --asked scripts/e2e_window.py 2>&1); then say "$only"; exit 3; fi
export KITOKEY_E2E_ONLY="$only"
# The run's logs and screenshots, where NapkinStack reads them after it (nstack e2e shell).
evidence=${NSTACK_EVIDENCE:-$PWD/.evidence/e2e}
evidence=${NSTACK_EVIDENCE:-$PWD/$out/e2e}
mkdir -p "$evidence"

watcher="org.kde.StatusNotifierWatcher /StatusNotifierWatcher org.kde.StatusNotifierWatcher"
Expand All @@ -35,7 +39,7 @@ fi
if [ -z "${KITOKEY_HEADLESS:-}" ] && pgrep -x kitokey >/dev/null; then
say "e2e: NOT RUN — a Kitokey is running outside the run: quit it first (its menu, Quitter Kitokey)"; exit 1
fi
cargo build --locked 2>&1 | tail -1 >> .evidence/e2e.txt
cargo build --locked 2>&1 | tail -1 >> "$out/e2e.txt"
bin=${CARGO_TARGET_DIR:-$PWD/target}/debug/kitokey
work=$(mktemp -d)
app=""
Expand Down Expand Up @@ -107,11 +111,20 @@ start() { # the language, the settings file's text, the recorder
say " (started: its tray item found in $((SECONDS - started_at)) s, $tries look(s))"
bus=${item%@*}; menu=${item#*@}/Menu
}
# This session's Kitokeys alone: a run cut in shards plays the same binary side by side, each session
# its own XDG_RUNTIME_DIR (#862).
ours() {
local pid count=0
for pid in $(pgrep -f "^$bin" || true); do
grep -qzx "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-}" "/proc/$pid/environ" 2>/dev/null && count=$((count + 1))
done
echo "$count"
}
again() { # a second start, the first one's settings: it hands over to the running one and leaves
LANGUAGE= LC_ALL=fr_FR.UTF-8 LANG=fr_FR.UTF-8 XDG_CONFIG_HOME="$work/config" XDG_DATA_HOME="$work/data" XDG_STATE_HOME="$work/state" \
KITOKEY_KEYRING=memory timeout 10 "$bin" "$@" 2>> "$work/errors" && left=0 || left=$?
check is "$left" 0
check is "$(pgrep -c -f "^$bin" || true)" 1
check is "$(ours)" 1
}
greyed() { busctl --user call "$bus" "$menu" com.canonical.dbusmenu GetLayout iias 0 -- -1 0 | grep -o '"enabled" b false' | wc -l; }
labels() { # busctl writes what is not ASCII as escapes: printf reads them back
Expand Down Expand Up @@ -141,7 +154,8 @@ failed=0
check() { if "$@"; then say " ok $*"; else say " FAIL $*"; failed=1; fi; }
is() { [ "$1" = "$2" ] || { say " got: $1"; return 1; }; }

if [ -z "$only" ]; then
# The tray's part: neither in a window shard (KITOKEY_E2E_ONLY) nor in main()'s own (#862).
if [ -z "$only" ] && [ "$part" != main ]; then
# The program that hears, played: it notes what it was given, fails, is slow or stuck when told.
cat > "$work/kitokey-transcription" <<'PLAYED'
#!/bin/sh
Expand Down Expand Up @@ -605,11 +619,13 @@ fi
scenario_done
# The window (cycle 03): Kitokey started with --window by tauri-driver, its page read over
# WebDriver (scripts/e2e_window.py). For a few seconds a window appears, twice.
if ! command -v WebKitWebDriver >/dev/null; then
if [ "$part" = tray ]; then
say "the window's scenarios: in the other shards"
elif ! command -v WebKitWebDriver >/dev/null; then
say "H. the window: NOT RUN — WebKitGTK's WebDriver is missing (Fedora: webkit2gtk4.1-devel)"
failed=1
else
command -v tauri-driver >/dev/null || cargo install tauri-driver --locked --version 2.1.0 2>&1 | tail -1 >> .evidence/e2e.txt
command -v tauri-driver >/dev/null || cargo install tauri-driver --locked --version 2.1.0 2>&1 | tail -1 >> "$out/e2e.txt"
mkdir -p "$work/window"
# Its pages as they were drawn, kept for an eye against D11's boards — the author's first.
# Kept where the user says, untouched; by default in the evidence, this run's alone.
Expand All @@ -622,24 +638,29 @@ else
quiet_module=$(pactl load-module module-null-sink sink_name=kitokey_e2e_quiet)
export PULSE_SINK=kitokey_e2e_quiet
# A window that stops answering fails the run, never holds it: it takes about 6 minutes.
if ! timeout --kill-after=30 1200 uv run --quiet --with selenium==4.51.0 --with jsonschema==4.23.0 --with rfc3339-validator==0.1.4 python -u scripts/e2e_window.py "$bin" "$work/window" | tee -a .evidence/e2e.txt; then
if ! timeout --kill-after=30 1200 uv run --quiet --with selenium==4.51.0 --with jsonschema==4.23.0 --with rfc3339-validator==0.1.4 python -u scripts/e2e_window.py "$bin" "$work/window" | tee -a "$out/e2e.txt"; then
failed=1
fi
fi

# Its log (ADR-0007): each recording and making under an id, never a path of the run's folder.
shell_log=$(cat "$work"/state/kitokey/logs/shell.*.jsonl 2>/dev/null || true)
if [ -z "$only" ]; then
if [ -z "$only" ] && [ "$part" != main ]; then
check grep -q '"message":"shell.recording_started"' <<< "$shell_log"
check grep -q '"message":"shell.minutes_written"' <<< "$shell_log"
fi
check test -z "$(grep -F "$work" <<< "$shell_log")"
check test ! -s "$work/errors"
# Every warning and error logged is one a scenario provokes, and each one named is logged: the
# logs read, not thrown away with the run's folder (D29).
logs_read() { uv run --quiet --with pyyaml==6.0.3 python scripts/e2e_logs.py "$work" "$evidence" MANIFEST.yaml | tee -a .evidence/e2e.txt; return "${PIPESTATUS[0]}"; }
logs_read() { uv run --quiet --with pyyaml==6.0.3 python scripts/e2e_logs.py "$work" "$evidence" MANIFEST.yaml | tee -a "$out/e2e.txt"; return "${PIPESTATUS[0]}"; }
check logs_read
[ -s "$work/errors" ] && sed 's/^/ /' "$work/errors" >> .evidence/e2e.txt
[ -s "$work/errors" ] && sed 's/^/ /' "$work/errors" >> "$out/e2e.txt"
if [ -n "$part" ]; then
# A shard, never the module's e2e: its status a partial run's, its last line says which.
[ "$failed" = 0 ] && say "e2e: shard ($part): its checks passed" || say "e2e: shard ($part): FAILED"
exit 3
fi
if [ -n "$only" ]; then
# Never read as the module's e2e: its status is a failure either way, its last line says which.
[ "$failed" = 0 ] && say "e2e: partial run ($only): its checks passed — not the module's e2e" || say "e2e: partial run ($only): FAILED"
Expand Down
5 changes: 3 additions & 2 deletions modules/shell/scripts/e2e_logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@
if not hit:
unknown.setdefault(entry.get("message", line), line)
unseen = [item for item in declared if item["pattern"] not in seen]
# A partial run (KITOKEY_E2E_ONLY) plays some scenarios: what the others provoke is not owed.
partial = bool(os.environ.get("KITOKEY_E2E_ONLY", "").strip())
# A partial run (KITOKEY_E2E_ONLY), or a shard (KITOKEY_E2E_PART, #862), plays some scenarios: what
# the others provoke is not owed — a sharded run judges every shard's logs together, once.
partial = bool(os.environ.get("KITOKEY_E2E_ONLY", "").strip() or os.environ.get("KITOKEY_E2E_PART", "").strip())
print(f" {lines} log lines kept in {kept}; {len(unknown)} warning or error message(s) not provoked, "
f"{len(unseen)} declared and never logged")
for line in unknown.values():
Expand Down
83 changes: 83 additions & 0 deletions modules/shell/scripts/e2e_sharded.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/usr/bin/env bash
# The module's e2e cut in shards played side by side (#862), each in a GNOME session of its own
# made by scripts/headless.sh: the tray's part; main()'s own scenarios, H to R; and the scenarios
# main() calls, in KITOKEY_E2E_SHARDS lists balanced by their measured times (scripts/e2e_times.txt).
# Their text put back in one, in the order a whole run says it; every shard's logs judged together,
# once, as a whole run's are (D29); one verdict. Run by headless.sh for a full run.
set -uo pipefail
here=$(cd "$(dirname "$0")" && pwd)
cd "$here/.."
out=.evidence
evidence=${NSTACK_EVIDENCE:-$PWD/$out/e2e}
mkdir -p "$out" "$evidence"
: > "$out/e2e.txt"
say() { echo "$*" | tee -a "$out/e2e.txt"; }
began=$SECONDS

# Built once, before the shards share it: each one's own build then finds it done.
cargo build --locked 2>&1 | tail -1 >> "$out/e2e.txt"
mapfile -t lists < <(python3 -I scripts/e2e_shards.py lists "${KITOKEY_E2E_SHARDS:-3}" scripts/e2e_window.py scripts/e2e_times.txt)
# Beside the sessions headless.sh makes, never under /tmp: the shards' screenshots and logs.
shards=$(mktemp -d "${XDG_CACHE_HOME:-$HOME/.cache}/kitokey-shards-XXXXXX")
trap 'rm -rf "$shards"' EXIT
say "e2e: in $((${#lists[@]} + 2)) shards side by side (#862): the tray's part, main()'s own scenarios, and the others in ${#lists[@]}"

# Each shard's environment: its folder for its text and evidence, and what it plays.
play() { # number, then the shard's own variables
local number=$1
shift
mkdir -p "$shards/$number"
env "$@" KITOKEY_E2E_SHARDS=1 KITOKEY_E2E_OUT="$shards/$number" NSTACK_EVIDENCE="$shards/$number/e2e" \
bash scripts/headless.sh > "$shards/$number/output.txt" 2>&1 &
pids+=("$!")
}
pids=()
play 0 KITOKEY_E2E_PART=tray KITOKEY_E2E_ONLY=
play 1 KITOKEY_E2E_PART=main KITOKEY_E2E_ONLY=
for number in "${!lists[@]}"; do
play $((number + 2)) KITOKEY_E2E_PART= KITOKEY_E2E_ONLY="${lists[$number]}"
done
statuses=()
for pid in "${pids[@]}"; do
status=0
wait "$pid" || status=$?
statuses+=("$status")
done

# The shards' text, in a whole run's order.
texts=()
for number in "${!pids[@]}"; do texts+=("$shards/$number/e2e.txt"); done
python3 -I scripts/e2e_shards.py merge scripts/e2e_window.py "${texts[@]}" | tee -a "$out/e2e.txt"

# Each shard's verdict: a shard ran whole when its last line says its checks passed. Not run on one
# — no GNOME, no WebDriver — is not run on all; any other end fails the run.
failed=0
not_run=0
for number in "${!pids[@]}"; do
last=$(tail -n 1 "$shards/$number/e2e.txt" 2>/dev/null || true)
case "${statuses[$number]}:$last" in
3:*"its checks passed"*) say " ok shard $number: $last" ;;
77:*) say " NOT RUN shard $number: $last"; not_run=1 ;;
*) say " FAIL shard $number (exit ${statuses[$number]}): $last"; failed=1 ;;
esac
done

# Their evidence together: every shard's logs judged at once, a whole run's (nothing « not owed »),
# their screenshots in one folder, each session's own log beside the evidence.
mkdir -p "$evidence/shots"
for number in "${!pids[@]}"; do
cp -r "$shards/$number/e2e/shots/." "$evidence/shots/" 2>/dev/null || true
cp "$shards/$number/headless-shell.log" "$out/headless-shell-$number.log" 2>/dev/null || true
done
logs=0
env -u KITOKEY_E2E_ONLY -u KITOKEY_E2E_PART uv run --quiet --with pyyaml==6.0.3 python scripts/e2e_logs.py "$shards" "$evidence" MANIFEST.yaml \
> "$shards/logs.txt" 2>&1 || logs=$?
tee -a "$out/e2e.txt" < "$shards/logs.txt"
if [ "$logs" = 0 ]; then say " ok logs_read (every shard's logs)"; else say " FAIL logs_read (every shard's logs)"; failed=1; fi
say " (the run, its shards side by side: $((SECONDS - began)) s)"
if [ "$not_run" = 1 ] && [ "$failed" = 0 ]; then
say "e2e: NOT RUN — a shard could not run here"
exit 77
fi
[ "$failed" = 0 ] && say "e2e: passed" || say "e2e: FAILED"
exit "$failed"
89 changes: 89 additions & 0 deletions modules/shell/scripts/e2e_shards.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
"""The window's scenarios cut in shards, and the shards' text put back in one (#862).

e2e_shards.py lists N e2e_window.py e2e_times.txt N lines, each a shard's names (KITOKEY_E2E_ONLY)
e2e_shards.py merge e2e_window.py SHARD... one text, the scenarios in main()'s order

The lists are balanced by each scenario's measured time, the longest first into the lightest shard,
as Bazel's shard_count and pytest-xdist's load groups do; a scenario not measured counts as the median.
"""
import pathlib
import re
import statistics
import sys

# Beside it, as e2e_window.py finds it, run with -I as the scripts are.
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import e2e_only # noqa: E402

TIMED = re.compile(r"^ \(([a-z_]+): [0-9.]+ s\)$")


def lists(shards: int, source: str, times: str) -> list[list[str]]:
"""The scenarios main() calls, in `shards` lists of about the same time, each in main()'s order."""
called = e2e_only.scenarios(source)
measured = {}
for line in times.splitlines():
if line.strip() and not line.startswith("#"):
name, seconds = line.split()
measured[name] = float(seconds)
median = statistics.median(measured.values()) if measured else 1.0
load = [0.0] * shards
given: list[list[str]] = [[] for _ in range(shards)]
for name in sorted(called, key=lambda name: (-measured.get(name, median), called.index(name))):
lightest = load.index(min(load))
given[lightest].append(name)
load[lightest] += measured.get(name, median)
return [[name for name in called if name in shard] for shard in given if shard]


def blocks(text: str) -> tuple[list[str], dict[str, list[str]]]:
"""A shard's text: its lines outside any scenario, and each scenario's own, by its name — every
line after the window's « partial run: … » or the scenario before it, up to the line that says
its time (scripts/e2e_window.py's `timed`)."""
lines = text.splitlines()
starts = [number for number, line in enumerate(lines) if line.startswith("partial run: ")]
begin = starts[-1] + 1 if starts else 0
outside = lines[:begin]
found: dict[str, list[str]] = {}
pending: list[str] = []
for line in lines[begin:]:
pending.append(line)
timed = TIMED.match(line)
if timed:
found[timed.group(1)] = pending
pending = []
# After the last scenario: the logs read, the shard's verdict; or a scenario cut short, unnamed.
outside.extend(pending)
return outside, found


def merge(source: str, shards: list[str]) -> str:
"""The tray's shard as it ran, then main()'s own scenarios and every one it calls, in its order,
whichever shard played them, then what each window shard said outside its scenarios."""
tray, *windows = shards
lines = tray.splitlines()
rest: list[tuple[int, list[str]]] = []
every: dict[str, list[str]] = {}
for number, text in enumerate(windows, start=1):
outside, found = blocks(text)
every.update(found)
rest.append((number, outside))
# main()'s own scenarios first, as main() plays them, then each it calls.
for name in ["main", *e2e_only.scenarios(source)]:
lines.extend(every.get(name, [f"{name}: NOT RUN — no shard said it"]))
for number, outside in rest:
lines.append(f"shard {number}:")
lines.extend(f" {line}" for line in outside)
return "\n".join(lines) + "\n"


if __name__ == "__main__":
if sys.argv[1] == "lists":
source = pathlib.Path(sys.argv[3]).read_text()
for shard in lists(int(sys.argv[2]), source, pathlib.Path(sys.argv[4]).read_text()):
print(",".join(shard))
elif sys.argv[1] == "merge":
source = pathlib.Path(sys.argv[2]).read_text()
print(merge(source, [pathlib.Path(path).read_text() for path in sys.argv[3:]]), end="")
else:
raise SystemExit(__doc__)
50 changes: 50 additions & 0 deletions modules/shell/scripts/e2e_times.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Each window scenario's time, in seconds, as a full run said it (scripts/e2e_window.py's « (name: N s) »):
# what scripts/e2e_shards.py balances the shards by. Measured at 9eae465 (#865), 2026-11-15; a scenario
# not listed counts as the median.
model_offered 22.7
models_listed 11.3
list_refreshed 1.5
speech_offered 4.2
service_named 11.9
models_guided 4.9
transcription_guided 3.7
ollama_pulled 13.1
first_run 29.5
context_written 19.6
confidential_kept 23.1
meeting_language 6.0
transcript_corrected 10.9
corrections_gated 3.7
press_on_the_page_left 35.8
context_gaps 4.7
two_panes 19.8
sidebar_folded 3.5
app_language 5.6
confidential_every_way 1.8
meeting_told_once 2.9
minutes_copied 3.7
nothing_silent 59.2
gestures_answer 24.0
window_looks 11.0
pages_seen 9.3
actions_rows 1.9
announced 37.1
heard_checked 25.1
microphone_chosen 169.9
microphone_latest_answer 2.2
microphone_rollback 4.4
sound_checked 32.5
bar_window 6.0
bar_opened 116.8
recording_found 1.0
recording_found_not_forgotten 6.8
context_before_start 11.7
own_models 2.3
cancelled_at_once 23.3
models_advanced 28.1
meeting_named 2.3
word_after_a_gap 1.3
minutes_checked 27.4
meeting_renamed 12.3
trial_over 17.1
trial_left 3.8
Loading
Loading