From 2081a294f069471a5241ab19196db92f91474fca Mon Sep 17 00:00:00 2001 From: Henrik Kurelid Date: Tue, 10 Nov 2026 17:59:28 +0100 Subject: [PATCH 1/2] test(cb-70mv): fleet-view-restart.sh presses g instead of waiting out the work pane's 30s refresh The start on cb-b2 and the guard's hold each need a board read taken after the pass ended; the suite waited for app::WORK_REFRESH_INTERVAL to give it one. It now drives the view through tests/lib/pty-drive and presses g while it waits, so both steps take a few seconds instead of up to thirty. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WCDKexJtfAU67v3sjW244j --- tests/fleet-view-restart.sh | 90 ++++++++++++++----------------------- 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/tests/fleet-view-restart.sh b/tests/fleet-view-restart.sh index 093d6bf2..468f468e 100644 --- a/tests/fleet-view-restart.sh +++ b/tests/fleet-view-restart.sh @@ -27,10 +27,11 @@ # The view ends a pass `END_GRACE_SECONDS' (30) after its state file's `since' says `waiting'. The # grace is the supervise table's to prove (tests/lib/supervise.cases), not R15's, so the pretend # pass dates its `waiting' file back past it and the view ends the pass on its next tick: the -# grace's real half minute is not paid twice (cb-n53p). What the suite still costs, about forty -# seconds, is mostly the work pane's own refresh (`app::WORK_REFRESH_INTERVAL', 30s) between the -# closing pass and the start on cb-b2. Every wait below is bounded, and a failure prints what the -# view logged and ran. +# grace's real half minute is not paid twice (cb-n53p). Nor is the work pane's own refresh +# (`app::WORK_REFRESH_INTERVAL', 30s): the start on cb-b2 needs a board read taken after cb-b1 +# closed, and the guard's hold one taken after cb-b2 was given back, so those two waits press `g', +# the view's refresh-everything key, rather than wait out the timer (cb-70mv). Every wait below is +# bounded, and a failure prints what the view logged and ran. # # bash tests/fleet-view-restart.sh @@ -207,53 +208,6 @@ exec bash -c 'while :; do sleep 1; done' "$(cerebro_marker_sentence "$name" "$CE STUB chmod +x "$stubs/bd" "$stubs/gh" "$scripts/launch" -# --- the pty ------------------------------------------------------------------------------------- -# -# python3's stdlib, because `script(1)' takes different arguments on util-linux and BSD and treats -# a closed stdin differently on each. The size is set on the slave before the child starts, so the -# first frame is drawn at 50x200. On SIGTERM it ends the view, which drops its sessions. -cat > "$fixture/pty.py" <<'PY' -import fcntl, os, signal, struct, sys, termios, time -capture, argv = sys.argv[1], sys.argv[2:] -master, slave = os.openpty() -fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 50, 200, 0, 0)) -pid = os.fork() -if pid == 0: - os.close(master) - os.setsid() - fcntl.ioctl(slave, termios.TIOCSCTTY, 0) - for fd in (0, 1, 2): - os.dup2(slave, fd) - if slave > 2: - os.close(slave) - os.execv(argv[0], argv) -os.close(slave) -def stop(*_): - try: - os.kill(pid, signal.SIGTERM) - for _ in range(50): - if os.waitpid(pid, os.WNOHANG)[0]: - break - time.sleep(0.1) - else: - os.kill(pid, signal.SIGKILL) - except (ProcessLookupError, ChildProcessError): - pass - sys.exit(0) -signal.signal(signal.SIGTERM, stop) -with open(capture, "ab", buffering=0) as out: - while True: - try: - data = os.read(master, 65536) - except OSError: - break - if not data: - break - out.write(data) -_, status = os.waitpid(pid, 0) -sys.exit(os.waitstatus_to_exitcode(status) if hasattr(os, "waitstatus_to_exitcode") else 1) -PY - view_pid="" suite_cleanup() { if [[ -n "$view_pid" ]] && kill -0 "$view_pid" 2>/dev/null; then @@ -280,6 +234,7 @@ explain() { # no transition log and the stub bd answers `[]'. Expected, and no part of what is asserted. echo "--- errors (tail; fleet-history, fleet-health and sweep errors are expected)" tail -n 8 "$consumer/.cerebro/state/errors.jsonl" 2>/dev/null || true + echo "--- view stderr"; tail -n 8 "$fixture/view.err" 2>/dev/null || true echo "--- view exited: $(kill -0 "$view_pid" 2>/dev/null && echo no || echo yes)" } >&2 } @@ -302,6 +257,22 @@ wait_for() { done } +press() { printf '%s' "$1" >&7; } + +# refreshing : the command, after a `g' - the view's refresh-everything key - at most +# once every two seconds. A wait_for over it waits on a board read the suite asked for rather than +# on the work pane's 30s timer. Pressed again rather than once: a `g' that lands while a board read +# is in flight is dropped, and a read begun before the pass closed or gave back its bead answers the +# board as it was. `g' only re-reads; it starts nothing the trigger would not. +next_g=0 +refreshing() { + if ((SECONDS >= next_g)); then + press g + next_g=$((SECONDS + 2)) + fi + "$@" +} + starts() { grep -c '^START ' "$fixture/launches" 2>/dev/null || true; } started_on() { grep -qx "START Rogue $1" "$fixture/launches" 2>/dev/null; } @@ -320,25 +291,30 @@ held_after_second_end() { # --- run it -------------------------------------------------------------------------------------- +# The view runs on tests/lib/pty-drive's 50x200 pty, which types into it whatever arrives on the +# keys fifo. The fifo is held open read-write for the whole run, so a key is one printf that never +# blocks. On SIGTERM the drive kills the view; suite_cleanup ends any session it left behind. export FIXTURE="$fixture" +mkfifo "$fixture/keys" ( cd "$consumer" export PATH="$stubs:$PATH" \ CEREBRO_CONSUMER_ROOT="$consumer" CEREBRO_CONSUMER_SHARED_ROOT="$consumer" \ CEREBRO_CONSUMER_MOUNT="$mount" CEREBRO_SCRIPTS="$scripts" TERM=xterm-256color - exec python3 "$fixture/pty.py" "$fixture/screen" "$tui" -) & + exec python3 -I "$repo_root/tests/lib/pty-drive" "$fixture/keys" "$fixture/screen" 50 200 -- "$tui" +) /dev/null 2>>"$fixture/view.err" & view_pid=$! +exec 7<>"$fixture/keys" wait_for 90 "the view starting Rogue on the first ready bead, cb-b1" started_on cb-b1 pass "the fleet view starts a standby producer on the first ready bead" -wait_for 150 "the view starting Rogue again on cb-b2 after the pass that closed cb-b1 (cb-r9ac)" \ - started_on cb-b2 +wait_for 60 "the view starting Rogue again on cb-b2 after the pass that closed cb-b1 (cb-r9ac)" \ + refreshing started_on cb-b2 pass "a pass that closed its bead changed the work, and the next start follows (cb-r9ac)" -wait_for 150 "an evaluation holding Rogue by the unchanged-work guard after cb-b2 was given back" \ - held_after_second_end +wait_for 60 "an evaluation holding Rogue by the unchanged-work guard after cb-b2 was given back" \ + refreshing held_after_second_end [[ "$(starts)" == 2 ]] || { explain; fail "expected two starts and no third, got: $(starts)"; } pass "a pass that gave its bead back untouched is held by the unchanged-work guard" From bbfb0fff6f63c6c7715794519aa1ab1e8ba45316 Mon Sep 17 00:00:00 2001 From: Henrik Kurelid Date: Tue, 10 Nov 2026 18:12:35 +0100 Subject: [PATCH 2/2] =?UTF-8?q?docs(cb-70mv):=20retrospective=20=E2=80=94?= =?UTF-8?q?=20local=20gate=20red=20in=20fleet-keys=20and=20a=20read-only?= =?UTF-8?q?=20view=20while=20CI=20was=20green?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WCDKexJtfAU67v3sjW244j --- docs/retrospectives/cb-70mv.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/retrospectives/cb-70mv.md diff --git a/docs/retrospectives/cb-70mv.md b/docs/retrospectives/cb-70mv.md new file mode 100644 index 00000000..52a6dcd7 --- /dev/null +++ b/docs/retrospectives/cb-70mv.md @@ -0,0 +1,17 @@ +# cb-70mv — retrospective + +- **Implementer:** Rogue +- **Date:** 2026-11-10 +- **PR:** #537 + +## The local gate was red twice in fleet-keys.sh, and one restart run stayed read-only, while CI was green + +**What happened.** While I was timing the changed `tests/fleet-view-restart.sh` alone, it failed 3 times in about 14 runs. Each time the header read `read-only; this window could not take charge of the fleet` for the whole 90 s first wait, and the run left no `supervisor.json`, `decisions.jsonl` or `errors.jsonl`. The next 28 runs were green. Both runs of `bash tests/gate` then failed only `tests/fleet-keys.sh`, on "timed out waiting for: the stub agent to start" (its 20 s bound). It passed twice alone. The same fleet-keys failure appears in cb-15cw's and cb-yf8h's suite logs from the same hour. The load average was 6 to 9. In CI, fleet-keys took 19 s and the restart suite 27 s, both green. + +**Why.** Not established for this run. It fits cb-8484's account: `main.rs` reads the lease endpoint once at startup (`read_supervisor_endpoint(...).ok()`), so a `fleet-supervisor` call that is slow under load leaves the view read-only for its whole life. + +**Cost.** About 35 minutes: two extra full gates and roughly 40 diagnostic suite runs. + +**Prevent by.** cb-8484's fix, which has not been filed as a bead as far as I can see: re-read the four `fleet-supervisor` answers on the supervision tick while any of them is missing, and print why a read failed. Also, I grepped `docs/retrospectives/` for "could not take charge" only after diagnosing, which cb-8dha and cb-8484 had already warned against. + +**Seen before.** cb-8484, cb-7jmm, cb-eguq, cb-gkhx, cb-lt16, cb-kmxs, cb-8dha, cb-n53p.