diff --git a/README.md b/README.md index 6c20da8..ebb7714 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Parameter | Description ``` * The option `-c` now disables the gui by default during command execution. * The new option `-a` allows to attach a host's tmux session. +* Every `-c` run writes its own log `_linbo-remote_.log` and appends one JSON line (hostname, mode, commands, dry_run, start, end, rc, log) to `linbo-remote_runs.jsonl`, both in `/var/log/linuxmuster/linbo/`. Full linbo-remote help: diff --git a/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_cli.py b/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_cli.py index 8fee8aa..8ab01c3 100644 --- a/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_cli.py +++ b/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_cli.py @@ -317,11 +317,17 @@ def sendCmds(hosts, commands, wait, secrets_uploaded, dry_run=False): ) session_name = lib.tmuxSessionName(host) - logfile = os.path.join(environment.LINBOLOGDIR, lib.runLogName(host)) + started = time.localtime() + logfile = os.path.join( + environment.LINBOLOGDIR, lib.runLogName(host, time.strftime('%Y%m%d%H%M%S', started)), + ) + record_prefix = lib.buildRunRecordPrefix(host, commands, started, logfile, dry_run=dry_run) script_path = os.path.join(TMPDIR, f'{os.getpid()}.{host}.sh') script_text = lib.renderRemoteScript( host, commands, script_path, secrets_uploaded=secrets_uploaded, dry_run=dry_run, + record_file=os.path.join(environment.LINBOLOGDIR, lib.RUN_RECORD_BASENAME), + record_prefix=record_prefix, ) with open(script_path, 'w') as f: f.write(script_text) diff --git a/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_lib.py b/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_lib.py index 51af1a4..02b7f40 100644 --- a/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_lib.py +++ b/src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_lib.py @@ -40,6 +40,7 @@ give a usable IP for those hosts at all. """ +import json import re import shlex import subprocess @@ -350,6 +351,28 @@ def runLogName(hostname, timestamp=None): return f'{hostname}_linbo-remote_{timestamp}.log' +RUN_RECORD_BASENAME = 'linbo-remote_runs.jsonl' + + +def buildRunRecordPrefix(hostname, commands, started, logfile, mode='direct', dry_run=False): + """ + The static part of one run's record (see renderRemoteScript()'s + record_file): a JSON object *without* its closing brace. The remote + script appends `,"end":...,"rc":...}` when it exits, because only it + knows the end time and exit status. `started` is a local + time.struct_time; the record is one line in LINBOLOGDIR/RUN_RECORD_BASENAME. + """ + record = { + 'hostname': hostname, + 'mode': mode, + 'commands': list(commands), + 'dry_run': bool(dry_run), + 'start': time.strftime('%Y-%m-%dT%H:%M:%S%z', started), + 'log': logfile, + } + return json.dumps(record, separators=(',', ':'))[:-1] + + def tmuxAttachTarget(hostname): """The actual tmux session name to use for `-a`/`-l` (attach/list) - see tmuxSessionName().""" return f'{hostname}_linbo-remote' @@ -363,7 +386,8 @@ def tmuxAttachTarget(hostname): _BACKGROUNDED_COMMAND_PREFIXES = ('start', 'reboot', 'halt', 'poweroff') -def renderRemoteScript(hostname, commands, script_path, secrets_uploaded=False, dry_run=False): +def renderRemoteScript(hostname, commands, script_path, secrets_uploaded=False, dry_run=False, + record_file=None, record_prefix=None): """ Build the per-host shell script executed inside a tmux session for -c (direct) mode: disables the GUI, runs each normalized command @@ -387,9 +411,32 @@ def renderRemoteScript(hostname, commands, script_path, secrets_uploaded=False, exactly where you don't want a real reboot while testing) so it just reports what it would do instead of actually doing it. See linbo_wrapper's own --dry-run handling. + record_file/record_prefix: when both are given, an EXIT trap appends one + JSON line to record_file - record_prefix (buildRunRecordPrefix()) + plus the end time and the script's exit status. A trap, so that an + aborted run (e.g. its tmux session being killed) leaves a record too, + with the 128+signal exit status. + The append is serialized with flock, as parallel runs share the file. """ wrapper = f'{WRAPPER} --dry-run' if dry_run else WRAPPER - lines = ['#!/bin/bash', f'{SSH_CMD} {hostname} gui_ctl disable', 'RC=0'] + lines = ['#!/bin/bash'] + if record_file and record_prefix: + lines += [ + f'RUN_RECORD={shlex.quote(record_prefix)}', + f'RUN_RECORD_FILE={shlex.quote(record_file)}', + 'writeRunRecord() {', + ' local rc=$?', + ' { flock 9 && printf \'%s,"end":"%s","rc":%d}\\n\' "$RUN_RECORD" ' + '"$(date +%Y-%m-%dT%H:%M:%S%z)" "$rc" >&9; } 2>/dev/null 9>>"$RUN_RECORD_FILE"', + '}', + # bash reports $?=0 in the EXIT trap after an unhandled fatal signal, which would + # record an aborted run as a success - turn the signal into its 128+n status first + "trap 'exit 129' HUP", + "trap 'exit 130' INT", + "trap 'exit 143' TERM", + 'trap writeRunRecord EXIT', + ] + lines += [f'{SSH_CMD} {hostname} gui_ctl disable', 'RC=0'] has_backgrounded = False for index, cmd in enumerate(commands): if index > 0: diff --git a/tests/python/test_linbo_remote_cli.py b/tests/python/test_linbo_remote_cli.py index dafb5d5..0b6dd56 100644 --- a/tests/python/test_linbo_remote_cli.py +++ b/tests/python/test_linbo_remote_cli.py @@ -182,6 +182,10 @@ def test_direct_dispatch_happy_path(monkeypatch, tmp_path, capsys): rf"cat > {re.escape(str(tmp_path))}/r100-pc01_linbo-remote_\d{{14}}\.log", pipe_args[0], ) assert re.search(r'Log see \S+r100-pc01_linbo-remote_\d{14}\.log\.', out) + # and the script appends the run record to the shared jsonl file + script_text = (tmp_path / f'{os.getpid()}.r100-pc01.sh').read_text() + assert f'RUN_RECORD_FILE={tmp_path}/linbo-remote_runs.jsonl' in script_text + assert '"hostname":"r100-pc01"' in script_text def test_direct_dispatch_dry_run_writes_dry_run_flag_into_script(monkeypatch, tmp_path, capsys): diff --git a/tests/python/test_linbo_remote_lib.py b/tests/python/test_linbo_remote_lib.py index faea472..d2a8a87 100644 --- a/tests/python/test_linbo_remote_lib.py +++ b/tests/python/test_linbo_remote_lib.py @@ -11,13 +11,21 @@ # import fnmatch +import json +import os import re +import signal +import shutil +import subprocess +import time import pytest from linbo_remote_lib import ( LinboRemoteError, + RUN_RECORD_BASENAME, buildOnbootCmds, + buildRunRecordPrefix, getMacFromAd, getIpFromAd, hostsInGroup, @@ -383,3 +391,102 @@ def test_resolve_wol_target_falls_back_to_dhcp_lease_when_ad_mac_invalid(): ) assert mac == '52:54:00:dd:ee:ff' assert ip == '10.16.100.1' + + +# --- per-run record (JSON line appended by the remote script) ---------------- + +STARTED = time.strptime('20261001120530', '%Y%m%d%H%M%S') + + +def test_run_record_prefix_is_open_json_object(): + prefix = buildRunRecordPrefix('r100-pc01', ['format:2', 'sync:1'], STARTED, '/log/x.log') + assert not prefix.endswith('}') + record = json.loads(prefix + '}') + assert record['hostname'] == 'r100-pc01' + assert record['mode'] == 'direct' + assert record['commands'] == ['format:2', 'sync:1'] + assert record['dry_run'] is False + assert record['log'] == '/log/x.log' + assert record['start'].startswith('2026-10-01T12:05:30') + + +def test_render_remote_script_without_record_has_no_trap(): + script = renderRemoteScript('r100-pc01', ['sync:1'], '/var/tmp/x.sh') + assert 'trap' not in script + assert 'RUN_RECORD' not in script + + +def test_render_remote_script_with_record_installs_exit_trap_first(): + script = renderRemoteScript( + 'r100-pc01', ['sync:1'], '/var/tmp/x.sh', + record_file='/log/' + RUN_RECORD_BASENAME, record_prefix='{"hostname":"r100-pc01"', + ) + lines = script.splitlines() + assert lines[0] == '#!/bin/bash' + assert 'trap writeRunRecord EXIT' in lines + assert lines.index('trap writeRunRecord EXIT') < next( + i for i, line in enumerate(lines) if 'gui_ctl disable' in line + ) + + +needs_bash_flock = pytest.mark.skipif( + not (shutil.which('bash') and shutil.which('flock')), reason='needs bash and flock', +) + + +def runRenderedScript(monkeypatch, tmp_path, ssh_cmd, commands): + """Run a rendered script for real, with linbo-ssh replaced by `true`/`false`.""" + import linbo_remote_lib + monkeypatch.setattr(linbo_remote_lib, 'SSH_CMD', ssh_cmd) + record_file = tmp_path / RUN_RECORD_BASENAME + prefix = buildRunRecordPrefix('r100-pc01', commands, STARTED, str(tmp_path / 'x.log')) + script = tmp_path / 'run.sh' + script.write_text(renderRemoteScript( + 'r100-pc01', commands, str(script), record_file=str(record_file), record_prefix=prefix, + )) + rc = subprocess.run(['bash', str(script)], check=False).returncode + return rc, record_file + + +@needs_bash_flock +def test_run_record_written_on_success(monkeypatch, tmp_path): + rc, record_file = runRenderedScript(monkeypatch, tmp_path, 'true', ['sync:1']) + assert rc == 0 + lines = record_file.read_text().splitlines() + assert len(lines) == 1 + record = json.loads(lines[0]) + assert record['rc'] == 0 + assert record['commands'] == ['sync:1'] + assert re.fullmatch(r'\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d[+-]\d{4}', record['end']) + + +@needs_bash_flock +def test_run_record_written_on_failure_and_appended(monkeypatch, tmp_path): + runRenderedScript(monkeypatch, tmp_path, 'true', ['sync:1']) + rc, record_file = runRenderedScript(monkeypatch, tmp_path, 'false', ['sync:1']) + assert rc == 1 + records = [json.loads(line) for line in record_file.read_text().splitlines()] + assert [r['rc'] for r in records] == [0, 1] + + +@needs_bash_flock +@pytest.mark.parametrize('sig, expected_rc', [ + (signal.SIGHUP, 129), # tmux kill-session / closed pane + (signal.SIGTERM, 143), + (signal.SIGINT, 130), +]) +def test_run_record_of_aborted_run_has_signal_exit_status(monkeypatch, tmp_path, sig, expected_rc): + import linbo_remote_lib + monkeypatch.setattr(linbo_remote_lib, 'SSH_CMD', 'sleep 30; true') + record_file = tmp_path / RUN_RECORD_BASENAME + prefix = buildRunRecordPrefix('r100-pc01', ['sync:1'], STARTED, str(tmp_path / 'x.log')) + script = tmp_path / 'run.sh' + script.write_text(renderRemoteScript( + 'r100-pc01', ['sync:1'], str(script), record_file=str(record_file), record_prefix=prefix, + )) + proc = subprocess.Popen(['bash', str(script)], start_new_session=True) + time.sleep(1) + os.killpg(proc.pid, sig) # whole group, like tmux does - kills the running child too + proc.wait(timeout=10) + record = json.loads(record_file.read_text().splitlines()[0]) + assert record['rc'] == expected_rc