feat(linbo-remote): append a JSON record per finished run

README.md

@@ -99,6 +99,7 @@ Parameter  |  Description
   ```
 * The option `-c` now disables the gui by default during command execution. 
 * The new option `-a` allows to attach a host's tmux session.
+* Every `-c` run writes its own log `<hostname>_linbo-remote_<YYYYmmddHHMMSS>.log` and appends one JSON line (hostname, mode, commands, dry_run, start, end, rc, log) to `linbo-remote_runs.jsonl`, both in `/var/log/linuxmuster/linbo/`.
 
 Full linbo-remote help:
 

src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_cli.py

@@ -317,11 +317,17 @@ def sendCmds(hosts, commands, wait, secrets_uploaded, dry_run=False):
             )
 
         session_name = lib.tmuxSessionName(host)
-        logfile = os.path.join(environment.LINBOLOGDIR, lib.runLogName(host))
+        started = time.localtime()
+        logfile = os.path.join(
+            environment.LINBOLOGDIR, lib.runLogName(host, time.strftime('%Y%m%d%H%M%S', started)),
+        )
+        record_prefix = lib.buildRunRecordPrefix(host, commands, started, logfile, dry_run=dry_run)
         script_path = os.path.join(TMPDIR, f'{os.getpid()}.{host}.sh')
 
         script_text = lib.renderRemoteScript(
             host, commands, script_path, secrets_uploaded=secrets_uploaded, dry_run=dry_run,
+            record_file=os.path.join(environment.LINBOLOGDIR, lib.RUN_RECORD_BASENAME),
+            record_prefix=record_prefix,
         )
         with open(script_path, 'w') as f:
             f.write(script_text)

src/serverfs/usr/share/linuxmuster/linbo/linbo_remote_lib.py

@@ -40,6 +40,7 @@
 give a usable IP for those hosts at all.
 """
 
+import json
 import re
 import shlex
 import subprocess
@@ -350,6 +351,28 @@ def runLogName(hostname, timestamp=None):
     return f'{hostname}_linbo-remote_{timestamp}.log'
 
 
+RUN_RECORD_BASENAME = 'linbo-remote_runs.jsonl'
+
+
+def buildRunRecordPrefix(hostname, commands, started, logfile, mode='direct', dry_run=False):
+    """
+    The static part of one run's record (see renderRemoteScript()'s
+    record_file): a JSON object *without* its closing brace. The remote
+    script appends `,"end":...,"rc":...}` when it exits, because only it
+    knows the end time and exit status. `started` is a local
+    time.struct_time; the record is one line in LINBOLOGDIR/RUN_RECORD_BASENAME.
+    """
+    record = {
+        'hostname': hostname,
+        'mode': mode,
+        'commands': list(commands),
+        'dry_run': bool(dry_run),
+        'start': time.strftime('%Y-%m-%dT%H:%M:%S%z', started),
+        'log': logfile,
+    }
+    return json.dumps(record, separators=(',', ':'))[:-1]
+
+
 def tmuxAttachTarget(hostname):
     """The actual tmux session name to use for `-a`/`-l` (attach/list) - see tmuxSessionName()."""
     return f'{hostname}_linbo-remote'
@@ -363,7 +386,8 @@ def tmuxAttachTarget(hostname):
 _BACKGROUNDED_COMMAND_PREFIXES = ('start', 'reboot', 'halt', 'poweroff')
 
 
-def renderRemoteScript(hostname, commands, script_path, secrets_uploaded=False, dry_run=False):
+def renderRemoteScript(hostname, commands, script_path, secrets_uploaded=False, dry_run=False,
+                       record_file=None, record_prefix=None):
     """
     Build the per-host shell script executed inside a tmux session for -c
     (direct) mode: disables the GUI, runs each normalized command
@@ -387,9 +411,32 @@ def renderRemoteScript(hostname, commands, script_path, secrets_uploaded=False,
         exactly where you don't want a real reboot while testing) so it just
         reports what it would do instead of actually doing it. See
         linbo_wrapper's own --dry-run handling.
+    record_file/record_prefix: when both are given, an EXIT trap appends one
+        JSON line to record_file - record_prefix (buildRunRecordPrefix())
+        plus the end time and the script's exit status. A trap, so that an
+        aborted run (e.g. its tmux session being killed) leaves a record too,
+        with the 128+signal exit status.
+        The append is serialized with flock, as parallel runs share the file.
     """
     wrapper = f'{WRAPPER} --dry-run' if dry_run else WRAPPER
-    lines = ['#!/bin/bash', f'{SSH_CMD} {hostname} gui_ctl disable', 'RC=0']
+    lines = ['#!/bin/bash']
+    if record_file and record_prefix:
+        lines += [
+            f'RUN_RECORD={shlex.quote(record_prefix)}',
+            f'RUN_RECORD_FILE={shlex.quote(record_file)}',
+            'writeRunRecord() {',
+            '    local rc=$?',
+            '    { flock 9 && printf \'%s,"end":"%s","rc":%d}\\n\' "$RUN_RECORD" '
+            '"$(date +%Y-%m-%dT%H:%M:%S%z)" "$rc" >&9; } 2>/dev/null 9>>"$RUN_RECORD_FILE"',
+            '}',
+            # bash reports $?=0 in the EXIT trap after an unhandled fatal signal, which would
+            # record an aborted run as a success - turn the signal into its 128+n status first
+            "trap 'exit 129' HUP",
+            "trap 'exit 130' INT",
+            "trap 'exit 143' TERM",
+            'trap writeRunRecord EXIT',
+        ]
+    lines += [f'{SSH_CMD} {hostname} gui_ctl disable', 'RC=0']
     has_backgrounded = False
     for index, cmd in enumerate(commands):
         if index > 0:

tests/python/test_linbo_remote_cli.py

@@ -182,6 +182,10 @@ def test_direct_dispatch_happy_path(monkeypatch, tmp_path, capsys):
         rf"cat > {re.escape(str(tmp_path))}/r100-pc01_linbo-remote_\d{{14}}\.log", pipe_args[0],
     )
     assert re.search(r'Log see \S+r100-pc01_linbo-remote_\d{14}\.log\.', out)
+    # and the script appends the run record to the shared jsonl file
+    script_text = (tmp_path / f'{os.getpid()}.r100-pc01.sh').read_text()
+    assert f'RUN_RECORD_FILE={tmp_path}/linbo-remote_runs.jsonl' in script_text
+    assert '"hostname":"r100-pc01"' in script_text
 
 
 def test_direct_dispatch_dry_run_writes_dry_run_flag_into_script(monkeypatch, tmp_path, capsys):

tests/python/test_linbo_remote_lib.py

@@ -11,13 +11,21 @@
 #
 
 import fnmatch
+import json
+import os
 import re
+import signal
+import shutil
+import subprocess
+import time
 
 import pytest
 
 from linbo_remote_lib import (
     LinboRemoteError,
+    RUN_RECORD_BASENAME,
     buildOnbootCmds,
+    buildRunRecordPrefix,
     getMacFromAd,
     getIpFromAd,
     hostsInGroup,
@@ -383,3 +391,102 @@ def test_resolve_wol_target_falls_back_to_dhcp_lease_when_ad_mac_invalid():
     )
     assert mac == '52:54:00:dd:ee:ff'
     assert ip == '10.16.100.1'
+
+
+# --- per-run record (JSON line appended by the remote script) ----------------
+
+STARTED = time.strptime('20261001120530', '%Y%m%d%H%M%S')
+
+
+def test_run_record_prefix_is_open_json_object():
+    prefix = buildRunRecordPrefix('r100-pc01', ['format:2', 'sync:1'], STARTED, '/log/x.log')
+    assert not prefix.endswith('}')
+    record = json.loads(prefix + '}')
+    assert record['hostname'] == 'r100-pc01'
+    assert record['mode'] == 'direct'
+    assert record['commands'] == ['format:2', 'sync:1']
+    assert record['dry_run'] is False
+    assert record['log'] == '/log/x.log'
+    assert record['start'].startswith('2026-10-01T12:05:30')
+
+
+def test_render_remote_script_without_record_has_no_trap():
+    script = renderRemoteScript('r100-pc01', ['sync:1'], '/var/tmp/x.sh')
+    assert 'trap' not in script
+    assert 'RUN_RECORD' not in script
+
+
+def test_render_remote_script_with_record_installs_exit_trap_first():
+    script = renderRemoteScript(
+        'r100-pc01', ['sync:1'], '/var/tmp/x.sh',
+        record_file='/log/' + RUN_RECORD_BASENAME, record_prefix='{"hostname":"r100-pc01"',
+    )
+    lines = script.splitlines()
+    assert lines[0] == '#!/bin/bash'
+    assert 'trap writeRunRecord EXIT' in lines
+    assert lines.index('trap writeRunRecord EXIT') < next(
+        i for i, line in enumerate(lines) if 'gui_ctl disable' in line
+    )
+
+
+needs_bash_flock = pytest.mark.skipif(
+    not (shutil.which('bash') and shutil.which('flock')), reason='needs bash and flock',
+)
+
+
+def runRenderedScript(monkeypatch, tmp_path, ssh_cmd, commands):
+    """Run a rendered script for real, with linbo-ssh replaced by `true`/`false`."""
+    import linbo_remote_lib
+    monkeypatch.setattr(linbo_remote_lib, 'SSH_CMD', ssh_cmd)
+    record_file = tmp_path / RUN_RECORD_BASENAME
+    prefix = buildRunRecordPrefix('r100-pc01', commands, STARTED, str(tmp_path / 'x.log'))
+    script = tmp_path / 'run.sh'
+    script.write_text(renderRemoteScript(
+        'r100-pc01', commands, str(script), record_file=str(record_file), record_prefix=prefix,
+    ))
+    rc = subprocess.run(['bash', str(script)], check=False).returncode
+    return rc, record_file
+
+
+@needs_bash_flock
+def test_run_record_written_on_success(monkeypatch, tmp_path):
+    rc, record_file = runRenderedScript(monkeypatch, tmp_path, 'true', ['sync:1'])
+    assert rc == 0
+    lines = record_file.read_text().splitlines()
+    assert len(lines) == 1
+    record = json.loads(lines[0])
+    assert record['rc'] == 0
+    assert record['commands'] == ['sync:1']
+    assert re.fullmatch(r'\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d[+-]\d{4}', record['end'])
+
+
+@needs_bash_flock
+def test_run_record_written_on_failure_and_appended(monkeypatch, tmp_path):
+    runRenderedScript(monkeypatch, tmp_path, 'true', ['sync:1'])
+    rc, record_file = runRenderedScript(monkeypatch, tmp_path, 'false', ['sync:1'])
+    assert rc == 1
+    records = [json.loads(line) for line in record_file.read_text().splitlines()]
+    assert [r['rc'] for r in records] == [0, 1]
+
+
+@needs_bash_flock
+@pytest.mark.parametrize('sig, expected_rc', [
+    (signal.SIGHUP, 129),    # tmux kill-session / closed pane
+    (signal.SIGTERM, 143),
+    (signal.SIGINT, 130),
+])
+def test_run_record_of_aborted_run_has_signal_exit_status(monkeypatch, tmp_path, sig, expected_rc):
+    import linbo_remote_lib
+    monkeypatch.setattr(linbo_remote_lib, 'SSH_CMD', 'sleep 30; true')
+    record_file = tmp_path / RUN_RECORD_BASENAME
+    prefix = buildRunRecordPrefix('r100-pc01', ['sync:1'], STARTED, str(tmp_path / 'x.log'))
+    script = tmp_path / 'run.sh'
+    script.write_text(renderRemoteScript(
+        'r100-pc01', ['sync:1'], str(script), record_file=str(record_file), record_prefix=prefix,
+    ))
+    proc = subprocess.Popen(['bash', str(script)], start_new_session=True)
+    time.sleep(1)
+    os.killpg(proc.pid, sig)    # whole group, like tmux does - kills the running child too
+    proc.wait(timeout=10)
+    record = json.loads(record_file.read_text().splitlines()[0])
+    assert record['rc'] == expected_rc