SAPOT-System · Adamskiee · Aug 7, 2026 · Aug 10, 2026
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -61,6 +61,7 @@ Run for each component actually touched — don't assume one component's green b
 | `mobile-app/sapot-mobile-app/` | `pnpm run testAll` (= test + typecheck + lint + expo-doctor), or the individual `pnpm test` / `pnpm run typecheck` / `pnpm run lint` |
 | `admin-frontend/sapot-admin/` | `pnpm run lint && pnpm run build` — **no test script exists in this component**; don't claim test coverage that isn't there |
 | `GSM-module/` | No automated tests exist — verify manually per `docs/getting-started/gsm-module-setup.md` |
+| `deploy/scripts/collect-status.py` | `python3 -m pytest tests` — **run from `deploy/`**. No CI job runs this; it is manual per CLAUDE.md's process. |
 
 If the change is release-relevant (server), `server/app/version.py` must match the git tag per `VERSIONING.md` before tagging — not typically a per-commit concern, but relevant if asked to prepare a release.
 

diff --git a/GSM-module/GSM-fastapi/main.py b/GSM-module/GSM-fastapi/main.py
@@ -13,7 +13,9 @@
 """
 
 import logging
+import os
 import sys
+from logging.handlers import RotatingFileHandler
 
 import uvicorn
 
@@ -23,12 +25,18 @@
 def setup_logging():
     level = getattr(logging, settings.log_level.upper(), logging.INFO)
     fmt   = "%(asctime)s [%(levelname)-8s] %(name)s: %(message)s"
+    log_dir = os.environ.get("GSM_LOG_DIR", ".")
+    os.makedirs(log_dir, exist_ok=True)
+    file_handler = RotatingFileHandler(
+        os.path.join(log_dir, "sapot.log"), maxBytes=10**6,
+        backupCount=3, encoding="utf-8",
+    )
     logging.basicConfig(
         level=level,
         format=fmt,
         handlers=[
             logging.StreamHandler(sys.stdout),
-            logging.FileHandler("sapot.log", encoding="utf-8"),
+            file_handler,
         ],
     )
     # Quiet down uvicorn's access log a little

diff --git a/deploy/config/nginx.prod.conf b/deploy/config/nginx.prod.conf
@@ -6,6 +6,8 @@ server {
     ssl_certificate_key /certs/server.key;
     ssl_protocols TLSv1.2 TLSv1.3;
     ssl_ciphers HIGH:!aNULL:!MD5;
+    location = /status             { alias /usr/share/nginx/status/status.html; }
+    location = /status/health.json { alias /usr/share/nginx/state/health.json; add_header Cache-Control no-store; }
     location /static/ { alias /usr/share/nginx/static/; autoindex off; expires 30d; add_header Cache-Control "public, immutable"; }
     location /tiles/ { proxy_pass http://tileserver:8080/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Path /tiles; gzip on; gzip_types application/json; gzip_min_length 256; }
     location ~ ^/(data|fonts|sprites)/ { proxy_pass http://tileserver:8080; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; }

diff --git a/deploy/config/status.env.example b/deploy/config/status.env.example
@@ -0,0 +1,9 @@
+# Read-only status collector credentials. This placeholder is intentionally
+# distinct from __GENERATE__, which substitutes the shared application secret.
+STATUS_DATABASE_URL=mysql+pymysql://sapot_status:__STATUS_DB_PASSWORD__@db:3306/sapot
+STATUS_REDIS_URL=redis://redis:6379
+STATUS_OUTPUT_DIR=/status
+STATUS_CERT_PATH=/certs/server.crt
+STATUS_INTEGRITY_PATH=/status/integrity.json
+STATUS_INTERVAL_SECONDS=60
+STATUS_PROBE_TIMEOUT=5
diff --git a/deploy/config/status.html b/deploy/config/status.html
@@ -0,0 +1,8 @@
+<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>SAPOT status</title>
+<style>body{font:16px system-ui;margin:2rem;background:#12151a;color:#e8ecf2}.card,li{background:#1c212b;border:1px solid #2c3340;border-radius:8px;padding:1rem;margin:.5rem 0}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:.5rem}.healthy{color:#2ea36b}.degraded{color:#d9a441}.failed{color:#d6455a}.unknown,.stale{color:#d9a441}ul{padding:0;list-style:none}</style>
+<h1>SAPOT status</h1><p id="sub">Loading...</p><p id="banner"></p><h2 id="overall"></h2><div id="counters" class="grid"></div><ul id="checks"></ul><footer id="integrity"></footer>
+<script>
+const labels={devicesConnected:'Devices online',messagesLastHour:'Messages / hour',mutatingRequestsLastHour:'Writes / hour',mutatingErrorRatePct:'Write errors',gsmSmsPending:'SMS to send',diskFreeBytes:'Disk free',certExpiresInDays:'Cert expires'};
+function render(d){let age=(Date.now()-Date.parse(d.generatedAt))/1000;banner.textContent=age>180?'Stale: collector may have stopped.':'';banner.className=age>180?'stale':'';overall.textContent=(d.overall||'unknown').toUpperCase();overall.className=d.overall||'unknown';sub.textContent='version '+(d.version||'unknown')+' · collected '+(d.generatedAt||'-');counters.innerHTML='';Object.entries(labels).forEach(([k,l])=>{let v=(d.counters||{})[k];if(k==='diskFreeBytes'&&v!=null)v=(v/1e9).toFixed(1)+' GB';if(k==='mutatingErrorRatePct'&&v!=null)v+='%';counters.innerHTML+='<div class="card">'+l+'<br><b>'+(v??'—')+'</b></div>'});checks.innerHTML=(d.checks||[]).map(c=>'<li>'+c.check+' <b class="'+c.status.toLowerCase()+'">'+c.status+'</b> '+(c.detail||'')+'</li>').join('');let i=d.integrity||{};integrity.textContent='Release file integrity: '+(i.status||'UNKNOWN')+(i.verifiedAt?' ('+i.verifiedAt+')':'')}
+function refresh(){fetch('/status/health.json',{cache:'no-store'}).then(r=>r.ok?r.json():Promise.reject(r.status)).then(render).catch(e=>{banner.textContent='Status not yet collected — '+e;overall.textContent='UNKNOWN';overall.className='unknown'})}refresh();setInterval(refresh,30000)
+</script></html>
diff --git a/deploy/scripts/collect-status.py b/deploy/scripts/collect-status.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Write a LAN-only, static status snapshot without using Docker's API."""
+import argparse
+import json
+import os
+import socket
+import sys
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+
+PRESENCE_KEY = "ws:online_users"
+TEMP_NAME = ".health.json.tmp"
+OUTPUT_NAME = "health.json"
+DEFAULT_SERVICE_URLS = {"api": "http://api:8000/version", "admin": "http://admin:3000", "tileserver": "http://tileserver:8080", "gsm": "http://gsm-fastapi:8001/health"}
+
+
+@dataclass
+class Config:
+    output_dir: str
+    cert_path: str
+    integrity_path: str
+    database_url: str
+    redis_url: str
+    probe_timeout: float
+    interval_seconds: int
+    service_urls: dict = field(default_factory=lambda: dict(DEFAULT_SERVICE_URLS))
+
+
+def config_from_env() -> Config:
+    database_url = os.environ.get("STATUS_DATABASE_URL")
+    if not database_url:
+        raise SystemExit("STATUS_DATABASE_URL is not set; refusing to start")
+    output_dir = os.environ.get("STATUS_OUTPUT_DIR", "/status")
+    return Config(output_dir, os.environ.get("STATUS_CERT_PATH", "/certs/server.crt"), os.environ.get("STATUS_INTEGRITY_PATH", os.path.join(output_dir, "integrity.json")), database_url, os.environ.get("STATUS_REDIS_URL", "redis://redis:6379"), float(os.environ.get("STATUS_PROBE_TIMEOUT", "5")), int(os.environ.get("STATUS_INTERVAL_SECONDS", "60")))
+
+
+def probe_http(url: str, timeout: float) -> tuple[bool, str]:
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as response:
+            code, body = response.status, response.read(4096)
+        return code < 500, f"HTTP {code}|{body.decode('utf-8', 'replace')[:200]}"
+    except urllib.error.HTTPError as error:
+        return error.code < 500, f"HTTP {error.code}"
+    except (urllib.error.URLError, socket.timeout, OSError) as error:
+        return False, f"unreachable: {error}"
+    except Exception as error:
+        return False, f"probe error: {error}"
+
+
+def collect_checks(config: Config):
+    checks, version = [], None
+    for name, url in config.service_urls.items():
+        ok, detail = probe_http(url, config.probe_timeout)
+        if name == "api" and ok:
+            try: version = json.loads(detail.split("|", 1)[1]).get("version")
+            except Exception: pass
+        checks.append({"check": name, "status": "PASS" if ok else "FAIL", "detail": ("reachable" if name in {"admin", "tileserver"} else "responding") if ok else detail.split("|", 1)[0]})
+    return checks, version
+
+
+def count_online_devices(config: Config):
+    try:
+        import redis
+        return int(redis.from_url(config.redis_url, socket_timeout=config.probe_timeout).zcount(PRESENCE_KEY, time.time(), "+inf"))
+    except Exception: return None
+
+
+def query_db_counters(config: Config):
+    counters = {"messagesLastHour": None, "mutatingRequestsLastHour": None, "mutatingErrorRatePct": None, "gsmSmsPending": None}
+    try:
+        from sqlalchemy import create_engine, text
+        engine = create_engine(config.database_url, pool_pre_ping=True, connect_args={"connect_timeout": int(config.probe_timeout)})
+        with engine.connect() as con:
+            try: counters["messagesLastHour"] = con.execute(text("SELECT COUNT(*) FROM message WHERE created_at > :cutoff"), {"cutoff": int(time.time() * 1000) - 3600000}).scalar()
+            except Exception: pass
+            try:
+                row = con.execute(text("SELECT COUNT(*) total, SUM(CASE WHEN JSON_EXTRACT(metadata_json, '$.status_code') >= 500 THEN 1 ELSE 0 END) errors FROM activity_logs WHERE created_at > :cutoff"), {"cutoff": (datetime.now(timezone.utc) - timedelta(hours=1)).replace(tzinfo=None)}).one()
+                total, errors = int(row.total or 0), int(row.errors or 0)
+                counters["mutatingRequestsLastHour"] = total
+                counters["mutatingErrorRatePct"] = round(errors / total * 100, 2) if total else 0.0
+            except Exception: pass
+            try: counters["gsmSmsPending"] = con.execute(text("SELECT COUNT(*) FROM sms_log WHERE status = 'pending'")).scalar()
+            except Exception: pass
+    except Exception: pass
+    return counters
+
+
+def disk_free_bytes(path):
+    try:
+        stat = os.statvfs(path); return int(stat.f_bavail * stat.f_frsize)
+    except Exception: return None
+
+
+def cert_expires_in_days(path):
+    try:
+        from cryptography import x509
+        with open(path, "rb") as f: cert = x509.load_pem_x509_certificate(f.read())
+        return int((cert.not_valid_after_utc - datetime.now(timezone.utc)).total_seconds() // 86400)
+    except Exception: return None
+
+
+def read_integrity(path):
+    try:
+        with open(path, encoding="utf-8") as f: result = json.load(f)
+        return {"status": result["status"], "verifiedAt": result.get("verifiedAt")} if isinstance(result, dict) and "status" in result else None
+    except Exception: return None
+
+
+def overall_status(checks):
+    if not checks: return "unknown"
+    states = {check["check"]: check["status"] for check in checks}
+    if states.get("api") != "PASS": return "failed"
+    return "degraded" if any(v != "PASS" for v in states.values()) else "healthy"
+
+
+def build_payload(checks, counters, integrity, version, generated_at, collector_age_seconds):
+    return {"generatedAt": generated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), "collectorAgeSeconds": collector_age_seconds, "version": version, "overall": overall_status(checks), "checks": checks, "counters": counters, "integrity": integrity or {"status": "UNKNOWN", "verifiedAt": None}}
+
+
+def write_atomic(output_dir, payload):
+    os.makedirs(output_dir, exist_ok=True)
+    temp = os.path.join(output_dir, TEMP_NAME)
+    with open(temp, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2); f.write("\n"); f.flush(); os.fsync(f.fileno())
+    os.replace(temp, os.path.join(output_dir, OUTPUT_NAME))
+
+
+def run_once(config):
+    started = datetime.now(timezone.utc)
+    try:
+        checks, version = collect_checks(config)
+        counters = query_db_counters(config)
+        counters.update({"devicesConnected": count_online_devices(config), "diskFreeBytes": disk_free_bytes(config.output_dir), "certExpiresInDays": cert_expires_in_days(config.cert_path)})
+        payload = build_payload(checks, counters, read_integrity(config.integrity_path), version, started, int((datetime.now(timezone.utc) - started).total_seconds()))
+    except Exception as error:
+        payload = {"generatedAt": started.strftime("%Y-%m-%dT%H:%M:%SZ"), "collectorAgeSeconds": 0, "version": None, "overall": "unknown", "reason": f"collector cycle failed: {error}", "checks": [], "counters": {}, "integrity": {"status": "UNKNOWN", "verifiedAt": None}}
+    try: write_atomic(config.output_dir, payload)
+    except Exception as error: print(f"collect-status: could not write output: {error}", file=sys.stderr, flush=True)
+    return payload
+
+
+def main():
+    parser = argparse.ArgumentParser(); parser.add_argument("--once", action="store_true"); args = parser.parse_args(); config = config_from_env()
+    if args.once: print(json.dumps(run_once(config), indent=2)); return 0
+    while True: run_once(config); time.sleep(config.interval_seconds)
+
+
+if __name__ == "__main__": raise SystemExit(main())
diff --git a/deploy/scripts/install.sh b/deploy/scripts/install.sh
@@ -8,7 +8,7 @@ if [ -L "$SAPOT_ROOT/releases/current" ] || [ -e "$SAPOT_ROOT/releases/current"
   current=$(manifest_value "$(readlink -f "$SAPOT_ROOT/releases/current")/manifest.json" version)
   log_error "already installed (v$current) - use upgrade.sh instead"; exit 1
 fi
-verify_checksums "$source_release" || { log_error "bundle checksum verification failed"; exit 1; }
+if verify_checksums "$source_release"; then integrity_status=PASS; else integrity_status=FAIL; log_error "bundle checksum verification failed"; exit 1; fi
 # Locate and validate the CA USB stick before anything is copied or started.
 # There is no self-signed fallback: the mobile app pins this CA, so a leaf it
 # did not issue leaves every production handset unable to connect.
@@ -17,18 +17,22 @@ disk_preflight "$(manifest_value "$source_manifest" requiredDiskBytes)"
 target="$SAPOT_ROOT/releases/v$version"; mkdir -p "$SAPOT_ROOT/releases"
 [ ! -e "$target" ] && cp -a "$source_release" "$target"
 prepare_env_files "$target"
+write_integrity "$integrity_status"
 ip=$($target/certs/detect-ip.sh 2>/dev/null || true)
 [ -n "$ip" ] || { read -r -p "LAN IP for TLS certificate: " ip; }
 ca_issue_leaf "$SAPOT_ROOT/shared/certs" "$ip" "$ca_dir" false
 log_info "certificate issued - the CA USB stick can be unplugged now"
 read -r -p "Is the GSM Arduino connected at $(grep '^GSM_ARDUINO_PORT=' "$SAPOT_ROOT/shared/gsm-arduino.env" | cut -d= -f2)? [y/N] " answer
 hardware=false; [[ "$answer" =~ ^[Yy]$ ]] && hardware=true
 for image in "$target"/images/*.tar; do docker load -i "$image"; done
+prepare_shared_dirs
 "$VERIFY_DIGESTS" "$target/manifest.json"
 compose "$target" up -d db redis; wait_healthy "$target" db; wait_healthy "$target" redis
 compose "$target" run --rm api alembic upgrade head
 compose "$target" up -d; for _ in {1..36}; do curl -kfs https://localhost/version >/dev/null 2>&1 && break; sleep 5; done
 curl -kfsS https://localhost/version >/dev/null || { log_error "nginx/api did not become ready"; exit 1; }
+provision_status_db_user "$target"
+compose "$target" run --rm --no-deps status-collector --once >/dev/null 2>&1 || log_error "initial status collection failed; /status will populate within one interval"
 ln -sfn "$target" "$SAPOT_ROOT/releases/current"; write_state install "" "$version" "$hardware"
 # Scheduled backups are the one part of this deployment that is useless when
 # left off, so install enables them rather than leaving it to a later manual

diff --git a/deploy/scripts/lib/deploy-common.sh b/deploy/scripts/lib/deploy-common.sh
@@ -103,14 +103,50 @@ provision_service_account() {
   chown "$user:$(id -gn "$user")" "$SAPOT_ROOT/shared/server.env" "$SAPOT_ROOT/shared/db-backups" "$SAPOT_ROOT/.lock"
 }
 prepare_env_files() {
-  local release=$1 generated secret mysql_password
-  mkdir -p "$SAPOT_ROOT/shared" "$SAPOT_ROOT/shared/certs" "$SAPOT_ROOT/shared/db-data" "$SAPOT_ROOT/shared/gsm-arduino-backups" "$SAPOT_ROOT/shared/db-backups"
-  secret=$(openssl rand -hex 32); mysql_password=$(openssl rand -hex 32)
-  for name in server admin gsm-fastapi gsm-arduino; do
+  local release=$1 generated secret mysql_password status_password
+  mkdir -p "$SAPOT_ROOT/shared" "$SAPOT_ROOT/shared/certs" "$SAPOT_ROOT/shared/db-data" "$SAPOT_ROOT/shared/gsm-arduino-backups"
+  secret=$(openssl rand -hex 32); mysql_password=$(openssl rand -hex 32); status_password=$(openssl rand -hex 32)
+  for name in server admin gsm-fastapi gsm-arduino status; do
     generated="$SAPOT_ROOT/shared/$name.env"
     [ -e "$generated" ] && continue
-    sed -e "s/__GENERATE__/$secret/g" -e "s/__FROM_SERVER_GSM_SECRET__/$secret/g" -e "s/__FROM_SERVER_MYSQL_PASSWORD__/$mysql_password/g" -e "s/mysql+pymysql:\/\/sapot:$secret@db/mysql+pymysql:\/\/sapot:$mysql_password@db/" "$release/config/$name.env.example" > "$generated"
+    sed -e "s/__GENERATE__/$secret/g" -e "s/__FROM_SERVER_GSM_SECRET__/$secret/g" -e "s/__FROM_SERVER_MYSQL_PASSWORD__/$mysql_password/g" -e "s/__STATUS_DB_PASSWORD__/$status_password/g" -e "s/mysql+pymysql:\/\/sapot:$secret@db/mysql+pymysql:\/\/sapot:$mysql_password@db/" "$release/config/$name.env.example" > "$generated"
     [ "$name" = server ] && sed -i "s|mysql+pymysql://sapot:$secret@db|mysql+pymysql://sapot:$mysql_password@db|" "$generated"
     chmod 600 "$generated"
   done
 }
+prepare_shared_dirs() {
+  local uid gid
+  mkdir -p "$SAPOT_ROOT/shared/status" "$SAPOT_ROOT/shared/logs/api" "$SAPOT_ROOT/shared/logs/gsm"
+  uid=$(docker run --rm --entrypoint id sapot/api:bundle -u 2>/dev/null || true)
+  gid=$(docker run --rm --entrypoint id sapot/api:bundle -g 2>/dev/null || true)
+  if [ -n "$uid" ] && [ -n "$gid" ]; then
+    chown -R "$uid:$gid" "$SAPOT_ROOT/shared/status" "$SAPOT_ROOT/shared/logs/api" 2>/dev/null || log_error "could not chown shared writable directories"
+  else log_error "could not determine api image uid"; fi
+}
+provision_status_db_user() {
+  local release=$1 root_password status_password attempts=24
+  root_password=$(grep '^MYSQL_ROOT_PASSWORD=' "$SAPOT_ROOT/shared/server.env" | cut -d= -f2-)
+  status_password=$(sed -n 's|^STATUS_DATABASE_URL=mysql+pymysql://sapot_status:\([^@]*\)@.*|\1|p' "$SAPOT_ROOT/shared/status.env")
+  [ -n "$root_password" ] && [ -n "$status_password" ] || { log_error "missing status database credentials"; return 1; }
+  compose "$release" exec -T -e MYSQL_PWD="$root_password" db mariadb -uroot <<SQL
+CREATE USER IF NOT EXISTS 'sapot_status'@'%' IDENTIFIED BY '$status_password';
+ALTER USER 'sapot_status'@'%' IDENTIFIED BY '$status_password';
+GRANT SELECT ON sapot.message TO 'sapot_status'@'%';
+GRANT SELECT ON sapot.activity_logs TO 'sapot_status'@'%';
+FLUSH PRIVILEGES;
+SQL
+  while [ "$attempts" -gt 0 ]; do
+    if compose "$release" exec -T -e MYSQL_PWD="$root_password" db mariadb -uroot -N -B -e "SELECT 1 FROM information_schema.tables WHERE table_schema='sapot' AND table_name='sms_log'" 2>/dev/null | grep -q 1; then
+      compose "$release" exec -T -e MYSQL_PWD="$root_password" db mariadb -uroot -e "GRANT SELECT ON sapot.sms_log TO 'sapot_status'@'%'; FLUSH PRIVILEGES;"
+      return 0
+    fi
+    sleep 5; attempts=$((attempts - 1))
+  done
+  log_error "sms_log did not appear; GSM pending count unavailable until reprovisioned"
+}
+write_integrity() {
+  local status=$1 dir="$SAPOT_ROOT/shared/status" tmp
+  mkdir -p "$dir"; tmp="$dir/.integrity.json.tmp"
+  printf '{"status": "%s", "verifiedAt": "%s"}\n' "$status" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$tmp"
+  mv -f "$tmp" "$dir/integrity.json"
+}
diff --git a/deploy/scripts/lib/retention.sh b/deploy/scripts/lib/retention.sh
@@ -4,6 +4,7 @@ set -euo pipefail
 root=${SAPOT_ROOT:-/opt/sapot}
 dry_run=false
 keep=${SAPOT_RELEASE_RETENTION:-3}
+log_days=${SAPOT_LOG_RETENTION_DAYS:-30}
 [ "${1:-}" = "--dry-run" ] && dry_run=true
 remove() { if "$dry_run"; then printf 'would remove %s\n' "$1"; else rm -rf -- "$1"; fi; }
 
@@ -25,3 +26,8 @@ done
 if [ -d "$root/shared/gsm-arduino-backups" ]; then
   find "$root/shared/gsm-arduino-backups" -type f -printf '%T@ %p\n' | sort -rn | tail -n +$((keep + 1)) | cut -d' ' -f2- | while read -r backup; do remove "$backup"; done
 fi
+if [ -d "$root/shared/logs" ]; then
+  while IFS= read -r stale; do remove "$stale"; done < <(
+    find "$root/shared/logs" -type f -mtime "+$log_days" -print
+  )
+fi