From ed174efa1b92bf5f724c0039a4a418c655de8f5d Mon Sep 17 00:00:00 2001 From: ash-iir <254348575+ash-iir@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:38:10 +0000 Subject: [PATCH 1/3] service: don't trust a stale scheduler pid after reboot The pid file survives reboot. When its pid was reused by another process, kill -0 succeeded and the scheduler loop never started for that boot. Match the loop's own command line, which carries the pid file path. Signed-off-by: ash-iir <254348575+ash-iir@users.noreply.github.com> (cherry picked from commit 619fbf6951b82c7899c13b16e85b85310d567258) --- module/service.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/module/service.sh b/module/service.sh index 3cbdf1f..70e03ad 100644 --- a/module/service.sh +++ b/module/service.sh @@ -48,8 +48,13 @@ mkdir -p "$LOGS" # honour the skip flag [ -f "$MODDIR/.scheduler.skip" ] && exit 0 -# single-instance guard: bail out if a scheduler loop is already alive -if [ -f "$LOOPPID" ] && kill -0 "$(cat "$LOOPPID" 2>/dev/null)" 2>/dev/null; then exit 0; fi +# single-instance guard: bail out if a scheduler loop is already alive. The pid file +# survives reboot and its pid can be reused by another process, so match the loop's +# own command line (it carries $LOOPPID) instead of trusting kill -0. +_lp="$(cat "$LOOPPID" 2>/dev/null)" +case "$_lp" in ''|*[!0-9]*) : ;; *) + case "$(tr '\0' ' ' < "/proc/$_lp/cmdline" 2>/dev/null)" in *"$LOOPPID"*) exit 0 ;; esac ;; +esac # recurring, detached heartbeat (survives boot, never blocks it): the loop # records its own pid for the guard, then runs scheduler-tick every INTERVAL. From 256b413979033c6932049b5a86a8ad32f73e7a2e Mon Sep 17 00:00:00 2001 From: ash-iir <254348575+ash-iir@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:39:22 +0000 Subject: [PATCH 2/3] worker: put the install verifier back after a batch restore Each batch item runs restore() in a subshell, which runs no EXIT trap, so verifier_restore never ran and the next item saved 0 as the old value. verifier_verify_adb_installs and package_verifier_enable stayed off. Turn the verifier off once in restore_batch and restore it there. Signed-off-by: ash-iir <254348575+ash-iir@users.noreply.github.com> (cherry picked from commit cb1454e4302063883d3142066c405ea6d34fc3e9) --- module/scripts/worker.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/module/scripts/worker.sh b/module/scripts/worker.sh index cc63ee9..dee15e3 100644 --- a/module/scripts/worker.sh +++ b/module/scripts/worker.sh @@ -1605,7 +1605,7 @@ restore() { # install APKs (decompress to /data/local/tmp, never leave raw behind) prog 25 installing "$PKG" TMP=/data/local/tmp/vdb.$$; mkdir -p "$TMP" - verifier_off + [ "${IN_BATCH:-0}" = 1 ] || verifier_off # a batch turns it off once in the parent (see restore_batch) RINST="$("$JQ" -r '.package.installer // empty' "$M" 2>/dev/null)" SESSION=""; want_item apk && SESSION="$(fw pm install-create $(pm_install_flags) $(installer_flag "$RINST") --user "$U" 2>/dev/null | sed -n 's/.*\[\([0-9]*\)\].*/\1/p')" if [ -n "$SESSION" ]; then @@ -2612,6 +2612,9 @@ restore_batch() { _base="$ARGS"; IN_BATCH=1; BATCH_I=0; _bok=0; _bfail=0; _bcodes="" _bu="$(jget "$_base" '.user // 0')"; case "$_bu" in ''|*[!0-9]*) _bu=0 ;; esac _BTO="$(cfg .batch_app_timeout)"; case "$_BTO" in ''|null|*[!0-9]*) _BTO=1800 ;; esac + # the verifier is switched off here, once, and put back by the parent: a subshell runs no + # EXIT trap, so per-item verifier_off would save "0" as the old value and never restore it. + verifier_off # each restore runs in a SUBSHELL (fds / shell state reclaimed per item) and reports back # through a status file; failure codes are tallied so a partial names why (issue #5, as backup). while IFS= read -r _r <&3; do @@ -2624,7 +2627,7 @@ restore_batch() { [ "$_lok" = true ] || _lok=false; [ -n "$_lcode" ] || _lcode=ERR_UNKNOWN if [ "$_lok" = true ]; then _bok=$((_bok + 1)); else _bfail=$((_bfail + 1)); _bcodes="$_bcodes $_lcode"; fi done 3< "$_lst" - rm -f "$_lst" "$_rcf"; ARGS="$_base"; IN_BATCH=0 + rm -f "$_lst" "$_rcf"; ARGS="$_base"; IN_BATCH=0; verifier_restore _bfails="$(printf '%s\n' $_bcodes | "$JQ" -R -s -c 'split("\n")|map(select(length>0))|group_by(.)|map({key:.[0],value:length})|from_entries' 2>/dev/null)" case "$_bfails" in ''|null) _bfails='{}' ;; esac _bfree="$(fs_free "$(backups_dir "$_bu")" 2>/dev/null)"; case "$_bfree" in ''|*[!0-9]*) _bfree=0 ;; esac From 7e4c5babeb7cdad70cdbeb17c99f62ff3b18626f Mon Sep 17 00:00:00 2001 From: ash-iir <254348575+ash-iir@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:15:45 +0000 Subject: [PATCH 3/3] scheduler: run each job as one batch op and prune after success scheduler-tick and the auto-backup observer started one detached worker per package, all at once, and pruned right after starting them. A job for all apps therefore ran every backup in parallel, and pruning counted the backups still being written, so a failed run could leave one good backup fewer than keep. - Each due job now starts one backup-batch (or restore-batch) op for its targets; the auto-backup observer starts one backup-batch. - Scheduled and auto batches carry keep; backup_batch prunes an app to keep finished backups only after that app's backup succeeded. - prune_pkg counts and removes only backups with a manifest.json. - scheduler-tick no longer reports pruned refs (pruning happens later, in the worker, and is still audited). Signed-off-by: ash-iir <254348575+ash-iir@users.noreply.github.com> (cherry picked from commit c5a219ed588149a2c670cc87a0b4d8178903c098) --- module/scripts/helper.sh | 64 ++++++++++++++++++++++++---------------- module/scripts/worker.sh | 5 ++++ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/module/scripts/helper.sh b/module/scripts/helper.sh index ec3518e..5a4f82d 100644 --- a/module/scripts/helper.sh +++ b/module/scripts/helper.sh @@ -211,8 +211,10 @@ prune_pkg() { # _d="$_bk/$1"; [ -d "$_d" ] || return 0 _keep="$2"; case "$_keep" in ''|*[!0-9]*) _keep=5 ;; esac _n=0 + # only finished backups count and get pruned: a dir without manifest.json is a backup still + # being written (or one that failed), and must neither push a good one out nor be deleted. for _sub in $(ls -1 "$_d" 2>/dev/null | sort -r); do - [ -d "$_d/$_sub" ] || continue + [ -f "$_d/$_sub/manifest.json" ] || continue _n=$(( _n + 1 )) [ "$_n" -gt "$_keep" ] || continue case "$_d/$_sub" in @@ -227,7 +229,7 @@ prune_pkg() { # # seeds the snapshot (never mass-backs-up an existing device). Honors the blocklist. # Framework-free degradation: without the dex helper only new installs are seen # (no versionCode), so updates are missed but installs still trigger. -auto_backup_scan() { # ; echoes detached opids +auto_backup_scan() { # ; echoes the detached opid _jq="$1"; _now="$2" [ -f "$STATE/config.json" ] || return 0 _on="$("$_jq" -r '.auto_backup // false' "$STATE/config.json" 2>/dev/null)" @@ -253,20 +255,23 @@ auto_backup_scan() { # ; echoes detached _changed="$("$_jq" -r --slurpfile seen "$_seen" ' ($seen[0] // {}) as $s | to_entries | map(select(($s[.key] // null) != .value)) | .[].key' "$_cur" 2>/dev/null)" - _n=0 + # one backup-batch op for all of them (one worker, apps in sequence, as backup-batch was built + # for); the worker prunes each app to after its backup succeeds. + _opid="$(new_opid)-auto"; _pkgs="" for _pkg in $_changed; do case "$_pkg" in ''|*..*|*/*|*' '*) continue ;; esac "$_jq" -e --arg p "$_pkg" '((.blocklist // []) | index($p)) != null' "$STATE/config.json" >/dev/null 2>&1 && continue - _j="$("$_jq" -nc --arg p "$_pkg" --argjson u "$_u" --arg m "$_mode" --arg d "$_dest" \ - '{package:$p,user:$u,mode:$m,dest:$d,items:["apk","data","data_de","obb","external","media"],auto:true}')" - _b64="$(printf '%s' "$_j" | base64 | tr -d '\n')" - _n=$(( _n + 1 )); _opid="$(new_opid)-auto$_n" - detach "$_opid" backup "$_b64" + _pkgs="$_pkgs$_pkg +" printf '{"ts":"%s","op":"auto-backup","opid":"%s","package":"%s"}\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(jstr "$_opid")" "$(jstr "$_pkg")" >> "$LOGS/audit.jsonl" - for _r in $(prune_pkg "$_u" "$_pkg" "$_keep"); do :; done - printf '%s\n' "$_opid" done + if [ -n "$_pkgs" ]; then + _j="$(printf '%s' "$_pkgs" | "$_jq" -R -s -c --argjson u "$_u" --arg m "$_mode" --arg d "$_dest" --argjson k "$_keep" \ + '{packages:(split("\n")|map(select(length>0))),user:$u,mode:$m,dest:$d,items:["apk","data","data_de","obb","external","media"],auto:true,keep:$k}')" + detach "$_opid" backup-batch "$(printf '%s' "$_j" | base64 | tr -d '\n')" + printf '%s\n' "$_opid" + fi cp "$_cur" "$_seen"; chmod 600 "$_seen"; rm -f "$_cur" } @@ -1278,7 +1283,7 @@ EOF # so skip the whole tick until the user has unlocked once. ce_ready || { printf '{"ok":true,"skipped":"ce_locked"}\n'; exit 0; } mkdir -p "$STATE/jobs" - RAN=""; PRUNED=""; checked=0; launchn=0 + RAN=""; checked=0; launchn=0 # a transient gate skips the job WITHOUT stamping last_run (retries next tick) + audits gate_skip() { printf '{"ts":"%s","op":"scheduler-gated","id":"%s","reason":"%s"}\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(jstr "$id")" "$(jstr "$1")" >> "$LOGS/audit.jsonl"; } @@ -1309,35 +1314,44 @@ EOF jmode="$("$JQ" -r '.mode // "full"' "$f" 2>/dev/null)" jdest="$("$JQ" -r '.dest // "local"' "$f" 2>/dev/null)" jref="$("$JQ" -r '.ref // empty' "$f" 2>/dev/null)" - fired=0 - # resolve this job's target packages (single / list / all, minus blocklist) and - # launch one detached op per target: a backup, or a restore of its latest backup. + fired=0; items="" + # resolve this job's target packages (single / list / all, minus blocklist) and launch ONE + # detached batch op for the job: backup-batch of the packages, or restore-batch of each + # one's latest backup. The worker runs them in sequence and prunes each app to + # after its backup succeeds. + launchn=$(( launchn + 1 )); opid="$(new_opid)-$launchn" for pkg in $(job_targets "$JQ" "$f"); do - launchn=$(( launchn + 1 )); fired=1 - opid="$(new_opid)-$launchn" if [ "$action" = restore ]; then ref="$jref"; [ -n "$ref" ] || ref="$(latest_ref "$juser" "$pkg")" - [ -n "$ref" ] || { launchn=$(( launchn - 1 )); continue; } # nothing to restore for this pkg - argj="$("$JQ" -nc --arg ref "$ref" '{ref:$ref}')" - detach "$opid" restore "$(printf '%s' "$argj" | base64 | tr -d '\n')" + [ -n "$ref" ] || continue # nothing to restore for this pkg + printf '%s' "$items" | grep -qxF -- "$ref" && continue # a pinned .ref is restored once + items="$items$ref +" else - argj="$("$JQ" -nc --arg p "$pkg" --argjson u "$juser" --arg m "$jmode" --arg d "$jdest" \ - '{package:$p,user:$u,mode:$m,dest:$d,items:["apk","data","data_de","obb","external","media"],sched:true}')" - detach "$opid" backup "$(printf '%s' "$argj" | base64 | tr -d '\n')" - for r in $(prune_pkg "$juser" "$pkg" "$keep"); do PRUNED="$PRUNED$("$JQ" -cn --arg r "$r" '$r'),"; done + items="$items$pkg +" fi + fired=1 RAN="$RAN$("$JQ" -cn --arg id "$id" --arg opid "$opid" --arg pkg "$pkg" --arg a "$action" '{id:$id,opid:$opid,package:$pkg,action:$a}')," printf '{"ts":"%s","op":"scheduler-run","id":"%s","opid":"%s","package":"%s","action":"%s"}\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(jstr "$id")" "$(jstr "$opid")" "$(jstr "$pkg")" "$(jstr "$action")" >> "$LOGS/audit.jsonl" done [ "$fired" = 1 ] || continue # due but no target resolved: do not stamp, retry next tick + if [ "$action" = restore ]; then + argj="$(printf '%s' "$items" | "$JQ" -R -s -c '{refs:(split("\n")|map(select(length>0)))}')" + detach "$opid" restore-batch "$(printf '%s' "$argj" | base64 | tr -d '\n')" + else + argj="$(printf '%s' "$items" | "$JQ" -R -s -c --argjson u "$juser" --arg m "$jmode" --arg d "$jdest" --argjson k "$keep" \ + '{packages:(split("\n")|map(select(length>0))),user:$u,mode:$m,dest:$d,items:["apk","data","data_de","obb","external","media"],sched:true,keep:$k}')" + detach "$opid" backup-batch "$(printf '%s' "$argj" | base64 | tr -d '\n')" + fi tmp="$STATE/jobs/.$id.tmp" "$JQ" --argjson now "$now" '.last_run=$now' "$f" > "$tmp" 2>/dev/null && mv "$tmp" "$f" && chmod 600 "$f" done AUTO="" for _ao in $(auto_backup_scan "$JQ" "$now"); do AUTO="$AUTO\"$(jstr "$_ao")\","; done - RAN="${RAN%,}"; PRUNED="${PRUNED%,}"; AUTO="${AUTO%,}" - printf '{"ok":true,"now":%s,"checked":%s,"ran":[%s],"pruned":[%s],"auto":[%s]}\n' "$now" "$checked" "$RAN" "$PRUNED" "$AUTO" ;; + RAN="${RAN%,}"; AUTO="${AUTO%,}" + printf '{"ok":true,"now":%s,"checked":%s,"ran":[%s],"auto":[%s]}\n' "$now" "$checked" "$RAN" "$AUTO" ;; companion-status) JQ="$(command -v "$BIN/jq" 2>/dev/null || echo jq)" diff --git a/module/scripts/worker.sh b/module/scripts/worker.sh index dee15e3..1df15aa 100644 --- a/module/scripts/worker.sh +++ b/module/scripts/worker.sh @@ -2554,6 +2554,9 @@ backup_batch() { _base="$ARGS"; IN_BATCH=1; BATCH_I=0; _bok=0; _bfail=0; _bcodes="" _bu="$(jget "$_base" '.user // 0')"; case "$_bu" in ''|*[!0-9]*) _bu=0 ;; esac _BTO="$(cfg .batch_app_timeout)"; case "$_BTO" in ''|null|*[!0-9]*) _BTO=1800 ;; esac + # scheduled/auto batches carry .keep: prune each app to finished backups (helper.sh + # prune) only after ITS backup succeeded, so a failed run never costs an older good backup. + _bkeep="$(jget "$_base" '.keep // empty')"; case "$_bkeep" in *[!0-9]*) _bkeep="" ;; esac # Pre-run space check (issue #5): a batch to LOCAL storage keeps every app's archive, so a big # run can fill the disk partway and end with a bare partial deep in. For a local destination, # sum the uncompressed app data (data + data_de, the always-present core footprint) up front; @@ -2596,6 +2599,8 @@ backup_batch() { _lok=false; _lcode=ERR_UNKNOWN; read -r _lok _lcode < "$_rcf" 2>/dev/null [ "$_lok" = true ] || _lok=false; [ -n "$_lcode" ] || _lcode=ERR_UNKNOWN if [ "$_lok" = true ]; then _bok=$((_bok + 1)); else _bfail=$((_bfail + 1)); _bcodes="$_bcodes $_lcode"; fi + [ "$_lok" = true ] && [ -n "$_bkeep" ] && "$MODDIR/scripts/helper.sh" prune \ + "$("$JQ" -nc --arg p "$_p" --argjson u "$_bu" --argjson k "$_bkeep" '{package:$p,user:$u,keep:$k}' | base64 | tr -d '\n')" >/dev/null 2>&1 done 3< "$_lst" rm -f "$_lst" "$_rcf"; ARGS="$_base"; IN_BATCH=0 _bfails="$(printf '%s\n' $_bcodes | "$JQ" -R -s -c 'split("\n")|map(select(length>0))|group_by(.)|map({key:.[0],value:length})|from_entries' 2>/dev/null)"