From 877581e23a2cc2ba1bad59638cd58298e542f3f5 Mon Sep 17 00:00:00 2001 From: Marc Bekhuis Date: Wed, 26 Aug 2026 13:49:47 +0200 Subject: [PATCH 1/4] Add --output-dir, --archive-name and --keep-dir to roll backup Lets a caller place the finished archive outside the project, name it, and keep the restorable directory form in place. Together these are what the Epartment build server currently needs its own backup command override for. Volumes are still staged in the project's .roll/backups/ and retention cleanup still only ever runs against that directory, so pointing --output-dir at a shared delivery directory cannot delete another environment's archives. The "latest" symlink is likewise only maintained when the archive stays in .roll/backups/, since a shared directory would have every environment fighting over one symlink. The destination is created and checked for writability before `env down` and before any volume is tarred, so an unusable path fails in seconds rather than after a multi-hour backup. --keep-dir suppresses the post-archive `rm -rf` of the backup directory. `roll restore` reads the directory form, so a workflow that archives the whole workspace after a backup only captures a restorable copy while that directory still exists. Co-Authored-By: Claude Opus 5 --- commands/backup.cmd | 104 ++++++++++++++++++++++++++++++++++--------- commands/backup.help | 17 ++++++- 2 files changed, 98 insertions(+), 23 deletions(-) diff --git a/commands/backup.cmd b/commands/backup.cmd index f99da51..34422e9 100755 --- a/commands/backup.cmd +++ b/commands/backup.cmd @@ -20,8 +20,16 @@ BACKUP_NAME="" BACKUP_DESCRIPTION="" BACKUP_DUPLICATE_NAME="" # New environment name for duplication BACKUP_DUPLICATE_DOMAIN="" # New domain for duplication +BACKUP_OUTPUT_DIR="" # Where the final archive is written (default: BACKUP_BASE_DIR) +BACKUP_ARCHIVE_NAME="" # Final archive basename, extension appended (default: backup__) +BACKUP_KEEP_DIR=0 # Keep the uncompressed backup directory alongside the archive PROGRESS=1 +## Volumes are always staged inside the project: that is where the disk space is budgeted, and +## retention cleanup must never be pointed at a shared drop directory holding other projects' +## archives. --output-dir only moves the finished archive. +BACKUP_BASE_DIR="$(pwd)/.roll/backups" + # Parse command line arguments POSITIONAL_ARGS=() # Start with any arguments passed from the main roll script @@ -99,6 +107,18 @@ while [[ $# -gt 0 ]]; do BACKUP_DUPLICATE_DOMAIN="${1#*=}" shift ;; + --output-dir=*) + BACKUP_OUTPUT_DIR="${1#*=}" + shift + ;; + --archive-name=*) + BACKUP_ARCHIVE_NAME="${1#*=}" + shift + ;; + --keep-dir) + BACKUP_KEEP_DIR=1 + shift + ;; --no-progress) PROGRESS=0 shift @@ -266,7 +286,7 @@ function detectEnabledServices() { function createBackupDirectory() { local timestamp=${1:-$(date +%s)} - local backup_dir="$(pwd)/.roll/backups/$timestamp" + local backup_dir="${BACKUP_BASE_DIR}/$timestamp" # Create backup directories mkdir -p "$backup_dir"/{volumes,config,metadata,logs} @@ -632,8 +652,32 @@ function verifyBackup() { fi } +## Resolve and validate the archive destination up front. This deliberately runs before +## `env down` and before a single volume is tarred: a drop directory that turns out to be +## missing or read-only after a multi-hour backup means the whole run is thrown away. +function resolveBackupOutputDir() { + if [[ -z "$BACKUP_OUTPUT_DIR" ]]; then + BACKUP_OUTPUT_DIR="${BACKUP_BASE_DIR}" + return 0 + fi + + if ! mkdir -p "$BACKUP_OUTPUT_DIR" 2>/dev/null; then + error "Cannot create backup output directory: $BACKUP_OUTPUT_DIR" + exit 1 + fi + + if [[ ! -w "$BACKUP_OUTPUT_DIR" ]]; then + error "Backup output directory is not writable: $BACKUP_OUTPUT_DIR" + exit 1 + fi + + ## Normalise to an absolute path: the final archive is written from inside a `cd` subshell, + ## where a relative destination would resolve against the staging directory instead. + BACKUP_OUTPUT_DIR="$(cd "$BACKUP_OUTPUT_DIR" && pwd)" +} + function cleanupOldBackups() { - local backup_base_dir="$(pwd)/.roll/backups" + local backup_base_dir="${BACKUP_BASE_DIR}" if [[ $BACKUP_RETENTION_DAYS -le 0 ]]; then return 0 @@ -657,6 +701,7 @@ function performBackup() { # Validate inputs validateCompression || exit 1 + resolveBackupOutputDir # Handle interactive password prompt if needed if [[ "$BACKUP_ENCRYPT" == "PROMPT" ]]; then @@ -802,32 +847,47 @@ function performBackup() { verifyBackup "$backup_dir" # Create compressed archive for the entire backup - local archive_name="backup_${ROLL_ENV_NAME}_${timestamp}$(getCompressionExtension)" + local archive_name="${BACKUP_ARCHIVE_NAME:-backup_${ROLL_ENV_NAME}_${timestamp}}$(getCompressionExtension)" + local archive_path="${BACKUP_OUTPUT_DIR}/${archive_name}" logMessage INFO "Creating final backup archive: $archive_name" - - # Suppress tar warnings when using --output-id + + ## The `cd` scopes tar's member paths to "/..." so the archive extracts straight into a + ## backups directory; the redirect uses the absolute destination so it can land elsewhere. + local archive_status=0 if [[ $BACKUP_OUTPUT_ID -eq 1 ]]; then - (cd "$(pwd)/.roll/backups" && tar -cf - "$timestamp" 2>/dev/null | $(getCompressionCommand) > "$archive_name") + (cd "${BACKUP_BASE_DIR}" && tar -cf - "$timestamp" 2>/dev/null | $(getCompressionCommand) > "$archive_path") || archive_status=$? else - (cd "$(pwd)/.roll/backups" && tar -cf - "$timestamp" | $(getCompressionCommand) > "$archive_name") + (cd "${BACKUP_BASE_DIR}" && tar -cf - "$timestamp" | $(getCompressionCommand) > "$archive_path") || archive_status=$? fi - - if [[ $? -eq 0 ]]; then - # Update latest symlink - (cd "$(pwd)/.roll/backups" && ln -sf "$archive_name" "latest$(getCompressionExtension)") - + + if [[ $archive_status -eq 0 ]]; then + ## Only maintain the "latest" pointer in the project's own backups directory. A shared + ## drop directory holds archives for many environments, where a single `latest` symlink + ## would be overwritten by whichever project finished last. + if [[ "${BACKUP_OUTPUT_DIR}" == "${BACKUP_BASE_DIR}" ]]; then + (cd "${BACKUP_BASE_DIR}" && ln -sf "$archive_name" "latest$(getCompressionExtension)") + fi + if [[ $BACKUP_OUTPUT_ID -eq 1 ]]; then # Only output the backup ID for programmatic use echo "$timestamp" else logMessage SUCCESS "Backup completed successfully!" logMessage INFO "Backup ID: $timestamp" - logMessage INFO "Archive: $archive_name ($(du -h "$(pwd)/.roll/backups/$archive_name" | cut -f1))" - logMessage INFO "Location: $(pwd)/.roll/backups/" + logMessage INFO "Archive: $archive_name ($(du -h "$archive_path" | cut -f1))" + logMessage INFO "Location: ${BACKUP_OUTPUT_DIR}/" + fi + + + ## Clean up the directory version, which the archive already contains. --keep-dir + ## retains it for a caller that needs a restorable backup left in place — restore + ## reads the directory form, so archiving the workspace after a backup only yields a + ## restorable copy while the directory is still there. + if [[ $BACKUP_KEEP_DIR -eq 0 ]]; then + rm -rf "$backup_dir" + else + logMessage INFO "Keeping uncompressed backup directory: $backup_dir" fi - - # Clean up directory version (keep archive) - rm -rf "$backup_dir" else logMessage ERROR "Failed to create final backup archive" exit 1 @@ -850,8 +910,8 @@ case "${BACKUP_COMMAND_PARAMS[0]}" in ;; list|ls) echo "Available backups:" - if [[ -d "$(pwd)/.roll/backups" ]]; then - ls -la "$(pwd)/.roll/backups/" | grep -E '^d.*[0-9]{10}$|^-.*backup_.*\.tar' + if [[ -d "${BACKUP_BASE_DIR}" ]]; then + ls -la "${BACKUP_BASE_DIR}/" | grep -E '^d.*[0-9]{10}$|^-.*backup_.*\.tar' else echo "No backups found." fi @@ -861,14 +921,14 @@ case "${BACKUP_COMMAND_PARAMS[0]}" in backup_id="${BACKUP_COMMAND_PARAMS[1]}" # First check if directory exists (uncompressed backup) - metadata_file="$(pwd)/.roll/backups/$backup_id/metadata/backup.json" + metadata_file="${BACKUP_BASE_DIR}/$backup_id/metadata/backup.json" if [[ -f "$metadata_file" ]]; then cat "$metadata_file" | jq '.' 2>/dev/null || cat "$metadata_file" else # Look for compressed archive archive_file="" for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do - potential_file="$(pwd)/.roll/backups/backup_${ROLL_ENV_NAME}_${backup_id}${ext}" + potential_file="${BACKUP_BASE_DIR}/backup_${ROLL_ENV_NAME}_${backup_id}${ext}" if [[ -f "$potential_file" ]]; then archive_file="$potential_file" break @@ -877,7 +937,7 @@ case "${BACKUP_COMMAND_PARAMS[0]}" in if [[ -z "$archive_file" ]]; then # Also check for generic archive names - archive_file=$(ls "$(pwd)/.roll/backups"/*"$backup_id"*.tar* 2>/dev/null | head -1) + archive_file=$(ls "${BACKUP_BASE_DIR}"/*"$backup_id"*.tar* 2>/dev/null | head -1) fi if [[ -n "$archive_file" ]]; then diff --git a/commands/backup.help b/commands/backup.help index 9ae2bf9..caefe89 100755 --- a/commands/backup.help +++ b/commands/backup.help @@ -38,6 +38,19 @@ ROLL_USAGE=$(cat <_). The 'latest' + symlink is only maintained when the archive stays in .roll/backups/. + --keep-dir Keep the uncompressed backup directory in .roll/backups/ as well as + the archive (removed by default). 'roll restore' reads the directory + form, so this is what leaves a restorable backup in the project. + \033[33mDuplication Options:\033[0m --duplicate-name=NAME Replace ROLL_ENV_NAME in backup for duplication --duplicate-domain=DOMAIN Replace TRAEFIK_DOMAIN in backup for duplication @@ -54,9 +67,11 @@ ROLL_USAGE=$(cat < Date: Wed, 26 Aug 2026 14:00:08 +0200 Subject: [PATCH 2/4] Fix infinite recursion in roll backup --help backup is on ROLL_CMD_ANYARGS, so roll's parser stops at --help and hands it to backup.cmd unchanged. The handler called `roll backup --help`, which re-entered the same branch and never terminated. Render the usage directly instead. The same pattern is present in restore.cmd, restore-full.cmd, duplicate.cmd, env.cmd, db.cmd and svc.cmd and is left untouched here. Co-Authored-By: Claude Opus 5 --- commands/backup.cmd | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/commands/backup.cmd b/commands/backup.cmd index 34422e9..c2ad636 100755 --- a/commands/backup.cmd +++ b/commands/backup.cmd @@ -40,8 +40,11 @@ fi while [[ $# -gt 0 ]]; do case "$1" in --help|-h) - roll backup --help - exit 0 + ## Do NOT re-invoke `roll backup --help` here. `backup` is on roll's ROLL_CMD_ANYARGS + ## list, so roll's own parser stops at --help and passes it straight through to this + ## script — re-invoking roll lands right back on this branch and recurses forever. + ## usage.cmd renders ROLL_CMD_HELP (backup.help) and exits on its own. + source "${ROLL_DIR}/commands/usage.cmd" ;; --compression=*) BACKUP_COMPRESSION="${1#*=}" From 5e4aee8f48982600e42a8aec4a82782179169dc7 Mon Sep 17 00:00:00 2001 From: Marc Bekhuis Date: Wed, 26 Aug 2026 14:11:58 +0200 Subject: [PATCH 3/4] Version bump --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index cbf93a1..7deb86f 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.7.0.6 \ No newline at end of file +0.7.1 \ No newline at end of file From d296282945c7c3ed42dbabaeac3556c762e8cf4d Mon Sep 17 00:00:00 2001 From: Marc Bekhuis Date: Fri, 28 Aug 2026 11:24:00 +0200 Subject: [PATCH 4/4] Address review feedback on the backup output flags - Set pipefail around the final tar|compress pipeline. roll sets `set -e` but never pipefail, so the pipeline reported the compressor's status: gzip exits 0 on a truncated stream, so a tar that ran out of disk was recorded as a successful backup and the uncompressed copy was then deleted. Verified: `(false | cat)` returns 0, with pipefail returns 1. - Reject a path separator in --archive-name. It names a file inside the output directory, so `--archive-name=../x` wrote outside it and past the writability checks. - Decide the "latest" symlink from whether --output-dir was given rather than by comparing it to the staging path. The same directory compares unequal as a string once a relative argument or a symlinked path is involved, which skipped the symlink for a backup that never left the project. - Document why BACKUP_BASE_DIR stays keyed on the working directory: it is pre-existing behaviour that restore.cmd shares, and changing it only here would make backups land where restore does not look. Co-Authored-By: Claude Opus 5 --- commands/backup.cmd | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/commands/backup.cmd b/commands/backup.cmd index c2ad636..4e25281 100755 --- a/commands/backup.cmd +++ b/commands/backup.cmd @@ -23,11 +23,18 @@ BACKUP_DUPLICATE_DOMAIN="" # New domain for duplication BACKUP_OUTPUT_DIR="" # Where the final archive is written (default: BACKUP_BASE_DIR) BACKUP_ARCHIVE_NAME="" # Final archive basename, extension appended (default: backup__) BACKUP_KEEP_DIR=0 # Keep the uncompressed backup directory alongside the archive +BACKUP_OUTPUT_REDIRECTED=0 # Set when --output-dir is given, whatever path it names PROGRESS=1 ## Volumes are always staged inside the project: that is where the disk space is budgeted, and ## retention cleanup must never be pointed at a shared drop directory holding other projects' ## archives. --output-dir only moves the finished archive. +## +## Keyed on the working directory rather than ROLL_ENV_PATH, so `roll backup` from a project +## SUB-directory stages into /.roll/backups. That is pre-existing behaviour and restore.cmd +## resolves the same way, so the two agree; changing it here alone would make backups land where +## restore does not look. Fix all of backup.cmd, restore.cmd and restore-full.cmd together, and +## note that restore-full.cmd has no ROLL_ENV_PATH at all — it never loads the env config. BACKUP_BASE_DIR="$(pwd)/.roll/backups" # Parse command line arguments @@ -112,6 +119,7 @@ while [[ $# -gt 0 ]]; do ;; --output-dir=*) BACKUP_OUTPUT_DIR="${1#*=}" + BACKUP_OUTPUT_REDIRECTED=1 shift ;; --archive-name=*) @@ -659,6 +667,13 @@ function verifyBackup() { ## `env down` and before a single volume is tarred: a drop directory that turns out to be ## missing or read-only after a multi-hour backup means the whole run is thrown away. function resolveBackupOutputDir() { + ## --archive-name names a file inside the output directory, so a path separator in it would + ## silently write somewhere else entirely (`--archive-name=../x`) and skip the checks below. + if [[ "$BACKUP_ARCHIVE_NAME" == */* ]]; then + error "--archive-name must be a filename, not a path: $BACKUP_ARCHIVE_NAME" + exit 1 + fi + if [[ -z "$BACKUP_OUTPUT_DIR" ]]; then BACKUP_OUTPUT_DIR="${BACKUP_BASE_DIR}" return 0 @@ -856,18 +871,24 @@ function performBackup() { ## The `cd` scopes tar's member paths to "/..." so the archive extracts straight into a ## backups directory; the redirect uses the absolute destination so it can land elsewhere. + ## + ## pipefail is essential and is NOT set globally in roll: without it the pipeline reports the + ## compressor's status, and gzip happily exits 0 on a truncated stream from a tar that ran out + ## of disk. That would report success and then delete the only uncompressed copy below. local archive_status=0 if [[ $BACKUP_OUTPUT_ID -eq 1 ]]; then - (cd "${BACKUP_BASE_DIR}" && tar -cf - "$timestamp" 2>/dev/null | $(getCompressionCommand) > "$archive_path") || archive_status=$? + (set -o pipefail; cd "${BACKUP_BASE_DIR}" && tar -cf - "$timestamp" 2>/dev/null | $(getCompressionCommand) > "$archive_path") || archive_status=$? else - (cd "${BACKUP_BASE_DIR}" && tar -cf - "$timestamp" | $(getCompressionCommand) > "$archive_path") || archive_status=$? + (set -o pipefail; cd "${BACKUP_BASE_DIR}" && tar -cf - "$timestamp" | $(getCompressionCommand) > "$archive_path") || archive_status=$? fi if [[ $archive_status -eq 0 ]]; then - ## Only maintain the "latest" pointer in the project's own backups directory. A shared - ## drop directory holds archives for many environments, where a single `latest` symlink - ## would be overwritten by whichever project finished last. - if [[ "${BACKUP_OUTPUT_DIR}" == "${BACKUP_BASE_DIR}" ]]; then + ## Only maintain the "latest" pointer when the archive was not redirected. A shared drop + ## directory holds archives for many environments, where a single `latest` symlink would + ## be overwritten by whichever project finished last. Keyed on whether --output-dir was + ## given rather than on comparing the two paths, which differ as strings for the same + ## directory once symlinks or a relative argument are involved. + if [[ $BACKUP_OUTPUT_REDIRECTED -eq 0 ]]; then (cd "${BACKUP_BASE_DIR}" && ln -sf "$archive_name" "latest$(getCompressionExtension)") fi