diff --git a/LaunchAgents/com.asiago.docs-tidy.plist b/LaunchAgents/com.asiago.docs-tidy.plist new file mode 100644 index 0000000..ca77d96 --- /dev/null +++ b/LaunchAgents/com.asiago.docs-tidy.plist @@ -0,0 +1,70 @@ + + + + + Label + com.asiago.docs-tidy + ProgramArguments + + /Users/andrewrich/.local/bin/docs-tidy + plan + --notify + + StartCalendarInterval + + + Month + 1 + Day + 1 + Hour + 10 + Minute + 0 + + + Month + 4 + Day + 1 + Hour + 10 + Minute + 0 + + + Month + 7 + Day + 1 + Hour + 10 + Minute + 0 + + + Month + 10 + Day + 1 + Hour + 10 + Minute + 0 + + + RunAtLoad + + LimitLoadToSessionType + Aqua + StandardOutPath + /Users/andrewrich/.local/state/com.asiago.docs-tidy.log + StandardErrorPath + /Users/andrewrich/.local/state/com.asiago.docs-tidy.log + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + + + diff --git a/docs-tidy.py b/docs-tidy.py new file mode 100755 index 0000000..4ced69e --- /dev/null +++ b/docs-tidy.py @@ -0,0 +1,1024 @@ +#!/usr/bin/env -S python3 -I +"""docs-tidy: quarterly ~/Documents tidy. Plan, then approve. + +Run `docs-tidy --help` for commands and env overrides. +Design: docs/plans/2026-10-21-docs-tidy-design.md +""" + +import argparse +import datetime +import errno +import glob +import hashlib +import json +import os +import plistlib +import re +import shutil +import sqlite3 +import stat +import subprocess +import sys +import time + +# Top-level Documents items that never archive. +KEEP = { + "army", + "DD214.pdf", + "ssn.pdf", + "Andrew Rich 1989 GED.pdf", + "Issue COE.pdf", + "1Password Emergency Kit A3-KCMDEC-my.pdf", +} + +POINTER = "ARCHIVED-TO-GOOGLE-DRIVE.txt" +TAX_POINTER = "ARCHIVED-YEARS.txt" +CREDIT = "credit-reports" +SNAPSHOT_PREFIX = "Documents - " +SKIPNAMES = {".DS_Store", ".localized"} +SECTIONS = ["archive", "snapshot-merge", "snapshot", "credit"] +MAX_PLAN_AGE_DAYS = 14 +FREE_RESERVE = 2 * 1024**3 +DUPE_MIN_SIZE = 1024 +DUPE_EXCLUDE_SUFFIXES = (".app", ".bundle", ".photoslibrary", ".rtfd", ".pages", ".numbers", ".key", ".workflow") +SF_DATALESS = 0x40000000 +TAG_XATTR = "com.apple.metadata:_kMDItemUserTags" +PEOPLE = { + "Andrew": {"ar", "amr", "andrew"}, + "Jen": {"jr", "jmr", "jen", "jennifer"}, +} +AGENCIES = ("Equifax", "Experian", "TransUnion") +CANON_RE = re.compile(r"^\d{4}-\d{2} (Andrew|Jen) (Equifax|Experian|TransUnion)\.pdf$") +REQUEST_RE = re.compile(r"^\d{4}-\d{2} (Andrew|Jen) Request Form\.pdf$") +BLANK_REQUEST = "Blank Annual Credit Report Request Form.pdf" +DATE_RE = re.compile(r"(?.""" + parts = rel.split("/") + if parts[0] == "tax" and len(parts) > 1: + return "/".join(parts[:2]) + return parts[0] + + +def under(rel, prefix): + return rel == prefix or rel.startswith(prefix + "/") + + +# ---------------------------------------------------------------- credit names + + +def read_tags(path): + """Raw Finder tags, colour suffix kept (e.g. "Important\\n6"). + + None when tags cannot be read on this system; [] when the file has none. + """ + if not (IS_MAC and shutil.which("xattr")): + return None + r = subprocess.run(["xattr", "-px", TAG_XATTR, path], capture_output=True, text=True) + if r.returncode != 0: + # A missing xattr means no tags; any other failure means unknown. + return [] if "No such xattr" in r.stderr else None + try: + tags = plistlib.loads(bytes.fromhex("".join(r.stdout.split()))) + except Exception: + return None + return [t for t in tags if isinstance(t, str)] + + +def tag_name(tag): + return tag.split("\n", 1)[0] + + +def merge_tags(have, canon): + """have, in order with colour suffixes, plus any canonical tag it lacks.""" + names = {tag_name(t) for t in have} + return list(have) + [t for t in canon if t not in names] + + +def write_tags(path, tags): + if not (IS_MAC and shutil.which("xattr")): + return False + hexval = plistlib.dumps(list(tags), fmt=plistlib.FMT_BINARY).hex() + subprocess.run(["xattr", "-wx", TAG_XATTR, hexval, path], check=True, capture_output=True) + return True + + +def canonical_tags(name): + m = CANON_RE.match(name) + if m: + return [m.group(1), m.group(2)] + m = REQUEST_RE.match(name) + if m: + return [m.group(1), "AnnualCreditReport"] + if name == BLANK_REQUEST: + return ["AnnualCreditReport"] + return None + + +def name_date(name, mtime): + m = DATE_RE.search(name) + if m: + return f"{m.group(1)}-{m.group(2)}" + return time.strftime("%Y-%m", time.localtime(mtime)) + + +def parse_credit(name, mtime): + """Return (new_name, tags) or (None, reason).""" + low = name.lower() + if not low.endswith(".pdf"): + return None, "not a PDF" + tokens = set(re.split(r"[^a-z]+", low[:-4])) + ym = name_date(name, mtime) + if low.startswith("fact act request form"): + people = [p for p, keys in PEOPLE.items() if tokens & keys] + if len(people) == 1: + return f"{ym} {people[0]} Request Form.pdf", [people[0], "AnnualCreditReport"] + if not people: + return BLANK_REQUEST, ["AnnualCreditReport"] + return None, "more than one person" + people = [p for p, keys in PEOPLE.items() if tokens & keys] + if len(people) != 1: + return None, "no person" if not people else "more than one person" + squashed = re.sub(r"[^a-z]", "", low) + agencies = [a for a in AGENCIES if a.lower() in squashed] + if len(agencies) != 1: + return None, "no agency" if not agencies else "more than one agency" + return f"{ym} {people[0]} {agencies[0]}.pdf", [people[0], agencies[0]] + + +# ---------------------------------------------------------------- plan + + +def build_plan(doc, drive, drivefs_dirs, day): + drivefs = load_drivefs(doc, drivefs_dirs) + files, skipped, unreadable = inventory(doc, drivefs) + cut = cutoff_ts(day) + date = day.isoformat() + arch_base = os.path.join(drive, "Documents Archive") + dupe_base = os.path.join(drive, "probably-dupes", date) + entries = [] + + def entry(section, rel, dst, **extra): + f = files[rel] + e = dict(section=section, src=os.path.join(doc, rel), dst=dst, rel=rel, md5=f["md5"], + size=f["size"], mtime=f["mtime"], dataless=f["dataless"]) + e.update(extra) + entries.append(e) + return e + + # 1. archive units + unit_names = [] + for name in sorted(os.listdir(doc)): + p = os.path.join(doc, name) + if (name.startswith(".") or name.startswith(SNAPSHOT_PREFIX) or name in (CREDIT, "tax", POINTER) + or name in KEEP or os.path.islink(p)): + continue + unit_names.append(name) + tax = os.path.join(doc, "tax") + if os.path.isdir(tax) and not os.path.islink(tax): + for y in sorted(os.listdir(tax)): + if y.startswith(".") or y == TAX_POINTER or os.path.islink(os.path.join(tax, y)): + continue + unit_names.append(f"tax/{y}") + units, archived = [], set() + for u in unit_names: + members = [r for r in files if under(r, u)] + if not members: + continue + # A unit we cannot fully read may hold recent files we cannot see. + if any(under(d, u) for d in unreadable): + continue + newest = max(files[r]["mtime"] for r in members) + if newest >= cut: + continue + units.append(dict(rel=u, newest=newest, files=len(members), bytes=sum(files[r]["size"] for r in members), + tax=u.startswith("tax/"))) + for r in sorted(members): + entry("archive", r, os.path.join(arch_base, r), unit=u) + archived.add(r) + + def in_snapshot(r): + return top(r).startswith(SNAPSHOT_PREFIX) + + local = {r for r in files if r not in archived and not in_snapshot(r)} + archived_units = {u["rel"] for u in units} + + # 2. snapshots + outside_md5 = {files[r]["md5"] for r in local if files[r]["md5"]} + merge_dst = {} + snapshots, needs_review = {}, [] + for host_dir in sorted({top(r) for r in files if in_snapshot(r)}): + s = dict(merge=[], stash=0, stash_bytes=0, superseded=[], review=0, files=0) + snapshots[host_dir] = s + for r in sorted(x for x in files if under(x, host_dir) and x != host_dir): + s["files"] += 1 + f = files[r] + inner = r[len(host_dir) + 1:] + stash_dst = os.path.join(dupe_base, r) + if not f["md5"]: + needs_review.append(r) + s["review"] += 1 + continue + if f["md5"] in outside_md5: + entry("snapshot", r, stash_dst, why="dupe") + s["stash"] += 1 + s["stash_bytes"] += f["size"] + continue + if unit_of(inner) in archived_units: + # Merging into a unit that is leaving would strand the file + # in a half-archived folder. + entry("snapshot", r, stash_dst, why="unit archived") + s["stash"] += 1 + s["stash_bytes"] += f["size"] + continue + if not os.path.lexists(os.path.join(doc, inner)) and inner not in merge_dst: + entry("snapshot-merge", r, os.path.join(doc, inner)) + merge_dst[inner] = f["md5"] + outside_md5.add(f["md5"]) + s["merge"].append(r) + continue + entry("snapshot", r, stash_dst, why="superseded") + s["stash"] += 1 + s["stash_bytes"] += f["size"] + s["superseded"].append(r) + + # 3. dupes (report only) + # Skip any file inside a git working tree: project files repeat by design. + git_cache = {} + + def in_work_tree(r): + parts = r.split("/")[:-1] + for i in range(len(parts) + 1): + d = "/".join(parts[:i]) + if d not in git_cache: + git_cache[d] = os.path.lexists(os.path.join(doc, d, ".git")) + if git_cache[d]: + return True + return False + + def dupe_ok(r): + f = files[r] + if f["size"] < DUPE_MIN_SIZE or not f["md5"]: + return False + for c in r.split("/"): + if c in (".git", "node_modules") or c.endswith(DUPE_EXCLUDE_SUFFIXES): + return False + return not in_work_tree(r) + + groups = {} + for r in local: + if dupe_ok(r): + groups.setdefault(files[r]["md5"], []).append(r) + # Report only, never moved: no rule can tell which copy is in use. + dupe_groups = [] + for md5, members in sorted(groups.items(), key=lambda kv: min(kv[1])): + # Copies inside one unit are deliberate (templates, renamed copies): + # only copies in different units count as dupes. + if len({unit_of(r) for r in members}) < 2: + continue + dupe_groups.append(dict(md5=md5, size=files[members[0]]["size"], copies=sorted(members))) + + # 4. credit + needs_manual, credit = [], [] + cdir = os.path.join(doc, CREDIT) + if os.path.isdir(cdir) and not os.path.islink(cdir): + names = sorted(os.listdir(cdir)) + existing = set(names) + proposals = [] + for n in names: + rel = f"{CREDIT}/{n}" + if n.startswith(".") or n in SKIPNAMES: + continue + if rel not in files: + if os.path.isdir(os.path.join(cdir, n)) and not os.path.islink(os.path.join(cdir, n)): + needs_manual.append((rel, "directory")) + continue + tags = canonical_tags(n) + have = read_tags(os.path.join(cdir, n)) + if tags is not None: + # Retag only when a canonical tag is missing; the user's own + # tags are kept. + if have is not None and merge_tags(have, tags) != have: + proposals.append((n, n, tags, have)) + continue + new, why = parse_credit(n, files[rel]["mtime"]) + if new is None: + needs_manual.append((rel, why)) + continue + proposals.append((n, new, why, have)) + targets = {} + for old, new, _tags, _have in proposals: + targets.setdefault(new, []).append(old) + for old, new, tags, have in proposals: + if old != new and (len(targets[new]) > 1 or new in existing): + needs_manual.append((f"{CREDIT}/{old}", f"would collide on {new!r}")) + continue + e = entry("credit", f"{CREDIT}/{old}", os.path.join(cdir, new), tags=tags, + retag_only=(old == new), old_tags=have) + credit.append(e) + needs_manual.sort() + + return dict( + version=1, + date=date, + created=datetime.datetime.now().isoformat(timespec="seconds"), + cutoff=cutoff_date(day).isoformat(), + doc=doc, + drive=drive, + drivefs=bool(drivefs), + inventory=dict(files=len(files), md5_drivefs=sum(1 for f in files.values() if f["md5_src"] == "drivefs"), + md5_local=sum(1 for f in files.values() if f["md5_src"] == "local"), + md5_unknown=sum(1 for f in files.values() if not f["md5"]), + dataless=sum(1 for f in files.values() if f["dataless"])), + units=units, + snapshots=snapshots, + dupe_groups=dupe_groups, + entries=entries, + skipped=[list(x) for x in skipped], + needs_review=needs_review, + needs_manual=[list(x) for x in needs_manual], + ) + + +# ---------------------------------------------------------------- summary + + +def human(n): + for unit in ("B", "KB", "MB", "GB"): + if abs(n) < 1000 or unit == "GB": + return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}" + n /= 1000 + return str(n) + + +def fmt_ts(ts): + return time.strftime("%Y-%m-%d", time.localtime(ts)) + + +def section_counts(plan): + c = {s: [0, 0] for s in SECTIONS} + for e in plan["entries"]: + if e["section"] not in c: + continue # a section this version no longer applies + c[e["section"]][0] += 1 + c[e["section"]][1] += e["size"] + return c + + +def summary(plan): + out = [] + w = out.append + inv = plan["inventory"] + w(f"docs-tidy plan {plan['date']} (cutoff {plan['cutoff']})") + w(f"Documents: {plan['doc']}") + w(f"Drive: {plan['drive']}") + w(f"Inventory: {inv['files']} files; md5 from DriveFS {inv['md5_drivefs']}, local {inv['md5_local']}," + f" unknown {inv['md5_unknown']}; dataless {inv['dataless']}") + w("") + w("Sections:") + for s, (n, b) in section_counts(plan).items(): + w(f" {s:15} {n:6} files {human(b):>10}") + w(f" {'skipped':15} {len(plan['skipped']):6}") + w(f" {'needs-review':15} {len(plan['needs_review']):6}") + w(f" {'needs-manual':15} {len(plan['needs_manual']):6}") + w("") + w(f"Archive units ({len(plan['units'])}):") + for u in plan["units"]: + w(f" {u['rel']} newest {fmt_ts(u['newest'])} {u['files']} files {human(u['bytes'])}") + w("") + w(f"Snapshots ({len(plan['snapshots'])}):") + for host, s in plan["snapshots"].items(): + w(f" {host}: {s['files']} files; merge {len(s['merge'])}, stash {s['stash']} ({human(s['stash_bytes'])})," + f" of which superseded {len(s['superseded'])}; needs review {s['review']}") + for r in s["merge"]: + w(f" merge {r}") + for r in s["superseded"]: + w(f" superseded {r}") + w("") + w(f"Duplicate files across folders, report only, never moved ({len(plan['dupe_groups'])} groups):") + for g in plan["dupe_groups"]: + w(f" {human(g['size'])} x{len(g['copies'])} {g['md5']}") + for r in g["copies"]: + w(f" {r}") + w("") + credit = [e for e in plan["entries"] if e["section"] == "credit"] + w(f"Credit renames ({len(credit)}):") + for e in credit: + tags = ", ".join(e["tags"]) + if e["retag_only"]: + w(f" retag {os.path.basename(e['src'])} [{tags}]") + else: + w(f" {os.path.basename(e['src'])} -> {os.path.basename(e['dst'])} [{tags}]") + w("") + w(f"Skipped, not regular files ({len(plan['skipped'])}):") + for rel, kind in plan["skipped"]: + w(f" {kind:8} {rel}") + w("") + w(f"Needs review, md5 unknown ({len(plan['needs_review'])}):") + for r in plan["needs_review"]: + w(f" {r}") + w("") + w(f"Needs manual ({len(plan['needs_manual'])}):") + for rel, why in plan["needs_manual"]: + w(f" {rel} ({why})") + return "\n".join(out) + "\n" + + +def post(title, msg): + """Post a macOS notification. Title and message go in as argv, so no + AppleScript string escaping is needed.""" + if IS_MAC and shutil.which("osascript"): + r = subprocess.run(["osascript", "-e", "on run argv", + "-e", "display notification (item 2 of argv) with title (item 1 of argv)", + "-e", "end run", title, msg], check=False, capture_output=True, text=True) + if r.returncode != 0: + log(f"notify failed (osascript exit {r.returncode}): {r.stderr.strip()}; {title}: {msg}") + else: + log(f"notify: {title}: {msg}") + + +def notify(plan): + c = section_counts(plan) + msg = (f"archive {c['archive'][0]}, merge {c['snapshot-merge'][0]}, snapshot {c['snapshot'][0]}," + f" credit {c['credit'][0]}; {len(plan['dupe_groups'])} dupe groups to review. Run: docs-tidy show") + post(f"docs-tidy plan {plan['date']}", msg) + + +# ---------------------------------------------------------------- commands + + +def newest_plan(sd): + hits = sorted(glob.glob(os.path.join(sd, "plan-*.json"))) + if not hits: + raise Refuse(f"no plan in {sd}; run: docs-tidy plan") + return hits[-1] + + +def load_plan(path): + path = path or newest_plan(state_dir()) + with open(path) as f: + return json.load(f), path + + +def cmd_plan(args): + day = today() + doc = doc_root() + if not os.path.isdir(doc): + raise Refuse(f"Documents root missing: {doc}") + plan = build_plan(doc, drive_root(), drivefs_dir(), day) + sd = state_dir() + os.makedirs(sd, exist_ok=True) + base = os.path.join(sd, f"plan-{plan['date']}") + with open(base + ".json", "w") as f: + json.dump(plan, f, indent=1) + f.write("\n") + text = summary(plan) + with open(base + ".txt", "w") as f: + f.write(text) + sys.stdout.write(text) + log(f"\nWrote {base}.json and {base}.txt") + if args.notify: + notify(plan) + return 0 + + +def cmd_show(args): + plan, _ = load_plan(args.plan) + sys.stdout.write(summary(plan)) + return 0 + + +def cleanup_one(d): + """rmdir d when it holds only .DS_Store/.localized; return True if removed. + + Never touches a directory that still holds anything else. + """ + try: + names = os.listdir(d) + except OSError: + return False + if any(n not in SKIPNAMES for n in names): + return False + for n in names: + p = os.path.join(d, n) + if os.path.isfile(p) and not os.path.islink(p): + os.remove(p) + try: + os.rmdir(d) + except OSError: + return False + return True + + +def cleanup_tree(root, stop): + """Bottom-up cleanup of a moved tree, then of its parents up to (not including) stop.""" + stop = os.path.abspath(stop) + root = os.path.abspath(root) + if os.path.isdir(root) and not os.path.islink(root): + for r, _dirs, _names in os.walk(root, topdown=False): + if r != root: + cleanup_one(r) + d = root + else: + d = os.path.dirname(root) + while d != stop and d.startswith(stop + os.sep) and cleanup_one(d): + d = os.path.dirname(d) + + +def move_file(e): + """Copy, verify, delete. Return (status, md5, note); status is OK or ERR.""" + src, dst = e["src"], e["dst"] + if not os.path.lexists(src): + if os.path.exists(dst): + return "OK", e.get("md5") or "", "already done" + return "ERR", e.get("md5") or "", "source missing and destination absent" + st = os.lstat(src) + if not stat.S_ISREG(st.st_mode): + return "ERR", "", "source is no longer a regular file" + h = md5_with_retry(src, is_dataless(st)) + if e.get("md5") and h != e["md5"]: + return "ERR", h, f"source changed since plan ({e['md5']} -> {h})" + if os.path.lexists(dst): + if not os.path.isfile(dst) or md5_file(dst) != h: + return "ERR", h, "destination exists with different content" + os.remove(src) + return "OK", h, "destination already had it" + os.makedirs(os.path.dirname(dst), exist_ok=True) + # dst did not exist (checked above), so any dst after a failure is this + # call's partial copy and is safe to remove. + try: + for attempt in range(4): + try: + shutil.copy2(src, dst) + break + except OSError as ex: + if ex.errno not in (60, errno.ETIMEDOUT) or attempt == 3: + raise + brctl_download(src) + time.sleep(1) + if md5_file(dst) != h: + raise RuntimeError("copy verification failed") + except BaseException: + if os.path.lexists(dst): + os.remove(dst) + raise + os.remove(src) + return "OK", h, "" + + +def unit_newest(doc, unit): + """Return (newest regular-file mtime or None, whether a directory was unreadable).""" + p = os.path.join(doc, unit) + if not os.path.lexists(p): + # Already moved (a re-run): not unreadable; move_file handles it. + return None, False + if os.path.isfile(p) and not os.path.islink(p): + return os.lstat(p).st_mtime, False + ts, bad = [], [] + for r, _d, names in os.walk(p, onerror=bad.append): + for n in names: + if n in SKIPNAMES: + continue + st = os.lstat(os.path.join(r, n)) + if stat.S_ISREG(st.st_mode): + ts.append(st.st_mtime) + return (max(ts) if ts else None), bool(bad) + + +def cmd_apply(args): + plan, path = load_plan(args.plan) + day = today() + skip = set(args.skip or []) + bad = skip - set(SECTIONS) + if bad: + raise Refuse(f"unknown section(s) for --skip: {sorted(bad)}; valid: {SECTIONS}") + plan_day = datetime.date.fromisoformat(plan["date"]) + age = (day - plan_day).days + if age > MAX_PLAN_AGE_DAYS: + raise Refuse(f"plan {path} is {age} days old (max {MAX_PLAN_AGE_DAYS}); re-run: docs-tidy plan") + doc, drive = plan["doc"], plan["drive"] + if not os.path.isdir(drive): + raise Refuse(f"Drive root missing: {drive}") + todo = [e for e in plan["entries"] if e["section"] not in skip] + need = sum(e["size"] for e in todo if e.get("dataless")) + FREE_RESERVE + free = shutil.disk_usage(doc).free + if free < need: + raise Refuse(f"free space {human(free)} on the Documents volume is below {human(need)}" + " (dataless sources plus 2 GiB)") + + cut = cutoff_ts(plan_day) + results = [] # (status, section, md5, src, dst, note) + skipped_units = [] + moved_roots = set() + date = day.isoformat() + credit_log = [] + t0 = time.time() + + for section in SECTIONS: + if section in skip: + log(f"skip section {section}") + continue + ents = [e for e in plan["entries"] if e["section"] == section] + if not ents: + continue + log(f"{section}: {len(ents)} entries") + bad_units = set() + if section == "archive": + for u in sorted({e["unit"] for e in ents}): + n, unreadable = unit_newest(doc, u) + if unreadable: + bad_units.add(u) + skipped_units.append((u, "has an unreadable directory")) + log(f" unit has an unreadable directory, skipped: {u}") + elif n is not None and n >= cut: + bad_units.add(u) + skipped_units.append((u, f"newest {fmt_ts(n)}")) + log(f" unit changed since plan, skipped: {u} (newest {fmt_ts(n)})") + for i, e in enumerate(ents): + if section == "archive" and e["unit"] in bad_units: + continue + try: + if section == "credit": + st, h, note = apply_credit(e, credit_log) + else: + st, h, note = move_file(e) + except Exception as ex: # record and continue; one file must not stop the run + st, h, note = "ERR", e.get("md5") or "", f"{type(ex).__name__}: {ex}" + results.append((st, section, h, e["src"], e["dst"], note)) + if st == "OK" and section != "credit": + if section == "archive": + moved_roots.add(os.path.join(doc, e["unit"])) + elif section in ("snapshot", "snapshot-merge"): + moved_roots.add(os.path.join(doc, top(e["rel"]))) + else: + # The file itself, so cleanup only walks upward from it and + # never sweeps sibling directories nothing moved from. + moved_roots.add(e["src"]) + if st == "ERR": + log(f" ERR {e['src']}: {note}") + if (i + 1) % 500 == 0: + log(f" {i + 1}/{len(ents)} {time.time() - t0:.0f}s") + + # The record of what moved goes to disk first, so a failure in the + # pointer or cleanup steps below cannot lose it. + sd = state_dir() + os.makedirs(sd, exist_ok=True) + with open(os.path.join(sd, f"apply-{date}.tsv"), "a") as f: + for st, section, h, src, dst, note in results: + f.write(f"{st}\t{section}\t{h}\t{src}\t{dst}" + (f"\t{note}" if note else "") + "\n") + if credit_log: + lp = os.path.join(doc, CREDIT, f".rename-log-{date}.tsv") + new = not os.path.exists(lp) + with open(lp, "a") as f: + if new: + f.write("old_name\tnew_name\ttags\n") + for row in credit_log: + f.write("\t".join(row) + "\n") + + ok = [r for r in results if r[0] == "OK"] + errs = [r for r in results if r[0] == "ERR"] + # Credit entries rename in place, so they never belong in the pointer. + moved = [r for r in ok if r[1] != "credit"] + # Pointers first: writing tax/ARCHIVED-YEARS.txt keeps tax/ from being + # removed as empty when its last year moves. + if moved: + write_pointer(plan, moved, date) + for root in sorted(moved_roots, key=len, reverse=True): + cleanup_tree(root, doc) + + by = {} + for st, section, *_ in results: + by.setdefault(section, [0, 0])[0 if st == "OK" else 1] += 1 + log("") + for s in SECTIONS: + if s in by: + log(f"{s:15} ok {by[s][0]:6} err {by[s][1]:6}") + for u, why in skipped_units: + log(f"skipped unit: {u} ({why})") + log(f"done in {time.time() - t0:.0f}s; log: {os.path.join(sd, f'apply-{date}.tsv')}") + if errs: + log(f"\n{len(errs)} error(s):") + for _st, section, _h, src, _dst, note in errs: + log(f" [{section}] {src}: {note}") + return 1 + return 0 + + +def apply_credit(e, credit_log): + src, dst = e["src"], e["dst"] + old, new = os.path.basename(src), os.path.basename(dst) + if not e["retag_only"]: + if not os.path.lexists(src): + if not os.path.exists(dst): + return "ERR", "", "source missing and destination absent" + else: + if os.path.lexists(dst): + return "ERR", "", "destination exists" + if e.get("md5") and md5_file(src) != e["md5"]: + return "ERR", "", "source changed since plan" + os.rename(src, dst) + credit_log.append((old, new, ",".join(e["tags"]))) + elif not os.path.exists(dst): + return "ERR", "", "file missing" + else: + credit_log.append((old, new, ",".join(e["tags"]))) + # The rename is done; a tag failure is a note on an OK result, not an ERR. + have = read_tags(dst) + if have is None: + have = e.get("old_tags") or [] + try: + tagged = write_tags(dst, merge_tags(have, e["tags"])) + except (OSError, subprocess.CalledProcessError) as ex: + return "OK", e.get("md5") or "", f"tags not written: {ex}" + return "OK", e.get("md5") or "", "" if tagged else "tags skipped (not macOS)" + + +def write_pointer(plan, ok, date): + doc, drive = plan["doc"], plan["drive"] + by = {} + for _st, section, _h, src, dst, _n in ok: + by.setdefault(section, []).append((src, dst)) + lines = [f"\nOn {date}, docs-tidy (plan {plan['date']}, cutoff {plan['cutoff']}) moved:\n"] + if "archive" in by: + done_src = {src for src, _ in by["archive"]} + done_units = sorted({e["unit"] for e in plan["entries"] + if e["section"] == "archive" and e["src"] in done_src}) + lines.append(f" archive: {len(by['archive'])} files in {len(done_units)} top-level items, unchanged since" + f" before {plan['cutoff']}, to\n {os.path.join(drive, 'Documents Archive')}\n") + for u in done_units: + lines.append(f" {u}\n") + if "snapshot-merge" in by: + lines.append(f" snapshot-merge: {len(by['snapshot-merge'])} files that existed only in a machine snapshot," + " merged into this folder at the same relative paths\n") + if "snapshot" in by: + lines.append(f" snapshot: {len(by['snapshot'])} machine-snapshot files to\n" + f" {os.path.join(drive, 'probably-dupes', plan['date'])}\n") + lines.append(" Every move was copy, verify checksum, then delete the original.\n") + with open(os.path.join(doc, POINTER), "a") as f: + f.writelines(lines) + done_src = {src for src, _ in by.get("archive", [])} + years = sorted({e["unit"][4:] for e in plan["entries"] + if e["section"] == "archive" and e["unit"].startswith("tax/") and e["src"] in done_src}) + if years: + tdir = os.path.join(doc, "tax") + if os.path.isdir(tdir): + with open(os.path.join(tdir, TAX_POINTER), "a") as f: + f.write(f"\nOn {date}, these tax years moved to Google Drive:\n {', '.join(years)}\n\n" + f"Location:\n {os.path.join(drive, 'Documents Archive', 'tax')}\n") + + +def main(argv=None): + p = argparse.ArgumentParser( + prog="docs-tidy", + description="Quarterly ~/Documents tidy: plan, then approve. `plan` changes nothing on disk except" + " the state dir; nothing moves until you run `apply`.", + epilog="Sections: " + ", ".join(SECTIONS) + ". State dir: $DOCS_TIDY_STATE or ~/.local/state/docs-tidy." + " Other overrides: DOCS_TIDY_DOC, DOCS_TIDY_DRIVE, DOCS_TIDY_DRIVEFS (none disables)," + " DOCS_TIDY_TODAY (YYYY-MM-DD).", + ) + sub = p.add_subparsers(dest="cmd", required=True) + sp = sub.add_parser("plan", help="inventory Documents and write plan-YYYY-MM-DD.{json,txt}") + sp.add_argument("--notify", action="store_true", help="post a macOS notification with the counts") + ss = sub.add_parser("show", help="print the text summary of a plan") + ss.add_argument("plan", nargs="?", help="plan JSON (default: newest in the state dir)") + sa = sub.add_parser("apply", help="execute a plan: copy, verify, delete") + sa.add_argument("plan", nargs="?", help="plan JSON (default: newest in the state dir)") + sa.add_argument("--skip", action="append", metavar="SECTION", choices=SECTIONS, + help="skip a section; repeat for more: " + ", ".join(SECTIONS)) + args = p.parse_args(argv) + # Under launchd nobody reads stderr, so a failed --notify plan must say so. + alert = args.cmd == "plan" and args.notify + try: + return {"plan": cmd_plan, "show": cmd_show, "apply": cmd_apply}[args.cmd](args) + except Refuse as e: + print(f"docs-tidy: refused: {e}", file=sys.stderr) + if alert: + post("docs-tidy plan FAILED", str(e)) + return 2 + except Exception as e: + if alert: + post("docs-tidy plan FAILED", f"{type(e).__name__}: {e}") + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/plans/2026-10-21-docs-tidy-design.md b/docs/plans/2026-10-21-docs-tidy-design.md new file mode 100644 index 0000000..edc55f5 --- /dev/null +++ b/docs/plans/2026-10-21-docs-tidy-design.md @@ -0,0 +1,196 @@ +# docs-tidy: quarterly Documents tidy (design) + +## Why + +On 2026-10-21 `~/Documents` (iCloud-synced, backed up to Google Drive as +machine ASIAGO) was tidied by hand: 9.1 GB of iCloud quota, about 4 GB of it +duplicates. Old items moved to Google Drive, an old machine snapshot was +de-duplicated, and credit reports were renamed to `YYYY-MM Person Agency.pdf` +with Finder tags. This tool repeats that each quarter. + +The run is **plan, then approve**. launchd writes a plan and sends a +notification. Nothing moves until Andrew runs `docs-tidy apply`. + +A launchd spike (2026-10-21) showed an Aqua LaunchAgent running +`/opt/homebrew/bin/python3` can read `~/Documents`, `~/Downloads`, the Drive +mount and the DriveFS databases with no TCC prompt, so the plan step runs +under launchd. + +## Files + +| Path | What | +| --- | --- | +| `docs-tidy.py` | Python entrypoint, mode 755, `#!/usr/bin/env -S python3 -I`. Symlinked as `~/.local/bin/docs-tidy`. | +| `LaunchAgents/com.asiago.docs-tidy.plist` | Runs `docs-tidy plan --notify` at 10:00 on Jan 1, Apr 1, Jul 1, Oct 1. `RunAtLoad` false, Aqua only. | +| `tests/docs-tidy.bats` | Fixture-tree tests; run on Ubuntu CI, so macOS-only calls are guarded. | + +Python, not Bash: the job reads SQLite, hashes files and writes binary +plists. Doing that in Bash would mean shelling out to Python anyway. + +## Commands + +- `docs-tidy plan [--notify]` writes `plan-YYYY-MM-DD.json` and + `plan-YYYY-MM-DD.txt` to the state dir, prints the text summary, and with + `--notify` posts a macOS notification with the counts. +- `docs-tidy show [PLAN]` prints the text summary of the newest plan. +- `docs-tidy apply [PLAN] [--skip SECTION ...]` executes a plan. + +## Configuration (env overrides exist for tests) + +| Setting | Default | Env | +| --- | --- | --- | +| Documents root | `~/Documents` | `DOCS_TIDY_DOC` | +| Drive root | glob `~/Library/CloudStorage/GoogleDrive-*/My Drive` (exactly one match, else error) | `DOCS_TIDY_DRIVE` | +| DriveFS dir | glob `~/Library/Application Support/Google/DriveFS/[0-9]*/` containing `mirror_sqlite.db`; `none` disables | `DOCS_TIDY_DRIVEFS` | +| State dir | `~/.local/state/docs-tidy` | `DOCS_TIDY_STATE` | +| Today | local date | `DOCS_TIDY_TODAY` (`YYYY-MM-DD`) | + +Archive destination: `/Documents Archive/`. +Snapshot stash destination: `/probably-dupes//`. + +`KEEP` is a visible list at the top of the script. These top-level items +never archive: `army`, `DD214.pdf`, `ssn.pdf`, `Andrew Rich 1989 GED.pdf`, +`Issue COE.pdf`, `1Password Emergency Kit A3-KCMDEC-my.pdf`. Plan counts +exclude them, so the counts in the notification are the real counts. + +## Cutoff + +The cutoff is 00:00 local on Jan 1 of `(today.year - 4)`. In Oct 2026 that is +2022-01-01; in Jan 2027 it is 2023-01-01. + +## Inventory + +1. List regular files only (`S_ISREG` on `lstat`). Sockets, FIFOs and + symlinks are reported as skipped and never planned. The 2026-10-21 run + failed on a git `fsmonitor--daemon.ipc` socket. + A directory that cannot be listed (for example mode 000) is reported as + skipped ("unreadable directory"). +2. Skip `.DS_Store` and `.localized` everywhere. +3. MD5 source, in order: + 1. the DriveFS mirror DB (`mirror_item.local_md5_checksum`), when its size + and mtime match `lstat`; + 2. a local hash, when the file is not dataless (macOS `st_flags & + 0x40000000`, `SF_DATALESS`); + 3. otherwise unknown. Never download a file just to plan. + +## Plan sections (applied in this order) + +1. **archive.** Units are the top-level entries of Documents, except + dotfiles, `Documents - *`, `credit-reports`, `tax`, the pointer file and + `KEEP`. Each `tax/` is its own unit. A unit archives when its newest + regular-file mtime is before the cutoff. A unit that contains an + unreadable directory never archives, because it may hold recent files + the plan cannot see. Every file in it maps to + `/Documents Archive/`. +2. **snapshot.** For each `Documents - ` folder, check every file + against the files that stay local after archive: + - If its MD5 matches a file outside the snapshots, stash it. + - If its merge target falls in a unit that this plan archives, stash it + (`why` = "unit archived"). Merging it would leave the file behind in a + folder that is moving to Drive. + - If it is unique and `/` is absent, + merge it there (section `snapshot-merge`). + - If it is unique but that path exists with other content, stash it and + list it as "superseded". + - If its MD5 is unknown, list it under "needs review" and plan nothing. +3. **dupes (report only).** Apply never moves these. Group the files that stay local (not archived, not in + snapshots) by MD5. Group only files of 1 KiB or larger. Exclude: + - any path with a `.git` or `node_modules` component, or a component + ending in `.app`, `.bundle`, `.photoslibrary`, `.rtfd`, `.pages`, + `.numbers`, `.key` or `.workflow`; + - any file inside a git working tree: a directory, at any depth, that + holds a `.git` entry (directory or file). Project files repeat by + design. + + Copies inside the same unit are deliberate (templates, renamed copies + such as `airship.png` and `dirigible.png`). A unit is a top-level entry + of Documents, or `tax/`. A group counts only when its copies span + two or more units. The summary lists every copy in each group. + + Why report only: no rule can tell which copy is the one in use. An + earlier version kept the earliest-mtime copy and stashed the rest. On + the 2026-10-21 dry run that would have kept a write-up in a scratch + `Codex` output folder and stashed the copy in `Beacon Biosignals`, and + taken 7 files out of a deliberately assembled HELOC packet. Real + duplicate clutter came from machine snapshots, which the snapshot + section handles. +4. **credit.** For files in `credit-reports` (not dotfiles) whose name does + not match `^\d{4}-\d{2} (Andrew|Jen) (Equifax|Experian|TransUnion)\.pdf$` + or the request-form names, propose a rename. Use the person and agency + rules from the 2026-10-21 `rename.py`: AR, AMR and Andrew mean Andrew; + JR, JMR, Jen and Jennifer mean Jen. + - The date comes from a `YYYYMMDD`, `YYYY-MM-DD` or `YYYY_MM_DD` in the + name, else the file's mtime. + - Unparseable names, or names that would collide, go to "needs manual". + Nothing is planned for them. + - A canonical name gets a retag-only entry only when one of its canonical + tags (`[Person, Agency]`) is missing. Tag names are compared with the + Finder colour suffix removed (`"Important\n6"` is `Important`). Extra + tags are the user's and never trigger a retag. + +The text summary shows counts and bytes per section, every archive unit with +its newest mtime, every dupe group (all copies), every +snapshot merge, every credit rename, and the skipped, needs-review and +needs-manual lists. + +## Apply + +Each guard below refuses the whole apply with a reason: + +- The plan is more than 14 days old. Fix: re-run `plan`. +- The Drive root is missing. +- Free space on the volume of Documents is below the bytes of the dataless + sources plus 2 GiB. + +Per file: copy, verify, delete. This is the pattern proven on 2026-10-21. + +1. Re-hash the source. If it differs from the planned MD5, record an error + and skip the file. +2. If the destination exists with the same MD5, delete the source. If it + exists with a different MD5, record an error. +3. Copy with `shutil.copy2`, re-hash the destination, and only then delete + the source. If the copy raises or the verification fails, remove the + partial destination (it did not exist before this copy) and record an + error. The source stays. +4. A missing source with an existing destination counts as already done. + +For archive units, first re-check that the unit's newest mtime is still +before the cutoff and that every directory in it can be read. If not, skip +the whole unit and report it. + +Before copying, call `brctl download` on each dataless source (macOS only). +On `ETIMEDOUT` (errno 60), call `brctl download` and retry, up to 3 times. +The 2026-10-21 run lost 9 files to this, and all 9 then copied in 5 s. + +Credit entries: `os.rename`, and record the rename-log row at once. Then +write tags with `xattr -wx com.apple.metadata:_kMDItemUserTags ` (macOS only; on other systems, log that tags were skipped). The tags +written are the file's current tags, in order and with colour suffixes, plus +any missing canonical tag. A tag-write failure is a note on an OK result, +not an error, because the rename already happened. Append one line per +change to `credit-reports/.rename-log-.tsv`. + +After the moves, in this order: + +- Write `apply-.tsv` (OK/ERR, section, md5, src, dst, note) to the + state dir first, so a failure in a later step cannot lose the record. +- Append a dated section to `/ARCHIVED-TO-GOOGLE-DRIVE.txt` + (create it if missing), unless only credit entries changed. When tax + years were archived, also append to `/tax/ARCHIVED-YEARS.txt`. + This runs before cleanup, so `tax/` is not removed as empty when its last + year moves. +- In each source tree that was moved, delete `.DS_Store` and `.localized`. + Then `rmdir` the empty directories, bottom-up. Never use `rm -rf`, and + never touch a directory that still holds other files. + +Exit 0 if every entry succeeded, else exit 1 and print a summary of the +errors. + +## Notifications + +`post(title, message)` runs `osascript -e 'on run argv' -e 'display +notification (item 2 of argv) with title (item 1 of argv)' -e 'end run' +TITLE MESSAGE`. The text goes in as argv, so it needs no AppleScript +escaping. A nonzero `osascript` exit is logged. Off macOS, the text is +printed instead. A `plan --notify` that fails posts "docs-tidy plan FAILED" +with the reason, because nobody reads the launchd log. diff --git a/tests/docs-tidy.bats b/tests/docs-tidy.bats new file mode 100755 index 0000000..467eac1 --- /dev/null +++ b/tests/docs-tidy.bats @@ -0,0 +1,620 @@ +#!/usr/bin/env bats +# bats tests for docs-tidy.py (plan / show / apply) against fixture trees. +# +# Run from repo root: bats tests/docs-tidy.bats +# +# Every env override is set in setup(), so no test can fall back to the real +# ~/Documents, Drive mount, DriveFS DB or state dir. A test that needs a +# different date passes DOCS_TIDY_TODAY per command (`run env ...`). + +setup() { + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:?must be run via bats}")/.." && pwd)" + SCRIPT="${REPO_ROOT}/docs-tidy.py" + DOC="${BATS_TEST_TMPDIR:?}/Documents" + DRIVE="${BATS_TEST_TMPDIR:?}/My Drive" + STATE="${BATS_TEST_TMPDIR:?}/state" + PLAN="${STATE}/plan-2026-10-21.json" + STASH="${DRIVE}/probably-dupes/2026-10-21" + ARCHIVE="${DRIVE}/Documents Archive" + mkdir -p "${DOC}" "${DRIVE}" + export DOCS_TIDY_DOC="${DOC}" + export DOCS_TIDY_DRIVE="${DRIVE}" + export DOCS_TIDY_STATE="${STATE}" + export DOCS_TIDY_DRIVEFS=none + export DOCS_TIDY_TODAY=2026-10-21 +} + +# Restore permissions that a test removed (chmod 000 / 555), so bats can +# clean up its temp dir. +teardown() { + chmod -R u+rwx "${BATS_TEST_TMPDIR:?}" +} + +# is_mac: true on macOS, where Finder tags and osascript exist. +is_mac() { + [[ "${OSTYPE}" == darwin* ]] +} + +# settags FILE TAG...: write Finder tags (macOS only). +settags() { + local file="$1" hex + shift + hex="$(python3 -I -c 'import plistlib, sys; print(plistlib.dumps(sys.argv[1:], fmt=plistlib.FMT_BINARY).hex())' "$@")" + xattr -wx com.apple.metadata:_kMDItemUserTags "${hex}" "${file}" +} + +# want_tags FILE EXPECTED: the file's Finder tags as a Python list repr. +want_tags() { + local hex got + hex="$(xattr -px com.apple.metadata:_kMDItemUserTags "$1")" + got="$(python3 -I -c 'import plistlib, sys; print(plistlib.loads(bytes.fromhex("".join(sys.argv[1].split()))))' "${hex}")" + if [ "${got}" != "$2" ]; then + printf 'tags: %s\n got: %s\nwant: %s\n' "$1" "${got}" "$2" >&2 + return 1 + fi +} + +# mk PATH STAMP [CONTENT]: create a file under DOC with an mtime of STAMP +# (touch -t format, CCYYMMDDhhmm). +mk() { + local path="${DOC}/$1" stamp="$2" content="${3:-content of $1}" + mkdir -p "${path%/*}" + printf '%s' "${content}" >"${path}" + touch -t "${stamp}" "${path}" +} + +# mkbig PATH STAMP CHAR: like mk, with 2 KiB of CHAR as the content, enough +# to pass the 1 KiB dupe threshold. +mkbig() { + local pad + pad="$(printf '%2048s' '')" + mk "$1" "$2" "${pad// /$3}" +} + +# mksock DIR NAME: create a Unix socket. Bind a relative name after cd, since +# macOS caps socket paths at 104 bytes and BATS_TEST_TMPDIR is long there. +mksock() { + (cd "${DOC}/$1" && python3 -I -c 'import socket, sys; socket.socket(socket.AF_UNIX).bind(sys.argv[1])' "$2") +} + +# want_in PLANFILE EXPR EXPECTED: EXPR (Python, plan bound to `p`) must +# print EXPECTED. +want_in() { + local got + got="$(python3 -I -c 'import json, sys; p = json.load(open(sys.argv[1])); print(eval(sys.argv[2]))' "$1" "$2")" + if [ "${got}" != "$3" ]; then + printf 'expr: %s\n got: %s\nwant: %s\n' "$2" "${got}" "$3" >&2 + return 1 + fi +} + +want() { + want_in "${PLAN}" "$1" "$2" +} + +# want_srcs SECTION EXPECTED: the sorted rel paths planned for SECTION. +want_srcs() { + want "sorted(e['rel'] for e in p['entries'] if e['section'] == '$1')" "$2" +} + +# want_file PATH EXPECTED: PATH holds exactly EXPECTED. +want_file() { + local got + got="$(<"$1")" + if [ "${got}" != "$2" ]; then + printf 'file: %s\n got: %s\nwant: %s\n' "$1" "${got}" "$2" >&2 + return 1 + fi +} + +@test "cutoff is Jan 1 four years back: 2026-10-21 -> 2022-01-01" { + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "p['cutoff']" "2022-01-01" +} + +@test "cutoff is Jan 1 four years back: 2027-01-05 -> 2023-01-01" { + run env DOCS_TIDY_TODAY=2027-01-05 "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_in "${STATE}/plan-2027-01-05.json" "p['cutoff']" "2023-01-01" +} + +@test "--help describes the commands" { + run "${SCRIPT}" --help + [ "${status}" -eq 0 ] + [[ "${output}" == *plan* && "${output}" == *show* && "${output}" == *apply* ]] +} + +@test "an old unit archives and a recent unit stays" { + mk "old/a.txt" 201901010000 + mk "old/sub/b.txt" 202012311200 + mk "oldfile.pdf" 201805050000 + mk "new/a.txt" 201901010000 + mk "new/b.txt" 202603010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs archive "['old/a.txt', 'old/sub/b.txt', 'oldfile.pdf']" + want "sorted(u['rel'] for u in p['units'])" "['old', 'oldfile.pdf']" + want "[e['dst'] for e in p['entries'] if e['rel'] == 'old/a.txt'][0]" "${ARCHIVE}/old/a.txt" + [ -f "${STATE}/plan-2026-10-21.txt" ] + [[ "${output}" == *"Archive units (2)"* ]] +} + +@test "KEEP items never archive and are not counted" { + mk "army/dd.pdf" 201001010000 + mk "DD214.pdf" 201001010000 + mk "ssn.pdf" 201001010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "len(p['entries'])" "0" + want "len(p['units'])" "0" +} + +@test "tax years split: an old year archives, a year with one recent file stays" { + mk "tax/2019/w2.pdf" 202002010000 + mk "tax/2021/w2.pdf" 202102010000 + mk "tax/2021/amended.pdf" 202203010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs archive "['tax/2019/w2.pdf']" + want "[u['rel'] for u in p['units']]" "['tax/2019']" +} + +@test "a socket and a symlink are skipped, not planned" { + mk "old/a.txt" 201901010000 + ln -s a.txt "${DOC}/old/link" + mksock old sock + [ -S "${DOC}/old/sock" ] + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs archive "['old/a.txt']" + want "sorted(p['skipped'])" "[['old/link', 'symlink'], ['old/sock', 'socket']]" +} + +@test "snapshot: merge a unique file, stash a dupe, stash a superseded file" { + mk "keep.txt" 202501010000 "same bytes" + mk "conflict.txt" 202501010000 "local version" + mk "Documents - HOST/copy-of-keep.txt" 202001010000 "same bytes" + mk "Documents - HOST/only/here.txt" 202001010000 "unique" + mk "Documents - HOST/conflict.txt" 202001010000 "snapshot version" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs snapshot-merge "['Documents - HOST/only/here.txt']" + want "[e['dst'] for e in p['entries'] if e['section'] == 'snapshot-merge'][0]" "${DOC}/only/here.txt" + want_srcs snapshot "['Documents - HOST/conflict.txt', 'Documents - HOST/copy-of-keep.txt']" + want "p['snapshots']['Documents - HOST']['superseded']" "['Documents - HOST/conflict.txt']" + want "[e['dst'] for e in p['entries'] if e['rel'] == 'Documents - HOST/conflict.txt'][0]" \ + "${STASH}/Documents - HOST/conflict.txt" + # Snapshot folders are never archive units, however old. + want_srcs archive "[]" +} + +@test "dupes: cross-folder copies are reported with every copy, never planned to move" { + mkbig "a/one.bin" 202405010000 x + mkbig "b/two.bin" 202305010000 x + mkbig "c/three.bin" 202305010000 x + mkbig "credit-reports/r.pdf" 202501010000 y + mkbig "other/r.pdf" 202301010000 y + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "[g['copies'] for g in p['dupe_groups']]" \ + "[['a/one.bin', 'b/two.bin', 'c/three.bin'], ['credit-reports/r.pdf', 'other/r.pdf']]" + want "len(p['entries'])" "0" + run "${SCRIPT}" show + [[ "${output}" == *"report only, never moved (2 groups)"* ]] +} + +@test "apply never moves duplicate files" { + mkbig "a/one.bin" 202301010000 x + mkbig "b/c/two.bin" 202401010000 x + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ -f "${DOC}/a/one.bin" ] + [ -f "${DOC}/b/c/two.bin" ] + [ ! -e "${DRIVE}/probably-dupes" ] +} + +@test "dupes: .git paths, small files and bundles are excluded" { + mkbig "repo/.git/objects/x" 202501010000 g + mkbig "other/x" 202501010000 g + mkbig "Thing.app/Contents/x" 202501010000 h + mkbig "loose/x" 202501010000 h + mkbig "Doc.pages/x" 202501010000 i + mkbig "loose2/y" 202501010000 i + mkbig "Search.workflow/Contents/document.wflow" 202501010000 j + mkbig "loose3/document.wflow" 202501010000 j + mk "small1/s.txt" 202501010000 "tiny" + mk "small2/s.txt" 202501010000 "tiny" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "len(p['dupe_groups'])" "0" +} + +@test "dupes: files inside a git working tree are excluded, at any depth" { + # A .git directory at the top of one unit, and a .git file (a worktree or + # submodule) two levels down in another. + mkdir -p "${DOC}/proj/.git" + mkbig "proj/src/data.bin" 202501010000 w + mkbig "outside/data.bin" 202501010000 w + mk "notes/deep/sub/.git" 202501010000 "gitdir: elsewhere" + mkbig "notes/deep/sub/lib/z.bin" 202501010000 z + mkbig "elsewhere/z.bin" 202501010000 z + # Control: the same layout with no .git entry is still a dupe group. + mkbig "plain/src/data.bin" 202501010000 v + mkbig "plain2/data.bin" 202601010000 v + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "[g['copies'] for g in p['dupe_groups']]" "[['plain/src/data.bin', 'plain2/data.bin']]" +} + +@test "dupes: node_modules paths are excluded" { + mkbig "site/node_modules/pkg/index.js" 202501010000 n + mkbig "other/index.js" 202501010000 n + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "len(p['dupe_groups'])" "0" +} + +@test "dupes: copies inside one unit are deliberate and stay" { + mkbig "Resume/_base/template.md" 202301010000 t + mkbig "Resume/_archived/acme/template.md" 202401010000 t + mkbig "icons/airship.png" 202301010000 p + mkbig "icons/dirigible.png" 202401010000 p + mkbig "tax/2024/form.pdf" 202401010000 f + mkbig "tax/2025/form.pdf" 202501010000 f + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + # Only the tax pair spans two units (tax/2024 and tax/2025). + want "[g['copies'] for g in p['dupe_groups']]" "[['tax/2024/form.pdf', 'tax/2025/form.pdf']]" +} + +@test "credit: names are parsed into renames; an unparseable name needs manual" { + mk "credit-reports/20240615 AMR Experian report.pdf" 202406150000 + mk "credit-reports/JR trans union.pdf" 202302100000 + mk "credit-reports/2024-10 Jen Equifax.pdf" 202410010000 + mk "credit-reports/mystery.pdf" 202401010000 + mk "credit-reports/.rename-log-2026-10-21.tsv" 202601010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "sorted((e['rel'], e['dst'].rsplit('/', 1)[1], e['tags']) for e in p['entries'] if e['section'] == 'credit' and not e['retag_only'])" \ + "[('credit-reports/20240615 AMR Experian report.pdf', '2024-06 Andrew Experian.pdf', ['Andrew', 'Experian']), ('credit-reports/JR trans union.pdf', '2023-02 Jen TransUnion.pdf', ['Jen', 'TransUnion'])]" + want "p['needs_manual']" "[['credit-reports/mystery.pdf', 'no person']]" +} + +@test "credit: a rename that would collide needs manual" { + mk "credit-reports/2024-06 Andrew Experian.pdf" 202406010000 + mk "credit-reports/AMR experian.pdf" 202406200000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "[e['rel'] for e in p['entries'] if e['section'] == 'credit' and not e['retag_only']]" "[]" + want "p['needs_manual'][0][0]" "credit-reports/AMR experian.pdf" +} + +@test "apply moves, verifies, deletes the source, rmdirs empty dirs and appends the pointer" { + mk "old/a.txt" 201901010000 "alpha" + mk "old/sub/b.txt" 201901010000 "beta" + mk "old/sub/.DS_Store" 201901010000 "junk" + mk "tax/2018/w2.pdf" 201902010000 "w2" + mk "recent.txt" 202601010000 "stay" + printf 'earlier notes\n' >"${DOC}/ARCHIVED-TO-GOOGLE-DRIVE.txt" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + want_file "${ARCHIVE}/old/a.txt" "alpha" + want_file "${ARCHIVE}/old/sub/b.txt" "beta" + want_file "${ARCHIVE}/tax/2018/w2.pdf" "w2" + [ ! -e "${DOC}/old" ] + [ ! -e "${DOC}/tax/2018" ] + [ -f "${DOC}/recent.txt" ] + run head -1 "${DOC}/ARCHIVED-TO-GOOGLE-DRIVE.txt" + [ "${output}" = "earlier notes" ] + grep -q "On 2026-10-21, docs-tidy" "${DOC}/ARCHIVED-TO-GOOGLE-DRIVE.txt" + grep -q "2018" "${DOC}/tax/ARCHIVED-YEARS.txt" + run grep -c '^OK' "${STATE}/apply-2026-10-21.tsv" + [ "${output}" = "3" ] + run grep -c '^ERR' "${STATE}/apply-2026-10-21.tsv" + [ "${output}" = "0" ] +} + +@test "apply keeps a directory that still holds other files" { + mk "old/a.txt" 201901010000 "alpha" + mksock old sock + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ ! -e "${DOC}/old/a.txt" ] + [ -S "${DOC}/old/sock" ] +} + +@test "apply merges and stashes a snapshot" { + mk "keep.txt" 202501010000 "same bytes" + mk "Documents - HOST/copy.txt" 202001010000 "same bytes" + mk "Documents - HOST/only/here.txt" 202001010000 "unique" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + want_file "${DOC}/only/here.txt" "unique" + [ -f "${STASH}/Documents - HOST/copy.txt" ] + [ ! -e "${DOC}/Documents - HOST" ] +} + +@test "apply refuses a plan older than 14 days" { + mk "old/a.txt" 201901010000 + run env DOCS_TIDY_TODAY=2026-10-01 "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run env DOCS_TIDY_TODAY=2026-10-20 "${SCRIPT}" apply "${STATE}/plan-2026-10-01.json" + [ "${status}" -eq 2 ] + [[ "${output}" == *"19 days old"* ]] + [ -f "${DOC}/old/a.txt" ] + [ ! -e "${ARCHIVE}" ] +} + +@test "apply refuses when the Drive root is missing" { + mk "old/a.txt" 201901010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + rmdir "${DRIVE}" + run "${SCRIPT}" apply + [ "${status}" -eq 2 ] + [[ "${output}" == *"Drive root missing"* ]] + [ -f "${DOC}/old/a.txt" ] +} + +@test "plan --notify posts a FAILED notification when plan refuses" { + # A stub osascript records the call, so no real notification pops up. On + # Linux the script logs the notification instead; accept either. + local bin="${BATS_TEST_TMPDIR:?}/bin" + mkdir -p "${bin}" + printf '#!/bin/sh\nprintf "%%s\\n" "$*" >>"%s/osascript.log"\n' "${BATS_TEST_TMPDIR}" >"${bin}/osascript" + chmod +x "${bin}/osascript" + rmdir "${DOC}" + run env PATH="${bin}:${PATH}" "${SCRIPT}" plan --notify + [ "${status}" -eq 2 ] + local seen="${output}" + if [[ -f "${BATS_TEST_TMPDIR}/osascript.log" ]]; then + seen="$(<"${BATS_TEST_TMPDIR}/osascript.log")" + fi + [[ "${seen}" == *"docs-tidy plan FAILED"* ]] +} + +@test "apply records an error and keeps the source when it changed after planning" { + mkbig "a/keep.bin" 202301010000 x + mkbig "Documents - HOST/a/keep.bin" 202401010000 x + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs snapshot "['Documents - HOST/a/keep.bin']" + printf 'edited' >>"${DOC}/Documents - HOST/a/keep.bin" + run "${SCRIPT}" apply + [ "${status}" -eq 1 ] + [ -f "${DOC}/Documents - HOST/a/keep.bin" ] + [ ! -e "${STASH}/Documents - HOST/a/keep.bin" ] + grep -q '^ERR snapshot' "${STATE}/apply-2026-10-21.tsv" + [[ "${output}" == *"source changed since plan"* ]] +} + +@test "apply skips an archive unit that changed after planning" { + mk "old/a.txt" 201901010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + mk "old/new.txt" 202605010000 + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ -f "${DOC}/old/a.txt" ] + [[ "${output}" == *"unit changed since plan, skipped: old"* ]] +} + +@test "a missing source with an existing destination counts as done" { + mk "old/a.txt" 201901010000 "alpha" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + mkdir -p "${ARCHIVE}/old" + cp -p "${DOC}/old/a.txt" "${ARCHIVE}/old/a.txt" + rm "${DOC}/old/a.txt" + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + grep -q "^OK archive .*already done" "${STATE}/apply-2026-10-21.tsv" +} + +@test "apply errors when the destination exists with other content" { + mk "old/a.txt" 201901010000 "alpha" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + mkdir -p "${ARCHIVE}/old" + printf 'different' >"${ARCHIVE}/old/a.txt" + run "${SCRIPT}" apply + [ "${status}" -eq 1 ] + [ -f "${DOC}/old/a.txt" ] + want_file "${ARCHIVE}/old/a.txt" "different" +} + +@test "apply renames credit reports and writes the rename log" { + mk "credit-reports/20240615 AMR Experian report.pdf" 202406150000 "pdf" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ -f "${DOC}/credit-reports/2024-06 Andrew Experian.pdf" ] + [ ! -e "${DOC}/credit-reports/20240615 AMR Experian report.pdf" ] + grep -q "20240615 AMR Experian report.pdf 2024-06 Andrew Experian.pdf Andrew,Experian" \ + "${DOC}/credit-reports/.rename-log-2026-10-21.tsv" + # A credit-only apply moved nothing to Drive, so it writes no pointer. + [ ! -e "${DOC}/ARCHIVED-TO-GOOGLE-DRIVE.txt" ] +} + +@test "--skip repeats to skip more than one section" { + mk "old/a.txt" 201901010000 + mk "credit-reports/20240615 AMR Experian report.pdf" 202406150000 "pdf" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply --skip credit --skip archive + [ "${status}" -eq 0 ] + [ -f "${DOC}/old/a.txt" ] + [ -f "${DOC}/credit-reports/20240615 AMR Experian report.pdf" ] +} + +@test "--skip dupes is rejected: dupes are report only" { + run "${SCRIPT}" apply --skip dupes + [ "${status}" -ne 0 ] + [[ "${output}" == *"invalid choice"* ]] +} + +@test "--skip rejects an unknown section" { + run "${SCRIPT}" apply --skip bogus + [ "${status}" -ne 0 ] + [[ "${output}" == *"invalid choice"* ]] +} + +@test "show prints the newest plan" { + mk "old/a.txt" 201901010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" show + [ "${status}" -eq 0 ] + [[ "${output}" == *"docs-tidy plan 2026-10-21"* ]] + [[ "${output}" == *"old newest 2019-01-01"* ]] +} + +@test "credit retag: only a missing canonical tag triggers it, and the user's tags are kept" { + if ! is_mac; then skip "Finder tags need macOS"; fi + mk "credit-reports/2024-10 Jen Equifax.pdf" 202410010000 + mk "credit-reports/2024-06 Andrew Experian.pdf" 202406010000 + settags "${DOC}/credit-reports/2024-10 Jen Equifax.pdf" $'Important\n6' Jen + # Canonical tags present (one with a colour suffix) plus a user tag: no retag. + settags "${DOC}/credit-reports/2024-06 Andrew Experian.pdf" Andrew $'Experian\n4' Mine + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want "[e['rel'] for e in p['entries'] if e['section'] == 'credit']" "['credit-reports/2024-10 Jen Equifax.pdf']" + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + want_tags "${DOC}/credit-reports/2024-10 Jen Equifax.pdf" "['Important\\n6', 'Jen', 'Equifax']" + want_tags "${DOC}/credit-reports/2024-06 Andrew Experian.pdf" "['Andrew', 'Experian\\n4', 'Mine']" +} + +@test "credit rename keeps the file's existing tags and adds the canonical ones" { + if ! is_mac; then skip "Finder tags need macOS"; fi + mk "credit-reports/AMR experian.pdf" 202406200000 + settags "${DOC}/credit-reports/AMR experian.pdf" $'Important\n6' + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + want_tags "${DOC}/credit-reports/2024-06 Andrew Experian.pdf" "['Important\\n6', 'Andrew', 'Experian']" +} + +@test "credit: a tag write failure is a note on an OK rename, and the rename is logged" { + mk "credit-reports/AMR experian.pdf" 202406200000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + # Read-only file: the rename still works (the directory is writable) but + # xattr cannot write the tags. + chmod 444 "${DOC}/credit-reports/AMR experian.pdf" + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ -f "${DOC}/credit-reports/2024-06 Andrew Experian.pdf" ] + grep -q "AMR experian.pdf 2024-06 Andrew Experian.pdf Andrew,Experian" \ + "${DOC}/credit-reports/.rename-log-2026-10-21.tsv" + grep -q "^OK credit" "${STATE}/apply-2026-10-21.tsv" + if is_mac; then + grep -q "^OK credit.*tags not written" "${STATE}/apply-2026-10-21.tsv" + fi +} + +@test "snapshot: a file whose unit is being archived is stashed, not merged" { + mk "old/a.txt" 201901010000 "alpha" + mk "Documents - HOST/old/extra.txt" 201801010000 "only in snapshot" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs archive "['old/a.txt']" + want_srcs snapshot-merge "[]" + want "[e['why'] for e in p['entries'] if e['section'] == 'snapshot']" "['unit archived']" +} + +@test "notify passes title and message to osascript as argv and logs a failure" { + mkdir -p "${BATS_TEST_TMPDIR}/bin" + cat >"${BATS_TEST_TMPDIR}/bin/osascript" <<'SH' +#!/usr/bin/env bash +printf '<%s>\n' "$@" >"${BATS_TEST_TMPDIR}/osascript.args" +exit 3 +SH + chmod +x "${BATS_TEST_TMPDIR}/bin/osascript" + run env PATH="${BATS_TEST_TMPDIR}/bin:${PATH}" "${SCRIPT}" plan --notify + [ "${status}" -eq 0 ] + if is_mac; then + grep -qx '' "${BATS_TEST_TMPDIR}/osascript.args" + grep -qx '' "${BATS_TEST_TMPDIR}/osascript.args" + grep -q '^.tsv before the pointer step, so a pointer failure keeps the record" { + mk "old/a.txt" 201901010000 "alpha" + mk "credit-reports/20240615 AMR Experian report.pdf" 202406150000 "pdf" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + # A directory where the pointer file should be makes write_pointer fail. + mkdir "${DOC}/ARCHIVED-TO-GOOGLE-DRIVE.txt" + run "${SCRIPT}" apply + [ "${status}" -ne 0 ] + grep -q "^OK archive .*old/a.txt" "${STATE}/apply-2026-10-21.tsv" + # The credit rename log is written before the pointer step too. + grep -q "2024-06 Andrew Experian.pdf" "${DOC}/credit-reports/.rename-log-2026-10-21.tsv" +} + +@test "re-applying a finished plan counts moved archive units as done" { + mk "old/a.txt" 201901010000 "alpha" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ ! -e "${DOC}/old" ] + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [[ "${output}" != *"unreadable"* ]] +} + +@test "an unreadable directory is skipped and its unit never archives" { + if [[ "${EUID}" -eq 0 ]]; then skip "root can read a chmod 000 directory"; fi + mk "old/a.txt" 201901010000 + mk "old/locked/b.txt" 201901010000 + mk "old2/a.txt" 201901010000 + chmod 000 "${DOC}/old/locked" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + want_srcs archive "['old2/a.txt']" + want "p['skipped']" "[['old/locked', 'unreadable directory']]" +} + +@test "apply skips a unit that gained an unreadable directory after planning" { + if [[ "${EUID}" -eq 0 ]]; then skip "root can read a chmod 000 directory"; fi + mk "old/a.txt" 201901010000 + mk "old/sub/b.txt" 201901010000 + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + chmod 000 "${DOC}/old/sub" + run "${SCRIPT}" apply + [ "${status}" -eq 0 ] + [ -f "${DOC}/old/a.txt" ] + [[ "${output}" == *"unit has an unreadable directory, skipped: old"* ]] +} + +@test "a failed copy records an error, keeps the source and leaves no destination" { + if [[ "${EUID}" -eq 0 ]]; then skip "root can write a read-only directory"; fi + mk "old/a.txt" 201901010000 "alpha" + run "${SCRIPT}" plan + [ "${status}" -eq 0 ] + mkdir -p "${ARCHIVE}/old" + chmod 555 "${ARCHIVE}/old" + run "${SCRIPT}" apply + [ "${status}" -eq 1 ] + want_file "${DOC}/old/a.txt" "alpha" + [ ! -e "${ARCHIVE}/old/a.txt" ] + grep -q "^ERR archive" "${STATE}/apply-2026-10-21.tsv" +}