diff --git a/.github/contracts/linker_relink_timeouts_v1.json b/.github/contracts/linker_relink_timeouts_v1.json new file mode 100644 index 000000000..354ca914b --- /dev/null +++ b/.github/contracts/linker_relink_timeouts_v1.json @@ -0,0 +1,13 @@ +{ + "contractVersion": 1, + "workflow": "relink.yml", + "timeouts": { + "relink": { + "kaggle": 90, + "local": 2880, + "server": 480 + }, + "resolve": 480, + "publish": 30 + } +} diff --git a/.github/host/otzaria-pipeline.tmpfiles.conf b/.github/host/otzaria-pipeline.tmpfiles.conf new file mode 100644 index 000000000..5e4239d59 --- /dev/null +++ b/.github/host/otzaria-pipeline.tmpfiles.conf @@ -0,0 +1,6 @@ +# Install on every shared heavy host as /etc/tmpfiles.d/otzaria-pipeline.conf. +d /run/lock/otzaria 1777 root root - +f /run/lock/otzaria/host-heavy.lock 0666 root root - +# Linker writes linker-ner.scope.json dynamically as 0644. The sticky +# directory preserves its ownership; this repo only removes it via sudo after +# verifying the complete /proc identity while holding host-heavy.lock. diff --git a/.github/scripts/assemble_split_asset.sh b/.github/scripts/assemble_split_asset.sh new file mode 100755 index 000000000..21e056593 --- /dev/null +++ b/.github/scripts/assemble_split_asset.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Join the parts split_release_asset.sh wrote, verifying every part and the whole. +# A copy of Otzaria/otzaria tool/release/assemble_split_asset.sh; change both together. +set -euo pipefail + +manifest=${1:?usage: assemble_split_asset.sh [output-file]} +[ -f "$manifest" ] || { echo "manifest not found: $manifest" >&2; exit 1; } + +manifest_dir=$(cd "$(dirname "$manifest")" && pwd) +archive_name=$(jq -er '.archive' "$manifest") +[ "$archive_name" = "$(basename "$archive_name")" ] || { + echo "unsafe archive name in manifest: $archive_name" >&2 + exit 1 +} +output=${2:-$manifest_dir/$archive_name} +[ ! -e "$output" ] || { echo "output already exists: $output" >&2; exit 1; } + +expected_count=$(jq -er '.parts | length' "$manifest") +[ "$expected_count" -gt 0 ] || { echo "manifest contains no parts" >&2; exit 1; } + +hash_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +while IFS=$'\t' read -r part_name expected_sha; do + [ "$part_name" = "$(basename "$part_name")" ] || { + echo "unsafe part name in manifest: $part_name" >&2 + exit 1 + } + part="$manifest_dir/$part_name" + [ -f "$part" ] || { echo "missing part: $part_name" >&2; exit 1; } + actual_sha=$(hash_file "$part") + [ "$actual_sha" = "$expected_sha" ] || { + echo "checksum mismatch: $part_name" >&2 + exit 1 + } +done < <(jq -r '.parts[] | [.name, .sha256] | @tsv' "$manifest") + +touch "$output" +for part_name in $(jq -r '.parts[].name' "$manifest"); do + cat "$manifest_dir/$part_name" >> "$output" +done + +expected_archive_sha=$(jq -er '.sha256' "$manifest") +actual_archive_sha=$(hash_file "$output") +[ "$actual_archive_sha" = "$expected_archive_sha" ] || { + echo "archive checksum mismatch: $output" >&2 + exit 1 +} +echo "Reassembled and verified: $output" diff --git a/.github/scripts/blacklist/books_blacklist_check.py b/.github/scripts/blacklist/books_blacklist_check.py new file mode 100644 index 000000000..590d734ae --- /dev/null +++ b/.github/scripts/blacklist/books_blacklist_check.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Check books_blacklist.txt against the titles the importer will actually see. + +The importer drops a Sefaria book when normalizeTitleKey(heTitle or enTitle) +equals the key of a blacklist line (SefariaBlacklists.kt). Nothing complains +when a line stops matching: Sefaria renames a book, or a line was typed with +different punctuation ("תלמוד מהו" vs "תלמוד מהו?"), and the book silently +comes back unless the author blacklist happens to catch it too. Ten lines had +drifted like that by October 2026. + +Two title sources are compared: + +* export: titles.json of the latest Otzaria/SefariaExport release. That is + the dump the next build imports, so a line must match it. +* live: sefaria.org/api/index. A rename shows up here first, and the export + follows days later. In October 2026 "Learning to Read Midrash" was + "לפשוטו של מדרש" live and still "ללמוד לקרוא מדרש" in the export. + +books_blacklist_titles.tsv (next to this script) maps every line to the +English title it blocked when it last matched. The English title is the +book's key in Sefaria and survives Hebrew renames, so a renamed book is found +for certain even after both sources moved on. + +With --fix, only certain changes are written: + +* add the book's current Hebrew title (export or live) under a line that + blocks it by an old one. The old line stays: it is harmless, and it still + holds if Sefaria reverts. +* replace a line that matches nothing but differs from exactly one title only + in punctuation or spacing (these never matched anything). +* drop a later line whose key duplicates an earlier one. + +Anything else is reported with close-match suggestions and fails the run. +Exit codes: 0 clean (possibly after fixes), 1 lines need a decision, +2 a title source could not be fetched (nothing is written). +""" + +import argparse +import difflib +import json +import os +import re +import sys +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +BLACKLIST = ROOT / "generator/sefariasqlite/src/jvmMain/resources/books_blacklist.txt" +LOCK = Path(__file__).resolve().parent / "books_blacklist_titles.tsv" +EXPORT_TITLES_URL = "https://github.com/Otzaria/SefariaExport/releases/latest/download/titles.json" +LIVE_INDEX_URL = "https://www.sefaria.org/api/index/" +# Both sources list ~6,600 books. A response far below that is truncated, and +# checking against it would call half the blacklist stale. +MIN_TITLES = 5000 + +LOCK_HEADER = ( + "# Maintained by books_blacklist_check.py: blacklist line the Sefaria English\n" + "# title it blocked when it last matched. Used to follow Hebrew renames.\n" +) + + +def normalize_key(value): + """normalizeTitleKey from SefariaImportText.kt, character for character.""" + if value is None or not value.strip(): + return None + for quote in ('"', "'", "׳", "״"): + value = value.replace(quote, "") + return re.sub(r"\s+", " ", value.lower()).replace("_", " ").strip() + + +def loose_key(value): + """Letters and digits only: what a punctuation-only typo cannot change.""" + key = normalize_key(value) or "" + return re.sub(r"[\W_]+", "", key) + + +def unescape(value): + return value.replace('\\"', '"').replace("\\'", "'") + + +@dataclass +class Entry: + index: int # position in the file's line list + title: str # as the importer reads it (trimmed, unescaped) + + @property + def key(self): + return normalize_key(self.title) + + +def parse_blacklist(text): + entries = [] + for i, raw in enumerate(text.split("\n")): + line = raw.removeprefix("").strip() + if line and not line.startswith("#"): + entries.append(Entry(i, unescape(line))) + return entries + + +@dataclass +class Result: + adds: list = field(default_factory=list) # (entry, new title, source) + replaces: list = field(default_factory=list) # (entry, new title) + duplicates: list = field(default_factory=list) # (entry, earlier entry) + unresolved: list = field(default_factory=list) # (entry, [suggestions]) + obsolete: list = field(default_factory=list) # (entry, en title) + skipped_paths: list = field(default_factory=list) + lock: dict = field(default_factory=dict) + + @property + def changed(self): + return bool(self.adds or self.replaces or self.duplicates) + + +def analyze(entries, export_titles, live_titles, lock): + """export_titles / live_titles: {English title: Hebrew title}.""" + by_key = {} + for source in (export_titles, live_titles): + for en, he in source.items(): + for title in (en, he): + k = normalize_key(title) + if k: + by_key.setdefault(k, en) + hebrew_titles = sorted({he for s in (export_titles, live_titles) for he in s.values() if he}) + by_loose = {} + for he in hebrew_titles: + by_loose.setdefault(loose_key(he), set()).add(he) + + result = Result() + seen = {} + covered = {e.key for e in entries if e.key} + for entry in entries: + if "/" in entry.title or "\\" in entry.title: + result.skipped_paths.append(entry) + continue + if entry.key in seen: + result.duplicates.append((entry, seen[entry.key])) + continue + seen[entry.key] = entry + + en = by_key.get(entry.key) + if en is None and entry.title in lock and (lock[entry.title] in export_titles or lock[entry.title] in live_titles): + en = lock[entry.title] + if en is not None: + result.lock[entry.title] = en + if normalize_key(en) in covered: + continue # the English line blocks it under any Hebrew name + for source_name, source in (("export", export_titles), ("live", live_titles)): + he = source.get(en) + if he and normalize_key(he) not in covered: + result.adds.append((entry, he, source_name)) + covered.add(normalize_key(he)) + if entry.key not in by_key: + result.obsolete.append((entry, en)) + continue + + candidates = by_loose.get(loose_key(entry.title), set()) + if len(candidates) == 1: + new = next(iter(candidates)) + if normalize_key(new) in covered: + result.duplicates.append((entry, seen.get(normalize_key(new), entry))) + else: + result.replaces.append((entry, new)) + covered.add(normalize_key(new)) + en = by_key.get(normalize_key(new)) + if en: + result.lock[new] = en + continue + + result.unresolved.append((entry, suggest(entry.title, hebrew_titles))) + return result + + +def suggest(title, hebrew_titles, limit=3): + target = loose_key(title) + loose = {} + for he in hebrew_titles: + loose.setdefault(loose_key(he), he) + picks = [loose[k] for k in difflib.get_close_matches(target, list(loose), n=limit, cutoff=0.6)] + if len(target) >= 6: + for k, he in loose.items(): + if len(k) >= 6 and (k in target or target in k) and he not in picks: + picks.append(he) + return picks[:limit] + + +def apply_fixes(text, result): + lines = text.split("\n") + drop = {entry.index for entry, _ in result.duplicates} + replace = {entry.index: new for entry, new in result.replaces} + after = {} + for entry, new, _ in result.adds: + after.setdefault(entry.index, []).append(new) + out = [] + for i, line in enumerate(lines): + if i in drop: + continue + out.append(replace.get(i, line)) + out.extend(after.get(i, [])) + return "\n".join(out) + + +def render_lock(entries_text, lock): + rows = [f"{e.title}\t{lock[e.title]}" for e in parse_blacklist(entries_text) if e.title in lock] + return LOCK_HEADER + "".join(row + "\n" for row in rows) + + +def read_lock(path): + lock = {} + if path.exists(): + for line in path.read_text(encoding="utf-8").splitlines(): + if line and not line.startswith("#") and "\t" in line: + he, en = line.split("\t", 1) + lock[he] = en + return lock + + +def fetch_json(url): + request = urllib.request.Request(url, headers={"User-Agent": "seforim-library-ci/1.0"}) + with urllib.request.urlopen(request, timeout=120) as response: + return json.loads(response.read().decode("utf-8")) + + +def fetch_live_titles(): + titles = {} + + def walk(node): + if isinstance(node, list): + for item in node: + walk(item) + elif isinstance(node, dict): + if "contents" in node: + walk(node["contents"]) + elif node.get("title"): + titles[node["title"]] = node.get("heTitle") or "" + + walk(fetch_json(LIVE_INDEX_URL)) + return titles + + +def render_report(result, export_count, live_count): + out = [f"Checked against {export_count} export titles and {live_count} live Sefaria titles.", ""] + if result.adds or result.replaces or result.duplicates: + out += ["### Fixed", ""] + for entry, new, source in result.adds: + out.append(f"- added `{new}` under `{entry.title}` (the book's current title, {source})") + for entry, new in result.replaces: + out.append(f"- `{entry.title}` → `{new}` (punctuation/spacing only)") + for entry, earlier in result.duplicates: + out.append(f"- removed duplicate `{entry.title}` (same key as `{earlier.title}`)") + out.append("") + if result.unresolved: + out += ["### Needs a decision", "", + "These lines match no book in the export or on Sefaria, so they block nothing.", "", + "| line | suggestions |", "| --- | --- |"] + for entry, picks in result.unresolved: + shown = "
".join(f"`{p}`" for p in picks) or "no close title" + out.append(f"| `{entry.title}` (line {entry.index + 1}) | {shown} |") + out.append("") + if result.obsolete: + out += ["### Old names kept", ""] + for entry, en in result.obsolete: + out.append(f"- `{entry.title}` no longer matches; the book ({en}) is blocked by its current title") + out.append("") + if result.skipped_paths: + out.append(f"{len(result.skipped_paths)} path line(s) not checked.") + if not (result.changed or result.unresolved): + out.append("All lines match a book.") + return "\n".join(out).rstrip() + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--fix", action="store_true", help="write the certain fixes and the lock file") + parser.add_argument("--report", type=Path, help="write the Markdown report here") + args = parser.parse_args() + + try: + export_titles = fetch_json(EXPORT_TITLES_URL) + live_titles = fetch_live_titles() + except Exception as e: # noqa: BLE001 - any network/decoding failure means: do not judge + print(f"::error::could not fetch a title source: {e}") + return 2 + if not isinstance(export_titles, dict) or len(export_titles) < MIN_TITLES or len(live_titles) < MIN_TITLES: + print(f"::error::title source looks truncated (export {len(export_titles)}, live {len(live_titles)})") + return 2 + + text = BLACKLIST.read_text(encoding="utf-8") + old_lock = read_lock(LOCK) + result = analyze(parse_blacklist(text), export_titles, live_titles, old_lock) + report = render_report(result, len(export_titles), len(live_titles)) + print(report) + if args.report: + args.report.write_text(report, encoding="utf-8") + + changed = False + if args.fix: + new_text = apply_fixes(text, result) if result.changed else text + lock = dict(old_lock) + lock.update(result.lock) + new_lock = render_lock(new_text, lock) + if new_text != text: + BLACKLIST.write_text(new_text, encoding="utf-8") + changed = True + if not LOCK.exists() or LOCK.read_text(encoding="utf-8") != new_lock: + LOCK.write_text(new_lock, encoding="utf-8") + changed = True + + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"changed={'true' if changed else 'false'}\n") + f.write(f"unresolved={len(result.unresolved)}\n") + return 1 if result.unresolved else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/blacklist/books_blacklist_titles.tsv b/.github/scripts/blacklist/books_blacklist_titles.tsv new file mode 100644 index 000000000..7c6abda80 --- /dev/null +++ b/.github/scripts/blacklist/books_blacklist_titles.tsv @@ -0,0 +1,235 @@ +# Maintained by books_blacklist_check.py: blacklist line the Sefaria English +# title it blocked when it last matched. Used to follow Hebrew renames. +הגיוני עוזיאל Hegyonei Uziel +שיג ושיח; קריאות חדשות בפרשת השבוע, בראשית Covenant and Conversation; Genesis; The Book of the Beginnings +שיג ושיח; קריאות חדשות בפרשת השבוע, שמות Covenant and Conversation; Exodus; The Book of Redemption +שיג ושיח; קריאות חדשות בפרשת השבוע, ויקרא Covenant and Conversation; Leviticus; The Book of Holiness +שיג ושיח; קריאות חדשות בפרשת השבוע, במדבר Covenant and Conversation; Numbers; The Wilderness Years +שיג ושיח; קריאות חדשות בפרשת השבוע, דברים Covenant and Conversation; Deuteronomy; Renewal of the Sinai Covenant +מאמרים על אתיקה; קריאות חדשות בפרשת השבוע Essays in Ethics; A Weekly Reading of the Jewish Bible +אני מאמין; קריאה שבועית בפרשת השבוע I Believe; A Weekly Reading of the Jewish Bible +שיעורים במנהיגות; קריאות חדשות בפרשת השבוע Lessons in Leadership; A Weekly Reading of the Jewish Bible +שיעורים ברוחניות; קריאות חדשות בפרשת השבוע Studies in Spirituality; A Weekly Reading of the Jewish Bible +ביאור שטיינזלץ, הקדמות לתנך Steinsaltz Introductions to Tanakh +ביאור שטיינזלץ על בראשית Steinsaltz on Genesis +ביאור שטיינזלץ על שמות Steinsaltz on Exodus +ביאור שטיינזלץ על ויקרא Steinsaltz on Leviticus +ביאור שטיינזלץ על במדבר Steinsaltz on Numbers +ביאור שטיינזלץ על דברים Steinsaltz on Deuteronomy +ביאור שטיינזלץ על יהושע Steinsaltz on Joshua +ביאור שטיינזלץ על שופטים Steinsaltz on Judges +ביאור שטיינזלץ על שמואל א Steinsaltz on I Samuel +ביאור שטיינזלץ על שמואל ב Steinsaltz on II Samuel +ביאור שטיינזלץ על מלכים א Steinsaltz on I Kings +ביאור שטיינזלץ על מלכים ב Steinsaltz on II Kings +ביאור שטיינזלץ על ישעיהו Steinsaltz on Isaiah +ביאור שטיינזלץ על ירמיהו Steinsaltz on Jeremiah +ביאור שטיינזלץ על יחזקאל Steinsaltz on Ezekiel +ביאור שטיינזלץ על הושע Steinsaltz on Hosea +ביאור שטיינזלץ על יואל Steinsaltz on Joel +ביאור שטיינזלץ על עמוס Steinsaltz on Amos +ביאור שטיינזלץ על עובדיה Steinsaltz on Obadiah +ביאור שטיינזלץ על יונה Steinsaltz on Jonah +ביאור שטיינזלץ על מיכה Steinsaltz on Micah +ביאור שטיינזלץ על נחום Steinsaltz on Nahum +ביאור שטיינזלץ על חבקוק Steinsaltz on Habakkuk +ביאור שטיינזלץ על צפניה Steinsaltz on Zephaniah +ביאור שטיינזלץ על חגי Steinsaltz on Haggai +ביאור שטיינזלץ על זכריה Steinsaltz on Zechariah +ביאור שטיינזלץ על מלאכי Steinsaltz on Malachi +ביאור שטיינזלץ על תהילים Steinsaltz on Psalms +ביאור שטיינזלץ על משלי Steinsaltz on Proverbs +ביאור שטיינזלץ על איוב Steinsaltz on Job +ביאור שטיינזלץ על שיר השירים Steinsaltz on Song of Songs +ביאור שטיינזלץ על רות Steinsaltz on Ruth +ביאור שטיינזלץ על איכה Steinsaltz on Lamentations +ביאור שטיינזלץ על קהלת Steinsaltz on Ecclesiastes +ביאור שטיינזלץ על אסתר Steinsaltz on Esther +ביאור שטיינזלץ על דניאל Steinsaltz on Daniel +ביאור שטיינזלץ על עזרא Steinsaltz on Ezra +ביאור שטיינזלץ על נחמיה Steinsaltz on Nehemiah +ביאור שטיינזלץ על דברי הימים א Steinsaltz on I Chronicles +ביאור שטיינזלץ על דברי הימים ב Steinsaltz on II Chronicles +התורה: פירושן של נשים The Torah; A Women's Commentary +לפשוטו של מדרש Learning to Read Midrash +ללמוד לקרוא מדרש Learning to Read Midrash +ביאור שטיינזלץ, הקדמות למשנה תורה Steinsaltz Introductions to Mishneh Torah +ביאור שטיינזלץ על משנה תורה, הלכות יסודי התורה Steinsaltz on Mishneh Torah, Foundations of the Torah +ביאור שטיינזלץ על משנה תורה, הלכות דעות Steinsaltz on Mishneh Torah, Human Dispositions +ביאור שטיינזלץ על משנה תורה, הלכות תלמוד תורה Steinsaltz on Mishneh Torah, Torah Study +ביאור שטיינזלץ על משנה תורה, הלכות עבודה זרה וחוקות הגויים Steinsaltz on Mishneh Torah, Foreign Worship and Customs of the Nations +ביאור שטיינזלץ על משנה תורה, הלכות תשובה Steinsaltz on Mishneh Torah, Repentance +ביאור שטיינזלץ על משנה תורה, הלכות קריאת שמע Steinsaltz on Mishneh Torah, Reading the Shema +ביאור שטיינזלץ על משנה תורה, הלכות תפילה וברכת כהנים Steinsaltz on Mishneh Torah, Prayer and the Priestly Blessing +ביאור שטיינזלץ על משנה תורה, הלכות תפילין ומזוזה וספר תורה Steinsaltz on Mishneh Torah, Tefillin, Mezuzah and the Torah Scroll +ביאור שטיינזלץ על משנה תורה, הלכות ציצית Steinsaltz on Mishneh Torah, Fringes +ביאור שטיינזלץ על משנה תורה, הלכות ברכות Steinsaltz on Mishneh Torah, Blessings +ביאור שטיינזלץ על משנה תורה, הלכות מילה Steinsaltz on Mishneh Torah, Circumcision +ביאור שטיינזלץ על משנה תורה, סדר התפילה Steinsaltz on Mishneh Torah, The Order of Prayer +ביאור שטיינזלץ על משנה תורה, הלכות שבת Steinsaltz on Mishneh Torah, Sabbath +ביאור שטיינזלץ על משנה תורה, הלכות עירובין Steinsaltz on Mishneh Torah, Eruvin +ביאור שטיינזלץ על משנה תורה, הלכות שביתת עשור Steinsaltz on Mishneh Torah, Rest on the Tenth of Tishrei +ביאור שטיינזלץ על משנה תורה, הלכות שביתת יום טוב Steinsaltz on Mishneh Torah, Rest on a Holiday +ביאור שטיינזלץ על משנה תורה, הלכות חמץ ומצה Steinsaltz on Mishneh Torah, Leavened and Unleavened Bread +ביאור שטיינזלץ על משנה תורה, הלכות שופר וסוכה ולולב Steinsaltz on Mishneh Torah, Shofar, Sukkah and Lulav +ביאור שטיינזלץ על משנה תורה, הלכות שקלים Steinsaltz on Mishneh Torah, Sheqel Dues +ביאור שטיינזלץ על משנה תורה, הלכות קידוש החודש Steinsaltz on Mishneh Torah, Sanctification of the New Month +ביאור שטיינזלץ על משנה תורה, הלכות תעניות Steinsaltz on Mishneh Torah, Fasts +ביאור שטיינזלץ על משנה תורה, הלכות מגילה וחנוכה Steinsaltz on Mishneh Torah, Scroll of Esther and Hanukkah +ביאור שטיינזלץ על משנה תורה, הלכות אישות Steinsaltz on Mishneh Torah, Marriage +ביאור שטיינזלץ על משנה תורה, הלכות גירושין Steinsaltz on Mishneh Torah, Divorce +ביאור שטיינזלץ על משנה תורה, הלכות יבום וחליצה Steinsaltz on Mishneh Torah, Levirate Marriage and Release +ביאור שטיינזלץ על משנה תורה, הלכות נערה בתולה Steinsaltz on Mishneh Torah, Virgin Maiden +ביאור שטיינזלץ על משנה תורה הלכות סוטה Steinsaltz on Mishneh Torah, Woman Suspected of Infidelity +ביאור שטיינזלץ על משנה תורה, הלכות איסורי ביאה Steinsaltz on Mishneh Torah, Forbidden Intercourse +ביאור שטיינזלץ על משנה תורה, הלכות מאכלות אסורות Steinsaltz on Mishneh Torah, Forbidden Foods +ביאור שטיינזלץ על משנה תורה, הלכות שחיטה Steinsaltz on Mishneh Torah, Ritual Slaughter +ביאור שטיינזלץ על משנה תורה, הלכות שבועות Steinsaltz on Mishneh Torah, Oaths +ביאור שטיינזלץ על משנה תורה, הלכות נדרים Steinsaltz on Mishneh Torah, Vows +ביאור שטיינזלץ על משנה תורה, הלכות נזירות Steinsaltz on Mishneh Torah, Nazariteship +ביאור שטיינזלץ על משנה תורה, הלכות ערכים וחרמין Steinsaltz on Mishneh Torah, Appraisals and Devoted Property +ביאור שטיינזלץ על משנה תורה, הלכות כלאים Steinsaltz on Mishneh Torah, Diverse Species +ביאור שטיינזלץ על משנה תורה, הלכות מתנות עניים Steinsaltz on Mishneh Torah, Gifts to the Poor +ביאור שטיינזלץ על משנה תורה, הלכות תרומות Steinsaltz on Mishneh Torah, Heave Offerings +ביאור שטיינזלץ על משנה תורה, הלכות מעשרות Steinsaltz on Mishneh Torah, Tithes +ביאור שטיינזלץ על משנה תורה, הלכות מעשר שני ונטע רבעי Steinsaltz on Mishneh Torah, Second Tithes and Fourth Year's Fruit +ביאור שטיינזלץ על משנה תורה, הלכות ביכורים ושאר מתנות כהונה שבגבולין Steinsaltz on Mishneh Torah, First Fruits and other Gifts to Priests Outside the Sanctuary +ביאור שטיינזלץ על משנה תורה, הלכות שמיטה ויובל Steinsaltz on Mishneh Torah, Sabbatical Year and the Jubilee +ביאור שטיינזלץ על משנה תורה, הלכות בית הבחירה Steinsaltz on Mishneh Torah, The Chosen Temple +ביאור שטיינזלץ על משנה תורה, הלכות כלי המקדש והעובדין בו Steinsaltz on Mishneh Torah, Vessels of the Sanctuary and Those who Serve Therein +ביאור שטיינזלץ על משנה תורה, הלכות ביאת מקדש Steinsaltz on Mishneh Torah, Admission into the Sanctuary +ביאור שטיינזלץ על משנה תורה, הלכות איסורי המזבח Steinsaltz on Mishneh Torah, Things Forbidden on the Altar +ביאור שטיינזלץ על משנה תורה, הלכות מעשה הקרבנות Steinsaltz on Mishneh Torah, Sacrificial Procedure +ביאור שטיינזלץ על משנה תורה, הלכות תמידים ומוספין Steinsaltz on Mishneh Torah, Daily Offerings and Additional Offerings +ביאור שטיינזלץ על משנה תורה, הלכות פסולי המוקדשין Steinsaltz on Mishneh Torah, Sacrifices Rendered Unfit +ביאור שטיינזלץ על משנה תורה, הלכות עבודת יום הכפורים Steinsaltz on Mishneh Torah, Service on the Day of Atonement +ביאור שטיינזלץ על משנה תורה, הלכות מעילה Steinsaltz on Mishneh Torah, Trespass +ביאור שטיינזלץ על משנה תורה, הלכות קרבן פסח Steinsaltz on Mishneh Torah, Paschal Offering +ביאור שטיינזלץ על משנה תורה, הלכות חגיגה Steinsaltz on Mishneh Torah, Festival Offering +ביאור שטיינזלץ על משנה תורה, הלכות בכורות Steinsaltz on Mishneh Torah, Firstlings +ביאור שטיינזלץ על משנה תורה, הלכות שגגות Steinsaltz on Mishneh Torah, Offerings for Unintentional Transgressions +ביאור שטיינזלץ על משנה תורה, הלכות מחוסרי כפרה Steinsaltz on Mishneh Torah, Offerings for Those with Incomplete Atonement +ביאור שטיינזלץ על משנה תורה, הלכות תמורה Steinsaltz on Mishneh Torah, Substitution +ביאור שטיינזלץ על משנה תורה, הלכות טומאת מת Steinsaltz on Mishneh Torah, Defilement by a Corpse +ביאור שטיינזלץ על משנה תורה, הלכות פרה אדומה Steinsaltz on Mishneh Torah, Red Heifer +ביאור שטיינזלץ על משנה תורה, הלכות טומאת צרעת Steinsaltz on Mishneh Torah, Defilement by Leprosy +ביאור שטיינזלץ על משנה תורה, הלכות מטמאי משכב ומושב Steinsaltz on Mishneh Torah, Those Who Defile Bed or Seat +ביאור שטיינזלץ על משנה תורה, הלכות שאר אבות הטומאות Steinsaltz on Mishneh Torah, Other Sources of Defilement +ביאור שטיינזלץ על משנה תורה, הלכות טומאת אוכלים Steinsaltz on Mishneh Torah, Defilement of Foods +ביאור שטיינזלץ על משנה תורה, הלכות כלים Steinsaltz on Mishneh Torah, Vessels +ביאור שטיינזלץ על משנה תורה, הלכות מקואות Steinsaltz on Mishneh Torah, Immersion Pools +ביאור שטיינזלץ על משנה תורה, הלכות נזקי ממון Steinsaltz on Mishneh Torah, Damages to Property +ביאור שטיינזלץ על משנה תורה, הלכות גניבה Steinsaltz on Mishneh Torah, Theft +ביאור שטיינזלץ על משנה תורה, הלכות גזילה ואבידה Steinsaltz on Mishneh Torah, Robbery and Lost Property +ביאור שטיינזלץ על משנה תורה, הלכות חובל ומזיק Steinsaltz on Mishneh Torah, One Who Injures a Person or Property +ביאור שטיינזלץ על משנה תורה, הלכות רוצח ושמירת נפש Steinsaltz on Mishneh Torah, Murderer and the Preservation of Life +ביאור שטיינזלץ על משנה תורה, הלכות מכירה Steinsaltz on Mishneh Torah, Sales +ביאור שטיינזלץ על משנה תורה, הלכות זכייה ומתנה Steinsaltz on Mishneh Torah, Ownerless Property and Gifts +ביאור שטיינזלץ על משנה תורה, הלכות שכנים Steinsaltz on Mishneh Torah, Neighbors +ביאור שטיינזלץ על משנה תורה, הלכות שלוחין ושותפין Steinsaltz on Mishneh Torah, Agents and Partners +ביאור שטיינזלץ על משנה תורה, הלכות עבדים Steinsaltz on Mishneh Torah, Slaves +ביאור שטיינזלץ על משנה תורה, הלכות שכירות Steinsaltz on Mishneh Torah, Hiring +ביאור שטיינזלץ על משנה תורה, הלכות שאלה ופיקדון Steinsaltz on Mishneh Torah, Borrowing and Deposit +ביאור שטיינזלץ על משנה תורה, הלכות מלווה ולווה Steinsaltz on Mishneh Torah, Creditor and Debtor +ביאור שטיינזלץ על משנה תורה, הלכות טוען ונטען Steinsaltz on Mishneh Torah, Plaintiff and Defendant +ביאור שטיינזלץ על משנה תורה, הלכות נחלות Steinsaltz on Mishneh Torah, Inheritances +ביאור שטיינזלץ על משנה תורה, הלכות סנהדרין והעונשין המסורין להם Steinsaltz on Mishneh Torah, The Sanhedrin and the Penalties within their Jurisdiction +ביאור שטיינזלץ על משנה תורה, הלכות עדות Steinsaltz on Mishneh Torah, Testimony +ביאור שטיינזלץ על משנה תורה, הלכות ממרים Steinsaltz on Mishneh Torah, Rebels +ביאור שטיינזלץ על משנה תורה, הלכות אבל Steinsaltz on Mishneh Torah, Mourning +ביאור שטיינזלץ על משנה תורה, הלכות מלכים ומלחמות Steinsaltz on Mishneh Torah, Kings and Wars +מדריך להלכות טהרת המשפחה A Lifetime Companion to the Laws of Jewish Family Life +פניני הלכה, טהרת המשפחה Peninei Halakhah, Family Purity +פניני הלכה, אמונה ומצוותיה Peninei Halakhah, Emunah Umitzvoteha +פניני הלכה, גיור Peninei Halakhah, Giyur +ההלכה כוחה ותפקידה HaHalakhah; Kochah VeTafkidah +תנאי בנישואין ובגט Tenai BeNissuin Uveget +השפעה; הלימוד המרכזי של הקבלה Giving; The Essential Teaching of the Kabbalah +הגדת הרב יונתן זקס The Jonathan Sacks Haggadah +מחלוקת לשם שמים Arguments for the Sake of Heaven +מועדים לשיחה; קריאות חדשות בחגי ישראל Ceremony and Celebration; Introduction to the Holidays +משבר וברית; מחשבה יהודית מודרנית ופוסט מודרנית Crisis and Covenant; Jewish Thought after the Holocaust +מסורת בעידן לא מסורתי Tradition in an Untraditional Age +פסח על שום מה? הגדת הרב יונתן זקס; מאמרים The Jonathan Sacks Haggadah; Essays +תורת אמך Torah of the Mothers +שיג ושיח מהדורה משפחתית Covenant and Conversation Family Edition +שיג ושיח; מהדורה עברית Covenant and Conversation; Hebrew Edition +רעיונות משני חיים; קריאות חדשות בפרשת השבוע Judaism's Life Changing Ideas; A Weekly Reading of the Jewish Bible +בלשון עתיד; חזון ליהודים וליהדות בתרבות הגלובלית Future Tense; A Vision for Jews and Judaism in the Global Culture +משנת ארץ ישראל, מבוא Mishnat Eretz Yisrael, Preface +ספרי זוטא; מדרשה של לוד Sifrei Zuta; Midrashah shel Lod +תורת ההגיון בהלכה Torat HaHiggayon BaHalakhah +מורה נבוכי הזמן Moreh Nevukhei HaZeman +חשבון הנפש Cheshbon HaNefesh +מלחמות השם יחיא קאפח The Wars of God +אגרת אריסטיאס Letter of Aristeas +פירוש ישראלי חדש על פרקי אבות A New Israeli Commentary on Pirkei Avot +איש כלבבו; שמואל Ish Kilvavo; on Samuel +איש לשבטו; שופטים Ish Leshivto; on Judges +מעבור הארץ; יהושע MeAvur HaAretz; on Joshua +אדם מן האדמה Adam Min HaAdamah +אוהב גר Ohev Ger +ביאור ישר על התורה Reggio on Torah +היפותטיקה Hypothetica +מדרשי פילון The Midrash of Philo +על שכל אדם ישר הוא בן חורין Every Good Man is Free +קאסוטו על בראשית Cassuto on Genesis +קאסוטו על שמות Cassuto on Exodus +תְּפִלַּת מְנַשֶׁה Prayer of Manasseh +תלמוד מהו? What is the Talmud +חכמת שלמה The Wisdom of Solomon +מעשה טוביה Book of Tobit +ספר מקבים ב The Book of Maccabees II +צוואות השבטים The Testaments of the Twelve Patriarchs +תהילים קנא Psalm 151 +תהלים קנד Psalm 154 +נגד פלאקוס Against Flaccus +בפרדס החסידות והקבלה BePardes HaChasidut VeHakabbalah +בין השיטין על הלכה ואגדה Bein HaShitin on Halakhah and Aggadah +גילוי וכיסוי בלשון Revealment and Concealment in Language +גן נעול Gan Naul +דברי שלום ואמת Words of Peace and Truth +דרשוני כרך א Dirshuni I +דרשוני כרך ב Dirshuni II +הלכה ואגדה Halakhah and Aggadah +זכור ושמור Zakhor VeShamor +ספר יסודי התורה Sefer Yesodei HaTorah +עלי באר על גילוי וכיסוי בלשון Ali Be'er on Revealment and Concealment in Language +פירושים על גילוי וכיסוי בלשון Commentaries on Revealment and Concealment in Language +שיחות על פרשת השבוע (ליבוביץ) Talks of Yeshayahu Leibowitz on the Parasha +יין לבנון על אבות Yein Levanon on Avot +קראתי בכל לב Karati Bekhol Lev +עמנואל הרומי על אסתר Immanuel of Rome on Esther +קול דודי דופק Kol Dodi Dofek +אם למקרא Em LaMikra +נשים בדיון הלכתי Jewish Legal Writings by Women +השותפות הגדולה; הדת, המדע והחיפוש אחר משמעות The Great Partnership; God, Science, and the Search for Meaning +אם הבנים שמחה Em HaBanim Semecha +משפטי עוזיאל Mishpetei Uziel +נוסח הכתובה Ketubah Text +סידור אשכנז Siddur Ashkenaz +ספר היובלים Book of Jubilees +אמונה בעתיד Faith in the Future +הבית שאנו בונים ביחד; בנייה מחדש של החברה The Home We Build Together; Recreating Society +רדיקלית אז, רדיקלית עכשיו Radical Then, Radical Now +לא בשם האל; אל מול האלימות הדתית Not in God's Name; Confronting Religious Violence +לכבוד השוני; כיצד נוכל למנוע את התנגשות התרבויות? The Dignity of Difference; How to Avoid the Clash of Civilizations +לרפא עולם שבור; החיים כקריאה לאחריות To Heal a Fractured World; The Ethics of Responsibility +עומדות שוב בסיני; יהדות מנקודת מבט פמיניסטית Standing Again at Sinai; Judaism from a Feminist Perspective +קהילה של אמונה Community of Faith +חומש קהת; ביאורי חסידות The Kehot Chumash; A Chasidic Commentary +שות במראה הבזק חלק עשירי B'Mareh HaBazak Volume X +שות במראה הבזק חלק שני B'Mareh HaBazak Volume II +עמנואל הרומי על משלי Immanuel of Rome on Proverbs +שיעורים במנהיגות; מהדורה עברית Lessons in Leadership; Hebrew Edition +הערות שוליים על אורות Footnotes on Orot +יזכור Yizkor +ביאור שטיינזלץ על קידושין Steinsaltz on Kiddushin +עין אי"ה Ein Ayah +שד"ל על בראשית Shadal on Genesis +ספר פוע"ה Sefer Puah +קול התור Kol HaTor +חידושי רמב"ן על כתובות Chiddushei Ramban on Ketubot +מעשה רוקח על המשנה Maaseh Rokeach on Mishnah +מהר"י ברונא Mahari Bruna +תיבת גמא Tevat Gome diff --git a/.github/scripts/blacklist/test_books_blacklist_check.py b/.github/scripts/blacklist/test_books_blacklist_check.py new file mode 100644 index 000000000..aa2c8f40e --- /dev/null +++ b/.github/scripts/blacklist/test_books_blacklist_check.py @@ -0,0 +1,101 @@ +"""Offline tests for books_blacklist_check.py (no network).""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import books_blacklist_check as check # noqa: E402 + + +def run(text, export, live, lock=None): + result = check.analyze(check.parse_blacklist(text), export, live, lock or {}) + return result, check.apply_fixes(text, result) + + +class NormalizeKeyTest(unittest.TestCase): + def test_matches_the_kotlin_rules(self): + self.assertEqual(check.normalize_key(' שו"ת מהר״ם_א '), "שות מהרם א") + self.assertEqual(check.normalize_key("Ish Kilvavo"), "ish kilvavo") + self.assertIsNone(check.normalize_key(" ")) + + def test_keeps_punctuation_the_importer_keeps(self): + self.assertNotEqual(check.normalize_key("תלמוד מהו"), check.normalize_key("תלמוד מהו?")) + + +class AnalyzeTest(unittest.TestCase): + EXPORT = {"Learning to Read Midrash": "ללמוד לקרוא מדרש", "What is the Talmud": "תלמוד מהו?"} + LIVE = {"Learning to Read Midrash": "לפשוטו של מדרש", "What is the Talmud": "תלמוד מהו?"} + + def test_clean_file_is_left_alone(self): + text = "# c\nללמוד לקרוא מדרש\nלפשוטו של מדרש\nתלמוד מהו?\n" + result, fixed = run(text, self.EXPORT, self.LIVE) + self.assertFalse(result.changed) + self.assertEqual(result.unresolved, []) + self.assertEqual(fixed, text) + + def test_live_rename_adds_the_new_title_and_keeps_the_old(self): + result, fixed = run("ללמוד לקרוא מדרש\n", self.EXPORT, self.LIVE) + self.assertEqual(fixed, "ללמוד לקרוא מדרש\nלפשוטו של מדרש\n") + self.assertEqual(result.lock, {"ללמוד לקרוא מדרש": "Learning to Read Midrash"}) + + def test_title_missing_from_the_export_is_restored(self): + # The October 2026 case: the line was moved to the live name while the + # export (= the next build) still carried the old one. + _, fixed = run("לפשוטו של מדרש\n", self.EXPORT, self.LIVE) + self.assertEqual(fixed, "לפשוטו של מדרש\nללמוד לקרוא מדרש\n") + + def test_rename_in_both_sources_is_followed_through_the_lock(self): + moved = {"Learning to Read Midrash": "לפשוטו של מדרש"} + lock = {"ללמוד לקרוא מדרש": "Learning to Read Midrash"} + result, fixed = run("ללמוד לקרוא מדרש\n", moved, moved, lock) + self.assertEqual(fixed, "ללמוד לקרוא מדרש\nלפשוטו של מדרש\n") + self.assertEqual([e.title for e, _ in result.obsolete], ["ללמוד לקרוא מדרש"]) + self.assertEqual(result.unresolved, []) + + def test_english_line_needs_no_hebrew_alias(self): + result, _ = run("Learning to Read Midrash\n", self.EXPORT, self.LIVE) + self.assertFalse(result.changed) + + def test_punctuation_typo_is_replaced(self): + result, fixed = run("א\nתלמוד מהו\nב\n", self.EXPORT, self.LIVE) + self.assertEqual(fixed, "א\nתלמוד מהו?\nב\n") + self.assertEqual([t for _, t in result.replaces], ["תלמוד מהו?"]) + + def test_duplicate_key_is_dropped(self): + result, fixed = run('תלמוד מהו?\nתלמוד מהו?\n', self.EXPORT, self.LIVE) + self.assertEqual(fixed, "תלמוד מהו?\n") + self.assertEqual(len(result.duplicates), 1) + + def test_unknown_line_is_reported_with_suggestions_not_changed(self): + export = {"Ish Kilvavo; on Samuel": "איש כלבבו; שמואל"} + text = "הערות על איש כלבבו; שמואל\n" + result, fixed = run(text, export, export) + self.assertEqual(fixed, text) + self.assertEqual(len(result.unresolved), 1) + self.assertIn("איש כלבבו; שמואל", result.unresolved[0][1]) + + def test_lock_pointing_at_a_vanished_book_does_not_resolve(self): + result, _ = run("ספר שנמחק\n", self.EXPORT, self.LIVE, {"ספר שנמחק": "Gone Book"}) + self.assertEqual(len(result.unresolved), 1) + + def test_path_lines_are_skipped(self): + result, _ = run("תנך/ספר\n", self.EXPORT, self.LIVE) + self.assertEqual(len(result.skipped_paths), 1) + self.assertEqual(result.unresolved, []) + + def test_escaped_quotes_are_read_like_the_importer(self): + export = {"Ein Ayah": 'עין אי"ה'} + result, _ = run('עין אי\\"ה\n', export, export) + self.assertEqual(result.unresolved, []) + + +class LockTest(unittest.TestCase): + def test_lock_follows_file_order_and_drops_removed_lines(self): + text = "ב\nא\n" + lock = {"א": "A", "ב": "B", "ג": "C"} + self.assertEqual(check.render_lock(text, lock), check.LOCK_HEADER + "ב\tB\nא\tA\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/bootstrap_host_lock.sh b/.github/scripts/bootstrap_host_lock.sh new file mode 100644 index 000000000..8b88814d1 --- /dev/null +++ b/.github/scripts/bootstrap_host_lock.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Idempotently provision the shared cross-repository flock on a self-hosted host. +set -euo pipefail + +source_config="${1:?tmpfiles source config is required}" +destination=/etc/tmpfiles.d/otzaria-pipeline.conf +expected=$'d /run/lock/otzaria 1777 root root -\nf /run/lock/otzaria/host-heavy.lock 0666 root root -' +actual="$(awk 'NF && $1 !~ /^#/ {print}' "$source_config")" +[ "$actual" = "$expected" ] || { + echo "::error::refusing to install an unexpected host-lock tmpfiles policy" + exit 2 +} + +if ! sudo test -f "$destination" || ! sudo cmp -s "$source_config" "$destination"; then + sudo install -o root -g root -m 0644 "$source_config" "$destination" +fi +sudo systemd-tmpfiles --create "$destination" + +lock=/run/lock/otzaria/host-heavy.lock +test -d /run/lock/otzaria +test -f "$lock" && test ! -L "$lock" && test -w "$lock" +[ "$(stat -c '%a:%U:%G' /run/lock/otzaria)" = '1777:root:root' ] +[ "$(stat -c '%a:%U:%G' "$lock")" = '666:root:root' ] +echo "durable host lease provisioned: $lock" diff --git a/.github/scripts/bounded_docker_run.sh b/.github/scripts/bounded_docker_run.sh new file mode 100644 index 000000000..fba7e8a6a --- /dev/null +++ b/.github/scripts/bounded_docker_run.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# bounded_run — build-library-index.yml's only way to run a container. +# +# A bound that reaches only the docker CLIENT is half a bound. SIGTERM +# is forwarded to PID 1 of the container — a non-interactive bash, +# which installs no handler and, as PID 1, is not killed by default +# action — and the SIGKILL that --kill-after sends afterwards goes to +# the client and cannot be proxied. The step would fail loudly while +# the container kept running, kept holding -v "$WORK":/work, and kept +# writing into an index the job had already given up on. So every run +# records its id and the bound disposes of it explicitly. +bounded_run() { + local bound="$1" tag="$2"; shift 2 + local cid="$WORK/cid-$tag" + local rc=0 + rm -f "$cid" + timeout --kill-after=60 "$bound" docker run --cidfile "$cid" "$@" || rc=$? + [ "$rc" -ne 0 ] && [ -s "$cid" ] || return "$rc" + # timeout exits 124 (TERM) or 137 (KILL) when the bound fired; any other failure is the container's own. + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "::warning::the bound of $bound expired; removing the container it left behind ($(cat "$cid"))" + elif docker inspect "$(cat "$cid")" > /dev/null 2>&1; then + echo "::warning::container $(cat "$cid") outlived its failed run; removing it" + else + return "$rc" + fi + docker rm -f "$(cat "$cid")" > /dev/null 2>&1 \ + || echo "::warning::that container could not be removed — it may still be running" + return "$rc" +} diff --git a/.github/scripts/check_note_body_engine.py b/.github/scripts/check_note_body_engine.py new file mode 100644 index 000000000..4dec91fb4 --- /dev/null +++ b/.github/scripts/check_note_body_engine.py @@ -0,0 +1,51 @@ +"""Refuses to index a DB whose line_note bodies are zstd frames with an engine that cannot read them. + +Usage: check_note_body_engine.py + +From the release that compresses line_note.body (schema 7, compressLineContent), only +otzaria_search_engine 5.0.1 and later decode a BLOB body; 5.0.0 skips it, and the index +would hold every book without its notes, stamped with an unchanged contentHash. +""" + +import re +import sqlite3 +import sys + +FIRST_DECODING_ENGINE = (5, 0, 1) + + +def blob_bodies(db_path): + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + try: + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'line_note'").fetchone(): + return False + return conn.execute("SELECT EXISTS(SELECT 1 FROM line_note WHERE typeof(body) = 'blob')").fetchone()[0] == 1 + finally: + conn.close() + + +def main(argv): + if len(argv) != 3: + print(__doc__.strip().splitlines()[2], file=sys.stderr) + return 2 + db_path, version = argv[1], argv[2] + if not blob_bodies(db_path): + print("line_note bodies are not compressed; any engine reads them") + return 0 + match = re.fullmatch(r"([0-9]+)\.([0-9]+)\.([0-9]+)", version) + if not match: + print(f"::error::otzaria_search_engine version '{version}' is not MAJOR.MINOR.PATCH; " + "cannot prove it decodes compressed line_note bodies") + return 1 + found = tuple(int(part) for part in match.groups()) + required = ".".join(map(str, FIRST_DECODING_ENGINE)) + if found < FIRST_DECODING_ENGINE: + print(f"::error::the database keeps line_note bodies as zstd frames, which otzaria_search_engine " + f"{version} skips; the index needs {required} or later") + return 1 + print(f"line_note bodies are zstd frames; otzaria_search_engine {version} decodes them (>= {required})") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/scripts/db_asset_names.sh b/.github/scripts/db_asset_names.sh new file mode 100644 index 000000000..ee807a01f --- /dev/null +++ b/.github/scripts/db_asset_names.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# The one definition of the full-DB release asset name. Sourced, not run. +# +# `seforim.db.zst` is reserved for DB schema <= 5 forever: released updaters, the +# first-install path and old app builds match that exact name, so a schema they +# cannot open must never appear under it. From schema 6 on the full DB ships as +# `seforim-schema.db.zst`, which those clients never see. +# validate_build_provenance.py:full_db_asset_name mirrors this (a test pins both). + +LEGACY_FULL_DB_ASSET=seforim.db.zst + +full_db_asset_name() { # -> the release asset name of that full DB + local schema="$1" + case "$schema" in + ''|*[!0-9]*|0*) + echo "::error::full_db_asset_name: db schema '$schema' is not a positive integer" >&2 + return 1 + ;; + esac + if [ "$schema" -ge 6 ]; then + printf 'seforim-schema%s.db.zst' "$schema" + else + printf '%s' "$LEGACY_FULL_DB_ASSET" + fi +} + +# jq over a release's JSON (`.assets[]`): the full-DB asset with the highest +# schema, the legacy name counting as schema 0; null when the release has none. +# A split DB (.part-NNN + .manifest.json) yields the archive name, .manifest +# and no .size/.digest; a single file wins over a split one of the same schema. +FULL_DB_ASSET_JQ='[.assets[] + | if (.name | test("^(seforim|seforim-schema[1-9][0-9]*)\\.db\\.zst\\.manifest\\.json$")) + then . + {name: (.name | rtrimstr(".manifest.json")), manifest: .name, size: null, digest: null} + else . end + | select(.name == "seforim.db.zst" or (.name | test("^seforim-schema[1-9][0-9]*\\.db\\.zst$")))] + | sort_by([(if .name == "seforim.db.zst" then 0 else (.name | ltrimstr("seforim-schema") | rtrimstr(".db.zst") | tonumber) end), + (if .manifest then 0 else 1 end)]) + | last' + +# Split only above 1.9 GiB, into 1900 MiB parts: GitHub refuses an asset of 2 GiB. +FULL_DB_SPLIT_THRESHOLD="${FULL_DB_SPLIT_THRESHOLD:-2040109465}" +FULL_DB_PART_SIZE="${FULL_DB_PART_SIZE:-1992294400}" +DB_ASSET_SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +stage_full_db() { # : the archive itself, or its parts + manifest when too big + local archive="$1" stage="$2" size work + size=$(stat -c '%s' "$archive" 2>/dev/null || stat -f '%z' "$archive") || return 1 + if [ "$size" -le "$FULL_DB_SPLIT_THRESHOLD" ]; then + cp "$archive" "$stage/" + return + fi + echo "$(basename "$archive") is $size bytes, above $FULL_DB_SPLIT_THRESHOLD - publishing it in parts" + work=$(mktemp -d "$stage/.split.XXXXXX") || return 1 + bash "$DB_ASSET_SCRIPTS_DIR/split_release_asset.sh" "$archive" "$work" "$FULL_DB_PART_SIZE" \ + && mv "$work"/* "$stage/" && rmdir "$work" +} + +download_full_db_by_name() { # [repo] -> / + # A split asset is assembled; every part and the whole are sha256-verified. + local tag="$1" name="$2" dest="$3" work part rc=0 + local -a repo=() + [ -z "${4:-}" ] || repo=(-R "$4") + work=$(mktemp -d "$dest/.fulldb.XXXXXX") || return 1 + if gh release download "$tag" "${repo[@]}" --pattern "$name" --dir "$dest" 2>"$work/single.err"; then + rm -rf "$work" + return 0 + fi + if ! gh release download "$tag" "${repo[@]}" --pattern "$name.manifest.json" --dir "$work" 2>/dev/null; then + cat "$work/single.err" >&2 + rm -rf "$work" + return 1 + fi + while IFS= read -r part; do + case "$part" in ''|*/*|.|..) echo "unsafe part name in $name.manifest.json: $part" >&2; rc=1; break ;; esac + gh release download "$tag" "${repo[@]}" --pattern "$part" --dir "$work" || { rc=1; break; } + done < <(jq -r '.parts[].name' "$work/$name.manifest.json") + if [ "$rc" = 0 ] && ! bash "$DB_ASSET_SCRIPTS_DIR/assemble_split_asset.sh" \ + "$work/$name.manifest.json" "$dest/$name" >/dev/null; then + rc=1 + rm -f "$dest/$name" + fi + rm -rf "$work" + return "$rc" +} + +split_full_db_meta() { # [repo] -> "\t" from its split manifest + local -a repo=() + [ -z "${3:-}" ] || repo=(-R "$3") + gh release download "$1" "${repo[@]}" --pattern "$2.manifest.json" -O - \ + | jq -er '[(.size | tostring), ("sha256:" + .sha256)] | @tsv' +} + +stream_full_db() { # [repo] -> the archive's bytes on stdout + # Parts are streamed unchecked; the consumer's zstd verifies the frame checksum. + local tag="$1" name="$2" manifest part + local -a repo=() + [ -z "${3:-}" ] || repo=(-R "$3") + if ! manifest=$(gh release download "$tag" "${repo[@]}" --pattern "$name.manifest.json" -O - 2>/dev/null); then + gh release download "$tag" "${repo[@]}" --pattern "$name" -O - + return + fi + for part in $(jq -r '.parts[].name' <<<"$manifest"); do + gh release download "$tag" "${repo[@]}" --pattern "$part" -O - || return 1 + done +} diff --git a/.github/scripts/extract_app_build_steps.py b/.github/scripts/extract_app_build_steps.py new file mode 100644 index 000000000..cc526fa4e --- /dev/null +++ b/.github/scripts/extract_app_build_steps.py @@ -0,0 +1,325 @@ +"""Write out, verbatim, the build_linux steps of Otzaria/otzaria that the index build replays. + +Every step of a raw build must be one this script knows as replayed, mirrored or skipped; any drift fails. + + extract_app_build_steps.py + extract_app_build_steps.py --fingerprints +""" + +import hashlib +import os +import re +import sys + +import yaml + +# file stem -> exact step name; run verbatim, in this order. +REPLAYED = { + "wpe-script": "Prepare WPE bundling script (shared)", + "pub-deps": "Prepare Flutter dependencies for Linux release", + "cargokit-options": "Build the search engine inside the compatibility container", + "patch-inappwebview": "Patch inappwebview_linux theme_color for WPE 2.48", + "verify-wpe-platform": "Verify WPE SDK (WPEPlatform) was used, not FDO/system", + "verify-glibc": "Verify search engine glibc compatibility", + "bundle-wpe": "Bundle WPE runtime into main bundle (raw + FULL)", + "verify-wpe-runtime": "Verify WPE runtime in packaged artifacts", +} + +FLUTTER_BUILD = "Build Flutter Linux app (with retry)" +WPE_DOWNLOAD = "Download prebuilt WPE runtime SDK" + +# Reproduced by build-library-index.yml itself; pinned by fingerprint of the comment-free run. +MIRRORED_RUN = { + "Prepare container base tools": "b996259b07b684be56e7cc529cfb4f1712c3381f19e732bc41e383b2ab63b67b", + "Set up Flutter (aarch64, from git)": "8518cce0a699547dc0517e1081e9709dad4bcff38defde7a22e4d02fb71225ce", + "Add cargo to PATH and verify toolchain": "ccd2ca002cdeb025b85eace91b76b9c430c8459f11d07fc991221d30f6a4f879", + "Install Linux build dependencies": "afad8660067a7909e955667c2cabd3782c2a5f889664f1bdc1604fa5daaa11cc", + WPE_DOWNLOAD: "291e7657cf03dd9b570975bf30f0bc9158bec15ed796b2eaf0887e7c30ede838", + "Find and prepare Linux packages": "21a48f84f5ae62a1db74bf93c6d9f81ab22f3cbcb80a5fd993ba76586c50c7b1", +} +MIRRORED_USES = { + "Clone repository": { + "uses": "actions/checkout@v7.0.1", + "with": {"ref": "${{ needs.bump_version.outputs.sha }}"}, + }, + "Set up Flutter": { + "if": "matrix.arch != 'aarch64'", + "uses": "subosito/flutter-action@v2", + "with": {"channel": "stable", "flutter-version": "3.47.7", "cache": True}, + }, + "Install Rust (stable) with rustup": {"uses": "dtolnay/rust-toolchain@stable"}, +} + +# The comment-free run of FLUTTER_BUILD; the index build runs it once, without the secret. +FLUTTER_BUILD_RUN = """set -eo pipefail +tries=0 +: > "$GITHUB_WORKSPACE/build_linux.log" +until [ $tries -ge 3 ] +do +if flutter build linux --verbose --dart-define=BIO_OBF_KEY=${{ secrets.BIO_OBF_KEY }} "--dart-define=OTZARIA_BUILD_COMMIT=$(git rev-parse --short=12 HEAD)" 2>&1 | tee -a "$GITHUB_WORKSPACE/build_linux.log"; then +break +fi +tries=$((tries+1)) +echo "flutter build linux failed. retry $tries/3 after short sleep..." +sleep 15 +done +if [ $tries -ge 3 ]; then +echo "::error::flutter build linux failed after 3 attempts" +exit 1 +fi""" +FLUTTER_BUILD_ENV = { + "CC": "clang", + "CXX": "clang++", + "CFLAGS": "-Wno-error=unknown-warning-option", + "CXXFLAGS": "-Wno-error=unknown-warning-option", +} + +# The job environment; the index build passes the /opt/wpe-sdk and arch-derived values itself. +JOB_ENV = { + "RUSTUP_HOME": "${{ github.workspace }}/.rustup", + "CARGO_HOME": "${{ github.workspace }}/.cargo", + "DEBIAN_FRONTEND": "noninteractive", + "FLUTTER_BUILD_DIR": "build/linux/${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}/release", + "PKG_CONFIG_PATH": "/opt/wpe-sdk/lib/pkgconfig", + "LD_LIBRARY_PATH": "/opt/wpe-sdk/lib", + "LIBRARY_PATH": "/opt/wpe-sdk/lib", + "JAVA_HOME": "/usr/lib/jvm/java-17-openjdk-${{ matrix.arch == 'aarch64' && 'arm64' || 'amd64' }}", +} +CONTAINER = {"image": "debian:bookworm-slim"} + +# The raw bundle is complete once this step has copied it; nothing after it is pinned. +RAW_BUNDLE_STEP = "Find and prepare Linux packages" + +# Every step up to RAW_BUNDLE_STEP that runs in a successful raw build, in order. +# R = replayed, M = mirrored, S = skipped (secrets, caches, data the index never reads). +STEPS = [ + ("Prepare container base tools", "M"), + ("Clone repository", "M"), + ("Check if commit message is a version tag", "S"), + ("Set up Flutter", "M"), + ("Set up Flutter (aarch64, from git)", "M"), + ("Create Google Calendar Credentials", "S"), + ("Install Rust (stable) with rustup", "M"), + ("Add cargo to PATH and verify toolchain", "M"), + ("Cache cargo registry and git", "S"), + ("Install Linux build dependencies", "M"), + (WPE_DOWNLOAD, "M"), + ("Prepare WPE bundling script (shared)", "R"), + ("Download bundled plugins for Linux packages", "S"), + ("Fetch biographies data", "S"), + ("Prepare Flutter dependencies for Linux release", "R"), + ("Build the search engine inside the compatibility container", "R"), + ("Patch inappwebview_linux theme_color for WPE 2.48", "R"), + (FLUTTER_BUILD, "M"), + ("Verify WPE SDK (WPEPlatform) was used, not FDO/system", "R"), + ("Verify search engine glibc compatibility", "R"), + ("Bundle WPE runtime into main bundle (raw + FULL)", "R"), + ("Verify WPE runtime in packaged artifacts", "R"), + (RAW_BUNDLE_STEP, "M"), +] + +ARCHES = ("x86_64", "aarch64") +_TOKEN = re.compile(r"\s*(?:(\|\||&&|==|!=|!|\(|\))|'([^']*)'|(matrix\.target|matrix\.arch)" + r"|(success|failure|always|cancelled)\(\))") +# A successful run: the only one that leaves a raw bundle behind. +_STATUS = {"success": True, "failure": False, "always": True, "cancelled": False} + + +def runs_for_raw(condition, arch): + """Evaluate a step `if:` for target raw on `arch` in a successful run; refuse anything else.""" + if condition is None: + return True + if isinstance(condition, bool): + return condition + text = str(condition).strip() + if text.startswith("${{") and text.endswith("}}"): + text = text[3:-2].strip() + tokens, pos = [], 0 + while pos < len(text): + match = _TOKEN.match(text, pos) + if not match or match.end() == pos: + fail(f"cannot classify the step condition '{condition}' for the raw build") + op, literal, variable, status = match.groups() + if op: + tokens.append(("op", op)) + elif literal is not None: + tokens.append(("val", literal)) + elif variable: + tokens.append(("val", "raw" if variable == "matrix.target" else arch)) + else: + tokens.append(("val", _STATUS[status])) + pos = match.end() + + def parse(i, level): + # level 0: ||, 1: &&, 2: == !=, 3: unary + if level == 3: + kind, value = tokens[i] if i < len(tokens) else (None, None) + if (kind, value) == ("op", "!"): + operand, i = parse(i + 1, 3) + return not operand, i + if (kind, value) == ("op", "("): + inner, i = parse(i + 1, 0) + if i >= len(tokens) or tokens[i] != ("op", ")"): + fail(f"cannot classify the step condition '{condition}' for the raw build") + return inner, i + 1 + if kind != "val": + fail(f"cannot classify the step condition '{condition}' for the raw build") + return value, i + 1 + left, i = parse(i, level + 1) + ops = {0: ("||",), 1: ("&&",), 2: ("==", "!=")}[level] + while i < len(tokens) and tokens[i][0] == "op" and tokens[i][1] in ops: + op = tokens[i][1] + right, i = parse(i + 1, level + 1) + if op == "||": + left = bool(left) or bool(right) + elif op == "&&": + left = bool(left) and bool(right) + elif not (isinstance(left, str) and isinstance(right, str)): + fail(f"cannot classify the step condition '{condition}' for the raw build") + else: + left = (left == right) if op == "==" else (left != right) + return left, i + + value, end = parse(0, 0) + if end != len(tokens) or not isinstance(value, bool): + fail(f"cannot classify the step condition '{condition}' for the raw build") + return value + + +def raw_steps(job): + """The steps of a successful raw build, up to and including RAW_BUNDLE_STEP.""" + names = [step.get("name") for step in job["steps"]] + if names.count(RAW_BUNDLE_STEP) != 1: + fail(f"build_linux has {names.count(RAW_BUNDLE_STEP)} steps named '{RAW_BUNDLE_STEP}', expected 1") + region = job["steps"][: names.index(RAW_BUNDLE_STEP) + 1] + return [s.get("name") for s in region if any(runs_for_raw(s.get("if"), a) for a in ARCHES)] + + +WPE_SHA_EXPR = re.compile( + r"^\$\{\{ matrix\.arch == 'aarch64' && '([0-9a-f]{64})' \|\| '([0-9a-f]{64})' \}\}$" +) + + +def fail(message): + print(f"::error::{message}", file=sys.stderr) + sys.exit(1) + + +def normalized(run): + lines = (line.strip() for line in (run or "").splitlines()) + return "\n".join(line for line in lines if line and not line.startswith("#")) + + +def fingerprint(run): + return hashlib.sha256(normalized(run).encode("utf-8")).hexdigest() + + +def pin(env_name): + value = os.environ.get(env_name, "") + if not value: + fail(f"{env_name} is not set") + return value + + +def build_linux(workflow): + with open(workflow, encoding="utf-8") as handle: + doc = yaml.safe_load(handle) + job = (doc.get("jobs") or {}).get("build_linux") + if not job or not job.get("steps"): + fail(f"{workflow} has no build_linux job") + return job + + +def check_structure(job): + names = raw_steps(job) + expected = [name for name, _ in STEPS] + if names != expected: + added = [n for n in names if n not in expected] + removed = [n for n in expected if n not in names] + fail(f"the steps of a raw build changed (added {added}, removed {removed}, or reordered); " + "classify them in extract_app_build_steps.py") + matrix = (job.get("strategy") or {}).get("matrix") or {} + if "raw" not in (matrix.get("target") or []) or not set(ARCHES) <= set(matrix.get("arch") or []): + fail(f"build_linux no longer builds raw for {ARCHES}: {matrix}") + if job.get("env") != JOB_ENV: + fail(f"build_linux job environment changed: {job.get('env')}") + if job.get("container") != CONTAINER: + fail(f"build_linux container changed: {job.get('container')}") + + +def check_mirrored(steps, arch): + for name, expected in MIRRORED_USES.items(): + found = {k: v for k, v in steps[name].items() if k != "name"} + if found != expected: + fail(f"step '{name}' changed: {found}") + for name, expected in MIRRORED_RUN.items(): + if fingerprint(steps[name].get("run")) != expected: + fail(f"step '{name}' changed (fingerprint {fingerprint(steps[name].get('run'))})") + + build = steps[FLUTTER_BUILD] + if normalized(build.get("run")) != FLUTTER_BUILD_RUN or build.get("env") != FLUTTER_BUILD_ENV: + fail(f"step '{FLUTTER_BUILD}' changed: the index build's flutter build no longer mirrors it") + if set(build) != {"name", "shell", "env", "run"} or build.get("shell") != "bash": + fail(f"step '{FLUTTER_BUILD}' changed shape: {sorted(build)}") + + flutter = pin("FLUTTER_VERSION") + if str(MIRRORED_USES["Set up Flutter"]["with"]["flutter-version"]) != flutter: + fail(f"the application builds with Flutter " + f"{MIRRORED_USES['Set up Flutter']['with']['flutter-version']}; the builder image pins {flutter}") + if f"--branch {flutter} " not in steps["Set up Flutter (aarch64, from git)"].get("run", ""): + fail(f"the application no longer clones Flutter {flutter} for aarch64") + + wpe = steps[WPE_DOWNLOAD] + wpe_env = wpe.get("env") or {} + if set(wpe_env) != {"WPE_RUNTIME_VERSION", "WPE_RUNTIME_SHA256"}: + fail(f"step '{WPE_DOWNLOAD}' environment changed: {sorted(wpe_env)}") + if str(wpe_env["WPE_RUNTIME_VERSION"]) != pin("WPE_VERSION"): + fail(f"the application builds with WPE {wpe_env['WPE_RUNTIME_VERSION']}; " + f"the builder image pins {pin('WPE_VERSION')}") + if f'tag="{pin("WPE_TAG")}"' not in (wpe.get("run") or ""): + fail(f"the application no longer downloads WPE from tag {pin('WPE_TAG')}") + digests = WPE_SHA_EXPR.match(str(wpe_env["WPE_RUNTIME_SHA256"])) + if not digests: + fail(f"WPE_RUNTIME_SHA256 is no longer a per-arch pair: {wpe_env['WPE_RUNTIME_SHA256']}") + app_digest = digests.group(1) if arch == "aarch64" else digests.group(2) + if app_digest != pin("WPE_SHA256"): + fail(f"the application pins WPE runtime {app_digest} for {arch}; the builder image pins {pin('WPE_SHA256')}") + + +def extract(steps, out_dir): + os.makedirs(out_dir, exist_ok=False) + for stem, name in REPLAYED.items(): + step = steps[name] + extra = sorted(set(step) - {"name", "shell", "run", "id"}) + if extra: + fail(f"step '{name}' now carries {extra}; it can no longer be replayed verbatim") + if step.get("shell") != "bash": + fail(f"step '{name}' is no longer a bash step") + run = step.get("run") or "" + if not run.strip() or "${{" in run: + fail(f"step '{name}' is empty or uses a workflow expression") + with open(os.path.join(out_dir, f"{stem}.sh"), "w", encoding="utf-8") as handle: + handle.write(run) + print(f"extracted '{name}' -> {stem}.sh") + + +def main(): + if len(sys.argv) == 3 and sys.argv[1] == "--fingerprints": + steps = {s.get("name"): s for s in build_linux(sys.argv[2])["steps"]} + for name in MIRRORED_RUN: + print(f"{fingerprint(steps[name].get('run'))} {name}") + return + if len(sys.argv) != 4: + fail("usage: extract_app_build_steps.py ") + workflow, out_dir, arch = sys.argv[1:] + if arch not in ("x86_64", "aarch64"): + fail(f"unsupported arch '{arch}'") + job = build_linux(workflow) + check_structure(job) + steps = {step["name"]: step for step in job["steps"]} + check_mirrored(steps, arch) + extract(steps, out_dir) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/host_lease.py b/.github/scripts/host_lease.py new file mode 100644 index 000000000..d822fd9f5 --- /dev/null +++ b/.github/scripts/host_lease.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Own the cross-step host flock and release it with PID-reuse protection.""" + +from __future__ import annotations + +import argparse +import fcntl +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + + +def start_ticks(pid: int) -> int: + tail = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].strip().split() + return int(tail[19]) + + +def proc_parent(pid: int) -> int: + tail = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].strip().split() + return int(tail[1]) + + +def proc_cmdline(pid: int) -> str: + return Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode( + "utf-8", "replace" + ) + + +def find_runner_owner(pid: int | None = None) -> tuple[int, int]: + current = os.getpid() if pid is None else pid + seen = set() + while current > 1 and current not in seen: + seen.add(current) + if "Runner.Worker" in proc_cmdline(current): + return current, start_ticks(current) + current = proc_parent(current) + raise RuntimeError("host lease must be started by a GitHub Runner.Worker descendant") + + +def owner_alive(pid: int, expected_start_ticks: int) -> bool: + try: + return start_ticks(pid) == expected_start_ticks and "Runner.Worker" in proc_cmdline(pid) + except (FileNotFoundError, ProcessLookupError, PermissionError): + return False + + +def write_state(path: Path, lock: str, owner_pid: int, owner_start_ticks: int) -> None: + value = { + "schema_version": 1, + "pid": os.getpid(), + "pgid": os.getpgrp(), + "start_ticks": start_ticks(os.getpid()), + "lock": lock, + "owner_pid": owner_pid, + "owner_start_ticks": owner_start_ticks, + } + path.parent.mkdir(parents=True, exist_ok=True) + fd, raw_tmp = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + tmp = Path(raw_tmp) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + finally: + try: + os.close(fd) + except OSError: + pass + tmp.unlink(missing_ok=True) + + +def hold(args) -> int: + fd = os.open(args.lock, os.O_WRONLY | os.O_APPEND) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f"host lease is busy: {args.lock}", file=sys.stderr) + return 75 + stop = False + + def request_stop(_signum, _frame): + nonlocal stop + stop = True + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + write_state(Path(args.state), args.lock, args.owner_pid, args.owner_start_ticks) + deadline = time.monotonic() + args.ttl + while ( + not stop + and time.monotonic() < deadline + and owner_alive(args.owner_pid, args.owner_start_ticks) + ): + time.sleep(1) + Path(args.state).unlink(missing_ok=True) + os.close(fd) + return 0 + + +def start(args) -> int: + state = Path(args.state) + state.unlink(missing_ok=True) + try: + owner_pid, owner_start = find_runner_owner() + except (OSError, RuntimeError) as exc: + print(f"cannot bind host lease to runner owner: {exc}", file=sys.stderr) + return 2 + command = [ + sys.executable, str(Path(__file__).resolve()), "hold", + "--lock", args.lock, "--state", args.state, "--ttl", str(args.ttl), + "--owner-pid", str(owner_pid), "--owner-start-ticks", str(owner_start), + ] + # Truly detach across GitHub steps: inheriting the step's stdout/stderr pipes + # can keep the run command open until the 25-hour holder exits even though + # this launcher returned. State-file readiness is the only IPC we need. + proc = subprocess.Popen( + command, + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + ) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if state.exists(): + value = json.loads(state.read_text()) + if value.get("pid") == proc.pid and value.get("pgid") == proc.pid: + print(f"host lease acquired: {args.lock} (holder {proc.pid})") + return 0 + rc = proc.poll() + if rc is not None: + return rc + time.sleep(0.1) + proc.terminate() + proc.wait(timeout=5) + print("host lease holder did not become ready", file=sys.stderr) + return 1 + + +def load_state(path: Path) -> dict: + def pairs(items): + out = {} + for key, value in items: + if key in out: + raise ValueError(f"duplicate key {key!r}") + out[key] = value + return out + raw = path.read_bytes() + value = json.loads(raw.decode("utf-8"), object_pairs_hook=pairs) + if ( + set(value) != { + "schema_version", "pid", "pgid", "start_ticks", "lock", + "owner_pid", "owner_start_ticks", + } + or type(value["schema_version"]) is not int + or value["schema_version"] != 1 + ): + raise ValueError("invalid lease-state schema") + for key in ("pid", "pgid", "start_ticks", "owner_pid", "owner_start_ticks"): + if type(value[key]) is not int or value[key] < 1: + raise ValueError(f"invalid lease-state {key}") + if not isinstance(value["lock"], str) or not value["lock"].startswith("/"): + raise ValueError("invalid lease-state lock path") + canonical = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + b"\n" + if raw != canonical: + raise ValueError("lease-state is not canonical JSON") + return value + + +def release(args) -> int: + path = Path(args.state) + if not path.exists(): + print("no local host lease state") + return 0 + try: + value = load_state(path) + pid = value["pid"] + cmdline = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace") + valid = ( + value["pgid"] == pid + and os.getpgid(pid) == pid + and start_ticks(pid) == value["start_ticks"] + and "host_lease.py hold" in cmdline + and value["lock"] in cmdline + ) + except (FileNotFoundError, ProcessLookupError): + valid = False + except Exception as exc: + print(f"invalid host lease state: {exc}", file=sys.stderr) + return 2 + if not valid: + path.unlink(missing_ok=True) + print("stale host lease state removed; no process signalled") + return 0 + os.killpg(pid, signal.SIGTERM) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + os.killpg(pid, signal.SIGKILL) + path.unlink(missing_ok=True) + print("host lease released") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + for name, func in (("start", start), ("hold", hold)): + p = sub.add_parser(name) + p.add_argument("--lock", required=True) + p.add_argument("--state", required=True) + p.add_argument("--ttl", type=int, default=90000) + if name == "hold": + p.add_argument("--owner-pid", type=int, required=True) + p.add_argument("--owner-start-ticks", type=int, required=True) + p.set_defaults(func=func) + p = sub.add_parser("release") + p.add_argument("--state", required=True) + p.set_defaults(func=release) + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/install-db-workflow-deps.sh b/.github/scripts/install-db-workflow-deps.sh new file mode 100644 index 000000000..359432fb3 --- /dev/null +++ b/.github/scripts/install-db-workflow-deps.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ -f /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release +fi + +OS_ID="${ID:-unknown}" +OS_ID_LIKE="${ID_LIKE:-}" +OS_FAMILY="$OS_ID $OS_ID_LIKE" + +# A self-hosted build machine is durable. Avoid package-index downloads and +# reinstall attempts on every weekly run once the complete toolset is already +# present; fresh/ephemeral runners still fall through to the installer below. +all_present=true +for tool in gh sqlite3 zstd unzstd jq curl unzip; do + command -v "$tool" >/dev/null 2>&1 || all_present=false +done +if [ "$all_present" = true ]; then + echo "DB workflow dependencies are already installed; skipping package-manager work." + exit 0 +fi + +if echo "$OS_FAMILY" | grep -qiE 'debian|ubuntu'; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends sqlite3 zstd jq curl unzip + if ! command -v gh >/dev/null 2>&1 && apt-cache show gh >/dev/null 2>&1; then + sudo apt-get install -y --no-install-recommends gh || true + fi +elif echo "$OS_FAMILY" | grep -qiE 'rhel|centos|fedora|oracle|almalinux|rocky'; then + sudo dnf install -y sqlite zstd jq curl unzip + if ! command -v gh >/dev/null 2>&1 && dnf info gh >/dev/null 2>&1; then + sudo dnf install -y gh || true + fi +else + echo "Unsupported OS for DB workflow dependencies: $OS_ID ($OS_ID_LIKE)" >&2 + exit 1 +fi + +if ! command -v gh >/dev/null 2>&1; then + echo "gh not found from package manager; installing GitHub CLI release binary" + GH_VERSION=$(curl -fsSL https://api.github.com/repos/cli/cli/releases/latest | jq -r '.tag_name | sub("^v"; "")') + case "$(uname -m)" in + x86_64) GH_ARCH=amd64 ;; + aarch64|arm64) GH_ARCH=arm64 ;; + *) + echo "Unsupported architecture for GitHub CLI: $(uname -m)" >&2 + exit 1 + ;; + esac + + curl -fL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${GH_ARCH}.tar.gz" -o /tmp/gh.tar.gz + rm -rf /tmp/gh + mkdir -p /tmp/gh + tar -xzf /tmp/gh.tar.gz -C /tmp/gh + GH_BIN=$(find /tmp/gh -type f -path '*/bin/gh' | head -n 1 || true) + if [ -z "$GH_BIN" ]; then + echo "Failed to locate gh binary after download" >&2 + exit 1 + fi + sudo install -m 0755 "$GH_BIN" /usr/local/bin/gh +fi + +command -v gh +command -v sqlite3 +command -v zstd +command -v unzstd +command -v jq +command -v unzip diff --git a/.github/scripts/library_announce/already_sent.sh b/.github/scripts/library_announce/already_sent.sh new file mode 100755 index 000000000..448afc048 --- /dev/null +++ b/.github/scripts/library_announce/already_sent.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 # the $ENV references are jq, not shell +# Prints the state of "$CHANNEL $TAG" across every run of this workflow, all attempts: +# sent a job with that name succeeded; +# partial ... none succeeded, but one failed, was cancelled or has not finished; +# none no such job ever started (dry runs skip channel jobs). +set -euo pipefail +: "${TAG:?}" "${CHANNEL:?}" "${GITHUB_REPOSITORY:?}" +export JOB_NAME="$CHANNEL $TAG" +# An announcing run cannot predate the release itself. +since="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" --jq .created_at)" +[ -n "$since" ] || { echo "::error::release $TAG has no created_at" >&2; exit 1; } +runs="$(gh api -X GET --paginate "repos/$GITHUB_REPOSITORY/actions/workflows/library-update-announce.yml/runs" \ + -f per_page=100 -f created=">=$since" --jq '.workflow_runs[].id')" +seen="" +for run in $runs; do + jobs="$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$run/jobs?filter=all&per_page=100" \ + --jq '.jobs[] | select(.name == $ENV.JOB_NAME) | "\(.status) \(.conclusion // "none")"')" + while read -r status conclusion; do + [ -n "${status:-}" ] || continue + seen+="$run $status $conclusion"$'\n' + done <<<"$jobs" +done +echo "$CHANNEL: $(grep -c . <<<"$seen" || true) '$JOB_NAME' job(s) among $(wc -w <<<"$runs") run(s) since $since" >&2 +if sent_by="$(awk '$3 == "success" {print $1; exit}' <<<"$seen")" && [ -n "$sent_by" ]; then + echo "sent $sent_by" + exit 0 +fi +partial="$(awk '$1 != "" && $3 != "skipped" {print $1}' <<<"$seen" | sort -un | tr '\n' ' ' | sed 's/ $//')" +if [ -n "$partial" ]; then + echo "partial $partial" +else + echo none +fi diff --git a/.github/scripts/library_announce/channel_decision.sh b/.github/scripts/library_announce/channel_decision.sh new file mode 100755 index 000000000..3b46da13d --- /dev/null +++ b/.github/scripts/library_announce/channel_decision.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Prints skip|proceed for "$CHANNEL $TAG" in this run, or fails. A partial send in +# another run must be resumed there (its recorded progress), never repeated here. +set -euo pipefail +: "${TAG:?}" "${CHANNEL:?}" "${GITHUB_RUN_ID:?}" "${GITHUB_REPOSITORY:?}" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +state="$(bash "$here/already_sent.sh")" +bad() { echo "::error::already_sent.sh printed '$state' for $CHANNEL $TAG" >&2; exit 1; } +[[ "$state" =~ ^(none|sent\ [0-9]+|partial\ [0-9]+(\ [0-9]+)*)$ ]] || bad +read -r kind runs <<<"$state" +case "$kind" in + none) echo proceed ;; + sent) + echo "::notice::$CHANNEL already announced $TAG in run $runs" >&2 + echo skip ;; + partial) + others=() + for run in $runs; do [ "$run" = "$GITHUB_RUN_ID" ] || others+=("$run"); done + if [ "${#others[@]}" -gt 0 ]; then + for run in "${others[@]}"; do + echo "::error::$CHANNEL for $TAG was started but not completed by run $run (failed, cancelled or still running)." \ + "Open https://github.com/$GITHUB_REPOSITORY/actions/runs/$run and use \"Re-run failed jobs\" there, so it" \ + "resumes from its recorded progress instead of sending a duplicate." >&2 + done + exit 1 + fi + echo proceed ;; + *) bad ;; +esac diff --git a/.github/scripts/library_announce/gate.sh b/.github/scripts/library_announce/gate.sh new file mode 100755 index 000000000..7ff6d39cd --- /dev/null +++ b/.github/scripts/library_announce/gate.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Decides, before anything is downloaded, which tag this run announces and on which +# channels. Writes tag, proceed, send and channels (JSON) to $GITHUB_OUTPUT. +set -euo pipefail +: "${EVENT_NAME:?}" "${GITHUB_REPOSITORY:?}" "${GITHUB_OUTPUT:?}" "${GITHUB_RUN_ID:?}" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +tag_re='^v[1-9][0-9]*-[0-9]{14}$' + +stop() { + echo "::notice::$1 — nothing to announce." + { echo "proceed=false"; echo "tag=${tag:-}"; echo "send=false"; echo "channels=[]"; } >> "$GITHUB_OUTPUT" + exit 0 +} + +# A release run's own handoff names the tag it published: pipeline-result-run--. +# Sets $tag; not run in a subshell, so stop() ends the whole gate. +set_tag_published_by_run() { + local handoff="pipeline-result-run-$WORKFLOW_RUN_ID-$WORKFLOW_RUN_ATTEMPT" dir err + dir="$(mktemp -d)" err="$dir/err" + if ! gh api "repos/$GITHUB_REPOSITORY/releases/tags/$handoff" --jq .tag_name > /dev/null 2> "$err"; then + grep -q 'HTTP 404' "$err" || { echo "::error::cannot read $handoff: $(tr '\n' ' ' < "$err")"; exit 1; } + stop "release run $WORKFLOW_RUN_ID:$WORKFLOW_RUN_ATTEMPT wrote no $handoff (it published nothing)" + fi + gh release download "$handoff" -R "$GITHUB_REPOSITORY" -D "$dir" -p pipeline-result.json -p pipeline-result.sha256 + [ "$(sha256sum "$dir/pipeline-result.json" | cut -d' ' -f1)" = "$(tr -d '[:space:]' < "$dir/pipeline-result.sha256")" ] || { + echo "::error::$handoff: pipeline-result.json does not match its sha256"; exit 1; } + local status + status="$(jq -er --argjson run "$WORKFLOW_RUN_ID" --argjson attempt "$WORKFLOW_RUN_ATTEMPT" \ + 'select(.child_run_id == $run and .child_run_attempt == $attempt) | .status' "$dir/pipeline-result.json")" || { + echo "::error::$handoff names another run"; exit 1; } + case "$status" in + reused) stop "release run $WORKFLOW_RUN_ID:$WORKFLOW_RUN_ATTEMPT reused an existing release" ;; + published) tag="$(jq -er .release_tag "$dir/pipeline-result.json")" ;; + *) echo "::error::$handoff has unknown status '$status'"; exit 1 ;; + esac +} + +case "$EVENT_NAME" in + workflow_dispatch) + tag="${INPUT_TAG:?}" send=true + if [ "${DRY_RUN:?}" = true ]; then send=false; fi ;; + workflow_run) send=true; set_tag_published_by_run ;; + *) echo "::error::unexpected event $EVENT_NAME"; exit 1 ;; +esac +[[ "$tag" =~ $tag_re ]] || { echo "::error::'$tag' is not a v- release tag"; exit 1; } + +state="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$tag" --jq '"\(.draft) \(.prerelease)"')" +if [ "$state" != "false false" ]; then + # Only a manual dispatch names the tag by hand; an automatic trigger just has nothing to do. + [ "$EVENT_NAME" = workflow_dispatch ] && { echo "::error::$tag is not a published (non-draft, non-pre-release) release"; exit 1; } + stop "$tag is not a published final release (draft prerelease: $state)" +fi + +channels=() +IFS=',' read -r -a requested <<< "${INPUT_CHANNELS:-forum,yemot,release}" +for channel in "${requested[@]}"; do + channel="$(tr -d '[:space:]' <<<"$channel")" + [ -n "$channel" ] || continue + case "$channel" in forum|yemot|release) ;; *) echo "::error::unknown channel '$channel' (allowed: forum,yemot,release)"; exit 1 ;; esac + if [ "$send" = true ]; then + # A plain assignment, so a failing check stops the gate instead of reading as "not sent". + decision="$(TAG="$tag" CHANNEL="$channel" bash "$here/channel_decision.sh")" + case "$decision" in + skip) continue ;; + proceed) ;; + *) echo "::error::channel_decision.sh printed '$decision' for $channel"; exit 1 ;; + esac + fi + channels+=("$channel") +done +[ "${#channels[@]}" -gt 0 ] || stop "every requested channel already announced $tag" + +json="$(printf '%s\n' "${channels[@]}" | jq -Rsc 'split("\n") | map(select(. != ""))')" +echo "announcing $tag on $json (send=$send)" +{ echo "proceed=true"; echo "tag=$tag"; echo "send=$send"; echo "channels=$json"; } >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/library_announce/library_update_announce.py b/.github/scripts/library_announce/library_update_announce.py new file mode 100644 index 000000000..550b6768e --- /dev/null +++ b/.github/scripts/library_announce/library_update_announce.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +"""Announce what a published Seforim DB release changed, book by book. + +The announcement compares the new release with the previous *published* one +(pre-releases never reach users, so they are skipped), and describes the two +databases themselves — Otzaria and Sefaria books alike — rather than any +source repository's file tree. + + previous-tag --current TAG tag names on stdin -> the previous published tag + catalog DB OUT seforim.db -> a small per-book JSON catalog + render --old A --new B --tag T --out DIR diff two catalogs -> forum posts + Yemot content + release notes + send --channel C --dir DIR post one rendered channel (forum / yemot) + +The release channel is uploaded by the workflow itself: RELEASE_NOTES_ASSET as is. +""" +import argparse +import hashlib +import html +import json +import os +import re +import sqlite3 +import sys +import time +from datetime import date, datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +TZ = ZoneInfo("Asia/Jerusalem") +TAG_RE = re.compile(r"^v([1-9][0-9]*)-[0-9]{14}$") +CATALOG_SCHEMA = 2 +FORUM_URL = "https://otzaria.org/forum" +FORUM_TOPIC_ID = 20 +# maximumPostLength from https://otzaria.org/forum/api/config (read 2026-09-29). +FORUM_MAX_POST_LENGTH = 32767 +CHANNELS = ("forum", "yemot", "release") +SENT_CHANNELS = ("forum", "yemot") +# The reuse scan in manual-generate-release.yml leaves exactly this name out of the asset count. +RELEASE_NOTES_ASSET = "library-changes.md" +YEMOT_PATH = "ivr2:/1" +TZINTUK_LIST = "books update" + +SECTIONS = ( + ("added", "התווספו הספרים הבאים:"), + ("removed", "נמחקו הספרים הבאים:"), + ("moved", "שונה מיקום/שם של הספרים הבאים:"), + ("changed", "השתנו הספרים הבאים:"), +) +ALSO_CHANGED = " (השתנה גם תוכנו)" +ALSO_MOVED = " (שונה גם מיקומו/שמו)" + +# Only real markup: literal text such as <חסר> or <בד"ה ...> is part of the book. +HTML_TAG_RE = re.compile(r"<[A-Za-z/!?][^>]*>") +WHITESPACE_RE = re.compile(r"\s+") + + +def db_version(tag): + match = TAG_RE.fullmatch(tag) + return int(match.group(1)) if match else None + + +def previous_published_tag(current, tags): + """The newest published tag strictly older than `current`, or None.""" + current_version = db_version(current) + if current_version is None: + raise ValueError(f"{current!r} is not a v- release tag") + older = [(v, t) for t in tags if (v := db_version(t)) is not None and v < current_version] + return max(older)[1] if older else None + + +def plain_text(content): + """A line as the reader sees it: tags stripped, entities decoded, whitespace collapsed.""" + # A tag is at most a word boundary: v29 turned spaces into
in 855 books. + # A space added or removed between two words, and any nikud/teamim change, IS a change. + return WHITESPACE_RE.sub(" ", html.unescape(HTML_TAG_RE.sub(" ", content))).strip() + + +def category_paths(conn): + rows = {cid: (parent, title) for cid, parent, title in conn.execute("SELECT id, parentId, title FROM category")} + paths = {} + for cid in rows: + parts, seen, node = [], set(), cid + # Roots have parentId NULL; a cycle or a dangling parent is corrupt data. + while node is not None: + if node in seen or node not in rows: + raise ValueError(f"category {cid}: broken parent chain at {node!r}") + seen.add(node) + parent, title = rows[node] + parts.append(title) + node = parent + paths[cid] = "/".join(reversed(parts)) + return paths + + +def book_editions(conn, split=False): + """bookId -> (versionTitle, display title) of the edition the book's own lines are. + + A book with only metadata rows (hasContent=0) has one version, and its lines + ARE that edition. Otherwise the edition is the one version whose every + version_line row equals the book's line and that covers every line any of the + book's editions covers. A mosaic of editions, or a tie, has no one edition: None. + In the split shape, a present version_line with NULL content inherits base text. + """ + covered = dict(conn.execute( + "SELECT bv.bookId, COUNT(DISTINCT vl.lineId) FROM book_version bv" + " JOIN version_line vl ON vl.versionId = bv.id GROUP BY bv.bookId")) + content = "lc.content" if split else "l.content" + version_content = f"COALESCE(vl.content, {content})" if split else "vl.content" + content_join = " LEFT JOIN line_content lc ON lc.id = l.id" if split else "" + versions = {} + for book_id, title, he_title, has_content, rows, equal in conn.execute( + "SELECT bv.bookId, bv.versionTitle, bv.heVersionTitle, bv.hasContent," + f" COUNT(vl.lineId), COALESCE(SUM({version_content} = {content}), 0)" + " FROM book_version bv LEFT JOIN version_line vl ON vl.versionId = bv.id" + " LEFT JOIN line l ON l.id = vl.lineId" + content_join + " GROUP BY bv.id"): + versions.setdefault(book_id, []).append((title, he_title, has_content, rows, equal)) + editions = {} + for book_id, rows in versions.items(): + if any(r[2] for r in rows): + rows = [r for r in rows if r[2] and 0 < r[3] == r[4] == covered[book_id]] + if len(rows) != 1: + editions[book_id] = None + continue + title, he_title = rows[0][:2] + # The application shows heVersionTitle when non-blank, else versionTitle. + editions[book_id] = (title, he_title if he_title and he_title.strip() else title) + return editions + + +def line_text_decoder(conn): + """A DB with zstd_dict stores line text as zstd frames of its one dictionary.""" + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='zstd_dict'").fetchone(): + return lambda value: value + import zstandard + + [(dictionary,)] = conn.execute("SELECT dict FROM zstd_dict") + decompressor = zstandard.ZstdDecompressor(dict_data=zstandard.ZstdCompressionDict(dictionary)) + return lambda value: decompressor.decompress(value).decode("utf-8") if isinstance(value, bytes) else value + + +def build_catalog(db_path): + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + try: + split = bool(conn.execute( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='line_content')" + " AND NOT EXISTS(SELECT 1 FROM pragma_table_info('line') WHERE name='content')" + ).fetchone()[0]) + paths = category_paths(conn) + sources = dict(conn.execute("SELECT id, name FROM source")) + editions = book_editions(conn, split) + books = {} + for book_id, title, category_id, source_id in conn.execute( + "SELECT id, title, categoryId, sourceId FROM book"): + edition = editions.get(book_id) + books[book_id] = {"id": book_id, "title": title, "path": paths[category_id], + "source": sources[source_id], "lines": 0, "chars": 0, "hash": None, + "edition": edition[0] if edition else None, + "edition_name": edition[1] if edition else None} + digest, current = None, None + + def flush(): + books[current]["hash"] = digest.hexdigest() + + decode = line_text_decoder(conn) + content_sql = "lc.content" if split else "l.content" + content_join = " JOIN line_content lc ON lc.id = l.id" if split else "" + for book_id, content in conn.execute( + f"SELECT l.bookId, {content_sql} FROM line l" + content_join + " ORDER BY l.bookId, l.lineIndex"): + if book_id != current: + if current is not None: + flush() + if book_id not in books: + raise ValueError(f"line rows reference missing book {book_id}") + current, digest = book_id, hashlib.sha256() + text = plain_text(decode(content)) + # Line breaks count as whitespace, so re-wrapping lines is not a change. + if text: + if books[book_id]["chars"]: + digest.update(b" ") + digest.update(text.encode("utf-8")) + books[book_id]["chars"] += len(text) + books[book_id]["lines"] += 1 + if current is not None: + flush() + return {"schema": CATALOG_SCHEMA, "books": sorted(books.values(), key=lambda b: b["id"])} + finally: + conn.close() + + +def display_name(book): + return f"{book['path']}/{book['title']}" + + +def entry(book, **extra): + value = {"name": display_name(book), "title": book["title"]} + value.update(extra) + return value + + +def diff_catalogs(old, new): + """Structured changes: every list is sorted by the (new) display name.""" + old_books = {b["id"]: b for b in old["books"]} + new_books = {b["id"]: b for b in new["books"]} + added = [new_books[i] for i in sorted(new_books.keys() - old_books.keys())] + removed = [old_books[i] for i in sorted(old_books.keys() - new_books.keys())] + + # A book's id is keyed by its title, so a rename surfaces as remove + add of identical text. + def by_hash(books): + index = {} + for book in books: + if book["chars"]: + index.setdefault(book["hash"], []).append(book) + return index + + removed_by_hash, added_by_hash = by_hash(removed), by_hash(added) + renames = [] + for text_hash in sorted(removed_by_hash.keys() & added_by_hash.keys()): + before, after = removed_by_hash[text_hash], added_by_hash[text_hash] + if len(before) != 1 or len(after) != 1: + raise ValueError("ambiguous rename: identical text in removed " + f"{[display_name(b) for b in before]} and added {[display_name(b) for b in after]}") + renames.append((before[0], after[0])) + paired = {id(book) for pair in renames for book in pair} + + moved, changed = [], [] + for before, after in renames: + moved.append(entry(after, old_name=display_name(before), old_title=before["title"], also_changed=False)) + for book_id in sorted(old_books.keys() & new_books.keys()): + before, after = old_books[book_id], new_books[book_id] + was_moved = display_name(before) != display_name(after) + text_changed = before["hash"] != after["hash"] + # Only a known edition replacing another known one: None is a mosaic, not an edition. + edition = (after["edition_name"] if before["edition"] and after["edition"] + and after["edition"] != before["edition"] else None) + if was_moved: + moved.append(entry(after, old_name=display_name(before), old_title=before["title"], + also_changed=text_changed or edition is not None)) + if text_changed or edition: + changed.append(entry(after, edition=edition, also_moved=was_moved)) + + def sort(entries): + return sorted(entries, key=lambda e: e["name"]) + + return { + "added": sort(entry(b) for b in added if id(b) not in paired), + "removed": sort(entry(b) for b in removed if id(b) not in paired), + "moved": sort(moved), + "changed": sort(changed), + } + + +def describe(key, item, spoken=False): + """One item as a line; Yemot reads titles, the forum full category paths.""" + name = item["title"] if spoken else item["name"] + if key == "moved": + old = item["old_title"] if spoken else item["old_name"] + text = f"{name} (לשעבר: {old})" if old != name else name + return text + (ALSO_CHANGED if item["also_changed"] else "") + if key == "changed": + text = f"{name} - שינוי מהדורה ל-{item['edition']}" if item["edition"] else name + return text + (ALSO_MOVED if item["also_moved"] else "") + return name + + +def forum_lines(changes): + return {key: [f"* {describe(key, item)}" for item in changes[key]] + for key, _ in SECTIONS if changes[key]} + + +def post_length(text): + # NodeBB compares JavaScript string length: UTF-16 code units. + return len(text.encode("utf-16-le")) // 2 + + +def build_forum_posts(changes, version, date_text, limit=FORUM_MAX_POST_LENGTH): + """The full list, split on line boundaries into consecutive posts of at most `limit`.""" + heading = f"# גרסת ספרייה {version}" + posts, current = [], f"{heading}\n\n**עדכון {date_text}**\n" + for key, lines in forum_lines(changes).items(): + title = dict(SECTIONS)[key] + block = f"\n## {title}\n" + for line in lines: + addition = block + line + "\n" + if post_length(current + addition) > limit: + posts.append(current) + current = f"{heading} (המשך)\n\n## {title}{' (המשך)' if not block else ''}\n" + addition = line + "\n" + if post_length(current + addition) > limit: + raise ValueError(f"a single line exceeds the forum limit {limit}: {line[:80]!r}") + current += addition + block = "" + posts.append(current) + return posts + + +def build_release_notes(changes, version, date_text): + """The whole list as one Markdown file, worded like the forum.""" + [notes] = build_forum_posts(changes, version, date_text, limit=float("inf")) + return notes + + +def build_yemot_content(changes): + """Heading -> the book names read out, one per line; same sections and items as the forum.""" + return {title: "\n".join(describe(key, item, spoken=True) for item in changes[key]) + for key, title in SECTIONS if changes[key]} + + +def heb_date(day): + from pyluach import dates + return dates.HebrewDate.from_pydate(day).hebrew_date_string() + + +class ForumRefused(RuntimeError): + # NodeBB's own error keys stay untranslated, so a locale change cannot hide the rate limit. + RATE_LIMIT_MARKERS = ("ניתן לפרסם פוסט רק פעם ב", "too-many-posts", "still-posting") + + @property + def retryable(self): + return any(marker in str(self) for marker in self.RATE_LIMIT_MARKERS) + + +def post_to_forum(text, token, session=None): + """One reply through NodeBB's Write API; raises ForumRefused if the forum refused it.""" + if session is None: + import requests + session = requests + response = session.post( + f"{FORUM_URL}/api/v3/topics/{FORUM_TOPIC_ID}", + json={"content": text}, + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + ) + try: + status = response.json().get("status") or {} + except ValueError: + raise ForumRefused(f"http-{response.status_code}: {response.text[:200]!r}") from None + if status.get("code") != "ok": + raise ForumRefused(f"{status.get('code') or response.status_code}: {status.get('message', '')}") + + +def send_forum(text, token, attempts=4, delay=5.0, post=post_to_forum): + for attempt in range(1, attempts + 1): + try: + post(text, token) + return + except ForumRefused as exc: + # Only the post-rate refusal is retried: after a transport error the + # post may already exist, and a duplicate is worse than a miss. + if not exc.retryable or attempt == attempts: + raise + time.sleep(delay * attempt) + + +def load_progress(path, payload): + """A channel's progress from an earlier attempt, bound to the exact payload bytes.""" + digest = hashlib.sha256(payload).hexdigest() + if path and Path(path).exists(): + state = json.loads(Path(path).read_text(encoding="utf-8")) + if state.get("payload_sha256") != digest: + raise SystemExit(f"::error::progress {path} was recorded for another announcement") + return state + return {"payload_sha256": digest} + + +def progress_saver(path): + def save(state): + if not path: + return + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(".tmp") + temporary.write_text(json.dumps(state, ensure_ascii=False, sort_keys=True), encoding="utf-8") + temporary.replace(target) + return save + + +def send_forum_posts(posts, token, state=None, save=None, send=None): + """Posts in order, from state["sent"] on; records each post the forum accepted.""" + send = send or send_forum + state = state if state is not None else {} + save = save or (lambda _: None) + start = state.get("sent", 0) + if not 0 <= start <= len(posts): + raise SystemExit(f"::error::progress says {start} posts were sent, but there are {len(posts)}") + if start: + print(f"forum: posts 1-{start} were sent by an earlier attempt") + for index in range(start, len(posts)): + text = posts[index] + try: + send(text, token) + except Exception as exc: + raise RuntimeError(f"forum post {index + 1}/{len(posts)} failed; posts 1-{index} were sent") from exc + state["sent"] = index + 1 + save(state) + print(f"forum post {index + 1}/{len(posts)} sent ({post_length(text)} chars)") + + +def send_yemot(content, heading, token, state=None, save=None): + from yemot import split_and_send + split_and_send(content, heading, token, YEMOT_PATH, TZINTUK_LIST, progress=state, save=save) + + +def parse_channels(text): + channels = [c.strip() for c in text.split(",") if c.strip()] + unknown = set(channels) - set(CHANNELS) + if unknown or not channels: + raise SystemExit(f"::error::unknown or empty channel list {text!r}; allowed: {', '.join(CHANNELS)}") + return channels + + +def load_catalog(path): + catalog = json.loads(Path(path).read_text(encoding="utf-8")) + if catalog.get("schema") != CATALOG_SCHEMA: + raise SystemExit(f"::error::unsupported catalog schema {catalog.get('schema')!r}") + return catalog + + +def render(args): + version = db_version(args.tag) + if version is None: + raise SystemExit(f"::error::{args.tag!r} is not a v- release tag") + channels = parse_channels(args.channels) + changes = diff_catalogs(load_catalog(args.old), load_catalog(args.new)) + editions = sum(1 for item in changes["changed"] if item["edition"]) + print("changes: " + " ".join(f"{k}={len(v)}" for k, v in changes.items()) + f" edition-changed={editions}") + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + (out / "changes.json").write_text(json.dumps(changes, ensure_ascii=False, indent=1), encoding="utf-8") + has_changes = any(changes.values()) + summary = {"tag": args.tag, "channels": channels, "has_changes": has_changes, + "counts": {k: len(v) for k, v in changes.items()}, "edition_changed": editions} + if has_changes: + date_text = heb_date(date.fromisoformat(args.as_of) if args.as_of else datetime.now(tz=TZ).date()) + posts = build_forum_posts(changes, version, date_text) + yemot = {"heading": f"גרסת ספרייה {version}\nעדכון {date_text}\n", "content": build_yemot_content(changes)} + (out / "forum_posts.json").write_text(json.dumps(posts, ensure_ascii=False, indent=1), encoding="utf-8") + (out / "yemot.json").write_text(json.dumps(yemot, ensure_ascii=False, indent=1), encoding="utf-8") + (out / RELEASE_NOTES_ASSET).write_text(build_release_notes(changes, version, date_text), encoding="utf-8") + summary["forum_posts"] = [post_length(p) for p in posts] + for index, text in enumerate(posts, 1): + print(f"----- forum post {index}/{len(posts)} ({post_length(text)} chars) -----") + print(text) + print("------------------------") + else: + print("⏭️ No book changed since the previous published release — nothing to announce.") + (out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8") + github_output = os.getenv("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a", encoding="utf-8") as sink: + sink.write(f"has_changes={'true' if has_changes else 'false'}\n") + sink.write(f"channels={json.dumps(channels)}\n") + return 0 + + +def send(args): + secret = {"forum": "FORUM_TOKEN", "yemot": "TOKEN_YEMOT"}[args.channel] + token = os.environ.get(secret, "") + if not token.strip(): + raise SystemExit(f"::error::{secret} is empty — set the repository secret before announcing to {args.channel}") + payload = (Path(args.dir) / ("forum_posts.json" if args.channel == "forum" else "yemot.json")).read_bytes() + state, save = load_progress(args.progress, payload), progress_saver(args.progress) + value = json.loads(payload.decode("utf-8")) + if args.channel == "forum": + send_forum_posts(value, token, state, save) + else: + send_yemot(value["content"], value["heading"], token, state, save) + print(f"✅ {args.channel} announced") + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + previous = commands.add_parser("previous-tag") + previous.add_argument("--current", required=True) + catalog = commands.add_parser("catalog") + catalog.add_argument("db") + catalog.add_argument("out") + rendered = commands.add_parser("render") + rendered.add_argument("--old", required=True) + rendered.add_argument("--new", required=True) + rendered.add_argument("--tag", required=True) + rendered.add_argument("--as-of", default="") + rendered.add_argument("--channels", default=",".join(CHANNELS)) + rendered.add_argument("--out", required=True) + sender = commands.add_parser("send") + sender.add_argument("--channel", required=True, choices=SENT_CHANNELS) + sender.add_argument("--dir", required=True) + sender.add_argument("--progress", default="") + args = parser.parse_args(argv) + + if args.command == "previous-tag": + tag = previous_published_tag(args.current, sys.stdin.read().split()) + print(tag or "") + return 0 + if args.command == "catalog": + value = build_catalog(args.db) + with open(args.out, "w", encoding="utf-8") as out: + json.dump(value, out, ensure_ascii=False, separators=(",", ":")) + print(f"catalog: {len(value['books'])} books -> {args.out}") + return 0 + if args.command == "render": + return render(args) + return send(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/library_announce/test_library_update_announce.py b/.github/scripts/library_announce/test_library_update_announce.py new file mode 100644 index 000000000..8a3096cd2 --- /dev/null +++ b/.github/scripts/library_announce/test_library_update_announce.py @@ -0,0 +1,1050 @@ +#!/usr/bin/env python3 +"""Tests for library_update_announce.py, yemot.py and their workflow.""" +import hashlib +import io +import json +import os +import shutil +import subprocess +import sqlite3 +import sys +import tempfile +import unittest +from contextlib import closing, redirect_stdout +from pathlib import Path +from unittest import mock + +try: + import yaml +except ImportError: # pragma: no cover - CI installs PyYAML explicitly + yaml = None + +ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import library_update_announce as lua # noqa: E402 +import yemot # noqa: E402 + +WORKFLOW = ROOT / ".github" / "workflows" / "library-update-announce.yml" +RELEASE = ROOT / ".github" / "workflows" / "manual-generate-release.yml" +CI = ROOT / ".github" / "workflows" / "ci.yml" + + +def make_db(path, books, lines, versions=(), split=False): + conn = sqlite3.connect(path) + conn.executescript(""" + CREATE TABLE category (id INTEGER PRIMARY KEY, parentId INTEGER, title TEXT NOT NULL); + CREATE TABLE source (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE book (id INTEGER PRIMARY KEY, categoryId INTEGER, sourceId INTEGER, title TEXT NOT NULL); + CREATE TABLE line (id INTEGER PRIMARY KEY, bookId INTEGER, lineIndex INTEGER, content TEXT NOT NULL); + CREATE TABLE book_version (id INTEGER PRIMARY KEY, bookId INTEGER NOT NULL, versionTitle TEXT NOT NULL, + heVersionTitle TEXT, hasContent INTEGER NOT NULL DEFAULT 0); + CREATE TABLE version_line (versionId INTEGER NOT NULL, lineId INTEGER NOT NULL, content TEXT NOT NULL); + -- Like the real DB: every root has parentId NULL, none is its own parent. + INSERT INTO category VALUES (1, NULL, 'תנך'), (2, 1, 'תורה'), (3, NULL, 'הלכה'); + INSERT INTO source VALUES (1, 'Sefaria'), (2, 'DictaToOtzaria'); + """) + conn.executemany("INSERT INTO book VALUES (?, ?, ?, ?)", books) + conn.executemany("INSERT INTO line (bookId, lineIndex, content) VALUES (?, ?, ?)", lines) + # versions: (bookId, versionTitle, heVersionTitle, {lineIndex: content} or None for metadata-only) + for book_id, title, he_title, texts in versions: + version_id = conn.execute( + "INSERT INTO book_version (bookId, versionTitle, heVersionTitle, hasContent) VALUES (?, ?, ?, ?)", + (book_id, title, he_title, int(texts is not None))).lastrowid + for index, content in (texts or {}).items(): + [(line_id,)] = conn.execute("SELECT id FROM line WHERE bookId = ? AND lineIndex = ?", (book_id, index)) + conn.execute("INSERT INTO version_line VALUES (?, ?, ?)", (version_id, line_id, content)) + if split: + conn.executescript(""" + CREATE TABLE line_content (id INTEGER PRIMARY KEY, content TEXT NOT NULL); + INSERT INTO line_content SELECT id, content FROM line; + ALTER TABLE line DROP COLUMN content; + ALTER TABLE version_line RENAME TO old_version_line; + CREATE TABLE version_line (versionId INTEGER NOT NULL, lineId INTEGER NOT NULL, content TEXT); + INSERT INTO version_line + SELECT v.versionId, v.lineId, + CASE WHEN CAST(v.content AS BLOB) = CAST(lc.content AS BLOB) THEN NULL ELSE v.content END + FROM old_version_line v JOIN line_content lc ON lc.id = v.lineId; + DROP TABLE old_version_line; + """) + conn.commit() + conn.close() + + +def catalog(books, lines, versions=()): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "seforim.db" + make_db(path, books, lines, versions) + return lua.build_catalog(path) + + +def names(entries): + return [e["name"] for e in entries] + + +class PreviousTagTest(unittest.TestCase): + def test_picks_the_newest_older_tag_and_ignores_others(self): + tags = ["v27-20260906092829", "v28-20260910220310", "v30-20261001000000", + "lines-snapshot-sha256-abc", "v9-20260706195857"] + self.assertEqual("v28-20260910220310", lua.previous_published_tag("v29-20260927072953", tags)) + + def test_first_release_has_no_previous(self): + self.assertIsNone(lua.previous_published_tag("v1-20260625232529", ["v1-20260625232529"])) + + def test_refuses_a_tag_that_is_not_a_db_release(self): + with self.assertRaises(ValueError): + lua.previous_published_tag("pipeline-result-run-1-1", []) + + +class PlainTextTest(unittest.TestCase): + def test_markup_entities_and_whitespace_are_not_text(self): + self.assertEqual("א ב & ג", lua.plain_text("א \n ב & ג ")) + self.assertEqual("<", lua.plain_text("&lt;")) + self.assertEqual("<", lua.plain_text("<")) + self.assertEqual("חסר <חסר> כאן <בד\"ה אמר>", lua.plain_text("חסר <חסר> כאן <בד\"ה אמר>")) + self.assertEqual("x", lua.plain_text("x")) + self.assertEqual(lua.plain_text("נקודה: הלכה"), lua.plain_text("נקודה:
הלכה")) + self.assertNotEqual(lua.plain_text("וירגה זכרונו"), lua.plain_text("וירגהזכרונו")) + + +class CatalogTest(unittest.TestCase): + def test_schema_split_preserves_catalog_with_inherited_different_and_missing_edition_rows(self): + books = [(i, 2, 1, f"book {i}") for i in range(1, 6)] + lines = [(i, j, text) for i in range(1, 6) for j, text in ((0, "alpha"), (1, "beta"))] + versions = [(1, "Read", None, {0: "alpha", 1: "beta"}), + (1, "Other", None, {0: "alpha", 1: "different"}), + (2, "Part A", None, {0: "alpha"}), (2, "Part B", None, {1: "beta"}), + (3, "Twin A", None, {0: "alpha", 1: "beta"}), + (3, "Twin B", None, {0: "alpha", 1: "beta"}), + (4, "Metadata", None, None), (5, "Empty", None, {})] + with tempfile.TemporaryDirectory() as tmp: + legacy, split = Path(tmp) / "legacy.db", Path(tmp) / "split.db" + make_db(legacy, books, lines, versions) + make_db(split, books, lines, versions, split=True) + with closing(sqlite3.connect(split)) as conn: + self.assertGreater(conn.execute("SELECT COUNT(*) FROM version_line WHERE content IS NULL").fetchone()[0], 0) + self.assertEqual(1, conn.execute("SELECT COUNT(*) FROM version_line WHERE content IS NOT NULL").fetchone()[0]) + before, after = lua.build_catalog(legacy), lua.build_catalog(split) + self.assertEqual(before, after) + self.assertEqual(["Read", None, None, "Metadata", None], [b["edition"] for b in after["books"]]) + self.assertFalse(any(lua.diff_catalogs(before, after).values())) + + def test_compressed_line_text_preserves_catalog(self): + try: + import zstandard + except ImportError: # pragma: no cover - CI installs zstandard explicitly + self.skipTest("zstandard is not installed") + dictionary = (ROOT / "generator" / "common" / "src" / "jvmMain" / "resources" / "zstd" + / "line_content.zdict").read_bytes() + compressor = zstandard.ZstdCompressor( + level=19, dict_data=zstandard.ZstdCompressionDict(dictionary), write_checksum=False) + books = [(1, 2, 1, "book 1"), (2, 2, 1, "book 2")] + lines = [(1, 0, "אמר רבא"), (1, 1, "בראשית"), (2, 0, "alpha")] + versions = [(1, "Read", None, {0: "אמר רבא", 1: "בראשית"}), (2, "Other", None, {0: "beta"})] + with tempfile.TemporaryDirectory() as tmp: + split, compressed = Path(tmp) / "split.db", Path(tmp) / "compressed.db" + make_db(split, books, lines, versions, split=True) + make_db(compressed, books, lines, versions, split=True) + with closing(sqlite3.connect(compressed)) as conn: + conn.create_function("zframe", 1, lambda t: None if t is None else compressor.compress(t.encode())) + conn.executescript(""" + UPDATE line_content SET content = zframe(content); + UPDATE version_line SET content = zframe(content) WHERE content IS NOT NULL; + CREATE TABLE zstd_dict (id INTEGER PRIMARY KEY, dict BLOB NOT NULL); + """) + conn.execute("INSERT INTO zstd_dict VALUES (1, ?)", (dictionary,)) + conn.commit() + self.assertEqual(lua.build_catalog(split), lua.build_catalog(compressed)) + + def test_paths_walk_up_to_a_null_parent_root(self): + value = catalog([(10, 2, 1, "בראשית")], [(10, 0, "א")]) + self.assertEqual("תנך/תורה", value["books"][0]["path"]) + self.assertEqual("Sefaria", value["books"][0]["source"]) + self.assertEqual(1, value["books"][0]["lines"]) + + def test_a_category_cycle_fails_loudly(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "seforim.db" + make_db(path, [(10, 2, 1, "בראשית")], []) + conn = sqlite3.connect(path) + conn.execute("UPDATE category SET parentId = 2 WHERE id = 1") + conn.commit() + conn.close() + with self.assertRaisesRegex(ValueError, "broken parent chain"): + lua.build_catalog(path) + + def test_hash_follows_line_order_not_insertion_order(self): + a = catalog([(10, 2, 1, "בראשית")], [(10, 0, "א"), (10, 1, "ב")]) + b = catalog([(10, 2, 1, "בראשית")], [(10, 1, "ב"), (10, 0, "א")]) + c = catalog([(10, 2, 1, "בראשית")], [(10, 0, "ב"), (10, 1, "א")]) + self.assertEqual(a["books"][0]["hash"], b["books"][0]["hash"]) + self.assertNotEqual(a["books"][0]["hash"], c["books"][0]["hash"]) + + def test_markup_and_line_wrapping_do_not_change_the_hash(self): + a = catalog([(10, 2, 1, "בראשית")], [(10, 0, "בראשית ברא"), (10, 1, "אלהים")]) + b = catalog([(10, 2, 1, "בראשית")], [(10, 0, "

בראשית

ברא אלהים"), (10, 1, "
")]) + c = catalog([(10, 2, 1, "בראשית")], [(10, 0, "בראשית ברא אלוהים")]) + self.assertEqual(a["books"][0]["hash"], b["books"][0]["hash"]) + self.assertNotEqual(a["books"][0]["hash"], c["books"][0]["hash"]) + + def test_book_without_lines_has_no_hash(self): + value = catalog([(10, 2, 1, "ריק")], []) + self.assertIsNone(value["books"][0]["hash"]) + + def test_edition_is_the_version_whose_text_the_book_lines_are(self): + value = catalog( + [(1, 2, 1, "א"), (2, 2, 1, "ב"), (3, 2, 1, "ג"), (4, 2, 2, "ד"), (5, 2, 1, "ה"), (6, 2, 1, "ו"), + (7, 2, 1, "ז")], + [(b, i, t) for b in range(1, 8) for i, t in ((0, "א"), (1, "ב"))], + [(1, "Only", None, None), + (2, "Other", "אחרת", {0: "א", 1: "שונה"}), (2, "Read", "הנקראת", {0: "א", 1: "ב"}), + (3, "Part A", "א", {0: "א"}), (3, "Part B", "ב", {1: "ב"}), + (5, "Blank", " ", {0: "א", 1: "ב"}), + (6, "Twin A", None, {0: "א", 1: "ב"}), (6, "Twin B", None, {0: "א", 1: "ב"}), + (7, "Not the text", "לא", {0: "א", 1: "אחר"})]) + books = {b["id"]: b for b in value["books"]} + self.assertEqual(("Only", "Only"), (books[1]["edition"], books[1]["edition_name"])) + self.assertEqual(("Read", "הנקראת"), (books[2]["edition"], books[2]["edition_name"])) + self.assertIsNone(books[3]["edition"], "a mosaic of editions is no one edition") + self.assertIsNone(books[4]["edition"], "no versions at all") + self.assertEqual("Blank", books[5]["edition_name"]) + self.assertIsNone(books[6]["edition"], "two identical editions are a tie") + self.assertIsNone(books[7]["edition"], "the only edition is not the book's text") + + +class DiffTest(unittest.TestCase): + def setUp(self): + self.old = catalog( + [(1, 2, 1, "בראשית"), (2, 2, 1, "שמות"), (3, 3, 2, "ישן"), (4, 3, 2, "נמחק"), (5, 2, 2, "נודד"), + (8, 2, 1, "ויקרא"), (9, 3, 1, "משנה")], + [(1, 0, "א"), (2, 0, "ב"), (3, 0, "תוכן ישן"), (4, 0, "ד"), (5, 0, "ה"), (8, 0, "ח"), (9, 0, "ט")], + [(8, "Old ed", "ישנה", {0: "ח"}), (8, "New ed", "מהדורה חדשה", {0: "ח ישן"}), (9, "Same", "אותה", None)]) + self.new = catalog( + [(1, 2, 1, "בראשית"), (2, 2, 1, "שמות"), (6, 3, 2, "חדש-בשם"), (7, 2, 1, "חדש"), (5, 3, 2, "נודד"), + (8, 2, 1, "ויקרא"), (9, 1, 1, "משנה")], + [(1, 0, "א"), (2, 0, "ב שונה"), (6, 0, "תוכן ישן"), (7, 0, "ז"), (5, 0, "ה"), (8, 0, "ח שונה"), + (9, 0, "ט שונה")], + [(8, "Old ed", "ישנה", {0: "ח"}), (8, "New ed", "מהדורה חדשה", {0: "ח שונה"}), + (9, "Same", "אותה", None)]) + self.changes = lua.diff_catalogs(self.old, self.new) + + def test_classifies_every_kind_of_change(self): + self.assertEqual(["תנך/תורה/חדש"], names(self.changes["added"])) + self.assertEqual(["הלכה/נמחק"], names(self.changes["removed"])) + self.assertEqual(["תנך/משנה", "תנך/תורה/ויקרא", "תנך/תורה/שמות"], names(self.changes["changed"])) + self.assertEqual(["הלכה/חדש-בשם", "הלכה/נודד", "תנך/משנה"], names(self.changes["moved"])) + + def test_markup_only_change_is_not_a_change(self): + self.assertNotIn("תנך/תורה/בראשית", names(self.changes["changed"])) + + def test_edition_change_is_one_changed_line(self): + lines = lua.forum_lines(self.changes)["changed"] + self.assertIn("* תנך/תורה/ויקרא - שינוי מהדורה ל-מהדורה חדשה", lines) + self.assertEqual(1, sum("ויקרא" in line for line in lines)) + + def test_an_edition_newly_identified_is_not_an_edition_change(self): + old = catalog([(1, 2, 1, "א")], [(1, 0, "א")], [(1, "Listed", None, {0: "אחר"})]) + new = catalog([(1, 2, 1, "א")], [(1, 0, "א")], [(1, "Listed", None, {0: "אחר"}), (1, "Read", None, {0: "א"})]) + self.assertIsNone(old["books"][0]["edition"]) + self.assertEqual("Read", new["books"][0]["edition"]) + self.assertFalse(any(lua.diff_catalogs(old, new).values())) + + def test_moved_and_changed_is_explicit_in_both_sections(self): + lines = lua.forum_lines(self.changes) + self.assertIn("* תנך/משנה (לשעבר: הלכה/משנה) (השתנה גם תוכנו)", lines["moved"]) + self.assertIn("* תנך/משנה (שונה גם מיקומו/שמו)", lines["changed"]) + self.assertIn("* הלכה/חדש-בשם (לשעבר: הלכה/ישן)", lines["moved"]) + + def test_identical_catalogs_have_no_changes(self): + self.assertFalse(any(lua.diff_catalogs(self.old, self.old).values())) + + def test_ambiguous_rename_fails_loudly(self): + old = catalog([(1, 2, 1, "א"), (2, 2, 1, "ב")], [(1, 0, "זהה"), (2, 0, "זהה")]) + new = catalog([(3, 2, 1, "ג"), (4, 2, 1, "ד")], [(3, 0, "זהה"), (4, 0, "זהה")]) + with self.assertRaisesRegex(ValueError, "ambiguous rename"): + lua.diff_catalogs(old, new) + + def test_a_single_twin_on_one_side_only_is_no_rename(self): + old = catalog([(1, 2, 1, "א"), (2, 2, 1, "ב")], [(1, 0, "זהה"), (2, 0, "זהה")]) + new = catalog([(3, 2, 1, "ג")], [(3, 0, "אחר")]) + changes = lua.diff_catalogs(old, new) + self.assertEqual(2, len(changes["removed"])) + self.assertEqual([], changes["moved"]) + + +class ForumPostTest(unittest.TestCase): + def changes(self, count, key="added"): + value = {k: [] for k, _ in lua.SECTIONS} + value[key] = [{"name": f"קטגוריה/ספר {i:04d}", "title": f"ספר {i:04d}"} for i in range(count)] + return value + + def test_post_lists_only_non_empty_sections(self): + changes = {"added": [{"name": "א/ב", "title": "ב"}], "removed": [], "moved": [], + "changed": [{"name": "ג/ד", "title": "ד", "edition": None, "also_moved": False}, + {"name": "ג/ה", "title": "ה", "edition": "מהד", "also_moved": False}]} + [post] = lua.build_forum_posts(changes, 30, "א' תשרי") + self.assertTrue(post.startswith("# גרסת ספרייה 30\n\n**עדכון א' תשרי**\n")) + self.assertIn("## התווספו הספרים הבאים:\n* א/ב\n", post) + self.assertIn("## השתנו הספרים הבאים:\n* ג/ד\n* ג/ה - שינוי מהדורה ל-מהד\n", post) + self.assertNotIn("נמחקו", post) + + def test_release_notes_hold_the_whole_list_in_one_file(self): + changes = self.changes(3000) + posts = lua.build_forum_posts(changes, 30, "א' תשרי") + self.assertGreater(len(posts), 1) + notes = lua.build_release_notes(changes, 30, "א' תשרי") + self.assertEqual([lua.build_forum_posts(changes, 30, "א' תשרי", limit=10**9)[0]], [notes]) + self.assertEqual(3000, notes.count("\n* ")) + self.assertNotIn("(המשך)", notes) + + def test_long_list_is_split_on_line_boundaries_without_losing_a_line(self): + changes = self.changes(1000) + changes["removed"] = [{"name": "x/נמחק", "title": "נמחק"}] + posts = lua.build_forum_posts(changes, 30, "x", limit=500) + self.assertGreater(len(posts), 10) + self.assertTrue(all(lua.post_length(p) <= 500 for p in posts)) + self.assertTrue(posts[0].startswith("# גרסת ספרייה 30\n")) + for post in posts[1:]: + self.assertTrue(post.startswith("# גרסת ספרייה 30 (המשך)\n\n## ")) + items = [line for post in posts for line in post.splitlines() if line.startswith("* ")] + self.assertEqual([f"* קטגוריה/ספר {i:04d}" for i in range(1000)] + ["* x/נמחק"], items) + self.assertEqual(posts, lua.build_forum_posts(changes, 30, "x", limit=500)) + + def test_a_continuation_mid_section_repeats_its_heading(self): + posts = lua.build_forum_posts(self.changes(100), 30, "x", limit=500) + self.assertIn("\n\n## התווספו הספרים הבאים: (המשך)\n* ", posts[1]) + + def test_default_limit_is_the_forum_maximum(self): + posts = lua.build_forum_posts(self.changes(3000), 30, "x") + self.assertGreater(len(posts), 1) + self.assertTrue(all(lua.post_length(p) <= 32767 for p in posts)) + + def test_length_counts_utf16_units_like_nodebb(self): + self.assertEqual(4, lua.post_length("אב😀")) + + def test_a_line_longer_than_a_post_fails_loudly(self): + changes = self.changes(1) + changes["added"][0]["name"] = "א" * 600 + with self.assertRaises(ValueError): + lua.build_forum_posts(changes, 30, "x", limit=500) + + +class YemotContentTest(unittest.TestCase): + def test_reads_titles_and_the_new_name_of_a_moved_book(self): + changes = { + "added": [{"name": "א/ב", "title": "ב"}], + "removed": [], + "moved": [{"name": "a/חדש", "title": "חדש", "old_name": "c/ישן", "old_title": "ישן", "also_changed": False}, + {"name": "a/נודד", "title": "נודד", "old_name": "c/נודד", "old_title": "נודד", + "also_changed": True}], + "changed": [{"name": "ג/ד", "title": "ד", "edition": "מהד", "also_moved": False}], + } + self.assertEqual({ + "התווספו הספרים הבאים:": "ב", + "שונה מיקום/שם של הספרים הבאים:": "חדש (לשעבר: ישן)\nנודד (השתנה גם תוכנו)", + "השתנו הספרים הבאים:": "ד - שינוי מהדורה ל-מהד", + }, lua.build_yemot_content(changes)) + + +class Response: + def __init__(self, body, status=200): + self.status_code, self.body, self.text = status, body, repr(body) + + def json(self): + if isinstance(self.body, Exception): + raise self.body + return self.body + + +class Session: + def __init__(self, max_file="005.tts", fail_on=None, fail_tzintuk=False): + self.max_file, self.fail_on, self.uploads, self.calls = max_file, fail_on, [], [] + self.fail_tzintuk = fail_tzintuk + + def get(self, url, params, timeout): + self.calls.append((url, timeout)) + if url.endswith("GetIVR2DirStats"): + return Response({"responseStatus": "OK", "maxFile": {"name": self.max_file}}) + if self.fail_tzintuk: + return Response({"responseStatus": "ERROR", "message": "busy"}) + return Response({"responseStatus": "OK"}) + + def post(self, url, data, timeout): + self.calls.append((url, timeout)) + if self.fail_on and self.fail_on in data["what"]: + return Response({"responseStatus": "ERROR", "message": "denied"}) + self.uploads.append((data["what"], data["contents"])) + return Response({"responseStatus": "OK"}) + + +class YemotTest(unittest.TestCase): + def test_split_never_loses_or_duplicates_text(self): + lines = [f"שורה {i}" + "x" * (i % 37) for i in range(400)] + content = "\n".join(lines) + parts = yemot.split_content(content, 100) + self.assertTrue(all(len(p) <= 100 for p in parts)) + self.assertEqual(lines, "\n".join(parts).split("\n")) + + def test_a_chunk_without_a_newline_is_cut_not_dropped(self): + content = "א" * 250 + "\nב" + parts = yemot.split_content(content, 100) + self.assertEqual(["א" * 100, "א" * 100, "א" * 50 + "\nב"], parts) + + def test_a_newline_at_the_chunk_start_does_not_loop(self): + content = "\n" + "א" * 150 + self.assertEqual(["א" * 99, "א" * 51], yemot.split_content(content, 100)) + + def test_uploads_continue_numbering_and_reverse_each_section(self): + session = Session(max_file="005.tts") + with mock.patch.object(yemot, "CHUNK_SIZE", 10): + yemot.split_and_send({"כותרת": "אאאא\nבבבב\nגגגג"}, "עדכון\n", "t", "ivr2:/1", "list", + session=session) + self.assertEqual([ + ("ivr2:/1/006.tts", "גגגג"), ("ivr2:/1/007.tts", "אאאא\nבבבב"), + ("ivr2:/1/007-Title.tts", "כותרת"), ("ivr2:/1/008.tts", "עדכון\n"), + ], session.uploads) + self.assertTrue(all(timeout for _, timeout in session.calls)) + self.assertTrue(session.calls[-1][0].endswith("RunTzintuk")) + + def test_unknown_numbering_fails_instead_of_restarting_at_000(self): + for max_file in (None, "", "Title.tts", "005-Title.tts"): + with self.subTest(max_file=max_file): + with self.assertRaises(yemot.YemotError): + yemot.get_file_num("t", "ivr2:/1", Session(max_file=max_file)) + + def test_a_refused_upload_fails_loudly(self): + session = Session(fail_on="-Title") + with self.assertRaisesRegex(yemot.YemotError, "UploadTextFile"): + yemot.split_and_send({"כותרת": "א"}, "עדכון\n", "t", "ivr2:/1", "list", session=session) + + def test_an_interrupted_send_resumes_without_reuploading(self): + content = {"כותרת": "אאאא\nבבבב\nגגגג"} + saved = [] + progress = {} + with mock.patch.object(yemot, "CHUNK_SIZE", 10): + with self.assertRaises(yemot.YemotError): + yemot.split_and_send(content, "עדכון\n", "t", "ivr2:/1", "list", session=Session(fail_on="007-Title"), + progress=progress, save=lambda state: saved.append(dict(state))) + self.assertEqual({"base": 5, "uploaded": 2, "tzintuk": False}, progress) + # The folder grew meanwhile: a resume must keep the recorded numbering. + second = Session(max_file="007.tts") + yemot.split_and_send(content, "עדכון\n", "t", "ivr2:/1", "list", session=second, + progress=progress, save=lambda state: saved.append(dict(state))) + self.assertEqual([("ivr2:/1/007-Title.tts", "כותרת"), ("ivr2:/1/008.tts", "עדכון\n")], second.uploads) + self.assertFalse(any(url.endswith("GetIVR2DirStats") for url, _ in second.calls)) + self.assertEqual({"base": 5, "uploaded": 4, "tzintuk": True}, saved[-1]) + + def test_tzintuk_runs_once_after_every_upload(self): + progress = {} + with self.assertRaises(yemot.YemotError): + yemot.split_and_send({"כ": "א"}, "ע\n", "t", "ivr2:/1", "list", session=Session(fail_tzintuk=True), + progress=progress) + self.assertEqual({"base": 5, "uploaded": 3, "tzintuk": False}, progress) + again = Session() + yemot.split_and_send({"כ": "א"}, "ע\n", "t", "ivr2:/1", "list", session=again, progress=progress) + self.assertEqual([], again.uploads) + self.assertEqual(["RunTzintuk"], [url.rsplit("/", 1)[-1] for url, _ in again.calls]) + done = Session() + yemot.split_and_send({"כ": "א"}, "ע\n", "t", "ivr2:/1", "list", session=done, progress=progress) + self.assertEqual([], done.calls) + + def test_upload_plan_is_deterministic(self): + content = {"א": "1\n2", "ב": "3"} + self.assertEqual([("006", "1\n2"), ("006-Title", "א"), ("007", "3"), ("007-Title", "ב"), ("008", "ד")], + yemot.upload_plan(content, "ד", 5)) + + def test_http_errors_and_non_json_fail_loudly(self): + with self.assertRaisesRegex(yemot.YemotError, "HTTP 500"): + yemot._checked(Response({}, status=500), "x") + with self.assertRaisesRegex(yemot.YemotError, "not JSON"): + yemot._checked(Response(ValueError("bad")), "x") + + +class RenderAndSendTest(unittest.TestCase): + def setUp(self): + self.old = catalog([(1, 2, 1, "בראשית")], [(1, 0, "א")]) + self.new = catalog([(1, 2, 1, "בראשית"), (2, 2, 1, "שמות")], [(1, 0, "א"), (2, 0, "ב")]) + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def render(self, old, new, argv=()): + paths = [] + for name, value in (("old.json", old), ("new.json", new)): + path = self.dir / name + path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + paths.append(str(path)) + out = io.StringIO() + with mock.patch.dict("os.environ", {"GITHUB_OUTPUT": str(self.dir / "gh_out")}), redirect_stdout(out): + code = lua.main(["render", "--old", paths[0], "--new", paths[1], "--tag", "v30-20261001000000", + "--as-of", "2026-10-01", "--out", str(self.dir / "out"), *argv]) + return code, out.getvalue() + + def test_nothing_changed_renders_nothing_to_send(self): + code, out = self.render(self.old, self.old) + self.assertEqual(0, code) + self.assertIn("nothing to announce", out) + self.assertFalse((self.dir / "out" / "forum_posts.json").exists()) + self.assertFalse((self.dir / "out" / lua.RELEASE_NOTES_ASSET).exists()) + self.assertIn("has_changes=false\n", (self.dir / "gh_out").read_text(encoding="utf-8")) + + def test_render_writes_every_channel_payload(self): + code, out = self.render(self.old, self.new) + self.assertEqual(0, code) + self.assertIn("תנך/תורה/שמות", out) + posts = json.loads((self.dir / "out" / "forum_posts.json").read_text(encoding="utf-8")) + self.assertTrue(posts[0].startswith("# גרסת ספרייה 30\n")) + spoken = json.loads((self.dir / "out" / "yemot.json").read_text(encoding="utf-8")) + self.assertEqual({"התווספו הספרים הבאים:": "שמות"}, spoken["content"]) + self.assertRegex(spoken["heading"], r"^גרסת ספרייה 30\nעדכון .*תשפ.*\n$") + self.assertEqual(posts, [(self.dir / "out" / lua.RELEASE_NOTES_ASSET).read_text(encoding="utf-8")]) + self.assertIn('channels=["forum", "yemot", "release"]\n', (self.dir / "gh_out").read_text(encoding="utf-8")) + + def test_unknown_channel_is_refused(self): + with self.assertRaises(SystemExit): + self.render(self.old, self.new, ["--channels", "forum,chat"]) + + def send(self, channel, env, progress=None): + argv = ["send", "--channel", channel, "--dir", str(self.dir / "out")] + if progress: + argv += ["--progress", str(progress)] + with mock.patch.dict("os.environ", env), redirect_stdout(io.StringIO()): + return lua.main(argv) + + def test_send_forum_posts_every_rendered_post_in_order(self): + self.render(self.old, self.new) + (self.dir / "out" / "forum_posts.json").write_text(json.dumps(["1", "2"]), encoding="utf-8") + with mock.patch.object(lua, "send_forum") as forum: + self.assertEqual(0, self.send("forum", {"FORUM_TOKEN": "t"})) + self.assertEqual([mock.call("1", "t"), mock.call("2", "t")], forum.call_args_list) + + def test_an_empty_secret_fails_before_sending(self): + self.render(self.old, self.new) + for channel, secret in (("forum", "FORUM_TOKEN"), ("yemot", "TOKEN_YEMOT")): + with self.subTest(channel=channel), mock.patch.object(lua, "send_forum") as forum, \ + mock.patch.object(lua, "send_yemot") as spoken: + with self.assertRaisesRegex(SystemExit, f"::error::{secret} is empty"): + self.send(channel, {secret: " "}) + forum.assert_not_called() + spoken.assert_not_called() + + def test_a_rerun_continues_the_forum_after_the_last_accepted_post(self): + self.render(self.old, self.new) + (self.dir / "out" / "forum_posts.json").write_text(json.dumps(["1", "2", "3"]), encoding="utf-8") + progress = self.dir / "progress" / "forum_progress.json" + with mock.patch.object(lua, "send_forum", side_effect=[None, lua.ForumRefused("403: no")]): + with self.assertRaisesRegex(RuntimeError, "post 2/3 failed; posts 1-1"): + self.send("forum", {"FORUM_TOKEN": "t"}, progress) + self.assertEqual(1, json.loads(progress.read_text(encoding="utf-8"))["sent"]) + with mock.patch.object(lua, "send_forum") as forum: + self.send("forum", {"FORUM_TOKEN": "t"}, progress) + self.assertEqual([mock.call("2", "t"), mock.call("3", "t")], forum.call_args_list) + self.assertEqual(3, json.loads(progress.read_text(encoding="utf-8"))["sent"]) + + def test_progress_for_another_announcement_is_refused(self): + self.render(self.old, self.new) + progress = self.dir / "forum_progress.json" + progress.write_text(json.dumps({"payload_sha256": "0" * 64, "sent": 1}), encoding="utf-8") + with mock.patch.object(lua, "send_forum") as forum, self.assertRaisesRegex(SystemExit, "another"): + self.send("forum", {"FORUM_TOKEN": "t"}, progress) + forum.assert_not_called() + + def test_a_failed_post_names_how_many_were_sent(self): + send = mock.Mock(side_effect=[None, lua.ForumRefused("403: no")]) + with redirect_stdout(io.StringIO()), self.assertRaisesRegex(RuntimeError, "post 2/3 failed; posts 1-1"): + lua.send_forum_posts(["a", "b", "c"], "t", send=send) + + def test_send_yemot_uses_the_rendered_content_and_heading(self): + self.render(self.old, self.new) + with mock.patch.object(lua, "send_yemot") as spoken: + self.send("yemot", {"TOKEN_YEMOT": "t"}, self.dir / "yemot_progress.json") + content, heading, token, state, _save = spoken.call_args.args + self.assertEqual({"התווספו הספרים הבאים:": "שמות"}, content) + self.assertTrue(heading.startswith("גרסת ספרייה 30\nעדכון ")) + self.assertEqual("t", token) + self.assertIn("payload_sha256", state) + + +class ForumTest(unittest.TestCase): + def test_the_rate_limit_refusal_is_retried(self): + post = mock.Mock(side_effect=[lua.ForumRefused("400: [[error:too-many-posts, 10]]"), None]) + with mock.patch.object(lua.time, "sleep"): + lua.send_forum("x", "t", post=post) + self.assertEqual(2, post.call_count) + + def test_any_other_refusal_is_not_retried(self): + post = mock.Mock(side_effect=lua.ForumRefused("403: [[error:no-privileges]]")) + with self.assertRaises(lua.ForumRefused): + lua.send_forum("x", "t", post=post) + post.assert_called_once() + + def test_the_token_is_sent_as_a_bearer_to_the_topic(self): + session = mock.Mock() + session.post.return_value.json.return_value = {"status": {"code": "ok"}} + lua.post_to_forum("טקסט", "secret", session=session) + url = session.post.call_args.args[0] + kwargs = session.post.call_args.kwargs + self.assertEqual(f"https://otzaria.org/forum/api/v3/topics/{lua.FORUM_TOPIC_ID}", url) + self.assertEqual("Bearer secret", kwargs["headers"]["Authorization"]) + self.assertEqual({"content": "טקסט"}, kwargs["json"]) + self.assertTrue(kwargs["timeout"]) + + def test_a_refusal_in_the_body_is_not_a_success(self): + session = mock.Mock() + session.post.return_value.json.return_value = {"status": {"code": "bad-request", "message": "x"}} + with self.assertRaises(lua.ForumRefused): + lua.post_to_forum("x", "t", session=session) + + +FAKE_GH = r"""#!/usr/bin/env python3 +import json, os, re, subprocess, sys +fixture = json.load(open(os.environ["FAKE_GH"])) +with open(os.environ["FAKE_GH"] + ".log", "a") as log: + log.write(" ".join(sys.argv[1:]) + "\n") +args, jq, endpoint = sys.argv[1:], None, None +if args[:2] == ["release", "download"]: + tag, folder = args[2], args[args.index("-D") + 1] + for name, text in fixture["assets"][tag].items(): + open(os.path.join(folder, name), "w").write(text) + sys.exit(0) +i = 1 +while i < len(args): + if args[i] in ("-X", "-f", "-R"): + i += 2 + elif args[i] == "--jq": + jq = args[i + 1]; i += 2 + elif args[i] == "--paginate": + i += 1 + else: + endpoint = args[i]; i += 1 +path = endpoint.split("?")[0] +if m := re.fullmatch(r"repos/[^/]+/[^/]+/releases/tags/(.+)", path): + if m.group(1) not in fixture["releases"]: + sys.stderr.write("gh: Not Found (HTTP 404)\n"); sys.exit(1) + value = fixture["releases"][m.group(1)] +elif path.endswith("/actions/workflows/library-update-announce.yml/runs"): + if fixture.get("runs_fail"): + sys.stderr.write("gh: Server Error (HTTP 502)\n"); sys.exit(1) + value = {"workflow_runs": [{"id": run} for run in fixture["jobs"]]} +elif m := re.fullmatch(r"repos/[^/]+/[^/]+/actions/runs/([0-9]+)/jobs", path): + value = {"jobs": fixture["jobs"][m.group(1)]} +else: + sys.stderr.write("unexpected endpoint " + endpoint + "\n"); sys.exit(2) +sys.stdout.write(subprocess.run(["jq", "-r", jq], input=json.dumps(value), text=True, + capture_output=True, check=True).stdout) +""" + +TAG = "v29-20260927072953" + + +@unittest.skipUnless(all(shutil.which(tool) for tool in ("bash", "jq", "sha256sum")), "gate needs bash, jq, sha256sum") +class GateTest(unittest.TestCase): + GATE = Path(__file__).resolve().parent / "gate.sh" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + (self.dir / "bin").mkdir() + gh = self.dir / "bin" / "gh" + gh.write_text(FAKE_GH, encoding="utf-8") + gh.chmod(0o755) + self.fixture = {"releases": {TAG: {"tag_name": TAG, "draft": False, "prerelease": False, + "created_at": "2026-09-27T07:30:00Z"}}, + "assets": {}, "jobs": {}} + + def tearDown(self): + self.tmp.cleanup() + + def handoff(self, status="published", run=100, attempt=1, tag=TAG, corrupt=False): + name = f"pipeline-result-run-{run}-{attempt}" + body = json.dumps({"status": status, "child_run_id": run, "child_run_attempt": attempt, + "release_tag": tag}) + "\n" + digest = hashlib.sha256(body.encode()).hexdigest() + self.fixture["releases"][name] = {"tag_name": name, "prerelease": True} + self.fixture["assets"][name] = {"pipeline-result.json": body, + "pipeline-result.sha256": ("0" * 64 if corrupt else digest) + "\n"} + + def gate(self, event, **env): + fixture = self.dir / "fixture.json" + fixture.write_text(json.dumps(self.fixture), encoding="utf-8") + output = self.dir / "out" + output.write_text("", encoding="utf-8") + values = {"PATH": f"{self.dir / 'bin'}{os.pathsep}{os.environ['PATH']}", "FAKE_GH": str(fixture), + "EVENT_NAME": event, "GITHUB_REPOSITORY": "Otzaria/SeforimLibrary", "GITHUB_OUTPUT": str(output), + "WORKFLOW_RUN_ID": "100", "WORKFLOW_RUN_ATTEMPT": "1", "GITHUB_RUN_ID": "500", **env} + result = subprocess.run(["bash", str(self.GATE)], env=values, capture_output=True, text=True) + outputs = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + log = Path(str(fixture) + ".log") + return result, outputs, log.read_text(encoding="utf-8") if log.exists() else "" + + def test_a_release_run_that_published_nothing_is_a_clean_skip(self): + result, out, _ = self.gate("workflow_run") + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("false", out["proceed"]) + self.assertIn("published nothing", result.stdout) + + def test_a_reuse_run_is_a_clean_skip(self): + self.handoff(status="reused") + result, out, _ = self.gate("workflow_run") + self.assertEqual((0, "false"), (result.returncode, out["proceed"])) + self.assertIn("reused", result.stdout) + + def test_a_release_run_announces_the_tag_it_published(self): + self.handoff() + result, out, _ = self.gate("workflow_run") + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual({"proceed": "true", "tag": TAG, "send": "true", + "channels": '["forum","yemot","release"]'}, out) + + def test_a_published_prerelease_is_a_clean_skip(self): + self.handoff() + self.fixture["releases"][TAG]["prerelease"] = True + result, out, _ = self.gate("workflow_run") + self.assertEqual((0, "false"), (result.returncode, out["proceed"])) + + def test_a_handoff_that_does_not_verify_fails(self): + for kwargs in ({"corrupt": True}, {"run": 999}, {"status": "odd"}): + with self.subTest(**kwargs): + self.fixture["releases"] = {TAG: self.fixture["releases"][TAG]} + run = kwargs.pop("run", 100) + self.handoff(run=run, **kwargs) + if run != 100: + self.fixture["releases"]["pipeline-result-run-100-1"] = self.fixture["releases"].pop( + f"pipeline-result-run-{run}-1") + self.fixture["assets"]["pipeline-result-run-100-1"] = self.fixture["assets"].pop( + f"pipeline-result-run-{run}-1") + result, out, _ = self.gate("workflow_run") + self.assertNotEqual(0, result.returncode) + self.assertNotIn("proceed", out) + + @staticmethod + def job(channel, conclusion, status="completed", tag=TAG): + return {"name": f"{channel} {tag}", "status": status, "conclusion": conclusion, "html_url": "u"} + + def send_gate(self): + return self.gate("workflow_dispatch", INPUT_TAG=TAG, DRY_RUN="false", INPUT_CHANNELS="forum,yemot") + + def test_channels_some_run_already_sent_are_dropped(self): + self.fixture["jobs"] = {"7": [self.job("forum", "failure"), self.job("forum", "success")], + "8": [self.job("yemot", "success", tag="v28-20260910220310"), + self.job("yemot", "skipped")]} + result, out, _ = self.send_gate() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual('["yemot"]', out["channels"]) + self.fixture["jobs"]["9"] = [self.job("yemot", "success")] + result, out, _ = self.send_gate() + self.assertEqual((0, "false"), (result.returncode, out["proceed"])) + self.assertIn("already announced", result.stdout) + + def test_a_channel_another_run_left_half_sent_fails_the_gate(self): + for job in (self.job("yemot", "failure"), self.job("yemot", "cancelled"), + self.job("yemot", None, status="in_progress")): + with self.subTest(job=job): + self.fixture["jobs"] = {"7": [job]} + result, out, _ = self.send_gate() + self.assertNotEqual(0, result.returncode) + self.assertNotIn("proceed", out) + self.assertIn("actions/runs/7", result.stderr) + self.assertIn('"Re-run failed jobs"', result.stderr) + + def test_a_rerun_of_the_run_that_left_it_half_sent_resumes(self): + self.fixture["jobs"] = {"500": [self.job("yemot", "failure")]} + result, out, _ = self.send_gate() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual('["forum","yemot"]', out["channels"]) + + def test_a_failing_history_check_fails_the_gate(self): + self.fixture["runs_fail"] = True + result, out, _ = self.send_gate() + self.assertNotEqual(0, result.returncode) + self.assertNotIn("proceed", out) + + def channel_step(self, **env): + doc = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + run = next(s["run"] for s in doc["jobs"]["forum"]["steps"] if s.get("id") == "dedup") + fixture = self.dir / "fixture.json" + fixture.write_text(json.dumps(self.fixture), encoding="utf-8") + output = self.dir / "step_out" + output.write_text("", encoding="utf-8") + values = {"PATH": f"{self.dir / 'bin'}{os.pathsep}{os.environ['PATH']}", "FAKE_GH": str(fixture), + "GITHUB_REPOSITORY": "Otzaria/SeforimLibrary", "GITHUB_OUTPUT": str(output), "GITHUB_RUN_ID": "500", + "TAG": TAG, "CHANNEL": "forum", **env} + # GitHub runs a step as `bash -e {0}`. + result = subprocess.run(["bash", "-e", "-c", run], cwd=ROOT, env=values, capture_output=True, text=True) + return result, output.read_text(encoding="utf-8") + + @unittest.skipIf(yaml is None, "PyYAML is required to parse the workflow") + def test_the_channel_step_skips_resumes_or_fails(self): + self.fixture["jobs"] = {"7": [self.job("forum", "success")]} + self.assertEqual((0, "sent=true\n"), (self.channel_step()[0].returncode, self.channel_step()[1])) + self.fixture["jobs"] = {"500": [self.job("forum", None, status="in_progress")]} + result, out = self.channel_step() + self.assertEqual((0, "sent=false\n"), (result.returncode, out), result.stderr) + self.fixture["jobs"] = {"7": [self.job("forum", "failure")], "500": [self.job("forum", None, "in_progress")]} + result, out = self.channel_step() + self.assertNotEqual(0, result.returncode) + self.assertEqual("", out) + self.fixture["jobs"], self.fixture["runs_fail"] = {}, True + result, out = self.channel_step() + self.assertNotEqual(0, result.returncode) + self.assertEqual("", out) + + def test_a_dry_run_checks_no_history_and_sends_nothing(self): + self.fixture["jobs"] = {"7": [self.job("forum", "success")]} + result, out, log = self.gate("workflow_dispatch", INPUT_TAG=TAG, DRY_RUN="true", INPUT_CHANNELS="yemot, forum") + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual({"proceed": "true", "tag": TAG, "send": "false", "channels": '["yemot","forum"]'}, out) + self.assertNotIn("/runs", log) + + def test_a_manual_run_for_an_unpublished_tag_fails(self): + self.fixture["releases"][TAG]["prerelease"] = True + result, _, _ = self.gate("workflow_dispatch", INPUT_TAG=TAG, DRY_RUN="false", INPUT_CHANNELS="forum") + self.assertNotEqual(0, result.returncode) + self.assertIn("not a published", result.stdout) + + def test_an_unknown_channel_fails(self): + result, _, _ = self.gate("workflow_dispatch", INPUT_TAG=TAG, DRY_RUN="true", INPUT_CHANNELS="forum,chat") + self.assertNotEqual(0, result.returncode) + + +@unittest.skipIf(yaml is None, "PyYAML is required to parse the workflow") +class WorkflowContractTest(unittest.TestCase): + def setUp(self): + self.text = WORKFLOW.read_text(encoding="utf-8") + self.doc = yaml.safe_load(self.text) + # PyYAML reads the bare `on:` key as boolean True. + self.on = self.doc.get("on", self.doc.get(True)) + + def test_follows_every_completed_release_run(self): + release = yaml.safe_load(RELEASE.read_text(encoding="utf-8")) + self.assertEqual([release["name"]], self.on["workflow_run"]["workflows"]) + self.assertEqual(["completed"], self.on["workflow_run"]["types"]) + # Whatever the run's conclusion: a publish followed by a failed step still announces. + self.assertNotIn("workflow_run.conclusion", self.text) + + def test_only_the_release_run_and_a_manual_dispatch_trigger_it(self): + # No `release` event: it would fire beside workflow_run every week and download twice. + self.assertEqual({"workflow_run", "workflow_dispatch"}, set(self.on)) + self.assertNotIn("github.event.release", self.text) + + def test_the_release_workflow_does_not_depend_on_the_announcement(self): + text = RELEASE.read_text(encoding="utf-8") + self.assertNotIn("library-update-announce", text) + self.assertNotIn("announce", yaml.safe_load(text)["jobs"]) + + def test_nothing_ties_the_release_run_to_the_announcement(self): + # workflow_run fires after the release run ends; hosted runners and own lock names only. + release = yaml.safe_load(RELEASE.read_text(encoding="utf-8")) + release_groups = {str((job.get("concurrency") or {}).get("group")) for job in release["jobs"].values()} + release_groups.add(str((release.get("concurrency") or {}).get("group"))) + for name, job in self.doc["jobs"].items(): + self.assertEqual("ubuntu-latest", job["runs-on"], name) + group = (job.get("concurrency") or {}).get("group") + if group: + self.assertTrue(group.startswith("library-update-announce-"), name) + self.assertNotIn(group, release_groups) + self.assertNotIn("concurrency", self.doc) + + def test_manual_run_is_a_dry_run_by_default(self): + inputs = self.on["workflow_dispatch"]["inputs"] + self.assertTrue(inputs["tag"]["required"]) + self.assertIs(True, inputs["dry_run"]["default"]) + self.assertEqual("forum,yemot,release", inputs["channels"]["default"]) + + def test_the_gate_decides_before_anything_is_downloaded(self): + jobs = self.doc["jobs"] + self.assertEqual(["gate", "prepare", "forum", "yemot", "release"], list(jobs)) + self.assertEqual("gate", jobs["prepare"]["needs"]) + self.assertEqual("needs.gate.outputs.proceed == 'true'", jobs["prepare"]["if"]) + gate = next(s for s in jobs["gate"]["steps"] if s.get("id") == "gate") + self.assertEqual("bash .github/scripts/library_announce/gate.sh", gate["run"]) + self.assertEqual("${{ github.event.workflow_run.id }}", gate["env"]["WORKFLOW_RUN_ID"]) + self.assertEqual("${{ github.event.workflow_run.run_attempt }}", gate["env"]["WORKFLOW_RUN_ATTEMPT"]) + + def test_each_channel_is_its_own_locked_resumable_job(self): + jobs = self.doc["jobs"] + for channel in ("forum", "yemot"): + job = jobs[channel] + self.assertEqual(f"{channel} ${{{{ needs.gate.outputs.tag }}}}", job["name"]) + self.assertEqual(["gate", "prepare"], job["needs"]) + self.assertEqual("read", job["permissions"]["actions"]) + self.assertIn(f"contains(fromJSON(needs.gate.outputs.channels), '{channel}')", job["if"]) + self.assertIn("needs.gate.outputs.send == 'true'", job["if"]) + self.assertEqual(f"library-update-announce-{channel}-${{{{ needs.gate.outputs.tag }}}}", + job["concurrency"]["group"]) + self.assertIs(False, job["concurrency"]["cancel-in-progress"]) + steps = {step.get("name") or step.get("uses"): step for step in job["steps"]} + dedup_step = steps["Skip if this tag was already announced here"]["run"] + self.assertIn('decision="$(bash .github/scripts/library_announce/channel_decision.sh)"', dedup_step) + self.assertNotIn("echo \"sent=$(", dedup_step) + self.assertIn("steps.dedup.outputs.sent != 'true'", steps["Announce"]["if"]) + self.assertIn(f"-n {channel}-progress", steps["Restore progress from an earlier attempt"]["run"]) + self.assertIn("--progress", steps["Announce"]["run"]) + upload = steps["actions/upload-artifact@v6"] + self.assertTrue(upload["if"].startswith("always()")) + self.assertEqual(f"{channel}-progress", upload["with"]["name"]) + dedup = (Path(__file__).resolve().parent / "already_sent.sh").read_text(encoding="utf-8") + self.assertIn("filter=all", dedup) + self.assertIn('JOB_NAME="$CHANNEL $TAG"', dedup) + + def test_release_job_attaches_the_notes_once_under_the_name_reuse_ignores(self): + job = self.doc["jobs"]["release"] + self.assertEqual("release ${{ needs.gate.outputs.tag }}", job["name"]) + self.assertEqual(["gate", "prepare"], job["needs"]) + self.assertEqual({"contents": "write", "actions": "read"}, job["permissions"]) + self.assertIn("contains(fromJSON(needs.gate.outputs.channels), 'release')", job["if"]) + self.assertIn("needs.gate.outputs.send == 'true'", job["if"]) + self.assertIn("needs.prepare.outputs.has_changes == 'true'", job["if"]) + self.assertEqual("library-update-announce-release-${{ needs.gate.outputs.tag }}", job["concurrency"]["group"]) + steps = {step.get("name") or step.get("uses"): step for step in job["steps"]} + self.assertIn('decision="$(bash .github/scripts/library_announce/channel_decision.sh)"', + steps["Skip if this tag was already announced here"]["run"]) + announce = steps["Announce"] + self.assertIn("steps.dedup.outputs.sent != 'true'", announce["if"]) + self.assertIn(f"out/{lua.RELEASE_NOTES_ASSET}\"", announce["run"]) + self.assertIn("gh release upload \"$TAG\" \"$notes\" --clobber", announce["run"]) + self.assertIn(f'select(.name == "{lua.RELEASE_NOTES_ASSET}")', announce["run"]) + # Every other asset still counts, so a reuse candidate must match its provenance exactly. + self.assertIn(f'[.assets[]|select(.name!="{lua.RELEASE_NOTES_ASSET}")]|length', + RELEASE.read_text(encoding="utf-8")) + + def test_no_google_chat_left(self): + script = (Path(__file__).resolve().parent / "library_update_announce.py").read_text(encoding="utf-8") + for text in (self.text, script): + self.assertNotIn("chat", text.lower()) + + def catalog_step(self): + return next(s["run"] for s in self.doc["jobs"]["prepare"]["steps"] if s.get("name") == "Build both catalogs") + + def test_databases_stream_into_zstd_with_the_long_window(self): + run = self.catalog_step() + self.assertIn('--jq "$FULL_DB_ASSET_JQ | .name // empty"', run) + self.assertIn('stream_full_db "$tag" "$asset" "$GITHUB_REPOSITORY"' + ' | zstd -d -q --long=31 --memory=2048MB -o "$db"', run) + # Saving helpers would put the .zst (or its parts) on disk beside the DB. + for saving in ("download_full_db_by_name", "gh release download", "--dir", "-D "): + self.assertNotIn(saving, run) + self.assertIn('rm -f "$db"', run) + names = (ROOT / ".github" / "scripts" / "db_asset_names.sh").read_text(encoding="utf-8") + stream = names[names.index("stream_full_db() {"):] + stream = stream[:stream.index("\n}\n")] + self.assertEqual(3, stream.count("-O -"), stream) + self.assertNotIn("--dir", stream) + + def run_catalog_step(self, releases): + """releases: {previous_tag: ..., current_tag: (asset, split_db, split_asset)} -> assets served.""" + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + root = Path(tmp) + binaries = root / "bin" + binaries.mkdir() + for tag, (name, split_db, split_asset) in releases.items(): + release = root / tag + release.mkdir() + (root / "built" / tag).mkdir(parents=True) + db = root / "built" / tag / name + make_db(db, [(1, 2, 1, "book")], [(1, 0, "text")], split=split_db) + if split_asset: + size = db.stat().st_size + subprocess.run([shutil.which("bash"), str(ROOT / ".github/scripts/split_release_asset.sh"), + str(db), str(release), str(size // 2 + 1)], check=True, capture_output=True) + self.assertFalse((release / name).exists()) + else: + shutil.copy(db, release / name) + assets = [{"name": p.name} for p in sorted(release.iterdir())] + assets.append({"name": "patch-v4-v6.db.zst"}) + if name != "seforim.db.zst": + assets.append({"name": "seforim.db.zst"}) + (release / "release.json").write_text(json.dumps({"assets": assets}), encoding="utf-8") + gh = binaries / "gh" + gh.write_text(f"#!{sys.executable}\n" + """ +import os, subprocess, sys +from pathlib import Path +args = sys.argv[1:] +root = Path(os.environ['FIXTURES']) +if args[0] == 'api': + tag = args[1].rsplit('/', 1)[-1] + query = args[args.index('--jq') + 1] + result = subprocess.run(['jq', '-r', query], input=(root / tag / 'release.json').read_text(), text=True) + sys.exit(result.returncode) +if args[:2] != ['release', 'download']: + sys.exit(f'stub gh: unexpected {args}') +tag, opts = args[2], args[3:] +opt = lambda flag: opts[opts.index(flag) + 1] if flag in opts else None +if opt('-R') != os.environ['GITHUB_REPOSITORY']: + sys.exit(f'stub gh: download without -R $GITHUB_REPOSITORY: {args}') +if opt('-O') != '-': + sys.exit(f'stub gh: the catalog step must stream, not save: {args}') +src = root / tag / opt('--pattern') +if not src.is_file(): + sys.exit(f'no assets match the file pattern {src.name}') +with (root / 'downloads.log').open('a') as log: + log.write(f'{tag}/{src.name}\\n') +sys.stdout.buffer.write(src.read_bytes()) +""", encoding="utf-8") + zstd = binaries / "zstd" + zstd.write_text(f"#!{sys.executable}\n" + """ +import sys +from pathlib import Path +args = sys.argv[1:] +for flag in ('-d', '--long=31', '--memory=2048MB', '-o'): + if flag not in args: + sys.exit(f'stub zstd: missing {flag} in {args}') +Path(args[args.index('-o') + 1]).write_bytes(sys.stdin.buffer.read()) +""", encoding="utf-8") + python = binaries / "python" + python.write_text(f"#!/bin/sh\nexec '{sys.executable}' \"$@\"\n", encoding="utf-8") + for path in (gh, zstd, python): + path.chmod(0o755) + previous, current = releases + result = subprocess.run([shutil.which("bash"), "-c", self.catalog_step()], cwd=ROOT, + env=dict(os.environ, PATH=f"{binaries}{os.pathsep}{os.environ['PATH']}", + RUNNER_TEMP=tmp, PREVIOUS=previous, TAG=current, + GITHUB_REPOSITORY="Otzaria/SeforimLibrary", FIXTURES=tmp), + capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + work = root / "announce" + self.assertEqual(["current.json", "previous.json"], sorted(p.name for p in work.iterdir())) + self.assertEqual(json.loads((work / "previous.json").read_text()), + json.loads((work / "current.json").read_text())) + return (root / "downloads.log").read_text().splitlines() + + @unittest.skipUnless(shutil.which("bash") and shutil.which("jq"), "bash and jq are required") + def test_catalog_workflow_downloads_the_full_asset_for_each_release_schema(self): + downloads = self.run_catalog_step({"v5": ("seforim.db.zst", False, False), + "v6": ("seforim-schema6.db.zst", True, False)}) + self.assertEqual(["v5/seforim.db.zst", "v6/seforim-schema6.db.zst"], downloads) + + @unittest.skipUnless(shutil.which("bash") and shutil.which("jq") and shutil.which("split") + and shutil.which("sha256sum"), "bash, jq, split and sha256sum are required") + def test_catalog_workflow_streams_a_split_full_asset_part_by_part(self): + downloads = self.run_catalog_step({"v5": ("seforim.db.zst", False, False), + "v6": ("seforim-schema6.db.zst", True, True)}) + self.assertEqual(["v5/seforim.db.zst", "v6/seforim-schema6.db.zst.manifest.json", + "v6/seforim-schema6.db.zst.part-000", "v6/seforim-schema6.db.zst.part-001"], + downloads) + + def test_hosted_python_is_set_up_before_pip(self): + checked = 0 + for job in self.doc["jobs"].values(): + uses = [step.get("uses", "") for step in job["steps"]] + runs = [step.get("run", "") for step in job["steps"]] + pips = [i for i, r in enumerate(runs) if "pip install" in r] + if not pips: + continue + checked += 1 + setup = next(i for i, u in enumerate(uses) if u.startswith("actions/setup-python@")) + self.assertLess(setup, pips[0]) + self.assertEqual(3, checked) + + def test_ci_installs_what_these_tests_import(self): + ci = yaml.safe_load(CI.read_text(encoding="utf-8")) + job = ci["jobs"]["announcer"] + install = next(s["run"] for s in job["steps"] if "pip install" in s.get("run", "")) + for package in ("requests", "pyluach", "tzdata", "pyyaml", "zstandard"): + self.assertIn(package, install) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/library_announce/yemot.py b/.github/scripts/library_announce/yemot.py new file mode 100644 index 000000000..4fcf461d5 --- /dev/null +++ b/.github/scripts/library_announce/yemot.py @@ -0,0 +1,121 @@ +import re + +BASE_URL = "https://www.call2all.co.il/ym/api/" +CHUNK_SIZE = 2000 +TIMEOUT = 60 +NUMBERED_FILE_RE = re.compile(r"^([0-9]+)\.[A-Za-z0-9]+$") + + +class YemotError(RuntimeError): + pass + + +def _session(session): + if session is not None: + return session + import requests + return requests.Session() + + +def _checked(response, action): + # Every Yemot call answers JSON with responseStatus; anything but OK is a failure. + if response.status_code != 200: + raise YemotError(f"{action}: HTTP {response.status_code}: {response.text[:200]!r}") + try: + body = response.json() + except ValueError: + raise YemotError(f"{action}: not JSON: {response.text[:200]!r}") from None + if not isinstance(body, dict) or body.get("responseStatus") != "OK": + raise YemotError(f"{action}: {body!r}"[:500]) + return body + + +def split_content(content: str, chunk_size: int = CHUNK_SIZE) -> list[str]: + """Cuts at the last newline that fits; a line longer than a chunk is cut hard.""" + parts = [] + start = 0 + while len(content) - start > chunk_size: + end = content.rfind("\n", start + 1, start + chunk_size + 1) + if end == -1: + parts.append(content[start:start + chunk_size]) + start += chunk_size + else: + parts.append(content[start:end]) + start = end + 1 + parts.append(content[start:]) + return [part for part in (p.strip() for p in parts) if part] + + +def upload_plan(content: dict, date_yemot: str, base: int) -> list[tuple[str, str]]: + """Every (file name, text) to upload after file `base`, in upload order.""" + plan, num = [], base + for key, value in content.items(): + all_partes = split_content(value, CHUNK_SIZE) + if not all_partes: + raise YemotError(f"section {key!r} has no content") + for chunk in all_partes[-1::-1]: + num += 1 + file_name = str(num).zfill(3) + plan.append((file_name, chunk)) + plan.append((f"{file_name}-Title", key)) + plan.append((str(num + 1).zfill(3), date_yemot)) + return plan + + +def split_and_send(content: dict, date_yemot: str, token: str, path: str, tzintuk_list_name: str, + session=None, progress=None, save=None): + """Uploads the plan, then one tzintuk; `progress` resumes an interrupted send exactly.""" + session = _session(session) + progress = progress if progress is not None else {} + save = save or (lambda _: None) + # The numbering is fixed once: a resumed send rewrites the same files, never new ones. + if "base" not in progress: + progress.update(base=get_file_num(token, path, session), uploaded=0, tzintuk=False) + save(progress) + plan = upload_plan(content, date_yemot, progress["base"]) + if not 0 <= progress["uploaded"] <= len(plan): + raise YemotError(f"progress says {progress['uploaded']} uploads, but the plan has {len(plan)}") + for file_name, text in plan[progress["uploaded"]:]: + send_to_yemot(text, token, path, file_name, session) + progress["uploaded"] += 1 + save(progress) + if not progress["tzintuk"]: + send_tzintuk(token, tzintuk_list_name, session) + progress["tzintuk"] = True + save(progress) + + +def send_to_yemot(content: str, token: str, path: str, file_name: str, session=None): + data = { + "token": token, + "what": f"{path}/{file_name}.tts", + "contents": content + } + response = _session(session).post(f"{BASE_URL}UploadTextFile", data=data, timeout=TIMEOUT) + _checked(response, f"UploadTextFile {file_name}") + + +def get_file_num(token: str, path: str, session=None) -> int: + data = { + "token": token, + "path": path + } + response = _session(session).get(f"{BASE_URL}GetIVR2DirStats", params=data, timeout=TIMEOUT) + body = _checked(response, "GetIVR2DirStats") + # Guessing a start number would overwrite the files already on the line. + max_file = body.get("maxFile") + name = max_file.get("name") if isinstance(max_file, dict) else None + match = NUMBERED_FILE_RE.fullmatch(name) if isinstance(name, str) else None + if not match: + raise YemotError(f"GetIVR2DirStats: no numbered file in {path} (maxFile={max_file!r}); refusing to " + "guess where numbering continues — an empty folder must be seeded by hand") + return int(match.group(1)) + + +def send_tzintuk(token: str, list_name: str, session=None): + data = { + "token": token, + "phones": f"tzl:{list_name}" + } + response = _session(session).get(f"{BASE_URL}RunTzintuk", params=data, timeout=TIMEOUT) + _checked(response, "RunTzintuk") diff --git a/.github/scripts/merge_private_books.py b/.github/scripts/merge_private_books.py new file mode 100644 index 000000000..d2bd0dcb3 --- /dev/null +++ b/.github/scripts/merge_private_books.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Lay a private books archive out as extra source folder(s) of the Otzaria extract. + +The archive (``private_books.zip``) carries, at its root: + +* ``אוצריא/...`` - the book files, in the same tree the public library uses; +* ``files_manifest.json`` - ``{"/אוצריא/": {"hash": ""}}`` + for every one of those files, exactly like the public ``files_manifest*.json``. + +Every file is copied under ``/אוצריא`` and every manifest entry is +merged into each ``files_manifest*.json`` present at the extract root, so the +generator assigns the books to their private source like any public one. +Fail closed: a missing or extra file, a hash mismatch, a path that already +exists in the public tree, or a source name the public manifest already uses. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import zipfile +from pathlib import Path, PurePosixPath + +LIBRARY_DIR = "אוצריא" +MANIFEST = "files_manifest.json" +PUBLIC_MANIFESTS = ("files_manifest.json", "files_manifest_new.json") + + +def _source_of(key: str) -> str: + parts = key.split("/") + if len(parts) < 3 or parts[1] != LIBRARY_DIR or not parts[0] or parts[0] == LIBRARY_DIR: + raise ValueError(f"manifest key {key!r} is not /{LIBRARY_DIR}/") + return parts[0] + + +def _public_sources(manifest: dict) -> set: + return {key.split("/")[0] for key in manifest if f"/{LIBRARY_DIR}/" in key} + + +def merge(archive: Path, extract_root: Path) -> dict: + """Merges [archive] into [extract_root]; returns {source name: book file count}.""" + with zipfile.ZipFile(archive) as zf: + manifest = json.loads(zf.read(MANIFEST).decode("utf-8-sig")) + if not isinstance(manifest, dict) or not manifest: + raise ValueError(f"{MANIFEST} must be a non-empty object") + by_rel = {} + counts: dict = {} + for key, entry in manifest.items(): + source = _source_of(key) + if not isinstance(entry, dict) or not isinstance(entry.get("hash"), str): + raise ValueError(f"manifest entry {key!r} has no hash") + rel = "/".join(key.split("/")[1:]) + if rel in by_rel: + raise ValueError(f"{rel!r} is listed twice") + by_rel[rel] = entry["hash"].lower() + counts[source] = counts.get(source, 0) + 1 + + files = {} + for info in zf.infolist(): + if info.is_dir() or info.filename == MANIFEST: + continue + path = PurePosixPath(info.filename) + if path.is_absolute() or ".." in path.parts or path.parts[0] != LIBRARY_DIR: + raise ValueError(f"unexpected archive entry {info.filename!r}") + files[info.filename] = info + if set(files) != set(by_rel): + raise ValueError( + f"archive files and {MANIFEST} differ: " + f"unlisted={sorted(set(files) - set(by_rel))[:5]} missing={sorted(set(by_rel) - set(files))[:5]}" + ) + + publics = [extract_root / name for name in PUBLIC_MANIFESTS if (extract_root / name).is_file()] + if not publics: + raise ValueError(f"no public files_manifest*.json under {extract_root}") + loaded = {p: json.loads(p.read_text(encoding="utf-8-sig")) for p in publics} + taken = set().union(*(_public_sources(m) for m in loaded.values())) + clash = sorted(set(counts) & taken) + if clash: + raise ValueError(f"private source name(s) already used by the public library: {clash}") + for path, public in loaded.items(): + overlap = set(public) & set(manifest) + if overlap: + raise ValueError(f"{path.name} already lists {sorted(overlap)[:5]}") + + for rel, info in sorted(files.items()): + data = zf.read(info) + if hashlib.sha256(data).hexdigest() != by_rel[rel]: + raise ValueError(f"hash mismatch for {rel!r}") + target = extract_root.joinpath(*PurePosixPath(rel).parts) + if target.exists(): + raise ValueError(f"{rel!r} already exists in the public library") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + + # Every check ran above, before the first file was written. + for path, public in loaded.items(): + public.update(manifest) + path.write_text(json.dumps(public, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return counts + + +def main(argv: list) -> int: + if len(argv) != 3: + print("usage: merge_private_books.py ", file=sys.stderr) + return 2 + try: + counts = merge(Path(argv[1]), Path(argv[2])) + except (OSError, ValueError, KeyError, zipfile.BadZipFile, json.JSONDecodeError) as exc: + print(f"private books merge failed: {exc}", file=sys.stderr) + return 1 + print("private books merged: " + ", ".join(f"{s}={n}" for s, n in sorted(counts.items()))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/.github/scripts/patch_anchor_schema.py b/.github/scripts/patch_anchor_schema.py new file mode 100644 index 000000000..313c9f2ad --- /dev/null +++ b/.github/scripts/patch_anchor_schema.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Pre-download patchability verdict for one patch-fan anchor. + +The release patch fan reconstitutes every anchor's DB from that release's +1.3 GB ``seforim.db.zst`` (110-135 s download plus ~7 s decompress) before +``PatchDbProducer`` ever gets to say the pair cannot be expressed as a delta at +all: run 33865604251 spent ~140 s on the v10 anchor only to be handed +``UnpatchableAnchorException``. This module answers the same question from a +few KB of metadata. + +It is strictly advisory, and asymmetric on purpose: + +* ``PROCEED`` is not a patchability claim. It means "not provably unpatchable + from metadata"; the producer keeps being the single authority and still says + so with exit code 3 plus the ``.unpatchable`` marker. +* ``UNPATCHABLE`` costs the fan at most one anchor, which the workflow already + treats as a warning. A release that loses *every* anchor still fails loudly + in the fan's final check. + +Two sources, cheapest first: + +1. :data:`LEGACY_UNPATCHABLE_DB_VERSIONS` - releases published before this + metadata existed that the repository's own history proves carry a dropped + column. +2. The anchor release's ``build_provenance.json`` ``db_schema`` block (written + by ``dump`` below, from db_version 27 on -- v26 was published 2026-09-04 + with ``schema_version`` 3 and no such block): the exact physical column set + of the DB that release published. Applying the producer's own first rule to it + - a column that exists in prev's table and no longer exists in new's - is + exact, not heuristic. + +The comparison is scoped exactly like `PatchDbProducer.planColumnMigrations`: +only the tables of the patch contract (`patch_tables_contract.json`, the same +fixture `PatchTablesContractTest` freezes) and, of those, only the ones the +producer would actually column-diff. A physical table outside the contract, or +one *promoted* into the contract by the schema bump, is skipped there and must +be skipped here too - otherwise a dropped column in such a table would cost the +fan a patch the producer would have produced. Whenever that scope cannot be +established, the verdict is PROCEED; it never widens to "every physical table". + +``dump`` is deliberately the *only* producer of that block, so what a build +publishes about itself and what a later build compares against are the same +bytes by construction. +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +# Anchors that predate the `db_schema` provenance block and are known - not +# guessed - to be unpatchable, so they can be skipped without a download: +# +# * commit 7755902 (2026-07-14) replaced `link.isDeclaredBase` with +# `link.baseProvenance` without bumping `db_schema_version`, so DBs +# released before it still claim schema 1 while carrying a column the +# current schema no longer declares - exactly the shape +# `PatchDbProducer.planColumnMigrations` calls unpatchable; +# * commit 849f754 ("feat(patch): add missing columns and indexes by +# migration instead of skipping the anchor") records the measured range: +# "Verified on real data: v9 (with its dropped link.isDeclaredBase removed) +# to v25 produced 15 migrations and passed the apply-hash check. v9 to v13 +# remain skipped until the dropped column is handled."; +# * run 33865604251 (building v26) re-confirmed it for the v10 anchor. +# +# The lower bound is reachability, not the start of the damage: v1-v8 predate +# 7755902 too (v8 was published 2026-07-06, v13 2026-07-09, v14 2026-07-19 -- +# the first release built after the swap), but no offset in {1,2,4,8,16} can +# reach below v11 from db_version 27 on, so listing them would be dead code. +# Deliberately a closed set, not an open-ended "everything below N": v14-v25 +# are known-good (v25 was produced from the same evidence) and must keep being +# offered to the producer. The whole list can be deleted once db_version 30 +# ships, when offset 16 no longer reaches v13. +LEGACY_UNPATCHABLE_DB_VERSIONS = frozenset(range(9, 14)) + +PROCEED = "PROCEED" +UNPATCHABLE = "UNPATCHABLE" +# Not a patch at all: the fan publishes the full-rebase barrier for the anchor. +# The line is "BARRIER ". +BARRIER = "BARRIER" + +# Mirrors PatchTables.kt LINE_CONTENT_SPLIT_SCHEMA_VERSION: no delta crosses it. +LINE_CONTENT_SPLIT_SCHEMA = 6 + + +def _one_line(text: str) -> str: + """Collapse a reason to the single line the workflow echoes.""" + return " ".join(str(text).split()) + + +def dump(db_path: Path) -> dict: + """The published DB's physical schema: version plus columns per table. + + Every table in seforim.db is declared in Database.sq and every one of them + is a patch table, so the physical table list is the patch table list. The + column lists are sorted the same way PatchTableColumnContractTest sorts + them, and the whole object is emitted as canonical JSON so it can be nested + inside build_provenance.json without breaking that file's byte-canonical + contract. + """ + if not db_path.is_file(): + raise ValueError(f"{db_path} is not a file") + try: + conn = sqlite3.connect(f"file:{db_path.resolve().as_posix()}?mode=ro", uri=True) + conn.execute("SELECT 1 FROM sqlite_master LIMIT 1") + except sqlite3.Error: + # A database left in WAL mode cannot always be opened read-only — + # SQLite wants to create the -shm file. build/ is writable and nothing + # below writes, so a normal connection is the safe fallback rather than + # a failed release step. + conn = sqlite3.connect(db_path) + try: + names = sorted( + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' " + "AND name NOT LIKE 'sqlite_%'" + ) + ) + tables = {} + for name in names: + quoted = name.replace('"', '""') + tables[name] = sorted( + row[1] for row in conn.execute(f'PRAGMA table_info("{quoted}")') + ) + row = conn.execute( + "SELECT value FROM schema_meta WHERE key='db_schema_version'" + ).fetchone() + finally: + conn.close() + if row is None or not str(row[0]).strip(): + raise ValueError( + f"db_schema_version row missing in {db_path} (release DB must be stamped)" + ) + if not tables: + raise ValueError(f"no tables found in {db_path}") + return {"db_schema_version": int(row[0]), "tables": tables} + + +def _anchor_db_schema(provenance_path: Path) -> tuple[dict | None, str]: + """The anchor's recorded db_schema block, or None plus why it is unusable.""" + if not provenance_path.exists(): + return None, f"{provenance_path.name} is not published by that release" + value = json.loads(provenance_path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + return None, "anchor provenance is not an object" + schema = value.get("db_schema") + if schema is None: + return None, "anchor provenance predates the db_schema block" + if not isinstance(schema, dict) or not isinstance(schema.get("tables"), dict): + return None, "anchor provenance carries an unreadable db_schema block" + if not all(isinstance(columns, list) for columns in schema["tables"].values()): + return None, "anchor provenance carries an unreadable db_schema block" + return schema, "" + + +def _this_schema_version(path: Path) -> int | None: + """This build's db_schema_version from its dump, or None while it does not exist yet.""" + try: + version = json.loads(path.read_text(encoding="utf-8")).get("db_schema_version") + except (OSError, UnicodeDecodeError, ValueError, AttributeError): + return None + return version if type(version) is int else None + + +def _contract(path: Path) -> tuple[list, int]: + """(table names in FK order, schemaVersion) from a patch-table contract fixture. + + The fixture is `generator/common/src/jvmTest/resources/patch_tables_contract.json` + of the payload checkout — the same commit whose PatchDbProducer runs — and + `PatchTablesContractTest` asserts byte-for-byte that it is the serialised + `PATCH_TABLES_IN_FK_ORDER`, so it cannot drift from the producer's list. + """ + value = json.loads(path.read_text(encoding="utf-8")) + tables = [entry["table"] for entry in value["fkOrder"]] + if not tables or not all(isinstance(table, str) and table for table in tables): + raise ValueError(f"{path.name} carries no usable fkOrder") + version = value["schemaVersion"] + if type(version) is not int: + raise ValueError(f"{path.name} carries no usable schemaVersion") + return tables, version + + +def _promoted_tables( + contract_path: Path, contract_tables: list, contract_version: int, anchor_schema_version +) -> tuple[frozenset | None, str]: + """Tables the producer refuses to column-diff because the bump promoted them. + + `PatchDbProducer` computes `promotedTables = patchTablesForSchemaVersion(to) + - patchTablesForSchemaVersion(from)` and `continue`s on each of them, so a + promoted table that happens to exist physically in the anchor is never + compared there. The same subtraction is available here from the frozen + per-schema fixtures that sit next to the current one + (`patch_tables_contract_schema_.json`). When the anchor's schema has no + frozen fixture the promoted set is not derivable, and an undecidable scope + means PROCEED rather than a comparison that might skip a good anchor. + """ + if type(anchor_schema_version) is not int: + return None, "the anchor's db_schema block does not state its db_schema_version" + if anchor_schema_version == contract_version: + return frozenset(), "" + if anchor_schema_version > contract_version: + return None, ( + f"the anchor claims db schema {anchor_schema_version}, newer than this build's " + f"{contract_version}" + ) + sibling = contract_path.with_name( + f"patch_tables_contract_schema_{anchor_schema_version}.json" + ) + if not sibling.is_file(): + return None, ( + f"no frozen patch-table contract for db schema {anchor_schema_version} " + f"({sibling.name}), so the promoted-table set is not derivable" + ) + older_tables, older_version = _contract(sibling) + if older_version != anchor_schema_version: + return None, f"{sibling.name} describes db schema {older_version}, not {anchor_schema_version}" + return frozenset(contract_tables) - frozenset(older_tables), "" + + +def check( + this_schema_path: Path, + anchor_version: int, + provenance_path: Path, + contract_path: Path | None, +) -> tuple[str, str]: + """Return (verdict, reason) for one anchor, without downloading its DB. + + The documented list is consulted before anything is read from disk, so a + known-unpatchable anchor is skipped even if this build's own schema dump is + unreadable for some reason. A build past the line_content split asks first + whether the anchor is on the other side of it: then no delta exists at all. + """ + this_version = _this_schema_version(this_schema_path) + if this_version is not None and this_version >= LINE_CONTENT_SPLIT_SCHEMA: + anchor_schema, why = _anchor_db_schema(provenance_path) + anchor_db_schema = anchor_schema.get("db_schema_version") if anchor_schema else None + if type(anchor_db_schema) is not int: + return PROCEED, ( + f"the anchor's db schema is unknown ({why or 'no db_schema_version'}) - the fan " + "reads it from the downloaded DB" + ) + if anchor_db_schema < LINE_CONTENT_SPLIT_SCHEMA: + return BARRIER, ( + f"{anchor_db_schema} the anchor is db schema {anchor_db_schema}; schema " + f"{this_version} keeps line text in line_content, so no delta reaches it - " + "publishing the full-rebase barrier" + ) + elif anchor_version in LEGACY_UNPATCHABLE_DB_VERSIONS: + return UNPATCHABLE, ( + f"db_version {anchor_version} predates the db_schema provenance block and is on " + "the documented unpatchable list (link.isDeclaredBase was dropped without a " + "db_schema_version bump, commit 7755902)" + ) + + anchor_schema, why = _anchor_db_schema(provenance_path) + if anchor_schema is None: + return PROCEED, f"no cheap evidence ({why}) - deferring to the producer" + anchor_tables = anchor_schema["tables"] + + if contract_path is None or not contract_path.is_file(): + return PROCEED, ( + "the patch-table contract fixture is unavailable, and the comparison must never " + "widen to every physical table - deferring to the producer" + ) + contract_tables, contract_version = _contract(contract_path) + + this_schema = json.loads(this_schema_path.read_text(encoding="utf-8")) + if this_schema.get("db_schema_version") != contract_version: + return PROCEED, ( + f"this build's DB is db schema {this_schema.get('db_schema_version')} but the " + f"contract fixture freezes db schema {contract_version} - deferring to the producer" + ) + new_tables = this_schema["tables"] + + promoted, why = _promoted_tables( + contract_path, contract_tables, contract_version, anchor_schema.get("db_schema_version") + ) + if promoted is None: + return PROCEED, f"{why} - deferring to the producer" + + # Exactly the producer's own loop: contract tables only, promoted ones + # skipped, and a table absent from either side never column-diffed (the + # producer (re)creates a table missing from prev from a full snapshot). + for table in contract_tables: + if table in promoted or table not in new_tables or table not in anchor_tables: + continue + prev_columns = anchor_tables[table] + dropped = [c for c in prev_columns if c not in set(new_tables[table])] + if dropped: + return UNPATCHABLE, ( + f"table '{table}' in the previous DB carries column(s) " + f"{', '.join(dropped)} that the new schema no longer declares" + ) + return PROCEED, ( + "every patch-contract column the anchor records survives in this build's schema" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + dump_parser = sub.add_parser("dump", help="print this DB's canonical schema JSON") + dump_parser.add_argument("db") + + check_parser = sub.add_parser("check", help="verdict for one patch-fan anchor") + check_parser.add_argument("--this-schema", required=True) + check_parser.add_argument("--anchor-version", required=True, type=int) + check_parser.add_argument("--anchor-provenance", required=True) + # Optional on purpose: an absent flag degrades to PROCEED like an absent + # file, so the fan can never be widened by forgetting it. + check_parser.add_argument("--contract-tables") + + args = parser.parse_args(argv) + + if args.command == "dump": + value = dump(Path(args.db)) + sys.stdout.write( + json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + ) + return 0 + + # The check must never be the reason a release fails: any surprise degrades + # to PROCEED and the producer decides as it always did. + try: + verdict, reason = check( + Path(args.this_schema), + args.anchor_version, + Path(args.anchor_provenance), + Path(args.contract_tables) if args.contract_tables else None, + ) + except (OSError, UnicodeDecodeError, ValueError, KeyError, TypeError) as exc: + verdict, reason = PROCEED, f"pre-check error ({exc.__class__.__name__}) - deferring to the producer" + print(f"{verdict} {_one_line(reason)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/patch_fan_anchors.sh b/.github/scripts/patch_fan_anchors.sh new file mode 100644 index 000000000..516834610 --- /dev/null +++ b/.github/scripts/patch_fan_anchors.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# The single derivation of "which prior releases this build's patch fan considers". +# +# Two places need the identical answer and must never drift: +# +# * "Prefetch patch-fan anchor DBs" starts the 1.3 GB downloads in parallel +# while "Generate Seforim Database" runs, and +# * "Produce + verify patch fan" walks the same anchors ~70 minutes later. +# +# A prefetch that resolved even one tag differently would download the wrong DB +# and silently degrade back to the serial path, so the loop lives here instead of +# being written twice. +# +# Usage: patch_fan_anchors.sh ... +# +# Prints one TSV row per offset — \t\t\t: +# +# ANCHOR the tag exists; the fan will try to produce this patch +# BELOW-ONE the offset reaches below db_version 1 — nothing to anchor on +# NO-RELEASE no prior release carries that db_version +# +# prior-versions.tsv is written by "Auto-discover prior releases" +# (cols: db_versionrelease_tag, newest db_version first). +set -euo pipefail + +if [ "$#" -lt 3 ]; then + echo "usage: ${0##*/} ..." >&2 + exit 2 +fi + +this_version="$1" +prior_versions="$2" +shift 2 + +case "$this_version" in + ''|*[!0-9]*) echo "::error::patch_fan_anchors: db version '$this_version' is not a number" >&2; exit 2 ;; +esac +[ -f "$prior_versions" ] || { echo "::error::patch_fan_anchors: $prior_versions does not exist" >&2; exit 2; } + +for offset in "$@"; do + case "$offset" in + ''|*[!0-9]*) echo "::error::patch_fan_anchors: offset '$offset' is not a number" >&2; exit 2 ;; + esac + target=$((this_version - offset)) + if [ "$target" -lt 1 ]; then + printf 'BELOW-ONE\t%s\t%s\t\n' "$offset" "$target" + continue + fi + tag=$(awk -F'\t' -v v="$target" '$1==v{print $2; exit}' "$prior_versions" || true) + if [ -z "$tag" ]; then + printf 'NO-RELEASE\t%s\t%s\t\n' "$offset" "$target" + continue + fi + printf 'ANCHOR\t%s\t%s\t%s\n' "$offset" "$target" "$tag" +done diff --git a/.github/scripts/patch_fan_lib.sh b/.github/scripts/patch_fan_lib.sh new file mode 100644 index 000000000..dbc356ea9 --- /dev/null +++ b/.github/scripts/patch_fan_lib.sh @@ -0,0 +1,449 @@ +#!/usr/bin/env bash +# Shared machinery for the weekly DB release's "Produce + verify patch fan" +# step: one anchor start to finish, the heartbeat that keeps the step from +# going dark, and the batch driver's drain. Sourced, not run. +# +# WHY THIS EXISTS +# --------------- +# GitHub Actions refuses to LOAD a workflow whose single `run:` template is +# longer than 21,000 characters. Run 34195296928 never started a job: +# "Invalid workflow file … (Line: 1974, Col: 14): Exceeded max expression +# length 21000" — the fan's body had grown to 22,592 characters. Taking the +# function definitions out leaves the step body as the driver alone. +# +# SHELL SEMANTICS +# --------------- +# Sourced into the step's own shell, so nothing about how any of this runs +# changes: the caller's `set -euo pipefail` is inherited verbatim (that is why +# this is sourced and not executed), the variables below are the step's own +# globals, and `produce_anchor` still forks from the step's shell when the +# driver backgrounds it. Deliberately NO `set` line here — re-issuing the +# options would be a no-op for the one caller and would silently impose them +# on anyone else who sources this. +# +# Read at SOURCE time: RUNNER_TEMP, GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT, the +# optional PATCH_FAN_HEARTBEAT_SECONDS (the heartbeat's cadence and paths) and +# the release credential preflight (RELEASE_TOKEN_KIND and what release_draft.sh +# reads for it). +# Read at CALL time, from the driver's globals: THIS_VER, THIS_SCHEMA, +# PREFETCH_DIR, PREFETCH_WAIT_SECONDS, PATCH_MAIN_CLASS, PATCH_JVM_ARGS, +# PATCH_CLASSPATH and the BATCH_* arrays. + +# This step runs after the relink wait, which can outlive the job's automatic +# GITHUB_TOKEN; every `gh` below uses the preflighted release credential instead. +# shellcheck source=release_draft.sh +. "$(dirname "${BASH_SOURCE[0]}")/release_draft.sh" +use_token "${RELEASE_TOKEN_KIND:-}" || { + echo "::error::patch fan: release credential preflight result is missing or invalid" >&2 + return 1 +} +# shellcheck source=db_asset_names.sh +. "$(dirname "${BASH_SOURCE[0]}")/db_asset_names.sh" + +# Mirrors PatchTables.kt requiresFullRebase: no delta crosses the line_content +# split (schema 6), because it would have to drop line.content. +requires_full_rebase() { # + [ "$1" -lt 6 ] && [ "$2" -ge 6 ] +} + +# The schema barrier for an anchor no delta can leave: a placeholder patch every +# applier refuses plus a "fullRebase" manifest, so that anchor's clients are sent +# to the full DB (or told to update the app) instead of finding no edge at all. +write_barrier() { # + local TARGET_VER="$1" ANCHOR_SCHEMA="$2" TAG="$3" OUT BARRIER_MAIN_CLASS RC=0 + OUT="$PWD/patches/patch-v${TARGET_VER}-v${THIS_VER}.db" + BARRIER_MAIN_CLASS=$(sed -n 's/^barrierMainClass=//p' \ + generator/common/build/patch-pipeline-launcher.properties 2>/dev/null || true) + if [ -n "$PATCH_MAIN_CLASS" ] && [ -n "$BARRIER_MAIN_CLASS" ]; then + java $PATCH_JVM_ARGS -cp "$PATCH_CLASSPATH" \ + -Dout="$OUT" \ + -DfromVersion="$TARGET_VER" -DtoVersion="$THIS_VER" \ + -DfromSchemaVersion="$ANCHOR_SCHEMA" -DtoSchemaVersion="$THIS_SCHEMA" \ + "$BARRIER_MAIN_CLASS" || RC=$? + else + "${GRADLE:?no Gradle toolchain selected}" :generator-common:writeSchemaBarrier \ + -Pout="$OUT" \ + -PfromVersion="$TARGET_VER" -PtoVersion="$THIS_VER" \ + -PfromSchemaVersion="$ANCHOR_SCHEMA" -PtoSchemaVersion="$THIS_SCHEMA" \ + --no-daemon --stacktrace || RC=$? + fi + if [ "$RC" -ne 0 ] || [ ! -s "$OUT.zst" ] || [ ! -s "$OUT.zst.manifest.json" ]; then + echo "::error::anchor v${TARGET_VER} ($TAG): could not write the schema $ANCHOR_SCHEMA → $THIS_SCHEMA barrier (exit $RC)" + rm -f "$OUT" "$OUT.zst" "$OUT.zst.manifest.json" + return 1 + fi + echo "anchor v${TARGET_VER} ($TAG): schema $ANCHOR_SCHEMA → $THIS_SCHEMA is a full rebase — published the schema barrier $(basename "$OUT.zst")" +} +announce_barriers() { # the driver's summary of the barriers staged in patches/ + local n + n=$({ grep -l '"fullRebase": true' patches/*.manifest.json 2>/dev/null || true; } | wc -l | tr -d ' ') + [ "$n" -eq 0 ] || echo "::notice::patch fan: $n anchor(s) get the schema $THIS_SCHEMA full-rebase barrier instead of a delta" +} + +# קורא db_schema_version מ-DB; טבלה/שורה חסרה → ERROR (release DB חייב מוטבע). +# sqlite3 זמין ברנרים ה-self-hosted (בשימוש גם בוורקפלואי delta-real-diff-*). +read_schema() { + local db="$1" v + if ! v=$(sqlite3 "$db" "SELECT value FROM schema_meta WHERE key='db_schema_version'" 2>/dev/null); then + echo "::error::schema_meta missing/unreadable in $db (release DB must be stamped)" >&2 + return 1 + fi + [ -n "$v" ] || { echo "::error::db_schema_version row missing in $db (release DB must be stamped)" >&2; return 1; } + printf '%s' "$v" +} + +# Why an anchor shipped no patch has to outlive its own subshell: the driver's +# final "no patch at all" check must tell a size-guard skip (still a +# publishable full-only release) from a broken fan. +SKIP_DIR="$RUNNER_TEMP/patch-fan-skips" +rm -rf "$SKIP_DIR" +mkdir -p "$SKIP_DIR" +record_skip() { # + printf 'anchor v%s: %s\n' "$2" "$3" > "$SKIP_DIR/v$2.$1" +} + +# One anchor, start to finish: pre-check, reconstitute, produce, +# verify, clean up. Every path it touches is scoped to its own offset +# so two of these can run side by side, it writes to its own log, and +# its exit status is the only channel back to the driver: 0 for a +# produced patch AND for every legitimate skip, non-zero only for a +# genuine failure — which fails the step exactly as `set -e` inside +# the serial loop always did. +produce_anchor() { # + local OFFSET="$1" TARGET_VER="$2" TAG="$3" + local ANCHOR_DIR="prev-dbs/anchor-$OFFSET" + local META_DIR="prev-meta/anchor-$OFFSET" + local PREV_DB="$ANCHOR_DIR/seforim.db" + local PATCH_OUT="$PWD/patches/patch-v${TARGET_VER}-v${THIS_VER}.db" + local PRECHECK WAIT_BUDGET PREFETCH_STATE PREFETCH_WAITED + local PREV_SCHEMA PRODUCE_RC REASON SKIP_KIND ANCHOR_SCHEMA DB_ASSETS DB_ASSET CANDIDATE + local T_START T_DOWNLOADED T_EXTRACTED T_DONE + echo "=== Producing patch v${TARGET_VER} → v${THIS_VER} (offset $OFFSET, tag=$TAG) ===" + + # Reconstituting an anchor costs 110–135 s of download plus a + # decompress; run 33865604251 paid ~140 s for the v10 anchor only + # to be told it was unpatchable. patch_anchor_schema.py answers the + # same question first, from the anchor's own build_provenance.json + # `db_schema` block (a few KB, exact from db_version 27 on — the + # published v26 provenance is schema_version 3 and carries none) or — + # for releases that predate that block — from a documented list of + # db_versions proven unpatchable by this repository's history. + # It is advisory in one direction only: PROCEED is NOT a + # patchability claim, the producer's exit-3 + ".unpatchable" + # marker below stays the sole authority, and the pre-check itself + # can never fail the release (any surprise degrades to PROCEED). + # --contract-tables scopes the column comparison to the producer's + # own table list. It comes from the PAYLOAD checkout — the same + # commit whose PatchDbProducer runs below, not the control-plane + # checkout — because a comparison wider than that list would skip + # anchors the producer would still have patched. + rm -rf "$META_DIR" + mkdir -p "$META_DIR" + # An anchor without provenance is normal (every release before + # v27 carries none) but gh's bare "no assets match the file + # pattern" never says which asset was missing — this does. + if ! gh release download "$TAG" \ + --pattern 'build_provenance.json' \ + --dir "$META_DIR" 2>"$META_DIR/gh.err"; then + echo "anchor v${TARGET_VER} ($TAG): release carries no build_provenance.json ($(tr -d '\r' < "$META_DIR/gh.err" | head -n1)) — the pre-check falls back to the documented unpatchable list" + fi + rm -f "$META_DIR/gh.err" + PRECHECK=$(python3 .pipeline-control/.github/scripts/patch_anchor_schema.py check \ + --this-schema build/db_schema.json \ + --anchor-version "$TARGET_VER" \ + --anchor-provenance "$META_DIR/build_provenance.json" \ + --contract-tables generator/common/src/jvmTest/resources/patch_tables_contract.json) \ + || PRECHECK="PROCEED pre-check did not run — deferring to the producer" + rm -rf "$META_DIR" + echo "pre-check: $PRECHECK" + if [ "${PRECHECK%% *}" = BARRIER ]; then + ANCHOR_SCHEMA="${PRECHECK#BARRIER }" + write_barrier "$TARGET_VER" "${ANCHOR_SCHEMA%% *}" "$TAG" + return + fi + if [ "${PRECHECK%% *}" = UNPATCHABLE ]; then + echo "::warning::anchor v${TARGET_VER} ($TAG): ${PRECHECK#* } — pre-download schema check declared the anchor unpatchable; skip anchor" + record_skip structural "$TARGET_VER" "${PRECHECK#* }" + return 0 + fi + + rm -rf "$ANCHOR_DIR" + mkdir -p "$ANCHOR_DIR" + # A1: this anchor was very probably downloaded and verified (size, + # and digest when GitHub published one) while the DB was being + # generated. Wait for THIS tag's marker; anything short of a + # verified hit — no marker inside the budget, a failed or + # unpatchable verdict, a vanished file — falls straight back to the + # unchanged serial download below. A missing marker also aborts the + # prefetch, so it cannot keep competing for the downlink. + # The "stop waiting" verdict has to outlive this anchor: the ones + # still to come run in their own subshells, where a shell variable + # could not reach them, so it is recorded as a file next to the + # prefetch state. + WAIT_BUDGET="$PREFETCH_WAIT_SECONDS" + if [ -f "$PREFETCH_DIR/.abandoned" ]; then + WAIT_BUDGET=0 + fi + PREFETCH_STATE=absent + if [ -f "$PREFETCH_DIR/.started" ]; then + PREFETCH_WAITED=0 + while [ ! -f "$PREFETCH_DIR/$TAG/.done" ] && [ ! -f "$PREFETCH_DIR/.abandoned" ] && [ "$PREFETCH_WAITED" -lt "$WAIT_BUDGET" ]; do + sleep 5 + PREFETCH_WAITED=$((PREFETCH_WAITED + 5)) + done + if [ -f "$PREFETCH_DIR/$TAG/.done" ]; then + PREFETCH_STATE=$(head -n1 "$PREFETCH_DIR/$TAG/.done") + sed -n '2,$p' "$PREFETCH_DIR/$TAG/.done" + else + echo "::warning::anchor v${TARGET_VER} ($TAG): no prefetch marker after ${PREFETCH_WAITED}s — falling back to the serial download" + bash .pipeline-control/.github/scripts/prefetch_patch_anchors.sh abort "$PREFETCH_DIR" \ + || echo "::warning::anchor v${TARGET_VER} ($TAG): the prefetch abort helper itself failed — continuing with the serial download" + : > "$PREFETCH_DIR/.abandoned" + fi + fi + T_START=$(date +%s) + if [ "$PREFETCH_STATE" = ok ] && [ -s "$PREFETCH_DIR/$TAG/seforim.db.zst" ]; then + mv "$PREFETCH_DIR/$TAG/seforim.db.zst" "$ANCHOR_DIR/seforim.db.zst" + echo "anchor v${TARGET_VER} ($TAG): reused the prefetched DB" + else + # Under `set -e` a failed download used to end this subshell with + # nothing but gh's own bare stderr, and the driver could only say + # "exit code 1". Name the release, the asset and gh's reason. + # This build's own schema name first (a delta needs an equal schema), + # then the legacy name every schema <= 5 release carries. + DB_ASSETS=$(full_db_asset_name "$THIS_SCHEMA") || return 1 + [ "$DB_ASSETS" = "$LEGACY_FULL_DB_ASSET" ] || DB_ASSETS="$DB_ASSETS $LEGACY_FULL_DB_ASSET" + DB_ASSET="" + for CANDIDATE in $DB_ASSETS; do + if download_full_db_by_name "$TAG" "$CANDIDATE" "$ANCHOR_DIR" \ + 2>"$ANCHOR_DIR/gh.err"; then + DB_ASSET="$CANDIDATE" + break + fi + done + if [ -z "$DB_ASSET" ]; then + echo "::error::anchor v${TARGET_VER} ($TAG): could not download ${DB_ASSETS// / or } from that release ($(tr -d '\r' < "$ANCHOR_DIR/gh.err" | head -n1)) — no patch can be produced against this anchor" + rm -rf "$ANCHOR_DIR" + return 1 + fi + rm -f "$ANCHOR_DIR/gh.err" + [ "$DB_ASSET" = seforim.db.zst ] || mv "$ANCHOR_DIR/$DB_ASSET" "$ANCHOR_DIR/seforim.db.zst" + fi + T_DOWNLOADED=$(date +%s) + if ! unzstd -c "$ANCHOR_DIR/seforim.db.zst" > "$PREV_DB"; then + echo "::error::anchor v${TARGET_VER} ($TAG): the full DB from that release could not be decompressed — no patch can be produced against this anchor" + rm -rf "$ANCHOR_DIR" + return 1 + fi + T_EXTRACTED=$(date +%s) + test -s "$PREV_DB" || { echo "::error::the full DB of release $TAG is empty"; return 1; } + + PREV_SCHEMA=$(read_schema "$PREV_DB") || return 1 + # Reached for an anchor whose provenance does not state its schema. + if requires_full_rebase "$PREV_SCHEMA" "$THIS_SCHEMA"; then + rm -rf "$ANCHOR_DIR" + write_barrier "$TARGET_VER" "$PREV_SCHEMA" "$TAG" + return + fi + + # Explicitly supported schema transitions. The producer derives + # contract promotions from both signed schema versions, rebuilds + # newly tracked tables from a full snapshot, and verifies a real + # apply. Unknown transitions remain fail-closed per anchor. + if [ "$PREV_SCHEMA" != "$THIS_SCHEMA" ]; then + if { [ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 3 ]; } || \ + { [ "$PREV_SCHEMA" = 1 ] && [ "$THIS_SCHEMA" = 4 ]; } || \ + { [ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 4 ]; } || \ + { [ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 4 ]; } || \ + { [ "$PREV_SCHEMA" = 1 ] && [ "$THIS_SCHEMA" = 5 ]; } || \ + { [ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 5 ]; } || \ + { [ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 5 ]; } || \ + { [ "$PREV_SCHEMA" = 4 ] && [ "$THIS_SCHEMA" = 5 ]; }; then + echo "schema $PREV_SCHEMA → $THIS_SCHEMA — producing the supported cross-schema delta" + else + echo "schema $PREV_SCHEMA → $THIS_SCHEMA is unsupported — skip anchor" + record_skip structural "$TARGET_VER" "schema $PREV_SCHEMA → $THIS_SCHEMA is unsupported" + rm -rf "$ANCHOR_DIR" + return 0 + fi + fi + + # A column added without a db_schema_version bump (e.g. + # category.heShortDesc, 2026-07-16, still schema 1) no longer costs + # us the anchor: PatchDbProducer emits an + # `ALTER TABLE … ADD COLUMN` migration — plain SQL that every + # released PatchApplier already runs before the upserts — and ships + # every row whose new value is non-NULL. The shell pre-check that + # compared PRAGMA table_info is therefore gone; the producer itself + # is now the single authority on what is patchable. + # + # It declares an anchor unpatchable only when no patch could ever + # express the change (a PRIMARY KEY column missing from prev, or a + # column dropped without a bump): PatchPipelineCli then exits 3 + # (UnpatchableAnchor) and drops ".unpatchable" carrying the + # reason, and this anchor is skipped for that one code. + # Every other non-zero exit still fails the release. + rm -f "$PATCH_OUT.unpatchable" + PRODUCE_RC=0 + if [ -n "$PATCH_MAIN_CLASS" ]; then + # Exactly what `gradle :generator-common:producePatchAndVerify` + # forks: main class, -Xmx/GC flags and runtime classpath all come + # from that task's own definition (see patchPipelineLauncher in + # generator/common/build.gradle.kts), and its -P properties are + # the -D system properties the task sets on the fork. ZSTD_LEVEL + # reaches the CLI through the step env either way. Keeping Gradle + # out of the loop is what lets two of these run at once. + java $PATCH_JVM_ARGS -cp "$PATCH_CLASSPATH" \ + -DprevDb=$PWD/$PREV_DB \ + -DnewDb=$PWD/build/seforim.db \ + -Dout=$PATCH_OUT \ + -DfromVersion=$TARGET_VER -DtoVersion=$THIS_VER \ + "$PATCH_MAIN_CLASS" || PRODUCE_RC=$? + else + "${GRADLE:?no Gradle toolchain selected}" :generator-common:producePatchAndVerify \ + -PprevDb=$PWD/$PREV_DB \ + -PnewDb=$PWD/build/seforim.db \ + -Pout=$PATCH_OUT \ + -PfromVersion=$TARGET_VER -PtoVersion=$THIS_VER \ + --no-daemon --stacktrace || PRODUCE_RC=$? + fi + # The Gradle task maps the CLI's exit 3 onto a warning and a + # successful build; run directly, that 3 arrives here. Both funnel + # into the one marker path below, and every OTHER non-zero exit + # fails this anchor — which fails the whole step, as before. + if [ "$PRODUCE_RC" -ne 0 ] && [ "$PRODUCE_RC" -ne 3 ]; then + echo "::error::anchor v${TARGET_VER} ($TAG): producePatchAndVerify failed with exit code $PRODUCE_RC" + rm -rf "$ANCHOR_DIR" + return "$PRODUCE_RC" + fi + if [ "$PRODUCE_RC" -eq 3 ] || [ -f "$PATCH_OUT.unpatchable" ]; then + REASON=$(cat "$PATCH_OUT.unpatchable" 2>/dev/null || true) + echo "::warning::anchor v${TARGET_VER} ($TAG): ${REASON:-see PatchPipelineCli output} — producer declared the anchor unpatchable; skip anchor" + # PatchSizeGuard.MARKER_REASON_TOKEN leads the marker when the + # delta was merely too big — a full-only release, not a defect. + SKIP_KIND=structural + case "$REASON" in oversized-delta:*) SKIP_KIND=oversized ;; esac + record_skip "$SKIP_KIND" "$TARGET_VER" "${REASON:-see PatchPipelineCli output}" + # Leave nothing behind: the marker plus the producer's half-built + # .tmp (and any stale .db) must not clutter patches/, which the + # release staging step globs. + rm -f "$PATCH_OUT.unpatchable" "$PATCH_OUT" "$PATCH_OUT.tmp" + rm -rf "$ANCHOR_DIR" + return 0 + fi + T_DONE=$(date +%s) + echo "anchor v${TARGET_VER} timings: download=$((T_DOWNLOADED - T_START))s extract=$((T_EXTRACTED - T_DOWNLOADED))s produce+verify+compress=$((T_DONE - T_EXTRACTED))s total=$((T_DONE - T_START))s" + + # Free disk for the anchors still to come — only the .zst + manifest + # need to ship; the .db was the verify ground-truth. + rm -rf "$ANCHOR_DIR" + rm -f "patches/patch-v${TARGET_VER}-v${THIS_VER}.db" \ + "patches/verify-patch-v${TARGET_VER}-v${THIS_VER}.db" + df -h / +} + +# Batch-buffered logging is why the two longest gaps in run +# 34024655297 (1318 s and 1076 s) were both inside this step with +# nothing on stdout: a hung producer looked exactly like a working +# one for 22 minutes. PatchPipelineCli emits nothing between +# "Producing patch" and its result — no progress to forward — so the +# heartbeat is the shell's: one line every HEARTBEAT_SECONDS naming +# the anchors still in flight, and one end line per anchor below. +# PATCH_FAN_HEARTBEAT_SECONDS exists so a test can drive this loop in +# seconds instead of in five-minute steps; the default is the one the +# runner uses. +HEARTBEAT_SECONDS="${PATCH_FAN_HEARTBEAT_SECONDS:-300}" +case "$HEARTBEAT_SECONDS" in ''|*[!0-9]*|0) HEARTBEAT_SECONDS=300 ;; esac +HEARTBEAT_POLL=5 +[ "$HEARTBEAT_POLL" -le "$HEARTBEAT_SECONDS" ] || HEARTBEAT_POLL="$HEARTBEAT_SECONDS" +HEARTBEAT_PID="" +HEARTBEAT_FLAG="$RUNNER_TEMP/patch-fan-heartbeat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" +# Which anchors the heartbeat is allowed to claim are still running. +# A file, because the heartbeat is a subshell: drain_batch rewrites it +# as each anchor lands, so the line never names an anchor that has +# already finished. +HEARTBEAT_INFLIGHT="$HEARTBEAT_FLAG.inflight" +heartbeat_inflight() { #