From 71176054303fcbd16abfc860896eee5e38c37b05 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 2 Nov 2026 17:52:06 -0800 Subject: [PATCH] feat(engines): drop the proton engine and its image The mirrors that copy the owner's accounts left the org and keep their own copies of the proton parts. lib now holds only what the rsync mirrors use. - Delete engines/proton.yml, docker/proton.Dockerfile, docker/s3.py and examples/proton/. - Drop the proton_drive_cli, age and python tables from the lock. Only the proton image read them. The rsync image takes python3 from apt. - ci and release build the rsync variant only. The guard steps run without a variant condition. - The chain-file fixture names katoptra/ctan. - README, CLAUDE.md and CONTRIBUTING.md describe one engine, one image and one example. toolbox.yml changes in comments only, and engines/rsync.yml does not change. Thus, the render of each public mirror stays the same. The release is v2.10.0. --- .github/chain-file.sh | 2 +- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 4 +- CLAUDE.md | 40 +-- CONTRIBUTING.md | 23 +- README.md | 377 +++++--------------- docker/lock.py | 4 +- docker/proton.Dockerfile | 85 ----- docker/s3.py | 50 --- engines/proton.yml | 274 -------------- examples/proton/.taskrc.yml | 3 - examples/proton/Taskfile.yml | 71 ---- examples/proton/fixtures/upload-failed.json | 2 - examples/proton/fixtures/upload-ok.json | 2 - examples/proton/op.env | 11 - examples/proton/render.txt | 140 -------- toolbox.yml | 17 +- toolchain.lock.toml | 55 +-- 18 files changed, 131 insertions(+), 1033 deletions(-) delete mode 100644 docker/proton.Dockerfile delete mode 100755 docker/s3.py delete mode 100644 engines/proton.yml delete mode 100644 examples/proton/.taskrc.yml delete mode 100644 examples/proton/Taskfile.yml delete mode 100644 examples/proton/fixtures/upload-failed.json delete mode 100644 examples/proton/fixtures/upload-ok.json delete mode 100644 examples/proton/op.env delete mode 100644 examples/proton/render.txt diff --git a/.github/chain-file.sh b/.github/chain-file.sh index c48aaa4..ba8335d 100644 --- a/.github/chain-file.sh +++ b/.github/chain-file.sh @@ -16,7 +16,7 @@ chain_file() { } if [ "${1-}" = --check ]; then - [ "$(chain_file 'katoptra/dropbox/.github/workflows/sync.yml@refs/heads/main')" = sync.yml ] + [ "$(chain_file 'katoptra/ctan/.github/workflows/sync.yml@refs/heads/main')" = sync.yml ] [ "$(chain_file 'katoptra/ctan/.github/workflows/hourly.yml@refs/heads/josh/topic')" = hourly.yml ] [ "$(chain_file 'o/r/.github/workflows/sync.yml@81fb6ee34abc7703087cfe4a303263febc912c6f')" = sync.yml ] echo "chain-file: PASS" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90eb9cc..70bd558 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,20 +11,18 @@ jobs: timeout-minutes: 30 strategy: matrix: - variant: [rsync, proton] + variant: [rsync] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/toolbox # The test cases of each guard show the inputs that the guard rejects. The build shows # an input that the guard accepts. - name: The security guards refuse what they should - if: matrix.variant == 'rsync' run: | sh .github/validate-vars.sh --check sh .github/chain-file.sh --check sh docker/gpg-gate-check.sh - name: The op verb tries again only when the command did not start - if: matrix.variant == 'rsync' run: sh .github/op-retry-check.sh - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # Actions keeps the layers in its cache, with one scope for each variant. release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84b6279..aab159f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ # On a semver tag, this workflow does these steps: -# 1. It makes each image for amd64 and arm64. +# 1. It makes the image for amd64 and arm64. # 2. It pushes the two tags. # 3. It moves the floating v tag. # 4. It publishes the GitHub release. @@ -19,7 +19,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - variant: [rsync, proton] + variant: [rsync] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 diff --git a/CLAUDE.md b/CLAUDE.md index 0d830e1..a924034 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,9 +3,9 @@ This repository is the toolbox that all katoptra mirrors include from a URL. `README.md` is the manual. It has sections about these parts of the system: -- The layers, the verbs and the engines +- The layers, the verbs and the engine - The secrets and the storage -- The images and the workflows. +- The image and the workflows. It also tells how a mirror uses these parts and how a mirror changes them. This file gives the items that a change must not break. Each item is a verified fact about go-task or a @@ -45,14 +45,14 @@ Read that section before you write. - Use only the keys `trusted-hosts` and `cache-expiry` in `.taskrc.yml`. Task ignores a key that it does not know, and it shows no warning. In that condition, it downloads the remote Taskfiles again each time that it runs. -- Each tool in an image is from `toolchain.lock.toml`. `docker/lock.py` reads that file - for the Dockerfiles and for the toolbox action, and the build examines each tool before +- Each tool in the image is from `toolchain.lock.toml`. `docker/lock.py` reads that file + for the Dockerfile and for the toolbox action, and the build examines each tool before it uses it. Most tools have a recorded sha256. The AWS CLI zip has no sha256, because upstream publishes only a detached PGP signature for it. Thus, the fetch stage verifies that signature with `docker/aws-cli.pub`, and it must find the fingerprint that `toolchain.lock.toml` pins. It uses the same GOODSIG and VALIDSIG awk check as the engine uses for TeX Live. -- The images set `TASK_REMOTE_OFFLINE=1`. In a run, task gets the included Taskfiles from +- The image sets `TASK_REMOTE_OFFLINE=1`. In a run, task gets the included Taskfiles from the `.task/remote` cache of the mirror. This cache goes into the container with the repository (a bind mount). Task does not get the Taskfiles from the network. - `run` and `render` put the git top level of the repository at `/work` (a bind mount). @@ -70,19 +70,13 @@ Read that section before you write. 3.53.1), but bash and dash print `on`. For a guard that must operate in all these shells, make each step exit manually. - The built-in shell of task has no `umask`. To make a file with the mode 0600, use - `install -m 600 /dev/null "$f"`. Then write the file. The `age` verb of the proton engine - does this. -- `pd` runs each command of the Proton CLI. After each command, with all exit codes, `pd` - examines the session, and it writes the session back to the bucket if the refresh token - changed. The token changes each time that the CLI uses the session. If a run keeps a - changed token and does not write it back, Proton rejects the token of the next run. A new - login is then necessary. For the same cause, two mirrors do not use one session. + `install -m 600 /dev/null "$f"`. Then write the file. - Pin each action to a full SHA, with the version in a comment at the end of the line. A mirror pins the two reusable workflows in this method. Each reusable workflow does a checkout of this repository at the commit of the workflow (`github.job_workflow_sha`) for the toolbox action and `toolchain.lock.toml`. Thus, the pin of the workflow is the only pin. The URLs in `includes:` use the git tag `v2`, and the image uses the tag - `-v2`. The two tags move. When they move, all mirrors get the release. + `rsync-v2`. The two tags move. When they move, all mirrors get the release. - Give `-r` to each `xargs` if a filter can make its input empty. GNU `xargs` runs its command one time on empty input. A guard on the file that a pipe reads is not a guard on the input of `xargs`. For example, the `SLASH` awk of `pages` removes the root, which has @@ -109,12 +103,12 @@ Read that section before you write. container branch and the decision-batch branch use `.run/tl` only if the batch has a `TL` path. - Do not make a task internal if a host verb starts it with its name in the image. go-task - does not start an internal task from the command line. For example, `empty-trash` starts - `empty-trash-pipeline` with `task op`. Thus, `empty-trash-pipeline` is not internal. + does not start an internal task from the command line. For example, `plan` starts + `plan-pipeline` with `task op`. Thus, `plan-pipeline` is not internal. - go-task reads the global vars of the Taskfiles that one Taskfile includes in a random sequence at each run (verified, 3.53.1 and 3.54.0). Thus, an engine global var that reads a toolbox var (`RUN`) can get that var with no value in the tasks of a mirror. For this - cause, `SESSION` and `UPLOAD` in `proton.yml` give `ROOT_DIR/.run` in full. Also, if a + cause, in an engine global var, write `ROOT_DIR/.run` in full, not `RUN`. Also, if a mirror sets `IMAGE` in its root vars, a run can use the default of the engine. If the toolbox entry of `includes:` sets `IMAGE`, the run always uses that value. - Do not put an empty field first in a line that `read` gets with a TAB in `IFS`. A TAB is @@ -146,14 +140,13 @@ Read that section before you write. ## Verifying a change ```sh -cd examples/rsync && task image-build && task run -- task tools && task check && task run -- task offline -cd examples/proton && task image-build && task run -- task tools && task check && task run -- task offline +cd examples/rsync && task image-build && task run -- task tools && task check && task run -- task offline sh .github/validate-vars.sh --check && sh .github/chain-file.sh --check && sh docker/gpg-gate-check.sh sh .github/op-retry-check.sh ``` -After a change to a verb, run `task render-update` to update the `render.txt` files. -`offline` is the check of each engine. +After a change to a verb, run `task render-update` to update `examples/rsync/render.txt`. +`offline` is the check of the engine. The rsync check runs on `examples/rsync/fixtures/`: @@ -172,11 +165,6 @@ The rsync check runs on `examples/rsync/fixtures/`: Make the tree again, with a new key, only to change its shape. There is no copy of the private half of the test key. -The proton check runs `confirm` on `examples/proton/fixtures/`: an upload summary that it -accepts and one that it rejects. Then it does a dry run of `empty-trash-pipeline` from the -command line. Last, it encrypts and decrypts a file with the `age` verb and a test -identity. - `task check` does not examine a change to `README.md`. For that change, the review is the check. The other repositories of the org link to headings of `README.md`. Do not change the text of these headings. If you change it, the links from those repositories cannot find them: @@ -187,8 +175,6 @@ these headings. If you change it, the links from those repositories cannot find - `#the-rsync-engine` - `#when-a-run-fails` - `#content-types` -- `#the-proton-engine` -- `#the-session` - `#r2-specifics` - `#monitoring` - `#rules-a-mirror-keeps`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7455d32..0aa526e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ are applicable to this repository. This repository adds three rules: sets.** If a `vars:` block has a default, the run uses that default, not the root value of the mirror. Put each default inline in the command: `{{.X | default N}}`. If two commands of a verb read the var, put the default in a task var with a different name - (`BATCH_SIZE: '{{.BATCH_GB | default 4}}'` in `split`). `IMAGE` is different: each + (`BATCH_SIZE: '{{.BATCH_GB | default 4}}'` in `split`). `IMAGE` is different: the engine sets `IMAGE: '{{.IMAGE | default "..."}}'` in `vars:`. Thus, if the toolbox entry of `includes:` gives `IMAGE`, the run uses that value. - **Do not change the name of a verb in `toolbox.yml` or in an engine.** A mirror includes @@ -15,30 +15,27 @@ are applicable to this repository. This repository adds three rules: one namespace. A new name for one of these verbs is a new major version. - **A change to a verb that a pipeline runs is a change to a `render.txt`.** `render` makes - a dry run of the `pipeline` of each example. A change to a verb changes these files: - - A toolbox verb: the `render.txt` of each example that runs it - - A verb of the rsync engine: `examples/rsync/render.txt` - - A verb of the proton engine: `examples/proton/render.txt`. - - A host verb, for example `run` or `session-seal`, is not in a render. Run - `task render-update` in each example. Commit the result with the change. The diff of the - pull request is the review. A verb with `silent: true` renders no command. Thus, for + a dry run of the `pipeline` of the example. A change to a toolbox verb or to a verb of + the rsync engine changes `examples/rsync/render.txt`. + + A host verb, for example `run` or `image-build`, is not in a render. Run + `task render-update` in `examples/rsync`. Commit the result with the change. The diff of + the pull request is the review. A verb with `silent: true` renders no command. Thus, for `report` and `report-engine`, the review of their rows is in the diff of the file that has the verb. ## Checking a change ```sh -cd examples/rsync && task image-build && task run -- task tools && task check && task run -- task offline -cd examples/proton && task image-build && task run -- task tools && task check && task run -- task offline +cd examples/rsync && task image-build && task run -- task tools && task check && task run -- task offline sh .github/validate-vars.sh --check && sh .github/chain-file.sh --check && sh docker/gpg-gate-check.sh ``` -CI does these steps for each image, on each pull request and on each push to `main`: +CI does these steps for the image, on each pull request and on each push to `main`: 1. It builds the image. 2. It runs `task run -- task tools`. 3. It runs `task check`. 4. It runs `task run -- task offline`. -It also runs the three guards one time, in the job of the `rsync` image. +It also runs the three guards before it builds the image. diff --git a/README.md b/README.md index b12cbfa..5eef9e0 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,8 @@ does these tasks: - It starts each run in a container. - It gets the secrets of the run. - It does a check of the pipeline, and it gives a report of each run. -- It moves the bytes. Each transport has one engine here: - - `engines/rsync.yml` copies an rsync upstream into a bucket. - - `engines/proton.yml` copies a staging tree into Proton Drive. +- It moves the bytes. The engine, `engines/rsync.yml`, copies an rsync upstream into a + bucket. A mirror contains only its identity, the hooks that it fills, and the verbs that no other mirror uses. @@ -38,34 +37,23 @@ that, you can read each section without the other sections. ```mermaid flowchart LR - subgraph up["Upstreams"] - u1["Trees over rsync, for example CTAN and GNU"] - u2["GitHub, over the API and git"] - u3["Dropbox, over the API"] - end + up["The upstream: a tree over rsync
for example CTAN or GNU"] sched["katoptra/dispatch
workflow_dispatch, on a schedule"] --> host subgraph job["One GitHub Actions job per run"] host["The runner: task sync"] --> box["The toolbox image: task pipeline"] end vault["A 1Password vault
one item per mirror"] -. "op run: values, by name" .-> host - u1 --> box - u2 --> box - u3 --> box - box --> r2["An S3-compatible bucket, R2
a public mirror: the mirror itself
a Proton mirror: the session, and state"] - box --> pd["Proton Drive
the private mirrors' sink"] + up --> box + box --> r2["An S3-compatible bucket, R2
the mirror itself"] r2 --> cf["A custom domain on the bucket"] --> cl["tlmgr, browsers, anyone"] box --> hc["healthchecks.io, one ping per run"] --> site["katoptra.org, live status"] ``` -Each mirror has the structure of this diagram, with one upstream and one sink. There are two -types of mirror: - -- A public mirror copies an rsync tree into a bucket, and a domain serves the bucket. The - bucket is the mirror. -- A private mirror copies an account into Proton Drive. Its bucket contains only the data - that the next run uses. +Each mirror has the structure of this diagram, with one upstream and one bucket. A mirror +copies an rsync tree into a bucket, and a domain serves the bucket. The bucket is the +mirror. -The two types of mirror do a run with the same steps: +Each run has these steps: 1. The scheduler starts a job on GitHub Actions. 2. The job downloads one image. @@ -86,8 +74,8 @@ flowchart TB end subgraph lib["katoptra/lib, at v2"] tb["toolbox.yml
menu, image, run, op, sync, plan, check
clock, due, reconciled, report, ping, failed"] - en["engines/rsync.yml, engines/proton.yml
the pipeline vocabulary, one file per transport"] - im["ghcr.io/katoptra/toolbox:rsync-v2, :proton-v2
docker/, toolchain.lock.toml"] + en["engines/rsync.yml
the pipeline vocabulary"] + im["ghcr.io/katoptra/toolbox:rsync-v2
docker/, toolchain.lock.toml"] wf[".github/workflows/sync.yml, check.yml
.github/actions/toolbox"] end tf -- "includes, flattened" --> tb @@ -158,7 +146,7 @@ The diagram does not show these four items: ## The toolbox -`toolbox.yml` contains the verbs that all mirrors have, with all engines. Half of these +`toolbox.yml` contains the verbs that all mirrors have, outside the engine. Half of these verbs run on the host and start the container. The other half run in the container: they are the first steps and the last steps of each pipeline. @@ -237,7 +225,7 @@ a failure. #### When a reconcile runs -A reconcile compares the mirror with its destination, not with its state. The time of the +A reconcile compares the bucket with upstream, not with the state. The time of the last reconcile gives the time of the next one. The start time of a run does not give it, because the scheduler is not in the mirror, and its times can change. A pipeline that does a reconcile runs `due` near its start. Its reconcile step runs only if `.run/reconcile` is @@ -249,8 +237,8 @@ there. | `RECONCILE_HOURS` | 24 | The number of full hours between two reconciles. For a different interval, a mirror sets it in its root vars | `.state/reconciled` contains the start epoch of the run that completed the last reconcile. -The `pull` and `push` of the engine read and write this file. Thus, with the rsync -engine, the file has xz compression, the same as the state. +The `pull` and `push` of the engine read and write this file. Thus, the file has xz +compression, the same as the state. The limit is half an hour less than `RECONCILE_HOURS`. A daily run starts some seconds before or after the time that is 24 h after the last run. With a limit of 24 h and no @@ -260,7 +248,7 @@ In these conditions, a run records no reconcile, and the reconcile is also neces next run: - The run stops with a failure before it completes its reconcile. -- An rsync run has batches that wait for a subsequent run. Its reconcile waits for the +- A run has batches that wait for a subsequent run. Its reconcile waits for the run of the chain that uploads the last batch. ### The parts that a mirror supplies @@ -280,11 +268,11 @@ next run: | `pull`, `push` | Tasks | The tasks that `due` and `reconciled` use to read and write `.state/reconciled`. An engine supplies the two tasks | | `excludes:` | Key in `includes:` | The verbs of lib that the mirror overrides. Refer to [Changing it](#changing-it) | -## The engines +## The engine -An engine is a second entry in `includes:`, with the verbs for one transport. The names of -these verbs are the same in all engines, and a mirror cannot use these names for a -different verb. Thus, the pipeline of a mirror has the same verbs with each engine. +An engine is a second entry in `includes:`, with the verbs for one transport. lib has one +engine, the rsync engine. A mirror cannot use the name of an engine verb for a different +verb. An engine reads the root vars of the mirror and the environment. It has no `vars:` default for a var that a mirror sets. With such a default, the run uses the default, not the value @@ -724,131 +712,13 @@ These procedures repair other problems: This procedure is safe, and it has no cost. If you do not do it, R2 aborts the upload after 7 days. -### The proton engine - -`engines/proton.yml` copies a staging tree into one Proton Drive folder, with the -`proton-drive` CLI of Proton. It is for a mirror that can copy its full upstream in one run. -The mirror fills `staging/`, and the engine does all the other steps. - -The bucket contains only the CLI session of the mirror. The engine keeps no other data -there, because of these two facts: - -- The CLI does not upload a file if Proton has its content. -- `-f create-new-revision` makes a new revision of a file that changed. - -Thus, the version history of Proton is the history of the mirror. - -```mermaid -flowchart LR - clock --> session --> destination --> stage --> upload --> confirm --> prune --> report --> ping - report --> re["report-engine"] --> rm["report-mirror"] - classDef hook stroke-dasharray: 5 5 - class stage,prune,re,rm hook -``` - -| Verb | Function | -|---|---| -| `pipeline`, `plan-pipeline` | The verbs in the diagram, in sequence. The read-only half runs to `stage`, and then prints the number of files and MB that the engine can upload | -| `session` | Downloads `.state/session.tar.age` from the bucket, decrypts it with `MIRROR_AGE_IDENTITY`, and puts the two session files from the tar into `.run/session` | -| `destination` | Makes a listing of the parent of `MIRROR_PROTON_DESTINATION`. It stops the run unless the parent has one folder of that name, and only one, with the UID `MIRROR_PROTON_DESTINATION_UID` | -| `stage` | A hook that each mirror writes. It fills `staging/` with the files that Proton must contain | -| `upload` | Runs one `filesystem upload -f create-new-revision -d merge -t --json` of `staging/*` into the destination. It writes the summary to `.run/upload.json` | -| `confirm` | In the summary, `transferredItems` and `skippedItems` together must be equal to the files and folders in staging, and `failedItems` must be 0. It writes the result to `.run/confirm.txt` | -| `prune` | A hook, empty here. A mirror that moves to the trash the items that its upstream deleted writes it, with `list-folder` and `trash` | -| `pd` | Runs each CLI call, and writes the stderr of the call to `.run/pd.err`. After each call, with all exit codes, it examines the session. If the token changed, it seals the session back into the bucket | -| `list-folder`, `trash` | `list-folder` writes the JSON listing of a folder to `OUT`. `trash` moves the nodes at `PATHS` to the trash of Proton | -| `session-seal -- ` | On the host: it encrypts the two files of a laptop login into the bucket | -| `empty-trash` | On the host: it deletes all the items in the trash of Proton, permanently, but only after you tell it to continue | -| `report-engine` | A hook: the rows of the engine in the run summary | - -The verbs in the table also run these verbs: `session-push`, `seal`, `age`, `pull`, `push` -and `empty-trash-pipeline`. `pull` and `push` move a key between the bucket and `.run`, -with no compression. A verb of a mirror can also run them. - -The stderr of `list-folder` goes to the log of the run, not to `.run/pd.err`. Its command -has `|| echo "[]" > OUT` at its end. Thus, the stderr redirect of `pd` goes to the `echo`, -not to the CLI. - -The engine reads no root var. It reads only the environment, with the names that the -`op.env` of each Proton mirror has: - -| Name | Value | -|---|---| -| `MIRROR_PROTON_DESTINATION` | The CLI path of the destination folder, for example `/my-files/GitHub` | -| `MIRROR_PROTON_DESTINATION_UID` | The UID of that folder. Each run compares it before it writes the first file | -| `MIRROR_R2_BUCKET` | The bucket that contains the session | -| `MIRROR_AGE_IDENTITY` | The `AGE-SECRET-KEY-...` line that encrypts and decrypts the session | -| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_ENDPOINT_URL_S3` | The credentials and the endpoint for `s3`. The proton mirrors use the `_S3` name for the endpoint. The AWS CLI and boto3 read the two names, `AWS_ENDPOINT_URL` and `AWS_ENDPOINT_URL_S3` | - -#### The session - -Only a sign-in in a browser can start a session of the CLI. Proton does not accept a new -sign-in from a datacenter address. Thus, you make the session one time on a laptop, and -each run gets it from the bucket, encrypted. The CLI changes the refresh token when it uses -the session. Thus, after each CLI call, `pd` examines the session. If the token changed, -`pd` seals the session back into the bucket. - -```mermaid -flowchart LR - subgraph laptop["Once, on a laptop"] - l1["proton-drive auth login
with the session as plain files under .run/pd"] --> l2["task session-seal -- .run/pd
tar, age-encrypt, push"] - end - l2 --> key[".state/session.tar.age
in the bucket"] - subgraph run["Every run"] - r1["session: pull, decrypt, extract to .run/session"] --> r2["pd: one CLI call"] --> r3{"token rotated?"} - r3 -- yes --> r4["session-push: seal it back"] - r3 -- no --> r2 - end - key --> r1 - r4 --> key -``` - -To make a session, do these steps on a laptop: - -1. On the Settings page of your Proton account, set telemetry to OFF. Do this for each - Proton account. -2. Run these commands: - - ```sh - export PROTON_DRIVE_CACHE_DIR=.run/pd PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file - proton-drive auth login # a browser opens; .run/pd holds two JSON files after - proton-drive filesystem create-folder /my-files # /my-files is the drive's own root, not a folder you make - proton-drive filesystem list -j /my-files # the entry named : its uid is the destination UID - task session-seal -- .run/pd # once the bucket and the vault exist - ``` - -Install the CLI at the version that `toolchain.lock.toml` pins. The format of the session -file changes with the version, and the Linux binary in the image must read the files that -the laptop wrote. Put `.run/pd` in the repository, because the container gets only the -repository. git ignores `.run/`, and `task clean` deletes it. - -Obey these three rules: - -- Do not let two mirrors use one session. The CLI changes the refresh token at each call. - Then Proton rejects the token that the other mirror has, and a new login is necessary. -- Do not run `task sync`, `task plan` or `task empty-trash` from a laptop while an Actions - run can be in progress. The three commands use the one session. If the laptop and the run - use it at the same time, a new login can be necessary. -- Do not keep a history of the session in the bucket. A previous copy contains a token that - the CLI changed, and the CLI cannot use that copy. - ### A pipeline of a mirror A mirror that uses a program for its work writes `pipeline` and `plan-pipeline` in its Taskfile. Their steps are the `clock`, `report` and `ping` of the toolbox, and the steps of the program. The mirror puts these two verbs in `excludes:` in the engine entry of -`includes:`. dropbox has this structure: - -- It includes the proton engine, which supplies `session`, `session-seal`, `empty-trash`, - `pull` and `push`. -- Each step is one `python -m migrator `. -- The Taskfile sets the sequence of the steps. -- The Python makes each decision, but the `due` of the toolbox finds if a run does a - reconcile. -- The `proton` image supplies the interpreter, boto3 and the CLI. - -A mirror of this structure fills `report-mirror` with its report. It changes no other part -of the toolbox. +`includes:`. A mirror of this structure fills `report-mirror` with its report. It changes +no other part of the toolbox. ## Secrets @@ -857,20 +727,20 @@ with its name. A repository contains only `op://` references. ```mermaid flowchart LR - item["A vault item, one per mirror
sections per service: r2, healthcheck, proton, age, ..."] --> ref["op.env
NAME=op://vault-uuid/item/section/field"] + item["A vault item, one per mirror
sections per service: r2, healthcheck"] --> ref["op.env
NAME=op://vault-uuid/item/section/field"] ref --> oprun["op run --env-file=op.env, on the host
values exported into its child's environment only"] sa["A service account that reads the vault
its token: the secret OP_SERVICE_ACCOUNT_TOKEN"] --> oprun oprun --> run["run: -e NAME for every name in op.env and PASS
values never on a command line, never in a log"] - run --> tool["the tool inside the image reads its environment
aws, boto3, curl, git"] + run --> tool["the tool inside the image reads its environment
aws, curl"] ``` ### The vault There is one vault, with one item for each mirror. Each item has the name of its -repository. An item has one section for each system that the mirror uses (`r2`, -`healthcheck`, `proton`, `age`, `github`, `dropbox`). The README of the mirror gives the -names of the fields. The name and the UUID of the vault are not secrets: without the token -of the service account, they give access to no item. +repository. An item has one section for each system that the mirror uses (`r2` and +`healthcheck`). The README of the mirror gives the names of the fields. The name and the +UUID of the vault are not secrets: without the token of the service account, they give +access to no item. The references give the UUID of the vault: `op:////
/`. Thus, if the vault gets a new name, the runs continue to operate. Also, a reference cannot contain a @@ -911,15 +781,10 @@ the variable, and it also puts the file into the container at that same path. | Name | The tool that reads it | Its source | |---|---|---| -| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | The AWS CLI and boto3 | The `r2` section | -| `AWS_ENDPOINT_URL` | The AWS CLI, in the `rsync` image | The `endpoint` field of the `r2` section | -| `AWS_ENDPOINT_URL_S3` | boto3, in the `proton` image. The AWS CLI also reads it | The same field. The proton mirrors use this name | -| `AWS_REGION` | The two tools | The value `auto`, in the image. R2 has one region. If `op run` removes the value `auto` from the output, the output is incorrect | -| `MIRROR_R2_BUCKET` | `s3`, in the proton engine | The `bucket` field of the `r2` section | -| `MIRROR_AGE_IDENTITY` | `age`, in the proton engine | The `age` section | -| `MIRROR_PROTON_DESTINATION`, `_UID` | `destination`, in the proton engine | The `proton` section | +| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | The AWS CLI | The `r2` section | +| `AWS_ENDPOINT_URL` | The AWS CLI | The `endpoint` field of the `r2` section | +| `AWS_REGION` | The AWS CLI | The value `auto`, in the image. R2 has one region. If `op run` removes the value `auto` from the output, the output is incorrect | | `HEALTHCHECK_URL` | `ping`, `ping-fail` | The `healthcheck` section. It is optional | -| `MIRROR_*` names that one mirror uses | That mirror | A section for that mirror: `github`, `dropbox` | ### Without a vault @@ -935,39 +800,35 @@ operates, but it puts the values in a second location. ## Storage -Each mirror has one S3-compatible bucket. The contents of the bucket are different for each -engine. +Each mirror has one S3-compatible bucket. The bucket contains these keys: -| Engine | Key | Contents | -|---|---|---| -| rsync | Each path of upstream, at the root | The mirror. A public domain serves the bucket | -| rsync | `.state/applied.txt.xz` | The state: the keys that the bucket contains, with the length and the mtime of upstream | -| rsync, with `INDEX` | `.state/indexed.txt.xz`, `/`, `` | The stamp and the page state that the directory pages showed last, and the pages with their two keys. With `INDEX_FILES`, `/` can be a copy of an index file | -| rsync, a key of the mirror | `index.html` | The landing page of tlnet. `reconcile` does not delete it, because it is in `OWN` | -| proton | `.state/session.tar.age` | The CLI session, encrypted. This is the only key | -| A mirror with its pipeline | `.state/state.sqlite.xz.age`, `.state/history/-